Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ class BuilderProcessor : AbstractProcessor() {
val paramType = TypeName.get(param.type)
if (param.isTaskInput) {
executeSpec.addStatement(
$$"$T $L = $T.bindInput(client, $T.class)",
$$"$T $L = $T.bindInput(context, client, $T.class)",
paramType,
param.local,
ARG_VALUES_TYPE,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ class BuilderTest {
public static final class Named implements Task {
@Override
public void execute(Context context, Client client) throws Exception {
TestExample.ScoreInput context_ = ArgValues.bindInput(client, TestExample.ScoreInput.class);
TestExample.ScoreInput context_ = ArgValues.bindInput(context, client, TestExample.ScoreInput.class);
new TestExample().named(context_);
}
}
Expand Down Expand Up @@ -814,7 +814,7 @@ class BuilderTest {
public static final class Score implements Task {
@Override
public void execute(Context context, Client client) throws Exception {
TestExample.ScoreInput input = ArgValues.bindInput(client, TestExample.ScoreInput.class);
TestExample.ScoreInput input = ArgValues.bindInput(context, client, TestExample.ScoreInput.class);
client.setXCom(new TestExample().score(client, input));
}
}
Expand Down
11 changes: 9 additions & 2 deletions java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Arg.kt
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,20 @@
package org.apache.airflow.sdk

/**
* A value a task can be given, of which [TaskRef] — the output of an upstream
* task — is the only form so far.
* A value a task can be given: the output of an upstream task, carried by the
* [TaskRef] that task's registration returned, or an inline constant.
*
* A constant has to be wrapped because a bare `Double` cannot implement this
* type: boxed types only, no primitives.
*
* @param T Type of the value.
*/
sealed class Arg<T>

internal class LiteralArg<T>(
internal val value: T?,
) : Arg<T>()

/**
* The output of a registered task, and the task's place in the flow.
*
Expand Down
10 changes: 8 additions & 2 deletions java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,14 @@ data class Context(
@JvmField val dagRun: DagRun,
@JvmField val ti: TaskInstance,
) {
/** Registration of the executing task; resolves wired data inputs. */
internal var taskDef: TaskDef? = null

internal companion object {
fun from(request: StartupDetails) =
fun from(
request: StartupDetails,
taskDef: TaskDef? = null,
): Context =
Context(
dagRun =
with(request.tiContext.dagRun) {
Expand All @@ -141,6 +147,6 @@ data class Context(
)
},
ti = with(request.ti) { TaskInstance(dagId, runId, taskId, mapIndex, tryNumber) },
)
).also { it.taskDef = taskDef }
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ class TaskDef(
}

internal val configValues = linkedMapOf<String, Any>()
internal val inputs = mutableListOf<Arg<*>>()
internal val upstreams = linkedSetOf<TaskDef>()
internal var owner: DagDef? = null

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ interface InputTask<I : TaskInput> : Task {
override fun execute(
context: Context,
client: Client,
) = execute(context, client, ArgValues.bindInput(client, inputType()))
) = execute(context, client, ArgValues.bindInput(context, client, inputType()))

/**
* Executes this task.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,10 @@ internal object TaskRunner {
request: StartupDetails,
client: Client,
): Any {
val definition =
bundle.taskDef(request.ti.dagId, request.ti.taskId)?.definition
val taskDef =
bundle.taskDef(request.ti.dagId, request.ti.taskId)
?: return TaskResult.of(TaskState.State.REMOVED)
val definition = taskDef.definition
val instance =
try {
definition.getDeclaredConstructor().newInstance()
Expand All @@ -103,7 +104,7 @@ internal object TaskRunner {
return TaskResult.failure(request.tiContext.shouldRetry)
}
return try {
instance.execute(Context.from(request), client)
instance.execute(Context.from(request, taskDef), client)
TaskResult.success()
} catch (e: CancellationException) {
throw e // Let coroutine cancellation propagate so the task coroutine unwinds.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,17 @@ package org.apache.airflow.sdk.internal

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.json.JsonMapper
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.runBlocking
import org.apache.airflow.sdk.Arg
import org.apache.airflow.sdk.Client
import org.apache.airflow.sdk.Context
import org.apache.airflow.sdk.LiteralArg
import org.apache.airflow.sdk.MissingXComException
import org.apache.airflow.sdk.TaskInput
import org.apache.airflow.sdk.TaskRef
import org.apache.airflow.sdk.execution.ArgBinding
import org.apache.airflow.sdk.execution.Logger
import java.lang.reflect.Field
Expand All @@ -42,6 +50,15 @@ import java.lang.reflect.Type
* graph the scheduler ordered the run by. Flat data parameters resolve the
* binding at their position (through [TaskArgs]); [TaskInput] fields resolve
* bindings by name.
*
* A natively authored Dag has no stub call site, so the supervisor sends no
* bindings for it and the inputs the Dag itself wired stand in. When the
* supervisor sends bindings they are used for every parameter; the Dag's own
* inputs are read only when it sends none.
*
* A count that does not match is fatal for flat parameters and a warning for a
* [TaskInput]: a position has no name to fall back on, while a field does, so
* the task still runs on what it can bind.
*/
object ArgValues {
private val mapper: ObjectMapper = JsonMapper.builder().build().findAndRegisterModules()
Expand All @@ -68,9 +85,18 @@ object ArgValues {
*/
@JvmStatic
fun <I : TaskInput> bindInput(
context: Context,
client: Client,
type: Class<I>,
): I {
// Runtime bindings carry argument names to match fields against. A wired
// input carries none, so it decodes into the whole input at once -- which
// is well defined because a TaskInput is a task's only data parameter.
wiredInputs(context, client)?.let { wired ->
warnWiredArity(client, type, wired.size)
return type.cast(decode(resolveWiredAll(wired.take(1), client).single(), type))
?: throw missingInput(wired[0], type.simpleName)
}
val input = newInput(type)
val arguments = ArgIndex(client.argBindings)
val unfilled = mutableListOf<String>()
Expand Down Expand Up @@ -147,6 +173,104 @@ object ArgValues {
type: Type,
): Any? = decode(client.resolveBinding(binding), type)

/**
* Reports a Dag that wired more inputs than a [TaskInput] can take. A
* [TaskInput] is a task's only data parameter, so exactly one input feeds
* it; the extras are ignored and the first input is used, leaving the task
* to run on what it can bind rather than failing the run outright.
*/
private fun warnWiredArity(
client: Client,
type: Class<*>,
wired: Int,
) {
if (wired <= 1) return
logger.warning(
"Dag's call passed argument(s) the task handler does not declare",
mapOf(
"task_id" to client.details.ti.taskId,
"input" to type.simpleName,
"declared" to 1,
"wired" to wired,
),
)
}

/**
* Resolves every wired input at once: each upstream is read once however
* many parameters it feeds, and several upstreams are read concurrently.
* The supervisor protocol matches responses to requests by id, so a task
* wired to several upstreams waits roughly one round trip rather than one
* per parameter.
*/
internal fun resolveWiredAll(
inputs: List<Arg<*>>,
client: Client,
): List<Any?> {
val upstreams = inputs.filterIsInstance<TaskRef<*>>().map { it.def.id }.distinct()
val fetched =
when (upstreams.size) {
0 -> emptyMap()
1 -> mapOf(upstreams[0] to client.getXCom(taskId = upstreams[0]))
else ->
runBlocking {
upstreams.map { taskId -> async(Dispatchers.IO) { taskId to client.getXCom(taskId = taskId) } }.awaitAll()
}.toMap()
}
return inputs.map { input ->
when (input) {
is TaskRef<*> -> fetched[input.def.id]
is LiteralArg<*> -> input.value
}
}
}

/** Decodes an already-resolved wired value into [type], passing null through. */
internal fun decodeWired(
value: Any?,
type: Type,
): Any? = decode(value, type)

/**
* The inputs the Dag wired for this task, or null when the run's arguments
* come from the stub call site. A task with no wired inputs reads the
* bindings, so a stub call that bound nothing keeps its own diagnostics.
*/
internal fun wiredInputs(
context: Context,
client: Client,
): List<Arg<*>>? = if (client.argBindings.isEmpty()) context.taskDef?.inputs?.takeIf { it.isNotEmpty() } else null

/**
* The failure for a wired argument that resolved to nothing where a value is
* required, naming [target] — the position of the parameter it feeds.
*/
internal fun missingWired(
input: Arg<*>,
target: String,
): MissingXComException =
when (input) {
is TaskRef<*> -> MissingXComException(input.def.id, target)
is LiteralArg<*> ->
MissingXComException(
"Task parameter '$target' is wired to a null literal, but has a primitive type that cannot " +
"be null; declare a boxed type (e.g. Integer instead of int) to receive null.",
)
}

/** The failure for a wired input that resolved to nothing for a [TaskInput]. */
private fun missingInput(
input: Arg<*>,
target: String,
): MissingXComException =
when (input) {
is TaskRef<*> ->
MissingXComException(
"Input '$target' requires an XCom from task '${input.def.id}', but none was pushed.",
)
is LiteralArg<*> -> MissingXComException("Input '$target' is wired to a null literal, so there is nothing to bind.")
}

/**
* Builds the failure for a binding that resolved to nothing where a value is
* required, naming [target] — the stub argument, or the [TaskInput] field
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@

package org.apache.airflow.sdk.internal

import org.apache.airflow.sdk.Arg
import org.apache.airflow.sdk.Client
import org.apache.airflow.sdk.Context
import org.apache.airflow.sdk.MissingXComException
import org.apache.airflow.sdk.execution.ArgBinding
import java.lang.reflect.Type

/**
* @suppress
Expand All @@ -46,6 +48,8 @@ class TaskArgs private constructor(
private val context: Context,
private val client: Client,
private val arguments: List<ArgBinding>,
private val wired: List<Arg<*>>?,
private val wiredValues: List<Any?>,
) {
companion object {
/**
Expand All @@ -71,6 +75,16 @@ class TaskArgs private constructor(
client: Client,
declared: Int,
): TaskArgs {
ArgValues.wiredInputs(context, client)?.let { wired ->
// Positional binding is strict in both directions: a parameter has no
// name to fall back on, so a count that does not match cannot be
// resolved and the run fails here rather than mid-task.
check(wired.size == declared) {
"Task '${context.ti.taskId}' declares $declared data parameter(s) " +
"but the Dag wired ${wired.size} argument(s)"
}
return TaskArgs(context, client, emptyList(), wired, ArgValues.resolveWiredAll(wired, client))
}
val bound = client.argBindings
val arguments = if (bound.size == declared) bound else bound.filterNot { it.fromDefault }
check(arguments.size == declared) {
Expand All @@ -83,7 +97,7 @@ class TaskArgs private constructor(
"Task '${context.ti.taskId}' declares $declared data parameter(s) " +
"but the stub call bound ${bound.size} argument(s)$defaults"
}
return TaskArgs(context, client, arguments)
return TaskArgs(context, client, arguments, null, emptyList())
}
}

Expand All @@ -96,7 +110,7 @@ class TaskArgs private constructor(
fun <T : Any> get(
position: Int,
type: Class<T>,
): T? = type.cast(ArgValues.valueAt(client, arguments[position], type))
): T? = type.cast(valueAt(position, type))

/**
* Resolves the argument bound at [position] into the generic [type], passing
Expand All @@ -108,7 +122,7 @@ class TaskArgs private constructor(
fun <T : Any> get(
position: Int,
type: TypeRef<T>,
): T? = ArgValues.valueAt(client, arguments[position], type.type) as T?
): T? = valueAt(position, type.type) as T?

/**
* Resolves the argument bound at [position] into [type], which must not be
Expand Down Expand Up @@ -136,7 +150,23 @@ class TaskArgs private constructor(
type: TypeRef<T>,
): T = get(position, type) ?: throw missingAt(position)

// The stub signature's own parameter name is the clearest label for a failure
// here: it is what the Dag author has to change.
private fun missingAt(at: Int) = ArgValues.missing(arguments[at], context.ti.taskId)
private fun valueAt(
position: Int,
type: Type,
): Any? =
if (wired != null) {
ArgValues.decodeWired(wiredValues[position], type)
} else {
ArgValues.valueAt(client, arguments[position], type)
}

// A bound argument is labelled with the stub signature's own parameter name,
// which is what the Dag author has to change. A wired one has no such name,
// so it is labelled by the position it feeds.
private fun missingAt(at: Int): MissingXComException =
if (wired != null) {
ArgValues.missingWired(wired[at], "#$at")
} else {
ArgValues.missing(arguments[at], context.ti.taskId)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,18 @@ internal fun taskContext(): Context =
dagRun = DagRun("d", "r", null, null, null, null, null, emptyMap()),
ti = TaskInstance("d", "r", "t", null, 1),
)

internal class NoopTask : Task {
override fun execute(
context: Context,
client: Client,
) = Unit
}

/** A context whose task was wired by its Dag with the given inputs. */
internal fun contextWiredWith(inputs: List<Arg<*>>): Context {
val def = TaskDef("t", NoopTask::class.java)
DagDef("d").addTask(def)
def.inputs += inputs
return taskContext().also { it.taskDef = def }
}
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ private fun <I : TaskInput> bind(
xcoms: Map<String, Any?> = emptyMap(),
): I {
val (client, _) = clientWith(bindings, xcoms)
return ArgValues.bindInput(client, type)
return ArgValues.bindInput(taskContext(), client, type)
}

private fun literal(
Expand Down
Loading
Loading