From 06a45fa8b48404f1b2b7e2422e79a0c3bf70f008 Mon Sep 17 00:00:00 2001 From: LIU ZHE YOU Date: Wed, 23 Sep 2026 06:54:13 +0000 Subject: [PATCH 1/3] Java SDK: Resolve a task's arguments from the Dag's own wiring A Dag authored in Java has no Python call site, so the supervisor sends no argument bindings for it and a task's data parameters had nothing to resolve against. Where the Dag wired inputs for a task, those stand in; a stub-backed task keeps reading its bindings, including when the call site bound none. The authoring surface that records those inputs lands in the next commit, so nothing declares them yet. --- .../apache/airflow/sdk/BuilderProcessor.kt | 2 +- .../org/apache/airflow/sdk/BuilderTest.kt | 4 +- .../main/kotlin/org/apache/airflow/sdk/Arg.kt | 11 +- .../kotlin/org/apache/airflow/sdk/Context.kt | 8 +- .../kotlin/org/apache/airflow/sdk/DagDef.kt | 1 + .../org/apache/airflow/sdk/InputTask.kt | 2 +- .../org/apache/airflow/sdk/execution/Task.kt | 7 +- .../apache/airflow/sdk/internal/ArgValues.kt | 133 +++++++++ .../apache/airflow/sdk/internal/TaskArgs.kt | 42 ++- .../org/apache/airflow/sdk/ArgTestSupport.kt | 15 + .../org/apache/airflow/sdk/ArgValuesTest.kt | 2 +- .../org/apache/airflow/sdk/InputTaskTest.kt | 90 ++++++ .../apache/airflow/sdk/execution/TaskTest.kt | 24 ++ .../airflow/sdk/internal/ArgValuesTest.kt | 266 ++++++++++++++++++ 14 files changed, 590 insertions(+), 17 deletions(-) create mode 100644 java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt diff --git a/java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt b/java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt index 049aba0d0601a..21b4165728e3b 100644 --- a/java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt +++ b/java-sdk/processor/src/main/kotlin/org/apache/airflow/sdk/BuilderProcessor.kt @@ -343,7 +343,7 @@ class BuilderProcessor : AbstractProcessor() { val paramType = TypeName.get(param.type) if (param.isTaskInput) { executeSpec.addStatement( - $$"$T $L = $T.bindInput(client, $T.class)", + $$"$T $L = $T.bindInput(context, client, $T.class)", paramType, param.local, ARG_VALUES_TYPE, diff --git a/java-sdk/processor/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt b/java-sdk/processor/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt index 6fd52e01b75a4..6decad1a90605 100644 --- a/java-sdk/processor/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt +++ b/java-sdk/processor/src/test/kotlin/org/apache/airflow/sdk/BuilderTest.kt @@ -427,7 +427,7 @@ class BuilderTest { public static final class Named implements Task { @Override public void execute(Context context, Client client) throws Exception { - TestExample.ScoreInput context_ = ArgValues.bindInput(client, TestExample.ScoreInput.class); + TestExample.ScoreInput context_ = ArgValues.bindInput(context, client, TestExample.ScoreInput.class); new TestExample().named(context_); } } @@ -814,7 +814,7 @@ class BuilderTest { public static final class Score implements Task { @Override public void execute(Context context, Client client) throws Exception { - TestExample.ScoreInput input = ArgValues.bindInput(client, TestExample.ScoreInput.class); + TestExample.ScoreInput input = ArgValues.bindInput(context, client, TestExample.ScoreInput.class); client.setXCom(new TestExample().score(client, input)); } } diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Arg.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Arg.kt index 8bedb4f4cb83f..556ed5a7d4a2c 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Arg.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Arg.kt @@ -20,13 +20,20 @@ package org.apache.airflow.sdk /** - * A value a task can be given, of which [TaskRef] — the output of an upstream - * task — is the only form so far. + * A value a task can be given: the output of an upstream task, carried by the + * [TaskRef] that task's registration returned, or an inline constant. + * + * A constant has to be wrapped because a bare `Double` cannot implement this + * type: boxed types only, no primitives. * * @param T Type of the value. */ sealed class Arg +internal class LiteralArg( + internal val value: T?, +) : Arg() + /** * The output of a registered task, and the task's place in the flow. * diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt index ece4d69b4f74b..75145e4141406 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt @@ -123,9 +123,14 @@ data class TaskInstance( data class Context( @JvmField val dagRun: DagRun, @JvmField val ti: TaskInstance, + /** Registration of the executing task; resolves wired data inputs. */ + internal val taskDef: TaskDef? = null, ) { internal companion object { - fun from(request: StartupDetails) = + fun from( + request: StartupDetails, + taskDef: TaskDef? = null, + ): Context = Context( dagRun = with(request.tiContext.dagRun) { @@ -141,6 +146,7 @@ data class Context( ) }, ti = with(request.ti) { TaskInstance(dagId, runId, taskId, mapIndex, tryNumber) }, + taskDef = taskDef, ) } } diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/DagDef.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/DagDef.kt index 6ba3ff7f165a2..c579ddd517be7 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/DagDef.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/DagDef.kt @@ -170,6 +170,7 @@ class TaskDef( } internal val configValues = linkedMapOf() + internal val inputs = mutableListOf>() internal val upstreams = linkedSetOf() internal var owner: DagDef? = null diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/InputTask.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/InputTask.kt index 52276f36f4ebf..c8dc446e40856 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/InputTask.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/InputTask.kt @@ -57,7 +57,7 @@ interface InputTask : Task { override fun execute( context: Context, client: Client, - ) = execute(context, client, ArgValues.bindInput(client, inputType())) + ) = execute(context, client, ArgValues.bindInput(context, client, inputType())) /** * Executes this task. diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/execution/Task.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/execution/Task.kt index 1c189061ae503..6490e221ee9ae 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/execution/Task.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/execution/Task.kt @@ -74,9 +74,10 @@ internal object TaskRunner { request: StartupDetails, client: Client, ): Any { - val definition = - bundle.taskDef(request.ti.dagId, request.ti.taskId)?.definition + val taskDef = + bundle.taskDef(request.ti.dagId, request.ti.taskId) ?: return TaskResult.of(TaskState.State.REMOVED) + val definition = taskDef.definition val instance = try { definition.getDeclaredConstructor().newInstance() @@ -103,7 +104,7 @@ internal object TaskRunner { return TaskResult.failure(request.tiContext.shouldRetry) } return try { - instance.execute(Context.from(request), client) + instance.execute(Context.from(request, taskDef), client) TaskResult.success() } catch (e: CancellationException) { throw e // Let coroutine cancellation propagate so the task coroutine unwinds. diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt index 80e42a259a2a8..7bd8d65f463cf 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt @@ -23,9 +23,17 @@ package org.apache.airflow.sdk.internal import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.json.JsonMapper +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import org.apache.airflow.sdk.Arg import org.apache.airflow.sdk.Client +import org.apache.airflow.sdk.Context +import org.apache.airflow.sdk.LiteralArg import org.apache.airflow.sdk.MissingXComException import org.apache.airflow.sdk.TaskInput +import org.apache.airflow.sdk.TaskRef import org.apache.airflow.sdk.execution.ArgBinding import org.apache.airflow.sdk.execution.Logger import java.lang.reflect.Field @@ -42,6 +50,15 @@ import java.lang.reflect.Type * graph the scheduler ordered the run by. Flat data parameters resolve the * binding at their position (through [TaskArgs]); [TaskInput] fields resolve * bindings by name. + * + * A natively authored Dag has no stub call site, so the supervisor sends no + * bindings for it and the inputs the Dag itself wired stand in. When the + * supervisor sends bindings they are used for every parameter; the Dag's own + * inputs are read only when it sends none. + * + * A count that does not match is fatal for flat parameters and a warning for a + * [TaskInput]: a position has no name to fall back on, while a field does, so + * the task still runs on what it can bind. */ object ArgValues { private val mapper: ObjectMapper = JsonMapper.builder().build().findAndRegisterModules() @@ -68,9 +85,18 @@ object ArgValues { */ @JvmStatic fun bindInput( + context: Context, client: Client, type: Class, ): I { + // Runtime bindings carry argument names to match fields against. A wired + // input carries none, so it decodes into the whole input at once -- which + // is well defined because a TaskInput is a task's only data parameter. + wiredInputs(context, client)?.let { wired -> + warnWiredArity(client, type, wired.size) + return type.cast(decode(resolveWired(wired[0], client), type)) + ?: throw missingInput(wired[0], type.simpleName) + } val input = newInput(type) val arguments = ArgIndex(client.argBindings) val unfilled = mutableListOf() @@ -147,6 +173,113 @@ object ArgValues { type: Type, ): Any? = decode(client.resolveBinding(binding), type) + /** + * Reports a Dag that wired more or fewer inputs than a [TaskInput] can take. + * A [TaskInput] is a task's only data parameter, so exactly one input feeds + * it; a mismatch warns and the first input is used, leaving the task to run + * on what it can bind rather than failing the run outright. + */ + private fun warnWiredArity( + client: Client, + type: Class<*>, + wired: Int, + ) { + if (wired == 1) return + logger.warning( + "Dag wired a different number of argument(s) than the task handler declares", + mapOf( + "task_id" to client.details.ti.taskId, + "input" to type.simpleName, + "declared" to 1, + "wired" to wired, + ), + ) + } + + /** + * Resolves every wired input at once: each upstream is read once however + * many parameters it feeds, and several upstreams are read concurrently. + * The supervisor protocol matches responses to requests by id, so a task + * wired to several upstreams waits roughly one round trip rather than one + * per parameter. + */ + internal fun resolveWiredAll( + inputs: List>, + client: Client, + ): List { + val upstreams = inputs.filterIsInstance>().map { it.def.id }.distinct() + val fetched = + when (upstreams.size) { + 0 -> emptyMap() + 1 -> mapOf(upstreams[0] to client.getXCom(taskId = upstreams[0])) + else -> + runBlocking { + upstreams.map { taskId -> async(Dispatchers.IO) { taskId to client.getXCom(taskId = taskId) } }.awaitAll() + }.toMap() + } + return inputs.map { input -> + when (input) { + is TaskRef<*> -> fetched[input.def.id] + is LiteralArg<*> -> input.value + } + } + } + + /** Decodes an already-resolved wired value into [type], passing null through. */ + internal fun decodeWired( + value: Any?, + type: Type, + ): Any? = decode(value, type) + + /** + * The inputs the Dag wired for this task, or null when the run's arguments + * come from the stub call site. A task with no wired inputs reads the + * bindings, so a stub call that bound nothing keeps its own diagnostics. + */ + internal fun wiredInputs( + context: Context, + client: Client, + ): List>? = if (client.argBindings.isEmpty()) context.taskDef?.inputs?.takeIf { it.isNotEmpty() } else null + + /** + * The failure for a wired argument that resolved to nothing where a value is + * required, naming [target] — the position of the parameter it feeds. + */ + internal fun missingWired( + input: Arg<*>, + target: String, + ): MissingXComException = + when (input) { + is TaskRef<*> -> MissingXComException(input.def.id, target) + is LiteralArg<*> -> + MissingXComException( + "Task parameter '$target' is wired to a null literal, but has a primitive type that cannot " + + "be null; declare a boxed type (e.g. Integer instead of int) to receive null.", + ) + } + + /** The failure for a wired input that resolved to nothing for a [TaskInput]. */ + private fun missingInput( + input: Arg<*>, + target: String, + ): MissingXComException = + when (input) { + is TaskRef<*> -> + MissingXComException( + "Input '$target' requires an XCom from task '${input.def.id}', but none was pushed.", + ) + is LiteralArg<*> -> MissingXComException("Input '$target' is wired to a null literal, so there is nothing to bind.") + } + + private fun resolveWired( + input: Arg<*>, + client: Client, + ): Any? = + when (input) { + is TaskRef<*> -> client.getXCom(taskId = input.def.id) + is LiteralArg<*> -> input.value + } + /** * Builds the failure for a binding that resolved to nothing where a value is * required, naming [target] — the stub argument, or the [TaskInput] field diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/TaskArgs.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/TaskArgs.kt index 31c840662c386..14c6404758b06 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/TaskArgs.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/TaskArgs.kt @@ -19,10 +19,12 @@ package org.apache.airflow.sdk.internal +import org.apache.airflow.sdk.Arg import org.apache.airflow.sdk.Client import org.apache.airflow.sdk.Context import org.apache.airflow.sdk.MissingXComException import org.apache.airflow.sdk.execution.ArgBinding +import java.lang.reflect.Type /** * @suppress @@ -46,6 +48,8 @@ class TaskArgs private constructor( private val context: Context, private val client: Client, private val arguments: List, + private val wired: List>?, + private val wiredValues: List, ) { companion object { /** @@ -71,6 +75,16 @@ class TaskArgs private constructor( client: Client, declared: Int, ): TaskArgs { + ArgValues.wiredInputs(context, client)?.let { wired -> + // Positional binding is strict in both directions: a parameter has no + // name to fall back on, so a count that does not match cannot be + // resolved and the run fails here rather than mid-task. + check(wired.size == declared) { + "Task '${context.ti.taskId}' declares $declared data parameter(s) " + + "but the Dag wired ${wired.size} argument(s)" + } + return TaskArgs(context, client, emptyList(), wired, ArgValues.resolveWiredAll(wired, client)) + } val bound = client.argBindings val arguments = if (bound.size == declared) bound else bound.filterNot { it.fromDefault } check(arguments.size == declared) { @@ -83,7 +97,7 @@ class TaskArgs private constructor( "Task '${context.ti.taskId}' declares $declared data parameter(s) " + "but the stub call bound ${bound.size} argument(s)$defaults" } - return TaskArgs(context, client, arguments) + return TaskArgs(context, client, arguments, null, emptyList()) } } @@ -96,7 +110,7 @@ class TaskArgs private constructor( fun get( position: Int, type: Class, - ): T? = type.cast(ArgValues.valueAt(client, arguments[position], type)) + ): T? = type.cast(valueAt(position, type)) /** * Resolves the argument bound at [position] into the generic [type], passing @@ -108,7 +122,7 @@ class TaskArgs private constructor( fun get( position: Int, type: TypeRef, - ): T? = ArgValues.valueAt(client, arguments[position], type.type) as T? + ): T? = valueAt(position, type.type) as T? /** * Resolves the argument bound at [position] into [type], which must not be @@ -136,7 +150,23 @@ class TaskArgs private constructor( type: TypeRef, ): T = get(position, type) ?: throw missingAt(position) - // The stub signature's own parameter name is the clearest label for a failure - // here: it is what the Dag author has to change. - private fun missingAt(at: Int) = ArgValues.missing(arguments[at], context.ti.taskId) + private fun valueAt( + position: Int, + type: Type, + ): Any? = + if (wired != null) { + ArgValues.decodeWired(wiredValues[position], type) + } else { + ArgValues.valueAt(client, arguments[position], type) + } + + // A bound argument is labelled with the stub signature's own parameter name, + // which is what the Dag author has to change. A wired one has no such name, + // so it is labelled by the position it feeds. + private fun missingAt(at: Int): MissingXComException = + if (wired != null) { + ArgValues.missingWired(wired[at], "#$at") + } else { + ArgValues.missing(arguments[at], context.ti.taskId) + } } diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt index 1fff219370b98..b94f193370a74 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt @@ -94,3 +94,18 @@ internal fun taskContext(): Context = dagRun = DagRun("d", "r", null, null, null, null, null, emptyMap()), ti = TaskInstance("d", "r", "t", null, 1), ) + +internal class NoopTask : Task { + override fun execute( + context: Context, + client: Client, + ) = Unit +} + +/** A context whose task was wired by its Dag with the given inputs. */ +internal fun contextWiredWith(inputs: List>): Context { + val def = TaskDef("t", NoopTask::class.java) + DagDef("d").addTask(def) + def.inputs += inputs + return taskContext().copy(taskDef = def) +} diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgValuesTest.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgValuesTest.kt index 16eaa161fbdd0..b6b5347ba4ec4 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgValuesTest.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgValuesTest.kt @@ -92,7 +92,7 @@ private fun bind( xcoms: Map = emptyMap(), ): I { val (client, _) = clientWith(bindings, xcoms) - return ArgValues.bindInput(client, type) + return ArgValues.bindInput(taskContext(), client, type) } private fun literal( diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt index b33abcfba0df1..6fa7f9f68d457 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt @@ -21,8 +21,12 @@ package org.apache.airflow.sdk +import org.apache.airflow.sdk.execution.Level +import org.apache.airflow.sdk.execution.LogSender import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test @@ -114,6 +118,92 @@ internal class InputTaskTest { assertEquals(0.5, input.threshold) } + @Test + @DisplayName("Should decode a TaskInput wholesale from its wired input when no bindings arrive") + fun shouldDecodeTaskInputFromWiredInput() { + // A native Dag has no stub call site, so there are no argument names to + // match fields against: the input the Dag wired to this task decodes into + // the whole TaskInput at once. + val context = contextWiredWith(listOf(LiteralArg(mapOf("region" to "emea", "threshold" to 0.5)))) + val (client, _) = clientWith(null) + val task = Summarize() + + task.execute(context, client) + + val input = requireNotNull(task.received) + assertEquals("emea", input.region) + assertEquals(0.5, input.threshold) + } + + @Test + @DisplayName("Should fail when the input wired to a TaskInput resolves to nothing") + fun shouldRejectNullWiredTaskInput() { + val context = contextWiredWith(listOf(LiteralArg>(null))) + val (client, _) = clientWith(null) + + val error = + assertThrows(MissingXComException::class.java) { Summarize().execute(context, client) } + + assertEquals( + "Input 'SummaryInput' is wired to a null literal, so there is nothing to bind.", + error.message, + ) + } + + @Test + @DisplayName("Should warn and bind the first input when the Dag wired more than a TaskInput takes") + fun shouldWarnWhenMoreInputsWiredThanTaskInputTakes() { + LogSender.messages.clear() + val context = + contextWiredWith( + listOf( + LiteralArg(mapOf("region" to "emea", "threshold" to 0.5)), + LiteralArg(mapOf("region" to "apac", "threshold" to 0.1)), + ), + ) + val (client, _) = clientWith(null) + val task = Summarize() + + task.execute(context, client) + + assertEquals("emea", requireNotNull(task.received).region) + val message = LogSender.messages.single { it.level == Level.WARNING } + assertEquals("Dag wired a different number of argument(s) than the task handler declares", message.event) + assertEquals(1, message.arguments["declared"]) + assertEquals(2, message.arguments["wired"]) + assertEquals("SummaryInput", message.arguments["input"]) + } + + @Test + @DisplayName("Should stay quiet when the Dag wired exactly one input for a TaskInput") + fun shouldNotWarnWhenOneInputWiredForTaskInput() { + LogSender.messages.clear() + val context = contextWiredWith(listOf(LiteralArg(mapOf("region" to "emea", "threshold" to 0.5)))) + val (client, _) = clientWith(null) + + Summarize().execute(context, client) + + assertTrue(LogSender.messages.none { it.level == Level.WARNING }) { + "unexpected warnings: ${LogSender.messages.map { it.event }}" + } + } + + @Test + @DisplayName("Should still match a TaskInput by name when the stub call bound no arguments") + fun shouldBindTaskInputWhenStubBoundNothing() { + // No bindings and no wired inputs: the name-matching path still owns this, + // so the fields take their defaults rather than the whole-input decode a + // wired TaskInput gets. + val (client, _) = clientWith(null) + val task = Summarize() + + task.execute(taskContext(), client) + + val input = requireNotNull(task.received) + assertNull(input.region) + assertEquals(0.0, input.threshold) + } + @Test @DisplayName("Should resolve the input type a superclass declared") fun shouldResolveInheritedInputType() { diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/execution/TaskTest.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/execution/TaskTest.kt index 5d16eda1ccb17..548a099bf81a0 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/execution/TaskTest.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/execution/TaskTest.kt @@ -169,6 +169,19 @@ class TaskTest { } } + @Test + @DisplayName("Should thread the task definition into the execution context") + fun shouldThreadTaskDefIntoContext() { + val result = + runTask( + bundleWith("asserting", TaskDefAssertingTask::class.java), + startupDetails(taskId = "asserting"), + noOpClient(), + ) + + Assertions.assertInstanceOf(SucceedTask::class.java, result) + } + private fun bundleWith( taskId: String, taskClass: Class, @@ -298,4 +311,15 @@ class TaskTest { client: Client, ): Unit = throw IllegalStateException("should not be reachable") } + + class TaskDefAssertingTask : Task { + override fun execute( + context: Context, + client: Client, + ) { + check(context.taskDef?.id == context.ti.taskId) { + "expected the runner to thread the task definition into the context" + } + } + } } diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt new file mode 100644 index 0000000000000..92e3b1db8bb43 --- /dev/null +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@file:Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") + +package org.apache.airflow.sdk.internal + +import org.apache.airflow.sdk.Arg +import org.apache.airflow.sdk.Client +import org.apache.airflow.sdk.Context +import org.apache.airflow.sdk.DagDef +import org.apache.airflow.sdk.DagRun +import org.apache.airflow.sdk.LiteralArg +import org.apache.airflow.sdk.MissingXComException +import org.apache.airflow.sdk.Task +import org.apache.airflow.sdk.TaskDef +import org.apache.airflow.sdk.TaskInstance +import org.apache.airflow.sdk.TaskRef +import org.apache.airflow.sdk.execution.comm.ConnectionResult +import org.apache.airflow.sdk.execution.comm.StartupDetails +import org.apache.airflow.sdk.execution.comm.VariableResult +import org.apache.airflow.sdk.execution.comm.XComResult +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.apache.airflow.sdk.execution.Client as Transport +import org.apache.airflow.sdk.execution.comm.TaskInstance as CommTaskInstance + +private class NoopArgTask : Task { + override fun execute( + context: Context, + client: Client, + ) = Unit +} + +/** Resolution of the inputs a Dag declared in Java, without runtime bindings. */ +internal class ArgValuesTest { + /** Upstream task ids read through the transport, in arrival order. */ + private val pulls = java.util.concurrent.CopyOnWriteArrayList() + + private fun clientWith(xcomsByTask: Map): Client = + Client( + StartupDetails().also { + it.ti = + CommTaskInstance().also { ti -> + ti.taskId = "consumer" + ti.dagId = "d" + ti.runId = "r" + ti.tryNumber = 1 + } + }, + object : Transport { + override fun getConnection(id: String): ConnectionResult = throw NotImplementedError() + + override fun getVariable(key: String): VariableResult = throw NotImplementedError() + + override fun setVariable( + key: String, + value: String, + description: String?, + ): Unit = throw NotImplementedError() + + override fun deleteVariable(key: String): Unit = throw NotImplementedError() + + override fun getXCom( + key: String, + dagId: String, + taskId: String, + runId: String, + mapIndex: Int?, + includePriorDates: Boolean, + ): XComResult { + pulls += taskId + arrival?.let { + it.countDown() + check(it.await(5, java.util.concurrent.TimeUnit.SECONDS)) { + "wired upstreams were read one at a time" + } + } + return XComResult().also { it.value = xcomsByTask[taskId] } + } + + override fun setXCom( + key: String, + value: Any, + dagId: String, + taskId: String, + runId: String, + mapIndex: Int, + ): Unit = throw NotImplementedError() + }, + ) + + private var arrival: java.util.concurrent.CountDownLatch? = null + + private fun contextFor(inputs: List>): Context { + val dag = DagDef("d") + inputs + .filterIsInstance>() + .map { it.def } + .distinct() + .forEach { dag.addTask(it) } + val def = TaskDef("consumer", NoopArgTask::class.java) + dag.addTask(def) + def.inputs += inputs + return contextWithoutTaskDef().copy(taskDef = def) + } + + private fun contextWithoutTaskDef(): Context = + Context( + dagRun = DagRun("d", "r", null, null, null, null, null, emptyMap()), + ti = TaskInstance("d", "r", "consumer", null, 1), + ) + + private fun handleFor(taskId: String): TaskRef = TaskRef(TaskDef(taskId, NoopArgTask::class.java)) + + @Test + @DisplayName("Should resolve a handle input from the upstream task's XCom") + fun shouldResolveHandleInputFromXCom() { + val context = contextFor(listOf(handleFor("producer"))) + + val args = TaskArgs.of(context, clientWith(mapOf("producer" to 42L)), 1) + + assertEquals(42L, args.require(0, java.lang.Long::class.java)) + } + + @Test + @DisplayName("Should resolve a literal input without touching the client") + fun shouldResolveLiteralInput() { + val context = contextFor(listOf(LiteralArg(7))) + + val args = TaskArgs.of(context, clientWith(emptyMap()), 1) + + assertEquals(7L, args.require(0, java.lang.Long::class.java)) + } + + @Test + @DisplayName("Should throw MissingXComException when a required upstream pushed no value") + fun shouldThrowForMissingRequiredValue() { + val context = contextFor(listOf(handleFor("producer"))) + val client = clientWith(mapOf("producer" to null)) + + val args = TaskArgs.of(context, client, 1) + + assertThrows(MissingXComException::class.java) { args.require(0, Integer::class.java) } + } + + @Test + @DisplayName("Should throw MissingXComException for a required null literal") + fun shouldThrowForRequiredNullLiteral() { + val context = contextFor(listOf(LiteralArg(null))) + val client = clientWith(emptyMap()) + + val args = TaskArgs.of(context, client, 1) + + val error = assertThrows(MissingXComException::class.java) { args.require(0, Integer::class.java) } + + assertEquals( + "Task parameter '#0' is wired to a null literal, but has a primitive type that cannot " + + "be null; declare a boxed type (e.g. Integer instead of int) to receive null.", + error.message, + ) + } + + @Test + @DisplayName("Should pass null through for optional inputs") + fun shouldPassNullThroughForOptionalInputs() { + val context = contextFor(listOf(handleFor("producer"))) + + val args = TaskArgs.of(context, clientWith(mapOf("producer" to null)), 1) + + assertNull(args.get(0, Integer::class.java)) + } + + @Test + @DisplayName("Should fail when the Dag wired fewer inputs than the task declares") + fun shouldFailOnUnwiredPosition() { + val context = contextFor(listOf(handleFor("producer"))) + + val error = + assertThrows(IllegalStateException::class.java) { + TaskArgs.of(context, clientWith(emptyMap()), 2) + } + + assertEquals( + "Task 'consumer' declares 2 data parameter(s) but the Dag wired 1 argument(s)", + error.message, + ) + } + + @Test + @DisplayName("Should read an upstream once when several parameters are wired to the same handle") + fun shouldReadEachUpstreamOnce() { + val producer = handleFor("producer") + val context = contextFor(listOf(producer, producer)) + + val args = TaskArgs.of(context, clientWith(mapOf("producer" to 42L)), 2) + + assertEquals(42L, args.require(0, java.lang.Long::class.java)) + assertEquals(42L, args.require(1, java.lang.Long::class.java)) + assertEquals(listOf("producer"), pulls) + } + + @Test + @DisplayName("Should read wired upstreams concurrently rather than one at a time") + fun shouldReadWiredUpstreamsConcurrently() { + val context = contextFor(listOf(handleFor("left"), handleFor("right"))) + // Each read blocks until both have arrived, so sequential resolution + // cannot satisfy it and the check inside the transport fails. + arrival = java.util.concurrent.CountDownLatch(2) + + val args = TaskArgs.of(context, clientWith(mapOf("left" to 1L, "right" to 2L)), 2) + + assertEquals(1L, args.require(0, java.lang.Long::class.java)) + assertEquals(2L, args.require(1, java.lang.Long::class.java)) + assertEquals(setOf("left", "right"), pulls.toSet()) + } + + @Test + @DisplayName("Should fail when the Dag wired more inputs than the task declares") + fun shouldFailOnSurplusWiredInput() { + val context = contextFor(listOf(handleFor("left"), handleFor("right"))) + + val error = + assertThrows(IllegalStateException::class.java) { + TaskArgs.of(context, clientWith(emptyMap()), 1) + } + + assertEquals( + "Task 'consumer' declares 1 data parameter(s) but the Dag wired 2 argument(s)", + error.message, + ) + } + + @Test + @DisplayName("Should report the stub call when nothing was wired and nothing was bound") + fun shouldFailWithoutTaskDef() { + val error = + assertThrows(IllegalStateException::class.java) { + TaskArgs.of(contextWithoutTaskDef(), clientWith(emptyMap()), 1) + } + + assertEquals( + "Task 'consumer' declares 1 data parameter(s) but the stub call bound 0 argument(s)", + error.message, + ) + } +} From 04855b4d68e84136cc3eb5cf8f8090470116ac35 Mon Sep 17 00:00:00 2001 From: ZHE YOU LIU Date: Thu, 1 Oct 2026 06:55:44 +0000 Subject: [PATCH 2/3] Keep taskDef out of Context's constructor and reuse resolveWiredAll --- .../src/main/kotlin/org/apache/airflow/sdk/Context.kt | 8 ++++---- .../org/apache/airflow/sdk/internal/ArgValues.kt | 11 +---------- .../kotlin/org/apache/airflow/sdk/ArgTestSupport.kt | 2 +- .../org/apache/airflow/sdk/internal/ArgValuesTest.kt | 2 +- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt index 75145e4141406..69badbd49f65c 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/Context.kt @@ -123,9 +123,10 @@ data class TaskInstance( data class Context( @JvmField val dagRun: DagRun, @JvmField val ti: TaskInstance, - /** Registration of the executing task; resolves wired data inputs. */ - internal val taskDef: TaskDef? = null, ) { + /** Registration of the executing task; resolves wired data inputs. */ + internal var taskDef: TaskDef? = null + internal companion object { fun from( request: StartupDetails, @@ -146,7 +147,6 @@ data class Context( ) }, ti = with(request.ti) { TaskInstance(dagId, runId, taskId, mapIndex, tryNumber) }, - taskDef = taskDef, - ) + ).also { it.taskDef = taskDef } } } diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt index 7bd8d65f463cf..62d9a2ecb7809 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt @@ -94,7 +94,7 @@ object ArgValues { // is well defined because a TaskInput is a task's only data parameter. wiredInputs(context, client)?.let { wired -> warnWiredArity(client, type, wired.size) - return type.cast(decode(resolveWired(wired[0], client), type)) + return type.cast(decode(resolveWiredAll(wired.take(1), client).single(), type)) ?: throw missingInput(wired[0], type.simpleName) } val input = newInput(type) @@ -271,15 +271,6 @@ object ArgValues { is LiteralArg<*> -> MissingXComException("Input '$target' is wired to a null literal, so there is nothing to bind.") } - private fun resolveWired( - input: Arg<*>, - client: Client, - ): Any? = - when (input) { - is TaskRef<*> -> client.getXCom(taskId = input.def.id) - is LiteralArg<*> -> input.value - } - /** * Builds the failure for a binding that resolved to nothing where a value is * required, naming [target] — the stub argument, or the [TaskInput] field diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt index b94f193370a74..1cc3ff3dbaef7 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/ArgTestSupport.kt @@ -107,5 +107,5 @@ internal fun contextWiredWith(inputs: List>): Context { val def = TaskDef("t", NoopTask::class.java) DagDef("d").addTask(def) def.inputs += inputs - return taskContext().copy(taskDef = def) + return taskContext().also { it.taskDef = def } } diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt index 92e3b1db8bb43..768ad066f5c2a 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/internal/ArgValuesTest.kt @@ -121,7 +121,7 @@ internal class ArgValuesTest { val def = TaskDef("consumer", NoopArgTask::class.java) dag.addTask(def) def.inputs += inputs - return contextWithoutTaskDef().copy(taskDef = def) + return contextWithoutTaskDef().also { it.taskDef = def } } private fun contextWithoutTaskDef(): Context = From b1a2e15ebdb4f88c1efb9a18056ddfdfb16347dc Mon Sep 17 00:00:00 2001 From: ZHE YOU LIU Date: Fri, 2 Oct 2026 09:51:13 +0000 Subject: [PATCH 3/3] Use ADR-0007 wording for the wired-arity warning --- .../org/apache/airflow/sdk/internal/ArgValues.kt | 12 ++++++------ .../kotlin/org/apache/airflow/sdk/InputTaskTest.kt | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt index 62d9a2ecb7809..559e975645be5 100644 --- a/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt +++ b/java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/ArgValues.kt @@ -174,19 +174,19 @@ object ArgValues { ): Any? = decode(client.resolveBinding(binding), type) /** - * Reports a Dag that wired more or fewer inputs than a [TaskInput] can take. - * A [TaskInput] is a task's only data parameter, so exactly one input feeds - * it; a mismatch warns and the first input is used, leaving the task to run - * on what it can bind rather than failing the run outright. + * Reports a Dag that wired more inputs than a [TaskInput] can take. A + * [TaskInput] is a task's only data parameter, so exactly one input feeds + * it; the extras are ignored and the first input is used, leaving the task + * to run on what it can bind rather than failing the run outright. */ private fun warnWiredArity( client: Client, type: Class<*>, wired: Int, ) { - if (wired == 1) return + if (wired <= 1) return logger.warning( - "Dag wired a different number of argument(s) than the task handler declares", + "Dag's call passed argument(s) the task handler does not declare", mapOf( "task_id" to client.details.ti.taskId, "input" to type.simpleName, diff --git a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt index 6fa7f9f68d457..ca557b9261158 100644 --- a/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt +++ b/java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/InputTaskTest.kt @@ -168,7 +168,7 @@ internal class InputTaskTest { assertEquals("emea", requireNotNull(task.received).region) val message = LogSender.messages.single { it.level == Level.WARNING } - assertEquals("Dag wired a different number of argument(s) than the task handler declares", message.event) + assertEquals("Dag's call passed argument(s) the task handler does not declare", message.event) assertEquals(1, message.arguments["declared"]) assertEquals(2, message.arguments["wired"]) assertEquals("SummaryInput", message.arguments["input"])