diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py b/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py index 81000c67d305a..9bbbaa96a06dc 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/datamodels/taskinstance.py @@ -491,6 +491,15 @@ class TIRunContext(BaseModel): ``None`` for regular tasks and for stub tasks that declare no parameters. """ + execution_timeout: float | None = None + """ + The task's ``execution_timeout`` in seconds, for tasks run by a language SDK. + + Covers ``@task.stub`` tasks and native language-SDK Dag tasks. Their runtime never sends + ``SetExecutionTimeout``, so the supervisor arms its deadline from this value. ``None`` for Python + tasks, which report their own timeout, and for language-SDK tasks without one. + """ + multi_team: bool = False """ Whether the deployment runs in multi-team mode. diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py b/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py index 40da0fcf8d90b..61212fbd56e3a 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/routes/task_instances.py @@ -92,6 +92,7 @@ from airflow.api_fastapi.execution_api.services.task_instances import ( client_supports_arg_bindings, get_arg_bindings, + get_language_sdk_execution_timeout, ) from airflow.api_fastapi.execution_api.versions.v2026_10_30 import IdentifyArchivedTaskStateUpdates from airflow.configuration import conf @@ -406,6 +407,12 @@ def ti_run( }, ) + # Stub and native language-SDK Dag tasks need the timeout from the server, not a parsed Dag. + if ( + execution_timeout := get_language_sdk_execution_timeout(dag_bag, ti, session=session) + ) is not None: + context.execution_timeout = execution_timeout + # Only set if they are non-null if ti.next_method: context.next_method = ti.next_method diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/services/task_instances.py b/airflow-core/src/airflow/api_fastapi/execution_api/services/task_instances.py index ab67aaa9814f0..bae418328ed92 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/services/task_instances.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/services/task_instances.py @@ -24,6 +24,7 @@ from sqlalchemy.orm import Session from airflow.models.dagbag import DBDagBag + from airflow.serialization.serialized_objects import SerializedOperator def client_supports_arg_bindings() -> bool: @@ -44,13 +45,11 @@ def client_supports_arg_bindings() -> bool: return AddArgBindingsToTIRunContext.is_applied -def get_arg_bindings(dag_bag: DBDagBag, ti: Any, *, session: Session) -> list | None: +def get_stub_task(dag_bag: DBDagBag, ti: Any, *, session: Session) -> SerializedOperator | None: """ - Extract the stub task's TaskFlow arg spec from its Dag version. + Return the TI's task from its Dag version if it runs in a language SDK, else ``None``. - Mapped (``.expand()``) stubs never capture a parse-time spec, so they resolve to - ``None`` here and keep the legacy ignored-args behavior; per-map-index delivery - lands in a follow-up. + Both ``@task.stub`` tasks and native language-SDK Dag tasks serialize with ``is_stub=True``. """ if ti.dag_version_id is None: return None @@ -58,4 +57,29 @@ def get_arg_bindings(dag_bag: DBDagBag, ti: Any, *, session: Session) -> list | return None if (task := dag.task_dict.get(ti.task_id)) is None or not task.is_stub: return None + return task + + +def get_arg_bindings(dag_bag: DBDagBag, ti: Any, *, session: Session) -> list | None: + """ + Extract the stub task's TaskFlow arg spec from its Dag version. + + Mapped (``.expand()``) stubs never capture a parse-time spec, so they resolve to + ``None`` here and keep the legacy ignored-args behavior; per-map-index delivery + lands in a follow-up. + """ + if (task := get_stub_task(dag_bag, ti, session=session)) is None: + return None return task.arg_bindings + + +def get_language_sdk_execution_timeout(dag_bag: DBDagBag, ti: Any, *, session: Session) -> float | None: + """ + Return ``execution_timeout`` in seconds for tasks run by a language SDK. + + These tasks need the timeout from the server; a Python task parses the Dag file and reports its own. + Return ``None`` for other tasks and when no timeout is set, including a zero timeout. + """ + if (task := get_stub_task(dag_bag, ti, session=session)) is None or not task.execution_timeout: + return None + return task.execution_timeout.total_seconds() diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py b/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py index 8936306bbf5a6..6df8e850b319e 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/versions/__init__.py @@ -56,6 +56,7 @@ AddArgBindingsToTIRunContext, AddCallbackRunEndpoint, AddDagRunNoteUpdateEndpoint, + AddExecutionTimeoutToTIRunContext, AddFirstTaskRescheduleStartDateField, AddLoopContext, AddMultiTeamToTIRunContext, @@ -77,6 +78,7 @@ AddArgBindingsToTIRunContext, AddCallbackRunEndpoint, AddDagRunNoteUpdateEndpoint, + AddExecutionTimeoutToTIRunContext, AddLoopContext, AddTerminalStateRetryReasonField, AddMultiTeamToTIRunContext, diff --git a/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2026_10_30.py b/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2026_10_30.py index ec4b6ddf12529..7b6b9746f914b 100644 --- a/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2026_10_30.py +++ b/airflow-core/src/airflow/api_fastapi/execution_api/versions/v2026_10_30.py @@ -92,6 +92,21 @@ def remove_arg_bindings_field(response: ResponseInfo) -> None: # type: ignore[m response.body.pop("arg_bindings", None) +class AddExecutionTimeoutToTIRunContext(VersionChange): + """Add ``execution_timeout`` so the supervisor enforces it for stub and native language-SDK Dag tasks.""" + + description = __doc__ + + instructions_to_migrate_to_previous_version = ( + schema(TIRunContext).field("execution_timeout").didnt_exist, + ) + + @convert_response_to_previous_version_for(TIRunContext) # type: ignore[arg-type] + def remove_execution_timeout_field(response: ResponseInfo) -> None: # type: ignore[misc] + """Strip ``execution_timeout`` from the run context for older clients.""" + response.body.pop("execution_timeout", None) + + class AddCallbackRunEndpoint(VersionChange): """Add the callbacks/{callback_id}/run endpoint a worker uses to exchange its single-use callback token.""" diff --git a/airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_task_instances.py b/airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_task_instances.py index b6c59388175ae..7125ea9ec5057 100644 --- a/airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_task_instances.py +++ b/airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_task_instances.py @@ -1049,6 +1049,44 @@ def transform(country: str, extracted: dict, limit: int = 10): ... assert response.status_code == 200 assert "arg_bindings" not in response.json() + def test_ti_run_returns_execution_timeout_for_stub_task_only(self, client, dag_maker): + """A stub task's execution_timeout is returned in seconds; other tasks report their own.""" + with dag_maker("test_stub_execution_timeout_dag", serialized=True): + + @task.stub(execution_timeout=timedelta(seconds=90)) + def with_timeout(): ... + + @task.stub + def without_timeout(): ... + + @task.stub(execution_timeout=timedelta(0)) + def zero_timeout(): ... + + @task(execution_timeout=timedelta(seconds=10)) + def python_task(): ... + + with_timeout() + without_timeout() + zero_timeout() + python_task() + + dr = dag_maker.create_dagrun() + tis = {ti.task_id: ti for ti in dr.get_task_instances()} + for ti in tis.values(): + ti.set_state(State.QUEUED) + dag_maker.session.flush() + + response = client.patch( + f"/execution/task-instances/{tis['with_timeout'].id}/run", json=self.RUN_PAYLOAD + ) + assert response.status_code == 200 + assert response.json()["execution_timeout"] == 90.0 + + for task_id in ("without_timeout", "zero_timeout", "python_task"): + response = client.patch(f"/execution/task-instances/{tis[task_id].id}/run", json=self.RUN_PAYLOAD) + assert response.status_code == 200 + assert "execution_timeout" not in response.json() + @mock.patch( "airflow.api_fastapi.execution_api.routes.task_instances.get_arg_bindings", autospec=True, diff --git a/airflow-core/tests/unit/api_fastapi/execution_api/versions/v2026_10_30/test_task_instances.py b/airflow-core/tests/unit/api_fastapi/execution_api/versions/v2026_10_30/test_task_instances.py index 513d15f6673dc..cb7cf9a02c019 100644 --- a/airflow-core/tests/unit/api_fastapi/execution_api/versions/v2026_10_30/test_task_instances.py +++ b/airflow-core/tests/unit/api_fastapi/execution_api/versions/v2026_10_30/test_task_instances.py @@ -17,6 +17,7 @@ from __future__ import annotations +from datetime import timedelta from uuid import uuid4 import pytest @@ -287,6 +288,39 @@ def test_head_version_includes_arg_bindings(self, client, stub_ti): ] +class TestExecutionTimeoutBackwardCompat: + def setup_method(self): + clear_db_runs() + + def teardown_method(self): + clear_db_runs() + + @pytest.fixture + def stub_ti(self, dag_maker): + with dag_maker("test_execution_timeout_compat_dag", serialized=True): + + @task.stub(execution_timeout=timedelta(seconds=90)) + def sleepy(): ... + + sleepy() + + dr = dag_maker.create_dagrun() + (ti,) = dr.get_task_instances() + ti.set_state(State.QUEUED) + dag_maker.session.flush() + return ti + + def test_old_version_strips_execution_timeout_even_when_set(self, old_ver_client, stub_ti): + response = old_ver_client.patch(f"/execution/task-instances/{stub_ti.id}/run", json=RUN_PATCH_BODY) + assert response.status_code == 200 + assert "execution_timeout" not in response.json() + + def test_head_version_includes_execution_timeout(self, client, stub_ti): + response = client.patch(f"/execution/task-instances/{stub_ti.id}/run", json=RUN_PATCH_BODY) + assert response.status_code == 200 + assert response.json()["execution_timeout"] == 90.0 + + @pytest.fixture def archived_attempt(client, exec_app, monkeypatch, create_task_instance, session): """A running attempt, authenticated with a signed token, that has since been retried.""" diff --git a/go-sdk/pkg/execution/genmodels/models.gen.go b/go-sdk/pkg/execution/genmodels/models.gen.go index 6e53729efe936..e9b1d6fca1968 100644 --- a/go-sdk/pkg/execution/genmodels/models.gen.go +++ b/go-sdk/pkg/execution/genmodels/models.gen.go @@ -1684,6 +1684,9 @@ type TIRunContext struct { // DagRun corresponds to the JSON schema field "dag_run". DagRun DagRun `msgpack:"dag_run"` + // ExecutionTimeout corresponds to the JSON schema field "execution_timeout". + ExecutionTimeout interface{} `msgpack:"execution_timeout,omitempty"` + // FirstTaskRescheduleStartDate corresponds to the JSON schema field // "first_task_reschedule_start_date". FirstTaskRescheduleStartDate interface{} `msgpack:"first_task_reschedule_start_date,omitempty"` @@ -1737,6 +1740,22 @@ type VariableResponse struct { type TaskIds []string +const TaskInstanceStateAwaitingInput TaskInstanceState = "awaiting_input" +const TaskInstanceStateDeferred TaskInstanceState = "deferred" +const TaskInstanceStateSkipped TaskInstanceState = "skipped" +const TaskInstanceStateUpstreamFailed TaskInstanceState = "upstream_failed" +const TaskInstanceStateUpForReschedule TaskInstanceState = "up_for_reschedule" +const TaskInstanceStateUpForRetry TaskInstanceState = "up_for_retry" +const TaskInstanceStateFailed TaskInstanceState = "failed" +const TaskInstanceStateRestarting TaskInstanceState = "restarting" +const TaskInstanceStateSuccess TaskInstanceState = "success" +const TaskInstanceStateRunning TaskInstanceState = "running" +const TaskInstanceStateQueued TaskInstanceState = "queued" +const TaskInstanceStateScheduled TaskInstanceState = "scheduled" +const TaskInstanceStateRemoved TaskInstanceState = "removed" + +type TaskInstanceState string + // Schema for TaskInstance model with minimal required fields needed for Runtime. type TaskInstance struct { // ContextCarrier corresponds to the JSON schema field "context_carrier". @@ -1776,22 +1795,6 @@ type TaskInstance struct { TryNumber int `msgpack:"try_number"` } -const TaskInstanceStateAwaitingInput TaskInstanceState = "awaiting_input" -const TaskInstanceStateDeferred TaskInstanceState = "deferred" -const TaskInstanceStateSkipped TaskInstanceState = "skipped" -const TaskInstanceStateUpstreamFailed TaskInstanceState = "upstream_failed" -const TaskInstanceStateUpForReschedule TaskInstanceState = "up_for_reschedule" -const TaskInstanceStateUpForRetry TaskInstanceState = "up_for_retry" -const TaskInstanceStateFailed TaskInstanceState = "failed" -const TaskInstanceStateRestarting TaskInstanceState = "restarting" -const TaskInstanceStateSuccess TaskInstanceState = "success" -const TaskInstanceStateRunning TaskInstanceState = "running" -const TaskInstanceStateQueued TaskInstanceState = "queued" -const TaskInstanceStateScheduled TaskInstanceState = "scheduled" -const TaskInstanceStateRemoved TaskInstanceState = "removed" - -type TaskInstanceState string - type TaskOutlets []AssetProfile type TaskBreadcrumbsResultBreadcrumbsElem map[string]interface{} diff --git a/go-sdk/schema/supervisor-schema.json b/go-sdk/schema/supervisor-schema.json index 77888c5adfdea..424db73976d9f 100644 --- a/go-sdk/schema/supervisor-schema.json +++ b/go-sdk/schema/supervisor-schema.json @@ -5325,6 +5325,18 @@ "default": null, "title": "Arg Bindings" }, + "execution_timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Timeout" + }, "multi_team": { "default": false, "title": "Multi Team", diff --git a/task-sdk/src/airflow/sdk/api/datamodels/_generated.py b/task-sdk/src/airflow/sdk/api/datamodels/_generated.py index 02d51e15b57e5..1de6f24e5d522 100644 --- a/task-sdk/src/airflow/sdk/api/datamodels/_generated.py +++ b/task-sdk/src/airflow/sdk/api/datamodels/_generated.py @@ -998,4 +998,5 @@ class TIRunContext(BaseModel): should_retry: Annotated[bool | None, Field(title="Should Retry")] = False start_date: Annotated[AwareDatetime | None, Field(title="Start Date")] = None arg_bindings: Annotated[list[TaskArgBinding] | None, Field(title="Arg Bindings")] = None + execution_timeout: Annotated[float | None, Field(title="Execution Timeout")] = None multi_team: Annotated[bool | None, Field(title="Multi Team")] = False diff --git a/task-sdk/src/airflow/sdk/execution_time/schema/schema.json b/task-sdk/src/airflow/sdk/execution_time/schema/schema.json index 77888c5adfdea..424db73976d9f 100644 --- a/task-sdk/src/airflow/sdk/execution_time/schema/schema.json +++ b/task-sdk/src/airflow/sdk/execution_time/schema/schema.json @@ -5325,6 +5325,18 @@ "default": null, "title": "Arg Bindings" }, + "execution_timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Timeout" + }, "multi_team": { "default": false, "title": "Multi Team", diff --git a/task-sdk/src/airflow/sdk/execution_time/schema/versions/__init__.py b/task-sdk/src/airflow/sdk/execution_time/schema/versions/__init__.py index 0f35e9f0e6f2b..44dbc7621345a 100644 --- a/task-sdk/src/airflow/sdk/execution_time/schema/versions/__init__.py +++ b/task-sdk/src/airflow/sdk/execution_time/schema/versions/__init__.py @@ -40,6 +40,7 @@ def get_bundle() -> VersionBundle: from airflow.sdk.execution_time.schema.versions.v2026_10_30 import ( AddArgBindingsToSupervisorTIRunContext, AddDagDefinitionsToDagFileParsingResult, + AddExecutionTimeoutToSupervisorTIRunContext, AddFirstTaskRescheduleStartDateToSupervisorTIRunContext, AddLoopRuntime, AddRegionCoordinatesToSupervisorTaskInstance, @@ -54,6 +55,7 @@ def get_bundle() -> VersionBundle: AddLoopRuntime, AddRetryReasonToTaskState, AddDagDefinitionsToDagFileParsingResult, + AddExecutionTimeoutToSupervisorTIRunContext, AddRegionCoordinatesToSupervisorTaskInstance, AddFirstTaskRescheduleStartDateToSupervisorTIRunContext, ), diff --git a/task-sdk/src/airflow/sdk/execution_time/schema/versions/v2026_10_30.py b/task-sdk/src/airflow/sdk/execution_time/schema/versions/v2026_10_30.py index d1efcf0b8c380..444eba6bbed96 100644 --- a/task-sdk/src/airflow/sdk/execution_time/schema/versions/v2026_10_30.py +++ b/task-sdk/src/airflow/sdk/execution_time/schema/versions/v2026_10_30.py @@ -111,3 +111,20 @@ class AddFirstTaskRescheduleStartDateToSupervisorTIRunContext(VersionChange): instructions_to_migrate_to_previous_version = ( schema(TIRunContext).field("first_task_reschedule_start_date").didnt_exist, ) + + +class AddExecutionTimeoutToSupervisorTIRunContext(VersionChange): + """ + Add the ``execution_timeout`` field carrying a language-SDK task's timeout in seconds. + + Language-SDK tasks are ``@task.stub`` tasks and native language-SDK Dag tasks. + + The supervisor-schema mirror of the execution API's ``AddExecutionTimeoutToTIRunContext``, + named apart so the two migrations are not confused. + """ + + description = __doc__ + + instructions_to_migrate_to_previous_version = ( + schema(TIRunContext).field("execution_timeout").didnt_exist, + ) diff --git a/task-sdk/src/airflow/sdk/execution_time/supervisor.py b/task-sdk/src/airflow/sdk/execution_time/supervisor.py index 92295bd4f8ddb..a7dc3c87c853f 100644 --- a/task-sdk/src/airflow/sdk/execution_time/supervisor.py +++ b/task-sdk/src/airflow/sdk/execution_time/supervisor.py @@ -1720,6 +1720,9 @@ def _on_child_started( ti_context = self.client.task_instances.start(ti.id, self.pid, datetime.now(tz=UTC)) self._should_retry = ti_context.should_retry self._last_successful_heartbeat = time.monotonic() + if ti_context.execution_timeout is not None: + # A lang-SDK runtime never sends ``SetExecutionTimeout``, so arm the deadline from the server value. + self._arm_execution_timeout(ti_context.execution_timeout) except Exception as e: # On any error kill that subprocess! self.kill(signal.SIGKILL) @@ -1972,13 +1975,15 @@ def _handle_execution_timeout_if_needed(self): """ Enforce ``execution_timeout`` from outside the task process. - The task process raises ``AirflowTaskTimeout`` itself when the timeout elapses, which is the only + A Python task process raises ``AirflowTaskTimeout`` itself when the timeout elapses, which is the only place ``on_kill``, the retry policy, callbacks and listeners can run. That needs a process that can still run Python signal handlers; one that has not reported a terminal state within the grace period is stuck (native code, SIGSEGV) and is sent SIGTERM, then SIGKILL. - The deadline is armed by ``SetExecutionTimeout``, which the task process sends right before - ``execute()``. Anything before that (bundle load, Dag parsing) is not covered here. + For Python tasks, ``SetExecutionTimeout`` arms the deadline right before ``execute()``; + bundle loading and Dag parsing are not covered. For language-SDK tasks (stub tasks and native + language-SDK Dag tasks), the server-provided timeout arms the deadline when the runtime + connects. In either case, enforcement starts after the timeout plus ``killed_task_cleanup_time``. """ if self._execution_timeout_due_in() > 0: return @@ -2125,10 +2130,13 @@ def _handle_skip_downstream_tasks( def _handle_set_execution_timeout( self, msg: SetExecutionTimeout, log: FilteringBoundLogger, req_id: int ) -> RequestResult: - self._execution_timeout_seconds = msg.timeout_seconds - self._execution_timeout_enforce_at = time.monotonic() + msg.timeout_seconds + KILLED_TASK_CLEANUP_TIME + self._arm_execution_timeout(msg.timeout_seconds) return None, {} + def _arm_execution_timeout(self, timeout_seconds: float) -> None: + self._execution_timeout_seconds = timeout_seconds + self._execution_timeout_enforce_at = time.monotonic() + timeout_seconds + KILLED_TASK_CLEANUP_TIME + def _handle_set_rendered_fields( self, msg: SetRenderedFields, log: FilteringBoundLogger, req_id: int ) -> RequestResult: diff --git a/task-sdk/tests/task_sdk/execution_time/schema/test_migrator.py b/task-sdk/tests/task_sdk/execution_time/schema/test_migrator.py index e8c4a16aff248..a3ebae6a7689c 100644 --- a/task-sdk/tests/task_sdk/execution_time/schema/test_migrator.py +++ b/task-sdk/tests/task_sdk/execution_time/schema/test_migrator.py @@ -652,3 +652,55 @@ def test_downgrade_strips_field_for_previous_version(self, real_migrator, startu def test_head_version_keeps_field(self, real_migrator, startup_details): out = real_migrator.downgrade(startup_details, "2026-10-30") assert out.ti_context.first_task_reschedule_start_date == self.FIRST_RESCHEDULE + + +class TestRealBundleExecutionTimeout: + """Drive the *real* supervisor bundle through the ``execution_timeout`` migration.""" + + @pytest.fixture + def startup_details(self): + now = datetime.datetime.now(datetime.UTC) + return StartupDetails( + ti=TaskInstance( + id=uuid.uuid4(), + task_id="wait", + dag_id="d", + run_id="r", + try_number=1, + dag_version_id=uuid.uuid4(), + ), + dag_rel_path="d.py", + bundle_info=BundleInfo(name="b", version=None), + start_date=now, + ti_context=TIRunContext( + dag_run=DagRun( + dag_id="d", + run_id="r", + logical_date=now, + data_interval_start=None, + data_interval_end=None, + start_date=now, + end_date=None, + run_type=DagRunType.MANUAL, + state=DagRunState.RUNNING, + run_after=now, + consumed_asset_events=[], + partition_key=None, + ), + max_tries=1, + execution_timeout=90.0, + ), + sentry_integration="", + ) + + @pytest.fixture + def real_migrator(self) -> SchemaVersionMigrator: + return get_schema_version_migrator() + + def test_downgrade_strips_field_for_previous_version(self, real_migrator, startup_details): + out = real_migrator.downgrade(startup_details, "2026-06-16").model_dump() + assert "execution_timeout" not in out["ti_context"] + + def test_head_version_keeps_field(self, real_migrator, startup_details): + out = real_migrator.downgrade(startup_details, "2026-10-30") + assert out.ti_context.execution_timeout == 90.0 diff --git a/task-sdk/tests/task_sdk/execution_time/test_supervisor.py b/task-sdk/tests/task_sdk/execution_time/test_supervisor.py index 433d6d11f3685..0287ebfba4e9c 100644 --- a/task-sdk/tests/task_sdk/execution_time/test_supervisor.py +++ b/task-sdk/tests/task_sdk/execution_time/test_supervisor.py @@ -1664,6 +1664,68 @@ def test_execution_timeout_kills_task_that_does_not_stop(self, mocker): proc._handle_execution_timeout_if_needed() mock_kill.assert_called_once() + def _start_activity_subprocess(self, mocker, server_timeout): + client = mocker.Mock() + client.task_instances.start.return_value = mocker.Mock( + should_retry=False, start_date=None, execution_timeout=server_timeout + ) + proc = ActivitySubprocess( + process_log=mocker.MagicMock(), + id=TI_ID, + pid=12345, + stdin=mocker.Mock(), + process=mocker.Mock(), + client=client, + ) + mocker.patch.object(ActivitySubprocess, "send_msg") + proc._on_child_started( + ti=TaskInstance( + id=TI_ID, + task_id="b", + dag_id="c", + run_id="d", + try_number=1, + dag_version_id=uuid7(), + queue="default", + ), + dag_rel_path="d.py", + bundle_info=FAKE_BUNDLE, + sentry_integration="", + ) + return proc + + def test_execution_timeout_from_server_kills_task_that_does_not_stop(self, mocker): + """A lang-SDK runtime never sends SetExecutionTimeout, so the supervisor arms it from the run context.""" + mocker.patch("airflow.sdk.execution_time.supervisor.KILLED_TASK_CLEANUP_TIME", 5.0) + monotonic = mocker.patch("time.monotonic", autospec=True, return_value=1.0) + mock_kill = mocker.patch("airflow.sdk.execution_time.supervisor.WatchedSubprocess.kill") + proc = self._start_activity_subprocess(mocker, server_timeout=30.0) + + monotonic.return_value += 34 + proc._handle_execution_timeout_if_needed() + mock_kill.assert_not_called() + + monotonic.return_value += 1 + proc._handle_execution_timeout_if_needed() + mock_kill.assert_called_once_with(signal.SIGTERM, force=True, escalation_delay=5.0) + proc.process_log.error.assert_called_once_with( + "Task did not stop before execution_timeout elapsed; terminating process", + timeout_seconds=30.0, + grace_period_seconds=5.0, + ) + proc._handle_execution_timeout_if_needed() + mock_kill.assert_called_once() + + def test_no_execution_timeout_from_server_never_kills_task(self, mocker): + mocker.patch("airflow.sdk.execution_time.supervisor.KILLED_TASK_CLEANUP_TIME", 5.0) + monotonic = mocker.patch("time.monotonic", autospec=True, return_value=1.0) + mock_kill = mocker.patch("airflow.sdk.execution_time.supervisor.WatchedSubprocess.kill") + proc = self._start_activity_subprocess(mocker, server_timeout=None) + + monotonic.return_value += 3600 + proc._handle_execution_timeout_if_needed() + mock_kill.assert_not_called() + @pytest.mark.parametrize( ("terminal_state", "exit_code"), [ diff --git a/ts-sdk/schema/supervisor-schema.json b/ts-sdk/schema/supervisor-schema.json index 77888c5adfdea..424db73976d9f 100644 --- a/ts-sdk/schema/supervisor-schema.json +++ b/ts-sdk/schema/supervisor-schema.json @@ -5325,6 +5325,18 @@ "default": null, "title": "Arg Bindings" }, + "execution_timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Execution Timeout" + }, "multi_team": { "default": false, "title": "Multi Team", diff --git a/ts-sdk/src/generated/supervisor.ts b/ts-sdk/src/generated/supervisor.ts index 7faf8c4877029..9e8c64e0cc7d5 100644 --- a/ts-sdk/src/generated/supervisor.ts +++ b/ts-sdk/src/generated/supervisor.ts @@ -265,6 +265,7 @@ export type TaskId1 = string; export type Name9 = string; export type Kind1 = "literal"; export type FromDefault = boolean; +export type ExecutionTimeout = number | null; export type MultiTeam = boolean; export type Type13 = "TaskCallbackRequest"; export type Filepath2 = string; @@ -1060,6 +1061,7 @@ export interface TIRunContext { should_retry?: ShouldRetry; start_date?: StartDate2; arg_bindings?: ArgBindings; + execution_timeout?: ExecutionTimeout; multi_team?: MultiTeam; } /**