Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions airflow-core/newsfragments/74830.significant.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Deadline Alert callback kwargs are decoded where the callback runs

The keyword arguments of Deadline Alert callbacks are no longer decoded by the scheduler. They
are kept in their serialized form and decoded where the callback runs: by the triggerer for an
``AsyncCallback``, and by the callback runner on the worker for a ``SyncCallback``.

Workers that run ``SyncCallback`` Deadline Alerts need a Task SDK that includes this change, so
upgrade ``apache-airflow`` and ``apache-airflow-task-sdk`` on workers together. A worker with
an upgraded ``apache-airflow`` and an older Task SDK fails such callbacks with an error instead
of running them. Callbacks created before the upgrade run as before.
8 changes: 8 additions & 0 deletions airflow-core/src/airflow/executors/base_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -912,8 +912,15 @@ def run_workload(
sentry_integration=getattr(workload, "sentry_integration", ""),
)
if isinstance(workload, ExecuteCallback):
from airflow.executors.workloads.callback import CALLBACK_KWARGS_ENCODING_KEY
from airflow.sdk.execution_time.callback_supervisor import supervise_callback

# Only passed when set, so callbacks stored without it run on any Task SDK, while
# encoded kwargs fail on a Task SDK that cannot decode them instead of reaching
# the callback encoded.
encoding_kwargs = {}
if kwargs_encoding := workload.callback.data.get(CALLBACK_KWARGS_ENCODING_KEY):
encoding_kwargs["callback_kwargs_encoding"] = kwargs_encoding
return supervise_callback(
id=workload.callback.id,
callback_path=workload.callback.data.get("path", ""),
Expand All @@ -923,6 +930,7 @@ def run_workload(
bundle_info=workload.bundle_info,
token=workload.token,
server=server,
**encoding_kwargs,
)
if isinstance(workload, TestConnection):
from airflow.sdk.execution_time.connection_test_supervisor import supervise_connection_test
Expand Down
6 changes: 6 additions & 0 deletions airflow-core/src/airflow/executors/workloads/callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ class CallbackFetchMethod(str, Enum):
IMPORT_PATH = "import_path"


# Key in ``CallbackDTO.data`` naming how ``data["kwargs"]`` is encoded. Without it the kwargs are
# passed to the callback as they are, as for callbacks stored before the key was introduced.
CALLBACK_KWARGS_ENCODING_KEY = "kwargs_encoding"
SERDE_KWARGS_ENCODING = "serde"


class CallbackDTO(BaseModel):
"""Schema for Callback with minimal required fields needed for Executors and Task SDK."""

Expand Down
6 changes: 4 additions & 2 deletions airflow-core/src/airflow/jobs/triggerer_job_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1503,8 +1503,10 @@ async def create_triggers(self):
else:
trigger_name = f"ID {trigger_id}"
trigger_instance = trigger_class(**deserialised_kwargs)
except TypeError as err:
self.log.error("Trigger failed to inflate", error=err)
except Exception as err:
# Kwargs are decoded here rather than when queued, so one trigger with undecodable
# kwargs (e.g. a class not in allowed_deserialization_classes) must not stop the runner.
self.log.error("Trigger failed to inflate", error=err, trigger_id=trigger_id)
self.failed_triggers.append((trigger_id, err))
continue
trigger_instance.trigger_id = trigger_id
Expand Down
52 changes: 41 additions & 11 deletions airflow-core/src/airflow/models/callback.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,19 @@
from sqlalchemy.orm import Mapped, mapped_column, relationship

# Re-exporting as _accepts_context for backward compatibility
from airflow._shared.module_loading import accepts_context as _accepts_context # noqa: F401
from airflow._shared.module_loading import (
accepts_context as _accepts_context, # noqa: F401
qualname,
)
from airflow._shared.observability.metrics import stats
from airflow._shared.timezones import timezone
from airflow.configuration import conf
from airflow.executors.workloads import BaseWorkload
from airflow.executors.workloads.callback import CallbackFetchMethod
from airflow.executors.workloads.callback import (
CALLBACK_KWARGS_ENCODING_KEY,
SERDE_KWARGS_ENCODING,
CallbackFetchMethod,
)
from airflow.models import Base
from airflow.models.base import StringID
from airflow.models.dagbundle import DagBundleModel
Expand Down Expand Up @@ -67,6 +74,12 @@ def __str__(self) -> str:
TERMINAL_STATES = frozenset((CallbackState.SUCCESS, CallbackState.FAILED))


def _format_kwargs_for_display(kwargs: dict | None) -> Any:
from airflow.serialization.decoders import format_encoded_value_for_display

return format_encoded_value_for_display(kwargs) if kwargs else ""


class CallbackType(str, Enum):
"""
Types of Callbacks.
Expand Down Expand Up @@ -167,10 +180,15 @@ def queue(self, *, session: Session) -> None:
def get_metric_info(self, status: CallbackState, result: Any, team_name: str | None = None) -> dict:
tags = {"result": result, **self.data}
tags.pop("prefix", None)
tags.pop(CALLBACK_KWARGS_ENCODING_KEY, None)

if "kwargs" in tags:
from airflow.serialization.decoders import format_encoded_value_for_display

# Remove the context (if exists) to keep the tags simple
tags["kwargs"] = {k: v for k, v in tags["kwargs"].items() if k != "context"}
tags["kwargs"] = format_encoded_value_for_display(
{k: v for k, v in tags["kwargs"].items() if k != "context"}
)

# Metric backends (statsd, OpenTelemetry) require tag values to be primitives.
# OTel's aggregation key is built via ``frozenset(attributes.items())``, which
Expand Down Expand Up @@ -237,22 +255,25 @@ def __init__(self, callback_def: ImportPathCallbackDefProtocol, **kwargs):
self.data |= callback_def.serialize()

def __repr__(self):
return f"{self.data['path']}({self.data['kwargs'] or ''}) on a triggerer"
return f"{self.data['path']}({_format_kwargs_for_display(self.data['kwargs'])}) on a triggerer"

def queue(self, *, session: Session) -> None:
from airflow.models.trigger import Trigger
from airflow.serialization.encoders import encode_queued_callback_kwargs
from airflow.triggers.callback import CallbackTrigger

team_name: str | None = None
if self.bundle_name and conf.getboolean("core", "multi_team"):
team_name = DagBundleModel.get_team_name(self.bundle_name, session=session)

self.trigger = Trigger.from_object(
CallbackTrigger(
callback_path=self.data["path"],
callback_kwargs=self.data["kwargs"],
queue=self.data.get("queue"),
)
# The kwargs stay encoded: the triggerer decodes them when it creates the CallbackTrigger.
self.trigger = Trigger.from_serialized_kwargs(
classpath=qualname(CallbackTrigger),
serialized_kwargs={
"callback_path": self.data["path"],
"callback_kwargs": encode_queued_callback_kwargs(self.data["kwargs"] or {}),
},
queue=self.data.get("queue"),
)
self.trigger.team_name = team_name
super().queue(session=session)
Expand Down Expand Up @@ -290,12 +311,21 @@ def __init__(
:param fetch_method: Method to fetch the callback at runtime
:param kwargs: Passed to parent Callback.__init__ (see base class for details)
"""
from airflow.serialization.encoders import encode_stored_callback_kwargs

super().__init__(**kwargs)
self.fetch_method = fetch_method
self.data |= callback_def.serialize()
# The kwargs are decoded by the callback runner on the worker, which needs the marker
# to tell them apart from the already-decoded kwargs of callbacks stored before it.
self.data["kwargs"] = encode_stored_callback_kwargs(self.data.get("kwargs") or {})
self.data[CALLBACK_KWARGS_ENCODING_KEY] = SERDE_KWARGS_ENCODING

def __repr__(self):
return f"{self.data['path']}({self.data['kwargs'] or ''}) on {self.data.get('executor', 'default')} executor"
return (
f"{self.data['path']}({_format_kwargs_for_display(self.data['kwargs'])}) "
f"on {self.data.get('executor', 'default')} executor"
)


class DagProcessorCallback(Callback):
Expand Down
22 changes: 20 additions & 2 deletions airflow-core/src/airflow/models/trigger.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,12 +154,16 @@ def kwargs(self, kwargs: dict[str, Any]) -> None:
@staticmethod
def encrypt_kwargs(kwargs: dict[str, Any]) -> str:
"""Encrypt the kwargs of the trigger."""
from airflow.sdk.serde import serialize

return Trigger._encrypt_serialized_kwargs(serialize(stringify_encoding_keys(kwargs)))

@staticmethod
def _encrypt_serialized_kwargs(serialized_kwargs: dict[str, Any]) -> str:
import json

from airflow.models.crypto import get_fernet
from airflow.sdk.serde import serialize

serialized_kwargs = serialize(stringify_encoding_keys(kwargs))
return get_fernet().encrypt(json.dumps(serialized_kwargs).encode("utf-8")).decode("utf-8")

@staticmethod
Expand Down Expand Up @@ -203,6 +207,20 @@ def from_object(cls, trigger: BaseTrigger) -> Trigger:
classpath, kwargs = trigger.serialize()
return cls(classpath=classpath, kwargs=kwargs, queue=trigger.queue)

@classmethod
def from_serialized_kwargs(
cls, classpath: str, serialized_kwargs: dict[str, Any], queue: str | None = None
) -> Trigger:
"""
Create a trigger row from kwargs that are already in their serde-encoded form.

The kwargs are stored exactly as given and decoded by the triggerer when it creates
the trigger, the same as kwargs stored by :meth:`from_object`.
"""
trigger = cls(classpath=classpath, kwargs={}, queue=queue)
trigger.encrypted_kwargs = cls._encrypt_serialized_kwargs(serialized_kwargs)
return trigger

@classmethod
@provide_session
def bulk_fetch(cls, ids: Iterable[int], *, session: Session = NEW_SESSION) -> dict[int, Trigger]:
Expand Down
89 changes: 75 additions & 14 deletions airflow-core/src/airflow/serialization/decoders.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,20 @@
from __future__ import annotations

import datetime
import math
from typing import TYPE_CHECKING, Any, TypeVar

import dateutil.relativedelta

from airflow._shared.module_loading import import_string
from airflow._shared.serialization import (
CLASSNAME,
DATA,
OLD_DATA,
OLD_DICT,
OLD_TYPE,
OLD_TYPE_TO_FULL_QUALNAME,
)
from airflow.serialization.definitions.assets import (
SerializedAsset,
SerializedAssetAlias,
Expand Down Expand Up @@ -250,20 +259,75 @@ def _decode_deadline_interval(raw_interval: Any) -> datetime.timedelta | Seriali
)


def _get_encoded_callback_kwargs(raw_kwargs: Any) -> dict[str, Any]:
"""
Return the encoded callback ``kwargs`` as a mapping, leaving every value encoded.

Only the container is normalised: the legacy ``{"__type": "dict", "__var": ...}`` spelling
is unwrapped so the result is always a plain mapping of argument name to encoded value.
"""
from airflow.sdk.serde import _convert

if not raw_kwargs:
return {}
kwargs = _convert(raw_kwargs) if isinstance(raw_kwargs, dict) else raw_kwargs
if not isinstance(kwargs, dict) or CLASSNAME in kwargs:
raise ValueError("Deadline callback kwargs is not a mapping.")
return kwargs


_DISPLAY_SEQUENCE_CLASSNAMES = frozenset({"builtins.tuple", "builtins.set", "builtins.frozenset"})


def format_encoded_value_for_display(value: Any) -> Any:
"""
Return a readable form of a serde-encoded value, for logs, metric tags and reprs.

Nothing the value names is imported or instantiated: ``timedelta`` and the built-in
collections are shown by value, any other class as ``classname(data)``. Values that are
not encoded are returned unchanged.

:meta private:
"""
if isinstance(value, list):
return [format_encoded_value_for_display(v) for v in value]
if not isinstance(value, dict):
return value
if OLD_TYPE in value and OLD_DATA in value:
if value[OLD_TYPE] == OLD_DICT:
return format_encoded_value_for_display(value[OLD_DATA])
classname = OLD_TYPE_TO_FULL_QUALNAME.get(value[OLD_TYPE], value[OLD_TYPE])
data = value[OLD_DATA]
elif CLASSNAME in value:
classname, data = value[CLASSNAME], value.get(DATA)
else:
return {str(k): format_encoded_value_for_display(v) for k, v in value.items()}

if (
classname == "datetime.timedelta"
and isinstance(data, (int, float))
and not isinstance(data, bool)
and math.isfinite(data)
):
try:
return str(datetime.timedelta(seconds=data))
except (OverflowError, ValueError, TypeError):
pass
if classname in _DISPLAY_SEQUENCE_CLASSNAMES and isinstance(data, list):
return [format_encoded_value_for_display(v) for v in data]
return f"{classname}({format_encoded_value_for_display(data)})"


def _decode_deadline_callback(raw_callback: Any) -> AsyncCallback | SyncCallback:
"""
Build the callback from its encoded form without importing what the payload names.

``kwargs`` is still passed through generic deserialization, and that is a deliberate,
documented limit rather than an oversight. Leaving it encoded would close a real
residual -- a legitimate callback can carry an arbitrary allow-listed class under its
kwargs, which serde constructs while the outer payload looks entirely valid -- but the
kwargs are consumed through two different paths that use two different encodings
(``BaseSerialization`` in the triggerer, serde here), and deferring the decode without
getting both exactly right silently hands user code an encoded dict in place of its
argument. That residual is not specific to deadlines: it is the general property of
deserializing Dag-author data, shared with every other serde call site. Closing it
belongs with that broader work, not smuggled in here.
``kwargs`` stays in its serde-encoded form. The scheduler and the API server only route the
callback and never call it, so they have no use for the decoded values. The kwargs are
decoded where the callback runs: by the triggerer when it loads the ``CallbackTrigger``
kwargs for an ``AsyncCallback``, and by the callback runner on the worker for a
``SyncCallback``. :func:`airflow.serialization.encoders.encode_deadline_alert` relies on
this to re-encode a decoded alert without serializing the kwargs a second time.
"""
from airflow.sdk.definitions.callback import (
DEADLINE_CALLBACK_TYPES,
Expand Down Expand Up @@ -297,10 +361,7 @@ def _decode_deadline_callback(raw_callback: Any) -> AsyncCallback | SyncCallback
if not isinstance(path, str):
raise ValueError("Deadline callback payload has no string path.")

from airflow.sdk.serde import deserialize

raw_kwargs = data.get("kwargs") or {}
fields: dict[str, Any] = {"kwargs": deserialize(raw_kwargs) if raw_kwargs else {}}
fields: dict[str, Any] = {"kwargs": _get_encoded_callback_kwargs(data.get("kwargs"))}
for optional in sorted(permitted_fields - {"path", "kwargs"}):
if optional in data:
value = data[optional]
Expand Down
Loading