Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
296 changes: 280 additions & 16 deletions airflow-core/docs/authoring-and-scheduling/event-scheduling.rst

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions airflow-core/newsfragments/67523.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Shared-stream triggers now support a producer-side ack channel so message-broker upstreams (Kafka, SQS, Pub/Sub, Service Bus) can commit, delete, or ack on the broker handle after all subscribers have accepted an event and the resulting trigger events have been persisted to the metadata database.
11 changes: 11 additions & 0 deletions airflow-core/src/airflow/config_templates/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2919,6 +2919,17 @@ triggerer:
type: integer
example: ~
default: "1024"
shared_stream_ack_timeout:
description: |
Per-event ack timeout in seconds for shared-stream triggers running in ack mode (triggers that
override ``BaseEventTrigger.create_shared_stream_producer``). A subscriber that has not finished
processing an event (moved past it and had its derived trigger events confirmed persisted) within
this window is force-failed. Other subscribers in the same group are unaffected; once they
resolve, the producer advances normally.
version_added: 3.3.0
type: float
example: ~
default: "300.0"
kerberos:
description: ~
options:
Expand Down
74 changes: 56 additions & 18 deletions airflow-core/src/airflow/jobs/triggerer_job_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from datetime import datetime
from socket import socket
from traceback import format_exception
from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, ClassVar, Literal, TextIO, TypedDict
from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, ClassVar, Literal, NamedTuple, TextIO, TypedDict
from uuid import uuid4

import anyio
Expand Down Expand Up @@ -279,6 +279,14 @@ def _execute(self) -> int | None:
log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__)


class TriggerEventEntry(NamedTuple):
"""A fired trigger event queued for dispatch to the Trigger model."""

trigger_id: int
event: DiscrimatedTriggerEvent
persist_seq: int | None # None when the event does not gate a broker advance


# Using this as a simple namespace
class messages:
class StartTriggerer(BaseModel):
Expand All @@ -295,7 +303,7 @@ class TriggerStateChanges(BaseModel):

type: Literal["TriggerStateChanges"] = "TriggerStateChanges"
events: Annotated[
list[tuple[int, DiscrimatedTriggerEvent]] | None,
list[TriggerEventEntry] | None,
# We have to specify a default here, as otherwise Pydantic struggles to deal with the discriminated
# union :shrug:
Field(default=None),
Expand All @@ -309,6 +317,10 @@ class TriggerStateSync(BaseModel):

to_create: list[workloads.RunTrigger]
to_cancel: set[int]
# Seqs of shared-stream trigger events the supervisor has persisted
# since the previous sync; the runner releases the matching broker
# advances on receipt.
events_persisted: list[int] | None = None


class HITLDetailResponseResult(HITLDetailResponse):
Expand Down Expand Up @@ -477,7 +489,11 @@ class TriggerRunnerSupervisor(WatchedSubprocess):
creating_triggers: deque[workloads.RunTrigger] = attrs.field(factory=deque, init=False)

# Outbound queue of events
events: deque[tuple[int, TriggerEvent]] = attrs.field(factory=deque, init=False)
events: deque[TriggerEventEntry] = attrs.field(factory=deque, init=False)

# Seqs of shared-stream trigger events persisted since the last state
# sync; drained into TriggerStateSync.events_persisted.
persisted_event_seqs: deque[int] = attrs.field(factory=deque, init=False)

# Outbound queue of failed triggers
failed_triggers: deque[tuple[int, list[str] | None]] = attrs.field(factory=deque, init=False)
Expand Down Expand Up @@ -546,9 +562,15 @@ def _handle_request(self, msg: ToTriggerSupervisor, log: FilteringBoundLogger, r
# handle leaks for every failed upload.
factory.close()

# Drain the persist confirmations accumulated since the last sync.
events_persisted: list[int] = []
while self.persisted_event_seqs:
events_persisted.append(self.persisted_event_seqs.popleft())

response = messages.TriggerStateSync(
to_create=[],
to_cancel=self.cancelling_triggers,
events_persisted=events_persisted or None,
)

# Pull out of these dequeues in a thread-safe manner
Expand Down Expand Up @@ -679,10 +701,14 @@ def load_triggers(self) -> None:
def handle_events(self):
"""Dispatch outbound events to the Trigger model which pushes them to the relevant task instances."""
while self.events:
# Get the event and its trigger ID
trigger_id, event = self.events.popleft()
entry = self.events.popleft()
# Tell the model to wake up its tasks
self.on_trigger_event(trigger_id=trigger_id, event=event)
self.on_trigger_event(trigger_id=entry.trigger_id, event=entry.event)
# Only reached when on_trigger_event returned, i.e. the event was
# persisted; a raise above leaves the seq unconfirmed so the
# bound shared-stream advance fails out and the broker redelivers.
if entry.persist_seq is not None:
self.persisted_event_seqs.append(entry.persist_seq)
# Emit stat event
stats.incr("triggers.succeeded")

Expand Down Expand Up @@ -1073,7 +1099,7 @@ class TriggerRunner:
to_cancel: deque[int]

# Outbound queue of events
events: deque[tuple[int, TriggerEvent]]
events: deque[TriggerEventEntry]

# Outbound queue of failed triggers
failed_triggers: deque[tuple[int, BaseException | None]]
Expand All @@ -1100,6 +1126,7 @@ def __init__(self):
self._shared_streams = SharedStreamManager(
log=self.log,
max_subscriber_queue=conf.getint("triggerer", "shared_stream_subscriber_queue_size"),
ack_timeout=conf.getfloat("triggerer", "shared_stream_ack_timeout"),
)
self.blocked_main_thread_warning_threshold = conf.getfloat(
"triggerer", "blocked_main_thread_warning_threshold"
Expand Down Expand Up @@ -1362,12 +1389,11 @@ async def cleanup_finished_triggers(self) -> list[int]:

def process_trigger_events(self, finished_ids: list[int]) -> messages.TriggerStateChanges:
# Copy out of our dequeues in threadsafe manner to sync state with parent
events_to_send: list[tuple[int, DiscrimatedTriggerEvent]] = []
events_to_send: list[TriggerEventEntry] = []
failures_to_send: list[tuple[int, list[str] | None]] = []

while self.events:
trigger_id, trigger_event = self.events.popleft()
events_to_send.append((trigger_id, trigger_event))
events_to_send.append(self.events.popleft())

while self.failed_triggers:
trigger_id, exc = self.failed_triggers.popleft()
Expand All @@ -1382,21 +1408,25 @@ def process_trigger_events(self, finished_ids: list[int]) -> messages.TriggerSta

def sanitize_trigger_events(self, msg: messages.TriggerStateChanges) -> messages.TriggerStateChanges:
req_encoder = _new_encoder()
events_to_send: list[tuple[int, DiscrimatedTriggerEvent]] = []
events_to_send: list[TriggerEventEntry] = []

if msg.events:
for trigger_id, trigger_event in msg.events:
for entry in msg.events:
try:
req_encoder.encode(trigger_event)
req_encoder.encode(entry.event)
except Exception as e:
logger.error(
"Trigger %s returned non-serializable result %r. Cancelling trigger.",
trigger_id,
trigger_event,
entry.trigger_id,
entry.event,
)
self.failed_triggers.append((trigger_id, e))
# The dropped event's seq (if any) is never confirmed, so
# the bound shared-stream advance times out as failed and
# the broker redelivers — the safe direction for an event
# that was never persisted.
self.failed_triggers.append((entry.trigger_id, e))
else:
events_to_send.append((trigger_id, trigger_event))
events_to_send.append(entry)

return messages.TriggerStateChanges(
events=events_to_send if events_to_send else None,
Expand All @@ -1417,6 +1447,8 @@ async def sync_state_to_supervisor(self, finished_ids: list[int]) -> None:
if resp:
self.to_create.extend(resp.to_create)
self.to_cancel.extend(resp.to_cancel)
if resp.events_persisted:
self._shared_streams.confirm_persisted(resp.events_persisted)
Comment thread
Lee-W marked this conversation as resolved.
Resolved

async def asend(self, msg: messages.TriggerStateChanges) -> messages.TriggerStateSync | None:
try:
Expand Down Expand Up @@ -1526,7 +1558,13 @@ async def run_trigger(
"Trigger fired event", name=self.triggers[trigger_id]["name"], result=event
)
self.triggers[trigger_id]["events"] += 1
self.events.append((trigger_id, event))
seq: int | None = None
if shared_key is not None:
# Bind and append back to back with no await in
# between, so no other coroutine can observe the
# event bound but not yet queued (or vice versa).
seq = self._shared_streams.bind_pending_event(trigger_id=trigger_id, key=shared_key)
self.events.append(TriggerEventEntry(trigger_id=trigger_id, event=event, persist_seq=seq))
span.set_status(Status(StatusCode.OK))
except asyncio.CancelledError as e:
# A trigger can be cancelled for two reasons:
Expand Down
72 changes: 50 additions & 22 deletions airflow-core/src/airflow/triggers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
from airflow.models.taskinstance import TaskInstance
from airflow.sdk.definitions.context import Context
from airflow.serialization.serialized_objects import SerializedBaseOperator
from airflow.triggers.shared_stream import SharedStreamProducer
Comment thread
Lee-W marked this conversation as resolved.
Resolved

Operator: TypeAlias = MappedOperator | SerializedBaseOperator

Expand Down Expand Up @@ -267,32 +268,30 @@ class BaseEventTrigger(BaseTrigger):
events. Called once per group in the triggerer.
* :meth:`filter_shared_stream` — convert the shared raw stream into this
trigger's own ``TriggerEvent`` instances, applying any per-trigger
filtering or transformation.
filtering or transformation. The stream has the same shape in ack
mode as on the fast path, so the filter needs no mode-specific
branches.

Triggers whose ``shared_stream_key`` returns ``None`` (the default)
keep the existing behavior: each trigger gets its own poll loop via
:meth:`run`.

**Suitable upstreams**

The shared-stream channel is **one-way** today: events flow from the
producer (``open_shared_stream``) to each subscriber's
``filter_shared_stream``, with no path back to tell the producer that a
subscriber accepted, dropped, or finished processing an event. That
restricts the pattern to upstreams whose consumption does **not** depend
on a side effect on a handle that only the producer holds:

* Idempotent / read-only reads (filesystem listings, polling REST APIs).
* Subscriber-side-effect cleanup, where the trigger's per-event action
(``unlink``, local marking, …) operates through APIs the subscriber
already owns, independent of the shared producer handle.

Upstreams **not** in scope include Kafka consumers (regardless of
commit mode), SQS with delete-on-process or visibility extension,
and any source where progress on the producer's handle is tied to
the subscriber's accept / reject decision. These sources need a way
for the subscriber to signal acceptance back to the producer, which
the current shared-stream API does not provide.
**Producer factory (ack mode)**

Override :meth:`create_shared_stream_producer` to enable ack mode. The
manager calls this classmethod once per group; the returned
:class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns the
broker connection, supplies events through its ``open_stream`` method
(instead of :meth:`open_shared_stream`), and is told to advance the
broker — commit, delete, or ack the messages — through per-lane batches
of events whose subscribers have all moved past them and whose derived
trigger events were persisted to the metadata database. Subscribers
receive raw events in both modes; ack mode changes only the bookkeeping
behind the stream, never the filter code.

See :mod:`airflow.triggers.shared_stream` for the full ack-mode design,
including snapshot-at-fan-out semantics, per-event timeout behavior, and
triggerer-restart redeliver notes.
"""

supports_triggerer_queue: bool = False
Expand Down Expand Up @@ -365,13 +364,37 @@ async def open_shared_stream(cls, kwargs: dict[str, Any]) -> AsyncIterator[Any]:
compose via ``super().open_shared_stream(kwargs)`` and reach
``cls`` for class-scoped state or diagnostics.

Required only when :meth:`shared_stream_key` returns non-``None``.
Required only when :meth:`shared_stream_key` returns non-``None``
and :meth:`create_shared_stream_producer` is not overridden. In ack
mode the events come from the producer's ``open_stream`` instead and
this method is not called.
"""
raise NotImplementedError(
f"{cls.__name__} declares a shared_stream_key but does not implement open_shared_stream"
)
yield # pragma: no cover - convince mypy this is an async iterator

@classmethod
def create_shared_stream_producer(cls, kwargs: dict[str, Any]) -> SharedStreamProducer:
"""
Build the broker-side producer for this trigger's shared stream (ack mode).

Overriding this classmethod opts the shared stream into ack mode.
The manager calls it once per shared-stream group; the returned
:class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns
the broker connection for the lifetime of one poll — it supplies
events through ``open_stream``, is told to advance the broker in
per-lane batches of events whose subscribers have all moved past
them and had their derived trigger events confirmed persisted, and
is closed when the poll ends. Do not open the broker connection
here; open it lazily inside the producer's ``open_stream``.

Triggers that do not override this method run the fast path:
subscribers receive raw events from :meth:`open_shared_stream`
exactly as before. The stream shape is the same in both modes.
"""
raise NotImplementedError(f"{cls.__name__} does not implement create_shared_stream_producer")

async def filter_shared_stream(self, shared_stream: AsyncIterator[Any]) -> AsyncIterator[TriggerEvent]:
"""
Transform the shared raw event stream into this trigger's events.
Expand All @@ -382,6 +405,11 @@ async def filter_shared_stream(self, shared_stream: AsyncIterator[Any]) -> Async
``yield`` a :class:`TriggerEvent` for each one that should fire this
trigger.

In ack mode, to have the broker terminally drop a raw event rather
than producing a trigger event from it, call
:func:`~airflow.triggers.shared_stream.reject_shared_stream_event`
while handling that raw event instead of yielding.

Required only when :meth:`shared_stream_key` returns non-``None``.
"""
raise NotImplementedError(
Expand Down
Loading
Loading