From 98304782994f86dbd7433fefd3031568390863da Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Tue, 26 May 2026 12:03:51 +0800 Subject: [PATCH 01/15] feat(triggers): add producer-side ack channel to shared-stream triggers Extend the SharedStreamManager + BaseEventTrigger shared-stream API from PR #66584 so it can also host upstreams that need a producer-held handle to advance (Kafka manual commit, SQS delete / visibility extend, Pub/Sub ack_id, Service Bus peek-lock). Producer triggers opt in by overriding the new `BaseEventTrigger.create_shared_stream_producer` classmethod to return a `SharedStreamProducer` (open_stream / advance / aclose) that owns the broker connection and yields `(event, broker_payload)` tuples. Subscribers receive an `AckToken` alongside each raw event; once every subscriber in scope at fan-out time has resolved (ack, nack, timeout, unsubscribe, or queue overflow), the manager calls `await producer.advance(broker_payload, outcome)` strictly in fan-out order, so cumulative-offset brokers can commit `offset + 1` directly. The `AdvanceOutcome` counts let producers distinguish clean delivery from nacks and failures. Triggers that don't override the factory keep the existing fast path, so `DirectoryFileDeleteTrigger` is unaffected. Slow subscribers are bounded by a per-event ack timeout (`[triggerer] shared_stream_ack_timeout`, default 300s); a force-failed subscriber is resolved out of all outstanding events via the existing `_PollFailure` path without blocking siblings. Broker advance failures are logged and in-order dispatch continues with the next event. Closes #67179 --- .../event-scheduling.rst | 180 +- airflow-core/newsfragments/67523.feature.rst | 1 + .../src/airflow/config_templates/config.yml | 10 + .../src/airflow/jobs/triggerer_job_runner.py | 1 + airflow-core/src/airflow/triggers/base.py | 69 +- .../src/airflow/triggers/shared_stream.py | 537 +++- .../tests/unit/jobs/test_triggerer_job.py | 90 + .../tests/unit/triggers/test_base_trigger.py | 11 + .../tests/unit/triggers/test_shared_stream.py | 2189 ++++++++++++++++- 9 files changed, 3005 insertions(+), 83 deletions(-) create mode 100644 airflow-core/newsfragments/67523.feature.rst diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index b0118bf0d8be5..821b2a7b0f79b 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -150,24 +150,180 @@ In other words, the savings is at the poll-loop and upstream-I/O layer, not at t Suitable upstreams ^^^^^^^^^^^^^^^^^^ -The shared-stream channel is **one-way** today: events flow from -``open_shared_stream`` out to each subscriber's ``filter_shared_stream``, -and there is no way for a subscriber to tell the producer "I accepted / -dropped / committed this event". That restricts the pattern to upstreams -whose consumption does **not** depend on a side effect on a handle that -only the producer holds. Good fits: +Good fits for the shared-stream pattern: * Idempotent / read-only reads — directory scans, polling REST APIs. * Subscriber-side-effect cleanup, where the trigger's per-event action (``unlink``, local marking, …) goes through APIs the subscriber owns independently of the shared producer handle. +* Message-broker upstreams (Kafka, SQS, Pub/Sub, Azure Service Bus) where + the producer must commit/delete/ack after all subscribers have processed + the message — use the ack channel described below. + +Producer-side ack channel +^^^^^^^^^^^^^^^^^^^^^^^^^ + +For upstreams where the producer must advance (commit, delete, or ack) only +after all subscribers have processed an event, override +:py:meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer` +to return a :py:class:`~airflow.triggers.shared_stream.SharedStreamProducer`. +When this factory is overridden, the manager enters **ack mode**: + +1. The manager calls ``create_shared_stream_producer(kwargs)`` once per + shared-stream group. The returned producer owns the broker connection for + the lifetime of one poll; open the connection lazily inside + ``open_stream``, not in the factory. +2. The producer's ``open_stream`` yields ``(event, broker_payload)`` tuples, + where ``broker_payload`` is whatever the producer needs later (e.g. an + SQS receipt handle, a Kafka offset, a Pub/Sub ack ID). +3. Each subscriber's ``filter_shared_stream`` receives ``(event, token)`` + pairs. The subscriber calls ``await token.ack()`` once it has accepted + the event, or ``await token.nack()`` to opt out. +4. Once every subscriber in the fan-out set has resolved an event (by + ``ack()``, ``nack()``, unsubscribing, timing out, or overflowing its + queue), the manager calls + ``await producer.advance(broker_payload, outcome)`` — commit the offset, + delete the SQS message, etc. The ``outcome`` is an + :py:class:`~airflow.triggers.shared_stream.AdvanceOutcome` carrying + per-event counts of how the subscribers resolved. + +**Ordering guarantee**: advance calls are dispatched strictly in event +order — ``advance`` for event N is awaited only after event N-1's +``advance`` returned (or failed and was logged). When the poll ends, the +manager calls ``await producer.aclose()`` once, best-effort. + +Example — SQS-like producer: -Currently **not** in scope: Kafka consumers (regardless of commit mode), -SQS with delete-on-process or visibility extension, and any source where -progress on the producer's handle is tied to the subscriber's accept / -reject decision. These sources need a way for the subscriber to signal -acceptance back to the producer, which the current shared-stream API does -not provide. +.. code-block:: python + + from collections.abc import AsyncIterator + from typing import Any + + from airflow.triggers.base import BaseEventTrigger, TriggerEvent + from airflow.triggers.shared_stream import AdvanceOutcome, SharedStreamProducer + + + class SqsSharedStreamProducer(SharedStreamProducer): + def __init__(self, queue_url: str): + self.queue_url = queue_url + self.client = None + + async def open_stream(self) -> AsyncIterator[tuple[Any, Any]]: + # Open the connection here, not in the trigger's factory. + self.client = await create_sqs_client() + while True: + messages = await poll_sqs(self.client, self.queue_url) + for msg in messages: + yield msg["Body"], msg["ReceiptHandle"] + + async def advance(self, broker_payload: Any, outcome: AdvanceOutcome) -> None: + # Called once all subscribers have resolved this message. + if outcome.is_clean: + await delete_sqs_message(self.client, self.queue_url, broker_payload) + # Otherwise leave the message for the visibility timeout to redeliver. + + async def aclose(self) -> None: + if self.client is not None: + await self.client.close() + + + class SqsSharedTrigger(BaseEventTrigger): + def __init__(self, *, queue_url: str, region: str | None = None): + super().__init__() + self.queue_url = queue_url + self.region = region + + def serialize(self): + return ( + f"{type(self).__module__}.{type(self).__qualname__}", + {"queue_url": self.queue_url, "region": self.region}, + ) + + def shared_stream_key(self): + return ("sqs", self.queue_url) + + @classmethod + def create_shared_stream_producer(cls, kwargs) -> SqsSharedStreamProducer: + return SqsSharedStreamProducer(kwargs["queue_url"]) + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + if self.region is None or raw.get("region") == self.region: + await token.ack() + yield TriggerEvent(raw) + else: + await token.nack() + + async def run(self): # pragma: no cover + yield TriggerEvent({}) + +Example — Kafka cumulative commit. A Kafka commit acknowledges every offset +up to the committed one, so it is only safe if no later event can be +committed while an earlier one is still pending. The ordering guarantee +above provides exactly that, which makes ``commit(offset + 1)`` correct: + +.. code-block:: python + + class KafkaSharedStreamProducer(SharedStreamProducer): + def __init__(self, topic: str): + self.topic = topic + self.consumer = None + + async def open_stream(self): + self.consumer = await create_kafka_consumer(self.topic) + async for message in self.consumer: + yield message.value, message.offset + + async def advance(self, broker_payload, outcome): + # Advance calls arrive in event order, so this cumulative commit + # can never skip past an event that is still pending. + await self.consumer.commit(broker_payload + 1) + + async def aclose(self): + if self.consumer is not None: + await self.consumer.stop() + +.. warning:: + Always call ``await token.ack()`` or ``await token.nack()`` **before** + ``yield``-ing the ``TriggerEvent``. If you yield first and the caller + abandons the generator (the common single-event case), the token is + never resolved and the producer will not advance until the per-event + timeout expires. + +**Snapshot-at-fan-out**: the set of subscribers that must ack a given event +is frozen at the moment the event is broadcast. A subscriber that joins +after the event was dispatched is not added to that event's pending set. + +**Per-event ack timeout**: if a subscriber has not called ``ack()`` or +``nack()`` within the ack timeout (default 5 minutes, configurable via the +``[triggerer] shared_stream_ack_timeout`` config option), the manager force-fails that +subscriber's trigger. Other subscribers are not affected; once their acks +arrive, the producer advances normally. The ack timeout is a manager-level +safety net and does not replace any native broker session or visibility timeout. +From the subscriber's perspective the force-fail surfaces as an ``AckTimeout`` +(importable from ``airflow.triggers.shared_stream``) raised by the +``shared_stream`` iterator inside ``filter_shared_stream``. Letting it propagate +is fine — the trigger fails through the standard trigger-failure path; catch it +only if the subscriber needs to run cleanup before failing. + +**Triggerer restart**: outstanding acks live in memory only. After a +triggerer restart, the broker redelivers messages that were not yet +acknowledged. Subscribers must therefore be idempotent. + +**nack() semantics**: ``nack()`` removes the subscriber from the pending +ack set and is counted in the ``nacked`` field of the event's +``AdvanceOutcome``. What that means for the broker is the producer's +decision inside ``advance`` — commit anyway, skip the commit, or trigger an +immediate redeliver (e.g. ``ChangeMessageVisibility(0)`` for SQS). + +**``shared_stream_subscriber_queue_size`` in ack mode**: the config bound +still governs unprocessed raw events per subscriber. The manager does +**not** wait for outstanding acks before pulling the next upstream event; +back-pressure is queue-bound — a subscriber whose queue is full is +force-failed. The queue primarily guards against burst delivery before a +subscriber's filter runs. Broker advances, by contrast, are dispatched +strictly in event order: an event whose acks are still pending delays the +advance of every later event, with the ack timeout bounding that wait. Verifying that sharing is active ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/newsfragments/67523.feature.rst b/airflow-core/newsfragments/67523.feature.rst new file mode 100644 index 0000000000000..fc83bf3b75171 --- /dev/null +++ b/airflow-core/newsfragments/67523.feature.rst @@ -0,0 +1 @@ +Shared-stream triggers now support a producer-side ack channel so message-broker upstreams (Kafka, SQS, Pub/Sub, Service Bus) can commit, delete, or ack on the broker handle after all subscribers have accepted an event. diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index 75985f64657dd..c000906b8b90f 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -2919,6 +2919,16 @@ triggerer: type: integer example: ~ default: "1024" + shared_stream_ack_timeout: + description: | + Per-event ack timeout in seconds for shared-stream triggers running in ack mode (triggers that + override ``BaseEventTrigger.create_shared_stream_producer``). A subscriber that has not called + ``token.ack()`` or ``token.nack()`` within this window is force-failed. Other subscribers in the + same group are unaffected; once their acks arrive the producer advances normally. + version_added: 3.3.0 + type: float + example: ~ + default: "300.0" kerberos: description: ~ options: diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py b/airflow-core/src/airflow/jobs/triggerer_job_runner.py index 326ab14d7a4b0..5e9a7c728dd23 100644 --- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py +++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py @@ -1100,6 +1100,7 @@ def __init__(self): self._shared_streams = SharedStreamManager( log=self.log, max_subscriber_queue=conf.getint("triggerer", "shared_stream_subscriber_queue_size"), + ack_timeout=conf.getfloat("triggerer", "shared_stream_ack_timeout"), ) self.blocked_main_thread_warning_threshold = conf.getfloat( "triggerer", "blocked_main_thread_warning_threshold" diff --git a/airflow-core/src/airflow/triggers/base.py b/airflow-core/src/airflow/triggers/base.py index 7cda8cc4272cd..f1117fff5414a 100644 --- a/airflow-core/src/airflow/triggers/base.py +++ b/airflow-core/src/airflow/triggers/base.py @@ -48,6 +48,10 @@ from airflow.sdk.definitions.context import Context from airflow.serialization.serialized_objects import SerializedBaseOperator + # Deferred to type-checking time: shared_stream imports this module at + # runtime, so a runtime import here would be circular. + from airflow.triggers.shared_stream import SharedStreamProducer + Operator: TypeAlias = MappedOperator | SerializedBaseOperator @@ -267,32 +271,30 @@ class BaseEventTrigger(BaseTrigger): events. Called once per group in the triggerer. * :meth:`filter_shared_stream` — convert the shared raw stream into this trigger's own ``TriggerEvent`` instances, applying any per-trigger - filtering or transformation. + filtering or transformation. In ack mode, the stream yields + ``(event, token)`` pairs, where ``token`` is an + :class:`~airflow.triggers.shared_stream.AckToken`; call + ``await token.ack()`` once the event is accepted. Triggers whose ``shared_stream_key`` returns ``None`` (the default) keep the existing behavior: each trigger gets its own poll loop via :meth:`run`. - **Suitable upstreams** - - The shared-stream channel is **one-way** today: events flow from the - producer (``open_shared_stream``) to each subscriber's - ``filter_shared_stream``, with no path back to tell the producer that a - subscriber accepted, dropped, or finished processing an event. That - restricts the pattern to upstreams whose consumption does **not** depend - on a side effect on a handle that only the producer holds: - - * Idempotent / read-only reads (filesystem listings, polling REST APIs). - * Subscriber-side-effect cleanup, where the trigger's per-event action - (``unlink``, local marking, …) operates through APIs the subscriber - already owns, independent of the shared producer handle. - - Upstreams **not** in scope include Kafka consumers (regardless of - commit mode), SQS with delete-on-process or visibility extension, - and any source where progress on the producer's handle is tied to - the subscriber's accept / reject decision. These sources need a way - for the subscriber to signal acceptance back to the producer, which - the current shared-stream API does not provide. + **Producer factory (ack mode)** + + Override :meth:`create_shared_stream_producer` to enable ack mode. The + manager calls this classmethod once per group; the returned + :class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns the + broker connection, supplies events through its ``open_stream`` method + (instead of :meth:`open_shared_stream`), and is told to advance the + broker — commit, delete, or ack the message — once all subscribers that + were online at broadcast time have resolved an event. When this factory + is not overridden, the fast path is taken: no ack tokens are issued and + subscribers receive raw events exactly as before. + + See :mod:`airflow.triggers.shared_stream` for the full ack-mode design, + including snapshot-at-fan-out semantics, per-event timeout behavior, and + triggerer-restart redeliver notes. """ supports_triggerer_queue: bool = False @@ -365,13 +367,36 @@ async def open_shared_stream(cls, kwargs: dict[str, Any]) -> AsyncIterator[Any]: compose via ``super().open_shared_stream(kwargs)`` and reach ``cls`` for class-scoped state or diagnostics. - Required only when :meth:`shared_stream_key` returns non-``None``. + Required only when :meth:`shared_stream_key` returns non-``None`` + and :meth:`create_shared_stream_producer` is not overridden. In ack + mode the events come from the producer's ``open_stream`` instead and + this method is not called. """ raise NotImplementedError( f"{cls.__name__} declares a shared_stream_key but does not implement open_shared_stream" ) yield # pragma: no cover - convince mypy this is an async iterator + @classmethod + def create_shared_stream_producer(cls, kwargs: dict[str, Any]) -> SharedStreamProducer: + """ + Build the broker-side producer for this trigger's shared stream (ack mode). + + Overriding this classmethod opts the shared stream into ack mode. + The manager calls it once per shared-stream group; the returned + :class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns + the broker connection for the lifetime of one poll — it supplies + events through ``open_stream``, is told to advance the broker once + all subscribers have resolved an event, and is closed when the poll + ends. Do not open the broker connection here; open it lazily inside + the producer's ``open_stream``. + + Triggers that do not override this method run the fast path: no ack + tokens are issued and subscribers receive raw events from + :meth:`open_shared_stream` exactly as before. + """ + raise NotImplementedError(f"{cls.__name__} does not implement create_shared_stream_producer") + async def filter_shared_stream(self, shared_stream: AsyncIterator[Any]) -> AsyncIterator[TriggerEvent]: """ Transform the shared raw event stream into this trigger's events. diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index f347a265524a1..f558669c3f751 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -27,28 +27,59 @@ instances. Triggers that opt out (the default) keep their independent ``run()``-based poll loops untouched. -Scope and the missing ack channel ---------------------------------- - -The shared-stream channel is **one-way**: events flow from -``open_shared_stream`` out to each subscriber's ``filter_shared_stream``, -with no path back. Subscribers cannot tell the producer "I accepted this -event; please advance / commit / ack". The pattern is therefore only safe -for upstreams whose consumption does not need a producer-side side effect -tied to a subscriber's accept / reject decision: - -* Idempotent / read-only reads (filesystem listings, polling REST APIs). -* Auto-commit Kafka consumers (``enable.auto.commit=true``). -* Subscriber-side-effect cleanup (``unlink``, local marking, …) where the - per-event action goes through APIs the subscriber owns independently. - -Kafka manual-commit consumers, SQS delete-on-process / visibility -extension, and similar message-broker patterns where progress is per-message -and tied to the subscriber's decision are **not** in scope here today. A -producer-side ack channel to cover them is a follow-up that should be -designed against a concrete Kafka or SQS consumer rather than against an -abstract API. See :class:`~airflow.triggers.base.BaseEventTrigger` for the -matching subclass-facing notes. +Producer-side ack channel +------------------------- + +When a trigger overrides +:meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer`, +the manager switches to **ack mode** for that stream. + +The factory is called once per group and returns a +:class:`SharedStreamProducer` that owns the broker connection for the +lifetime of one poll. The manager drives the producer's ``open_stream``, +which yields ``(raw_event, broker_payload)`` tuples, and hands +``(raw_event, token)`` pairs — where ``token`` is an :class:`AckToken` — to +each subscriber's queue instead of the raw event alone. A subscriber calls +``await token.ack()`` once it has accepted the event, or +``await token.nack()`` to opt out. When every subscriber that was online at +broadcast time has resolved the event (by ``ack()``, ``nack()``, +unsubscribing, or being force-failed), the manager calls +``await producer.advance(broker_payload, outcome)`` exactly once — this is +where the producer commits / deletes / acks on the broker. The +:class:`AdvanceOutcome` carries per-event counts of how the subscribers +resolved. Advance calls are dispatched by a single pump task strictly in +fan-out order: event N's ``advance`` is awaited only after event N-1's +``advance`` returned (or raised and was logged). When the poll ends, the +manager awaits ``producer.aclose()`` once, best-effort. + +**Snapshot-at-fan-out**: the set of subscribers that must ack an event is +frozen at broadcast time. A subscriber that joins after the event was +broadcast is not added to that event's pending set. + +**Per-event ack timeout**: a background task scans outstanding events. +Any subscriber that has not acknowledged within ``ack_timeout`` seconds is +force-failed via the existing :class:`_PollFailure` path (exception type +:class:`AckTimeout`). Other subscribers are not affected; once the +remaining acks arrive the producer advances normally. + +**Triggerer restart**: outstanding acks are in-memory only. After a +triggerer restart, the broker will redeliver unacknowledged events. +Subscribers must therefore be idempotent. + +**``shared_stream_subscriber_queue_size`` in ack mode**: the bound is +still "unprocessed raw events per subscriber". The manager does **not** +wait for outstanding acks before pulling the next event from the producer's +stream; back-pressure is purely queue-bound — a subscriber whose queue is +full is force-failed via :class:`_SubscriberOverflow`. The queue mainly +protects against burst delivery before a subscriber's filter has had a +chance to run. Broker advances, however, are dispatched by a single pump +task strictly in event order: while the head event's subscribers are still +pending, every later advance waits (the ack timeout is the backstop that +bounds this head-of-line wait). + +Triggers that do **not** override ``create_shared_stream_producer`` run the +**fast path**: no event IDs, no ack table, no AckToken — subscribers +receive raw events as before (backward-compatible). Lifecycle invariants -------------------- @@ -80,19 +111,25 @@ from __future__ import annotations import asyncio +import time +import weakref +from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, AsyncIterator, Callable, Hashable from contextlib import suppress -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal import structlog +from airflow.triggers.base import BaseEventTrigger + if TYPE_CHECKING: from structlog.stdlib import BoundLogger - from airflow.triggers.base import BaseEventTrigger - log = structlog.get_logger(__name__) +__all__ = ["AckTimeout", "AckToken", "AdvanceOutcome", "SharedStreamManager", "SharedStreamProducer"] + DEFAULT_SUBSCRIBER_QUEUE_MAX = 1024 """Default per-subscriber queue size for shared streams. @@ -106,6 +143,14 @@ ``[triggerer] shared_stream_subscriber_queue_size`` config option. """ +DEFAULT_ACK_TIMEOUT = 300.0 +"""Default per-event ack timeout in seconds (5 minutes). + +When ack mode is active, a subscriber that has not called ``token.ack()`` or +``token.nack()`` within this window is force-failed via :class:`AckTimeout`. +Override per-manager with ``SharedStreamManager(ack_timeout=...)``. +""" + class _PollTerminated(Exception): """ @@ -136,6 +181,159 @@ def __init__(self, exc: BaseException) -> None: self.exc = exc +class AckTimeout(Exception): + """ + Raised in a subscriber whose ack did not arrive within the per-event timeout. + + Treated the same as :class:`_SubscriberOverflow` — the subscriber's trigger + fails through the standard trigger-failure path. Other subscribers in the same + group are unaffected; their acks still advance the producer normally. + """ + + +@dataclass(frozen=True, slots=True) +class AdvanceOutcome: + """ + Per-event resolution counts handed to :meth:`SharedStreamProducer.advance`. + + Every subscriber that was online when the event was broadcast is counted + in exactly one field: + + * ``acked`` — called ``token.ack()``, or unsubscribed while the event was + outstanding (implicit ack-out). + * ``nacked`` — called ``token.nack()``. + * ``failed`` — force-failed by the manager (ack timeout or queue overflow). + + An event broadcast while no subscribers were online carries all-zero + counts and is clean. + """ + + acked: int + nacked: int + failed: int + + @property + def is_clean(self) -> bool: + """Whether every subscriber accepted the event (``nacked`` and ``failed`` are both zero).""" + return self.nacked == 0 and self.failed == 0 + + +class SharedStreamProducer(ABC): + """ + Broker-side half of a shared stream running in ack mode. + + Returned by + :meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer`; + one instance owns the broker connection for the lifetime of one poll. + """ + + @abstractmethod + def open_stream(self) -> AsyncIterator[tuple[Any, Any]]: + """ + Open the broker connection and yield ``(raw_event, broker_payload)`` pairs. + + Implement as an async generator. ``broker_payload`` is any opaque + object this producer needs later to advance the broker (e.g. a Kafka + offset, SQS receipt handle, Pub/Sub ack ID). Called once per poll; + open the broker connection here, not in the factory or ``__init__``. + + Implementations are expected to run for the lifetime of the group — + returning without raising is treated as an error and propagated to + every subscriber, so the contract is "yield forever, or raise". + """ + + @abstractmethod + async def advance(self, broker_payload: Any, outcome: AdvanceOutcome) -> None: + """ + Advance the broker for one fully resolved event. + + Called with each event's ``broker_payload`` strictly in fan-out + order, one at a time: the call for event N is awaited only after the + call for event N-1 returned (or raised and was logged). This makes + cumulative schemes such as a Kafka offset commit safe. ``outcome`` + carries the per-event resolution counts; use it to decide whether to + commit, skip, or trigger a broker-side redeliver. + """ + + async def aclose(self) -> None: + """ + Release broker resources; called once when the poll ends. + + Best-effort: a raised exception is logged and not propagated. The + default implementation does nothing. + """ + + +@dataclass +class _OutstandingEntry: + """State for one outstanding (broadcast-but-not-yet-advanced) event.""" + + pending: set[int] # trigger_ids still awaiting ack + created_at: float # monotonic clock at broadcast (via injectable _now) + broker_payload: Any + # Resolution counts; together they form the event's AdvanceOutcome. + acked: int = 0 + nacked: int = 0 + failed: int = 0 + + +class AckToken: + """ + Handed to subscribers alongside each shared-stream event in ack mode. + + Call ``await token.ack()`` once the event has been accepted, or + ``await token.nack()`` to opt out — the resolution is reported to the + producer through the event's :class:`AdvanceOutcome`. Repeated calls are + no-ops. After the group stops, calls silently do nothing. + """ + + __slots__ = ("_event_id", "_trigger_id", "_group_ref", "_resolved") + + def __init__( + self, + *, + event_id: int, + trigger_id: int, + group_ref: weakref.ref[_SharedStreamGroup], + ) -> None: + self._event_id = event_id + self._trigger_id = trigger_id + self._group_ref = group_ref + self._resolved = False + + def _resolve(self, resolution: Literal["acked", "nacked"]) -> None: + """Mark the token resolved and report it to the owning group, once.""" + if self._resolved: + return + self._resolved = True + group = self._group_ref() + if group is not None: + group._resolve_subscriber( + event_id=self._event_id, trigger_id=self._trigger_id, resolution=resolution + ) + + async def ack(self) -> None: + """ + Notify the producer that this subscriber has accepted the event. + + Declared ``async`` so the token API leaves room for awaitable + implementations; the in-process implementation resolves synchronously. + """ + self._resolve("acked") + + async def nack(self) -> None: + """ + Opt out of this event; counted as ``nacked`` in the event's :class:`AdvanceOutcome`. + + How the broker reacts is the producer's decision inside + :meth:`SharedStreamProducer.advance` — commit anyway, skip, or + trigger a redeliver. Declared ``async`` so the token API leaves room + for awaitable implementations; the in-process implementation + resolves synchronously. + """ + self._resolve("nacked") + + async def _drain(queue: asyncio.Queue) -> AsyncGenerator[Any, None]: """ Yield items from ``queue`` until a poll termination sentinel arrives. @@ -163,7 +361,9 @@ def __init__( kwargs: dict[str, Any], on_poll_terminate: Callable[[_SharedStreamGroup], None], max_subscriber_queue: int, + ack_timeout: float, log: BoundLogger, + _now: Callable[[], float] = time.monotonic, ) -> None: self.key = key self.trigger_class = trigger_class @@ -171,9 +371,23 @@ def __init__( self.log = log self._on_poll_terminate = on_poll_terminate self._max_subscriber_queue = max_subscriber_queue + self._ack_timeout = ack_timeout + self._now = _now self._subscribers: dict[int, asyncio.Queue] = {} - self._overflowed: set[int] = set() + # Subscribers already force-failed (queue overflow or ack timeout); + # excluded from subsequent broadcasts until they unsubscribe. + self._failed_subscribers: set[int] = set() self._poll_task: asyncio.Task | None = None + # Ack mode state — populated only when create_shared_stream_producer + # is overridden. + self._outstanding: dict[int, _OutstandingEntry] = {} + self._next_event_id: int = 0 + self._ack_timeout_task: asyncio.Task | None = None + # Advance pump: a single task dispatches broker advances strictly in + # fan-out order, woken through this event whenever an entry resolves. + self._advance_wakeup: asyncio.Event = asyncio.Event() + self._pump_stopping: bool = False + self._pump_task: asyncio.Task | None = None def start(self) -> None: """Start the underlying poll loop. Call exactly once per group.""" @@ -184,22 +398,122 @@ def start(self) -> None: name=f"shared-stream-poll[{self.key!r}]", ) + def _request_pump_stop(self) -> None: + """ + Ask the advance pump to drain the already-resolved prefix and exit. + + Synchronous (no await) so it can run inside ``_poll``'s terminal + section without yielding. + """ + self._pump_stopping = True + self._advance_wakeup.set() + + async def _run_advance_pump(self, producer: SharedStreamProducer) -> None: + """ + Dispatch broker advances strictly in fan-out order, one at a time. + + A single task harvests the resolved prefix of ``_outstanding`` (dict + insertion order is event-id order) and awaits ``producer.advance`` + for each entry in turn; an advance that raises is logged and the + pump moves on to the next entry. On stop the pump finishes the + resolved prefix, abandons unresolved entries to broker redelivery, + and exits. + """ + while True: + if not self._pump_stopping: + await self._advance_wakeup.wait() + self._advance_wakeup.clear() + while self._outstanding: + event_id = next(iter(self._outstanding)) + entry = self._outstanding[event_id] + if entry.pending: + break + del self._outstanding[event_id] + outcome = AdvanceOutcome(acked=entry.acked, nacked=entry.nacked, failed=entry.failed) + try: + await producer.advance(entry.broker_payload, outcome) + except Exception as exc: + self.log.error( + "Producer advance raised; broker advance failed", + key=self.key, + exc_info=exc, + ) + if self._pump_stopping: + return + + def _is_ack_required(self) -> bool: + # Check whether any class in the MRO (before BaseEventTrigger) defines + # create_shared_stream_producer — i.e. the subclass has overridden it. + for klass in self.trigger_class.__mro__: + if klass is BaseEventTrigger: + # Reached the base without finding an override — fast path. + return False + if "create_shared_stream_producer" in klass.__dict__: + return True + return False + async def _poll(self) -> None: + ack_required = self._is_ack_required() + producer: SharedStreamProducer | None = None terminal_exc: BaseException | None = None try: - async for raw_event in self.trigger_class.open_shared_stream(self.kwargs): - for trigger_id, queue in self._subscribers.items(): - if trigger_id in self._overflowed: - # Subscriber has been force-failed on a previous - # overflow; the failure sentinel is already in its - # queue and unsubscribe will drop it on next pass. + if ack_required: + # A factory failure flows through the terminal broadcast + # path below, like any other poll failure. + producer = self.trigger_class.create_shared_stream_producer(self.kwargs) + self._pump_task = asyncio.create_task( + self._run_advance_pump(producer), + name=f"shared-stream-advance-pump[{self.key!r}]", + ) + self._ack_timeout_task = asyncio.create_task( + self._run_ack_timeout_loop(), + name=f"shared-stream-ack-timeout[{self.key!r}]", + ) + event_source: AsyncIterator[Any] = producer.open_stream() + else: + event_source = self.trigger_class.open_shared_stream(self.kwargs) + async for item in event_source: + if ack_required: + raw_event, broker_payload = item + # Snapshot the subscriber set at fan-out time. + snapshot = set(self._subscribers.keys()) - self._failed_subscribers + event_id = self._next_event_id + self._next_event_id += 1 + self._outstanding[event_id] = _OutstandingEntry( + pending=snapshot.copy(), + created_at=self._now(), + broker_payload=broker_payload, + ) + if not snapshot: + # No subscribers to ack — the entry is born resolved; + # route it through the pump so broker advances stay + # in fan-out order. + self._advance_wakeup.set() continue - try: - queue.put_nowait(raw_event) - except asyncio.QueueFull: - self._fail_overflowed_subscriber(trigger_id, queue) + group_ref: weakref.ref[_SharedStreamGroup] = weakref.ref(self) + for trigger_id in snapshot: + queue = self._subscribers[trigger_id] + token = AckToken(event_id=event_id, trigger_id=trigger_id, group_ref=group_ref) + try: + queue.put_nowait((raw_event, token)) + except asyncio.QueueFull: + self._fail_overflowed_subscriber(trigger_id, queue) + # The dead subscriber will never resolve anything + # it still owes; fail it out of every outstanding + # entry (including this one) so it cannot + # head-block the ordered advance pump. + self._fail_subscriber_in_outstanding(trigger_id) + else: + raw_event = item + for trigger_id, queue in self._subscribers.items(): + if trigger_id in self._failed_subscribers: + continue + try: + queue.put_nowait(raw_event) + except asyncio.QueueFull: + self._fail_overflowed_subscriber(trigger_id, queue) terminal_exc = _PollTerminated( - f"open_shared_stream for {self.key!r} returned without raising; " + f"shared stream for {self.key!r} returned without raising; " "shared streams are expected to run for the lifetime of the group" ) except asyncio.CancelledError: @@ -210,6 +524,17 @@ async def _poll(self) -> None: terminal_exc = exc self.log.exception("Shared stream poll failed; propagating to subscribers", key=self.key) finally: + # Synchronous section — no yield until the terminal broadcast is + # done (see "Lifecycle invariants" in the module docstring), so + # no late subscriber can attach to the dead group. Awaiting the + # cancelled ack-timeout task, the pump drain, and the producer + # close are all deferred below the broadcast for the same reason. + cancelled_ack_task: asyncio.Task | None = None + if self._ack_timeout_task is not None and not self._ack_timeout_task.done(): + self._ack_timeout_task.cancel() + cancelled_ack_task = self._ack_timeout_task + # Synchronous: flags the pump to drain its resolved prefix and exit. + self._request_pump_stop() if terminal_exc is not None: # Synchronous: evict from the manager and broadcast the # sentinel before returning to the loop, so no coroutine can @@ -220,6 +545,102 @@ async def _poll(self) -> None: # Drain stale events then put the failure sentinel so every # subscriber wakes up even if its queue was at capacity. self._drain_and_offer_failure(queue, failure) + # End of synchronous section; yields are safe from here on. + if cancelled_ack_task is not None: + with suppress(asyncio.CancelledError): + await cancelled_ack_task + if self._pump_task is not None: + # The pump exits by itself once it has dispatched the already + # resolved prefix; the suppress is defensive. + with suppress(asyncio.CancelledError): + await self._pump_task + if producer is not None: + try: + await producer.aclose() + except Exception as exc: + self.log.warning("Producer aclose failed", key=self.key, exc_info=exc) + + def _resolve_subscriber( + self, + *, + event_id: int, + trigger_id: int, + resolution: Literal["acked", "nacked", "failed"], + ) -> None: + """ + Record one subscriber's resolution of one event. + + Removes ``trigger_id`` from the pending set of ``event_id`` and adds + it to the matching outcome count. If the set empties, wakes the + advance pump — deleting the entry and dispatching the broker advance + are the pump's job, so advances stay in fan-out order. Duplicate + calls for the same (event_id, trigger_id) are no-ops, which keeps + every subscriber counted exactly once per event. + """ + entry = self._outstanding.get(event_id) + if entry is None or trigger_id not in entry.pending: + return # already resolved, already advanced, or never existed + entry.pending.discard(trigger_id) + if resolution == "acked": + entry.acked += 1 + elif resolution == "nacked": + entry.nacked += 1 + else: + entry.failed += 1 + if not entry.pending: + self._advance_wakeup.set() + + def _fail_subscriber_in_outstanding(self, trigger_id: int) -> None: + """ + Resolve ``trigger_id`` as failed in every outstanding entry. + + A force-failed subscriber (queue overflow or ack timeout) will never + resolve the events it still owes; leaving it pending in older entries + would head-block the ordered advance pump until each entry's own ack + timeout. This is the retroactive counterpart of excluding the + subscriber from future broadcasts via ``_failed_subscribers``. + """ + for event_id in list(self._outstanding): + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="failed") + + async def _run_ack_timeout_loop(self) -> None: + """ + Background task: force-fail subscribers whose ack is overdue. + + Cadence is ``max(0.01, ack_timeout / 10)`` — runs ten times per timeout + window, floored at 10 ms to avoid burning CPU when ``ack_timeout`` is small. + """ + cadence = max(0.01, self._ack_timeout / 10) + while True: + await asyncio.sleep(cadence) + now = self._now() + for event_id, entry in list(self._outstanding.items()): + if now - entry.created_at < self._ack_timeout: + continue + timed_out = set(entry.pending) + for trigger_id in timed_out: + queue = self._subscribers.get(trigger_id) + if queue is not None: + self.log.warning( + "Ack timeout; force-failing subscriber", + key=self.key, + trigger_id=trigger_id, + event_id=event_id, + ) + self._drain_and_offer_failure( + queue, + _PollFailure( + AckTimeout( + f"shared stream {self.key!r} trigger {trigger_id} " + f"did not ack event {event_id} within {self._ack_timeout}s" + ) + ), + ) + self._failed_subscribers.add(trigger_id) + # Fail the subscriber out of every outstanding entry (not + # just this one); the resolution wakes the pump, which + # owns entry deletion and the ordered broker advance. + self._fail_subscriber_in_outstanding(trigger_id) def subscribe(self, trigger_id: int) -> AsyncIterator[Any]: """Register ``trigger_id`` as a subscriber and return its raw event stream.""" @@ -233,7 +654,12 @@ def unsubscribe(self, trigger_id: int) -> None: # Active subscribers exit through their consuming task being cancelled # (Airflow's standard idiom); dropping the queue is enough here. self._subscribers.pop(trigger_id, None) - self._overflowed.discard(trigger_id) + self._failed_subscribers.discard(trigger_id) + # Resolve in all outstanding entries — implicit ack-out (counted as + # acked) to prevent the producer waiting forever for a subscriber + # that has left. + for event_id in list(self._outstanding): + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") def _fail_overflowed_subscriber(self, trigger_id: int, queue: asyncio.Queue) -> None: """ @@ -263,7 +689,7 @@ def _fail_overflowed_subscriber(self, trigger_id: int, queue: asyncio.Queue) -> ) ), ) - self._overflowed.add(trigger_id) + self._failed_subscribers.add(trigger_id) def _drain_and_offer_failure(self, queue: asyncio.Queue, failure: _PollFailure) -> None: """ @@ -284,12 +710,27 @@ def is_empty(self) -> bool: return not self._subscribers async def stop(self) -> None: - """Cancel the poll task if it is still running and wait for it to exit.""" - if self._poll_task is None or self._poll_task.done(): - return - self._poll_task.cancel() - with suppress(asyncio.CancelledError): - await self._poll_task + """ + Cancel the poll and ack-timeout tasks if still running. + + The poll's ``finally`` drains the advance pump (in-flight and + already-resolved advances complete; unresolved events are abandoned + to broker redelivery) and closes the producer. + """ + if self._ack_timeout_task is not None and not self._ack_timeout_task.done(): + self._ack_timeout_task.cancel() + with suppress(asyncio.CancelledError): + await self._ack_timeout_task + if self._poll_task is not None and not self._poll_task.done(): + self._poll_task.cancel() + with suppress(asyncio.CancelledError): + await self._poll_task + # Defensive: if the poll never reached its finally, make sure the + # pump still drains and exits. + if self._pump_task is not None and not self._pump_task.done(): + self._request_pump_stop() + with suppress(asyncio.CancelledError): + await self._pump_task class SharedStreamManager: @@ -311,9 +752,13 @@ def __init__( *, log: BoundLogger | None = None, max_subscriber_queue: int = DEFAULT_SUBSCRIBER_QUEUE_MAX, + ack_timeout: float = DEFAULT_ACK_TIMEOUT, + _now: Callable[[], float] = time.monotonic, ) -> None: self.log = log or structlog.get_logger(__name__) self._max_subscriber_queue = max_subscriber_queue + self._ack_timeout = ack_timeout + self._now = _now self._groups: dict[Hashable, _SharedStreamGroup] = {} def subscribe( @@ -340,7 +785,9 @@ def subscribe( kwargs=kwargs, on_poll_terminate=self._handle_poll_terminate, max_subscriber_queue=self._max_subscriber_queue, + ack_timeout=self._ack_timeout, log=self.log, + _now=self._now, ) self._groups[key] = group group.start() diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py b/airflow-core/tests/unit/jobs/test_triggerer_job.py index 03add9408b294..04a8700388cbb 100644 --- a/airflow-core/tests/unit/jobs/test_triggerer_job.py +++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py @@ -907,6 +907,96 @@ async def _drive(): # Group is torn down on unsubscribe. assert trigger_runner._shared_streams._groups == {} + def test_shared_stream_ack_mode_integration(self, session) -> None: + """A BaseEventTrigger whose producer factory is overridden advances after all acks.""" + from airflow.triggers.base import BaseEventTrigger, TriggerEvent + from airflow.triggers.shared_stream import SharedStreamProducer + + advanced: list[Any] = [] + # Container lets _drive() write trigger_runner back for post-run assertions. + state: dict[str, Any] = {} + + class _AckSharedProducer(SharedStreamProducer): + async def open_stream(self): + yield {"region": "us"}, "receipt-handle-1" + # Stay alive so the manager can tear us down on unsubscribe. + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + advanced.append(broker_payload) + state["advance_called"].set() + + class _AckSharedTrigger(BaseEventTrigger): + def __init__(self, queue_url: str, region: str | None = None): + super().__init__() + self.queue_url = queue_url + self.region = region + + def serialize(self): + return ( + f"{type(self).__module__}.{type(self).__qualname__}", + {"queue_url": self.queue_url, "region": self.region}, + ) + + def shared_stream_key(self): + return ("ack-queue", self.queue_url) + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _AckSharedProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + if self.region is None or raw["region"] == self.region: + await token.ack() + yield TriggerEvent(raw) + else: + await token.nack() + + async def run(self): # pragma: no cover - replaced by filter_shared_stream + yield TriggerEvent({}) + + async def _drive(): + advance_called = asyncio.Event() + state["advance_called"] = advance_called + + trigger_runner = TriggerRunner() + trigger_runner.triggers = { + 1: {"task": MagicMock(spec=asyncio.Task), "is_watcher": True, "name": "us", "events": 0} + } + trigger = _AckSharedTrigger(queue_url="https://ack-q", region="us") + trigger.task_instance = MagicMock() + trigger.task_instance.map_index = -1 + state["trigger_runner"] = trigger_runner + + run_task = asyncio.create_task(trigger_runner.run_trigger(1, trigger)) + # Wait until advance has fired — unambiguous signal that ack round-trip completed. + await asyncio.wait_for(advance_called.wait(), timeout=2.0) + # Advance is dispatched from the pump task, so it can run while run_trigger is + # still suspended between yielding the event and appending it to the outbound + # queue. Wait for the event to land before cancelling, like the non-ack test above. + for _ in range(100): + if trigger_runner.events: + break + await asyncio.sleep(0.01) + run_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await run_task + + asyncio.run(_drive()) + trigger_runner = state["trigger_runner"] + + events = list(trigger_runner.events) + assert len(events) == 1 + trigger_id, event = events[0] + assert trigger_id == 1 + assert event.payload == {"region": "us"} + + # The producer's advance must have been called with the broker_payload. + assert advanced == ["receipt-handle-1"], "producer.advance must be called after the subscriber acks" + # Group is torn down on unsubscribe. + assert trigger_runner._shared_streams._groups == {} + def test_run_trigger_on_kill_timeout_does_not_block_cleanup(self, session) -> None: """A hanging on_kill() is interrupted after the timeout and cleanup still runs.""" trigger_runner = TriggerRunner() diff --git a/airflow-core/tests/unit/triggers/test_base_trigger.py b/airflow-core/tests/unit/triggers/test_base_trigger.py index 8e5690a3ef991..338168e7c022e 100644 --- a/airflow-core/tests/unit/triggers/test_base_trigger.py +++ b/airflow-core/tests/unit/triggers/test_base_trigger.py @@ -232,3 +232,14 @@ async def stream(): payloads = [event.payload async for event in us.filter_shared_stream(stream())] assert [p["region"] for p in payloads] == ["us", "us"] + + +def test_create_shared_stream_producer_raises_by_default(): + """A subclass that does not override create_shared_stream_producer gets NotImplementedError. + + The manager detects this via MRO inspection and takes the fast path (no ack + tokens issued). But calling the method directly must still raise, confirming it + is not accidentally a no-op on the base class. + """ + with pytest.raises(NotImplementedError, match="create_shared_stream_producer"): + _PlainEventTrigger.create_shared_stream_producer({}) diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index 28577ccbb528f..71d14bacdf954 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -17,13 +17,21 @@ from __future__ import annotations import asyncio +import time +import weakref from contextlib import suppress +from unittest.mock import MagicMock import pytest +import structlog from airflow.triggers.base import BaseEventTrigger, TriggerEvent from airflow.triggers.shared_stream import ( + AckTimeout, + AckToken, + AdvanceOutcome, SharedStreamManager, + SharedStreamProducer, _PollFailure, _SharedStreamGroup, _SubscriberOverflow, @@ -464,7 +472,7 @@ async def drain_fast(): ) # The group is still alive — only the slow subscriber was failed; fast is still subscribed. assert key in manager._groups - assert 1 in manager._groups[key]._overflowed + assert 1 in manager._groups[key]._failed_subscribers await manager.unsubscribe(1, key) await manager.unsubscribe(2, key) @@ -654,8 +662,6 @@ async def test_fail_overflowed_subscriber_drains_full_queue_before_putting_senti raise :exc:`asyncio.QueueFull` before any draining occurred and the subscriber would never receive its failure sentinel. """ - import structlog - cap = 3 queue: asyncio.Queue = asyncio.Queue(maxsize=cap) # Pre-fill the queue to capacity with stale events. @@ -670,6 +676,7 @@ async def test_fail_overflowed_subscriber_drains_full_queue_before_putting_senti kwargs={}, on_poll_terminate=lambda g: None, max_subscriber_queue=cap, + ack_timeout=300.0, log=structlog.get_logger("test"), ) trigger_id = 42 @@ -682,4 +689,2178 @@ async def test_fail_overflowed_subscriber_drains_full_queue_before_putting_senti sentinel = queue.get_nowait() assert isinstance(sentinel, _PollFailure), "sentinel must be a _PollFailure" assert isinstance(sentinel.exc, _SubscriberOverflow), "the wrapped exception must be _SubscriberOverflow" - assert trigger_id in group._overflowed, "trigger_id must be recorded in _overflowed" + assert trigger_id in group._failed_subscribers, "trigger_id must be recorded in _failed_subscribers" + + +class _RecordingProducer(SharedStreamProducer): + """Producer that records advance/aclose activity onto its trigger class.""" + + def __init__(self, trigger_cls, events_with_payloads: list[tuple]): + self._trigger_cls = trigger_cls + self._events_with_payloads = events_with_payloads + + async def open_stream(self): + for event, broker_payload in self._events_with_payloads: + yield event, broker_payload + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + self._trigger_cls.advanced.append(broker_payload) + self._trigger_cls.outcomes.append(outcome) + + async def aclose(self): + self._trigger_cls.aclose_calls += 1 + + +def _make_ack_required_trigger_class(events_with_payloads: list[tuple]): + """ + Return a fresh trigger class whose ``create_shared_stream_producer`` + returns a :class:`_RecordingProducer` yielding ``(event, broker_payload)`` + tuples and recording each advance call into class-level lists. + """ + + class _AckRequiredTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + outcomes: list[AdvanceOutcome] = [] + aclose_calls: int = 0 + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RecordingProducer(cls, events_with_payloads) + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + return _AckRequiredTrigger + + +async def _collect_ack_stream(trigger, stream, *, n: int) -> list: + """Collect ``n`` TriggerEvent items from an ack-aware filter_shared_stream.""" + out = [] + async for event in trigger.filter_shared_stream(stream): + out.append(event) + if len(out) >= n: + break + return out + + +@pytest.mark.parametrize( + ("override_level", "expected"), + [ + pytest.param("middle", True, id="middle-overrides-grandchild-inherits"), + pytest.param(None, False, id="no-override-anywhere"), + pytest.param("grandchild", True, id="grandchild-overrides-itself"), + ], +) +def test_is_ack_required_multi_level_inheritance(override_level, expected): + """``_is_ack_required`` walks the MRO down to ``BaseEventTrigger``: any class + along the way that defines ``create_shared_stream_producer`` — including one + a subclass merely inherits from — switches the group to ack mode. + """ + + class _Parent(_ProgrammableSharedStreamTrigger): + pass + + class _Middle(_Parent): + pass + + class _Grandchild(_Middle): + pass + + # Detection only — the poll never runs, so returning None is fine. + def create_shared_stream_producer(cls, kwargs): + return None + + if override_level == "middle": + _Middle.create_shared_stream_producer = classmethod(create_shared_stream_producer) + elif override_level == "grandchild": + _Grandchild.create_shared_stream_producer = classmethod(create_shared_stream_producer) + + group = _SharedStreamGroup( + key="test-key", + trigger_class=_Grandchild, + kwargs={}, + on_poll_terminate=lambda g: None, + max_subscriber_queue=8, + ack_timeout=300.0, + log=structlog.get_logger("test"), + ) + + assert group._is_ack_required() is expected + + +@pytest.mark.asyncio +async def test_ack_required_producer_advances_after_all_subscribers_ack(): + """Producer's advance hook fires exactly once after both subscribers ack.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "hello"}, "receipt-1"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + tokens: list[AckToken] = [] + + async def collect_token(stream, trigger): + async for _raw, token in stream: + tokens.append(token) + break + + # Collect the (event, token) tuples without acking yet. + await asyncio.gather( + asyncio.wait_for(collect_token(s1, t1), timeout=1.0), + asyncio.wait_for(collect_token(s2, t2), timeout=1.0), + ) + + assert len(cls.advanced) == 0, "advance must not fire before all acks" + + await tokens[0].ack() + assert len(cls.advanced) == 0, "advance must not fire after only one ack" + + await tokens[1].ack() + await asyncio.sleep(0) # let the advance pump run + assert cls.advanced == ["receipt-1"], "advance must fire exactly once after all acks" + finally: + await manager.unsubscribe(1, key) + await manager.unsubscribe(2, key) + + +@pytest.mark.asyncio +async def test_ack_required_producer_does_not_advance_on_partial_ack(): + """Producer does not advance when only one of two subscribers has acknowledged.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "partial"}, "receipt-partial"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + token1 = None + got_s2 = asyncio.Event() + + async def grab_token1(stream): + nonlocal token1 + async for _raw, token in stream: + token1 = token + break + + async def drain_s2(stream): + async for _ in stream: # pragma: no branch + got_s2.set() + break + + await asyncio.gather( + asyncio.wait_for(grab_token1(s1), timeout=1.0), + asyncio.wait_for(drain_s2(s2), timeout=1.0), + ) + + await token1.ack() + await asyncio.sleep(0) + assert cls.advanced == [], "producer must not advance with only one of two acks" + finally: + await manager.unsubscribe(1, key) + await manager.unsubscribe(2, key) + + +@pytest.mark.asyncio +async def test_ack_timeout_force_fails_slow_subscriber_only(): + """A slow subscriber is force-failed after timeout; the fast subscriber and advance are unaffected.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "timeout-test"}, "receipt-timeout"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t_fast, t_slow = cls(), cls() + key = t_fast.shared_stream_key() + + # Inject a fake clock so the test controls time without relying on wall-clock + # or time_machine (which does not affect time.monotonic). + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + # Very short timeout; cadence = max(0.01, 0.1/10) = 0.01s. + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + try: + s_fast = manager.subscribe(trigger_id=1, trigger=t_fast, key=key) + s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) + + token_fast = None + + async def grab_fast(stream): + nonlocal token_fast + async for _raw, token in stream: + token_fast = token + break + + async def drain_slow(stream): + # Collect from slow stream — we expect AckTimeout to appear. + async for _ in stream: + break + + await asyncio.gather( + asyncio.wait_for(grab_fast(s_fast), timeout=1.0), + asyncio.wait_for(drain_slow(s_slow), timeout=1.0), + ) + + # fast acks; slow doesn't. + await token_fast.ack() + + # Advance fake clock past ack_timeout; the timeout loop sees now() - created_at >= 0.1. + fake_clock[0] = 0.2 + slow_queue = manager._groups[key]._subscribers.get(2) + # Let _run_ack_timeout_loop tick through a few cadence cycles (cadence=0.01s real). + await asyncio.sleep(0.05) + + assert slow_queue is not None + sentinel = slow_queue.get_nowait() + assert isinstance(sentinel, _PollFailure) + assert isinstance(sentinel.exc, AckTimeout) + + # Fast subscriber (already acked) must not have received a spurious AckTimeout. + fast_queue = manager._groups[key]._subscribers.get(1) + assert fast_queue is not None + assert fast_queue.empty(), "fast (already-acked) subscriber must not receive a spurious AckTimeout" + + # After timeout the advance should have fired (pending emptied). + await asyncio.sleep(0) + assert cls.advanced == ["receipt-timeout"] + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_ack_timeout_at_cap_boundary_does_not_timeout(): + """Subscriber acking inside the window (< ack_timeout) must NOT be force-failed.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "boundary-test"}, "receipt-boundary"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + + # Inject a fake clock: created_at will be recorded at fake_clock[0] = 0.0. + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + token_ref = None + + async def grab_token(s): + nonlocal token_ref + async for _raw, token in s: + token_ref = token + break + + await asyncio.wait_for(grab_token(stream), timeout=1.0) + + # Advance fake clock to 99 ms — strictly inside the 100 ms ack_timeout window. + # now() - created_at = 0.099 < 0.1, so the timeout loop must NOT fire. + fake_clock[0] = 0.099 + # Let the timeout loop tick a few cadence cycles (cadence = max(0.01, 0.1/10) = 0.01s real). + await asyncio.sleep(0.05) + + subscriber_queue = manager._groups[key]._subscribers.get(1) + assert subscriber_queue is not None + assert subscriber_queue.empty(), "subscriber must not have been force-failed at t=99ms" + + # Now ack — advance should fire. + await token_ref.ack() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-boundary"] + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_ack_timeout_exactly_at_timeout_force_fails(): + """Subscriber whose ack is exactly ``ack_timeout`` old IS force-failed (elapsed >= timeout).""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "exact-boundary"}, "receipt-exact"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + + # Inject a fake clock: created_at will be recorded at fake_clock[0] = 0.0. + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + async def drain_one(s): + # Pull the (event, token) tuple but never ack. + async for _ in s: + break + + await asyncio.wait_for(drain_one(stream), timeout=1.0) + + # Advance fake clock to exactly the 100 ms ack_timeout. + # now() - created_at == 0.1 is not < 0.1, so the timeout loop must fire — + # this pins the boundary at >= rather than >. + fake_clock[0] = 0.1 + # Let the timeout loop tick a few cadence cycles (cadence = max(0.01, 0.1/10) = 0.01s real). + await asyncio.sleep(0.05) + + subscriber_queue = manager._groups[key]._subscribers.get(1) + assert subscriber_queue is not None + sentinel = subscriber_queue.get_nowait() + assert isinstance(sentinel, _PollFailure) + assert isinstance(sentinel.exc, AckTimeout) + + # The pending set emptied, so the producer advance fired. + await asyncio.sleep(0) + assert cls.advanced == ["receipt-exact"] + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("ack_timeout", "expected_cadence"), + [ + (0.05, 0.01), # 0.05/10 = 0.005 < 0.01 → floor clamps to 0.01 + (0.2, 0.02), # 0.2/10 = 0.02 >= 0.01 → exact computed value + ], + ids=["floor-clamp", "computed"], +) +async def test_ack_timeout_cadence_floor(ack_timeout, expected_cadence): + """Cadence floor: max(0.01, ack_timeout/10) is floored at 0.01 when ack_timeout/10 < 0.01.""" + cls = _make_ack_required_trigger_class([({"msg": "cadence-test"}, "receipt-cadence")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + + first_sleep: list[float] = [] + real_sleep = asyncio.sleep + + async def capturing_sleep(delay): + if not first_sleep: + first_sleep.append(delay) + # Stop after capturing so the test doesn't run real time. + raise asyncio.CancelledError + await real_sleep(delay) + + manager = SharedStreamManager(ack_timeout=ack_timeout) + try: + with mock.patch("airflow.triggers.shared_stream.asyncio.sleep", side_effect=capturing_sleep): + manager.subscribe(trigger_id=1, trigger=t1, key=key) + # Give the event loop a turn so the group starts and _run_ack_timeout_loop fires. + with suppress(asyncio.CancelledError): + await real_sleep(0.05) + finally: + await manager.stop_all() + + assert first_sleep, "capturing_sleep was never called — _run_ack_timeout_loop did not start" + assert first_sleep[0] == pytest.approx(expected_cadence), ( + f"cadence for ack_timeout={ack_timeout} should be {expected_cadence}, got {first_sleep[0]}" + ) + + +@pytest.mark.asyncio +async def test_late_subscriber_does_not_block_advance_of_earlier_event(): + """A subscriber joining after event N is broadcast is not in event N's ack set.""" + event_broadcast = asyncio.Event() + proceed = asyncio.Event() + + class _LateProducer(SharedStreamProducer): + def __init__(self, trigger_cls): + self._trigger_cls = trigger_cls + + async def open_stream(self): + event_broadcast.set() + await proceed.wait() + yield {"msg": "late-test"}, "receipt-late" + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + self._trigger_cls.advanced.append(broker_payload) + + class _LateTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _LateProducer(cls) + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + _LateTrigger.advanced.clear() + + t_early = _LateTrigger() + t_late = _LateTrigger() + key = t_early.shared_stream_key() + manager = SharedStreamManager() + try: + s_early = manager.subscribe(trigger_id=1, trigger=t_early, key=key) + + # Wait for open_shared_stream to reach the proceed gate, then unblock. + await asyncio.wait_for(event_broadcast.wait(), timeout=1.0) + proceed.set() + + # Give the poll task a few ticks to run past the yield and put the + # (event, token) tuple into the early subscriber's queue. + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if manager._groups[key]._outstanding: + break + await asyncio.sleep(0) + + # The event is now outstanding with only the early subscriber in + # its pending set. Subscribe the late subscriber AFTER fan-out. + manager.subscribe(trigger_id=2, trigger=t_late, key=key) + + # Confirm the late subscriber is NOT in the outstanding entry's pending set. + entry = next(iter(manager._groups[key]._outstanding.values()), None) + assert entry is not None + assert 2 not in entry.pending, "late subscriber must not be in the pending set of earlier event" + + # Collect from early subscriber (filter acks immediately). + events = await asyncio.wait_for(_collect_ack_stream(t_early, s_early, n=1), timeout=1.0) + assert len(events) == 1 + + await asyncio.sleep(0) + # The late subscriber did not participate in ack set; advance must have fired. + assert _LateTrigger.advanced == ["receipt-late"] + finally: + await manager.unsubscribe(1, key) + await manager.unsubscribe(2, key) + + +@pytest.mark.asyncio +async def test_nack_removes_subscriber_from_ack_set_without_redeliver(): + """nack() removes the subscriber from the pending set; no redeliver happens.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "nack-test"}, "receipt-nack"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + tokens: list[AckToken] = [] + + async def grab_token(stream): + async for _raw, token in stream: + tokens.append(token) + break + + await asyncio.gather( + asyncio.wait_for(grab_token(s1), timeout=1.0), + asyncio.wait_for(grab_token(s2), timeout=1.0), + ) + + # t1 acks, t2 nacks. + await tokens[0].ack() + await tokens[1].nack() + await asyncio.sleep(0) + + assert cls.advanced == ["receipt-nack"], "advance must fire after ack + nack" + + # No redelivery — the manager has no redeliver mechanism. + group = manager._groups.get(key) + if group: + assert len(group._outstanding) == 0 + finally: + await manager.unsubscribe(1, key) + await manager.unsubscribe(2, key) + + +@pytest.mark.asyncio +async def test_double_ack_is_noop(): + """Calling ack() twice on the same token must not raise or advance twice.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "double-ack"}, "receipt-double"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + token = None + + async def grab_token(stream): + nonlocal token + async for _raw, tk in stream: + token = tk + break + + await asyncio.wait_for(grab_token(s1), timeout=1.0) + assert token is not None + + await token.ack() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-double"] + + # Second ack — must be a no-op, not raise, not double-advance. + await token.ack() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-double"], "second ack must not trigger a second advance" + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("first", "second"), + [ + ("ack", "nack"), + ("nack", "ack"), + ], +) +async def test_cross_order_double_call_is_noop(first, second): + """ack-then-nack and nack-then-ack are both no-ops — _resolved blocks the second call.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "cross-order"}, "receipt-cross"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + token = None + + async def grab_token(stream): + nonlocal token + async for _raw, tk in stream: + token = tk + break + + await asyncio.wait_for(grab_token(s1), timeout=1.0) + assert token is not None + + # First call — should resolve the token and advance. + await getattr(token, first)() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-cross"] + + # Second call (opposite) — must be a no-op; advance must not fire again. + await getattr(token, second)() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-cross"], ( + f"{second}() after {first}() must not trigger a second advance" + ) + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_subscriber_unsubscribe_during_outstanding_ack(): + """When a subscriber leaves while its ack is pending, the group advances without it.""" + cls = _make_ack_required_trigger_class( + [ + ({"msg": "unsub-during-ack"}, "receipt-unsub"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + tokens: list[AckToken] = [] + + async def grab_token(stream): + async for _raw, token in stream: + tokens.append(token) + break + + await asyncio.gather( + asyncio.wait_for(grab_token(s1), timeout=1.0), + asyncio.wait_for(grab_token(s2), timeout=1.0), + ) + + assert len(tokens) == 2 + # t1 acks; t2 leaves without acking (implicit ack-out via unsubscribe). + await tokens[0].ack() + await manager.unsubscribe(2, key) + await asyncio.sleep(0) + + assert cls.advanced == ["receipt-unsub"], ( + "unsubscribe must trigger implicit ack-out so producer can advance" + ) + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_ack_mode_queue_full_during_fanout_does_not_break_iteration(): + """A full queue at fan-out time must not break the iteration of remaining subscribers.""" + cls = _make_ack_required_trigger_class([({"msg": "burst"}, "receipt-burst")]) + cls.advanced.clear() + cls.outcomes.clear() + + t_full, t_ok = cls(), cls() + key = t_full.shared_stream_key() + # Queue of size 1 — pre-filling it will force QueueFull on fan-out for trigger 1. + manager = SharedStreamManager(max_subscriber_queue=1) + try: + s_full = manager.subscribe(trigger_id=1, trigger=t_full, key=key) + s_ok = manager.subscribe(trigger_id=2, trigger=t_ok, key=key) + + # Pre-fill trigger 1's queue so put_nowait raises QueueFull at fan-out. + group = manager._groups[key] + group._subscribers[1].put_nowait( + ("filler", AckToken(event_id=-1, trigger_id=1, group_ref=weakref.ref(group))) + ) + + # t_ok: drive its ack stream and collect the real event. + ok_result: list = [] + + async def collect_ok(): + events = await _collect_ack_stream(t_ok, s_ok, n=1) + ok_result.extend(events) + + # t_full: its queue will be drained + replaced by _PollFailure(_SubscriberOverflow). + # _drain() raises the inner exception, so we expect _SubscriberOverflow here. + full_exc: list[BaseException] = [] + + async def collect_full(): + try: + async for _ in s_full: + pass + except Exception as exc: + full_exc.append(exc) + + await asyncio.gather( + asyncio.wait_for(collect_full(), timeout=2.0), + asyncio.wait_for(collect_ok(), timeout=2.0), + ) + + # Give the advance pump a tick to run. + await asyncio.sleep(0) + + # t_full got an overflow failure (not a RuntimeError from set mutation). + assert len(full_exc) == 1 + assert isinstance(full_exc[0], _SubscriberOverflow) + + # t_ok got the real event and acknowledged it → advance was triggered. + assert len(ok_result) == 1 + assert ok_result[0].payload == {"msg": "burst"} + assert cls.advanced == ["receipt-burst"] + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_no_subscriber_snapshot_advances_in_order(): + """Events broadcast while no subscribers are online still advance, in fan-out order. + + Empty-snapshot entries are born resolved and go through the same ordered + pump as everything else; their outcomes carry all-zero counts. + """ + cls = _make_ack_required_trigger_class( + [ + ({"msg": "no-sub-1"}, "p1"), + ({"msg": "no-sub-2"}, "p2"), + ({"msg": "no-sub-3"}, "p3"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + # Subscribe then immediately unsubscribe so the group exists but has zero subscribers + # when the events arrive. + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + manager._groups[key].unsubscribe(1) # remove without stopping the poll task + + # The poll task is still running; give it a few ticks to broadcast. + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(cls.advanced) >= 3: + break + await asyncio.sleep(0.01) + + assert cls.advanced == ["p1", "p2", "p3"], ( + "advance must be called for every event, in fan-out order, even with no subscribers online" + ) + assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * 3 + assert all(outcome.is_clean for outcome in cls.outcomes) + + del s1 # silence unused-variable warning + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_producer_advance_exception_is_logged(): + """When producer.advance raises, the error is logged and the poll group stays alive.""" + proceed_flag = asyncio.Event() + + class _RaisingAdvanceProducer(SharedStreamProducer): + async def open_stream(self): + yield ({"msg": "event-1"}, "payload-1") + await proceed_flag.wait() + yield ({"msg": "event-2"}, "payload-2") + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + raise RuntimeError("simulated broker failure") + + class _RaisingAdvanceTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RaisingAdvanceProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _RaisingAdvanceTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + # Intercept log.error on the group so we can assert it was called. + group = manager._groups[key] + mock_log_error = MagicMock() + group.log = MagicMock() + group.log.error = mock_log_error + + # Consume the first event — this triggers an ack and schedules advance. + events1 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) + assert len(events1) == 1 + + # Give the advance pump a tick to run and raise. + await asyncio.sleep(0.05) + + # log.error must have been called with a message about the failed advance. + assert mock_log_error.called, "log.error must be called when producer.advance raises" + call_args = mock_log_error.call_args + assert "broker advance failed" in call_args[0][0], ( + f"log.error message must mention 'broker advance failed', got: {call_args[0][0]}" + ) + + # The poll group is still alive — the exception must not have killed it. + assert key in manager._groups, "group must survive a producer.advance exception" + + # A second event can still be produced and consumed. + proceed_flag.set() + events2 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) + assert len(events2) == 1 + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_stop_drains_in_flight_advance(): + """stop() must await an advance already in flight so graceful shutdown does not drop it.""" + advance_started = asyncio.Event() + advance_may_finish = asyncio.Event() + advance_done_at: list[float] = [] + + class _SlowAdvanceProducer(SharedStreamProducer): + def __init__(self, trigger_cls): + self._trigger_cls = trigger_cls + + async def open_stream(self): + yield ({"msg": "slow"}, "receipt-slow") + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + advance_started.set() + await advance_may_finish.wait() + await asyncio.sleep(0.05) # ensure stop() must actually wait for us + self._trigger_cls.advanced.append(broker_payload) + advance_done_at.append(time.monotonic()) + + class _SlowAdvanceTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _SlowAdvanceProducer(cls) + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + _SlowAdvanceTrigger.advanced.clear() + + t1 = _SlowAdvanceTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + # Drive the one event through so the advance is dispatched. + events = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) + assert len(events) == 1 + + # Wait for producer.advance to actually start (confirms the advance is in-flight). + await asyncio.wait_for(advance_started.wait(), timeout=1.0) + + # Unblock the advance hook and stop; advance still has a 50 ms sleep so stop() + # must genuinely await the task rather than racing past it. + advance_may_finish.set() + await manager.stop_all() + stop_returned_at = time.monotonic() + + # stop_all() must have waited for the in-flight advance to complete. + assert _SlowAdvanceTrigger.advanced == ["receipt-slow"], ( + "stop() must drain the in-flight advance before returning" + ) + assert advance_done_at, "the in-flight advance must have completed" + assert advance_done_at[0] <= stop_returned_at, ( + "stop() must wait for in-flight advance to finish before returning" + ) + + +@pytest.mark.asyncio +async def test_ack_mode_late_subscriber_during_poll_failure_window_gets_fresh_group(): + """Ack-mode variant of the poll-failure-window race: tearing down the ack-timeout + task must not introduce a yield point before the group's eviction + sentinel + broadcast. A subscriber arriving in the same tick the poll died must create a + fresh group rather than attach to the dead one. + """ + invocations: list[int] = [] + + class _AckWindowProducer(SharedStreamProducer): + def __init__(self, trigger_cls): + self._trigger_cls = trigger_cls + + async def open_stream(self): + n = len(invocations) + invocations.append(n) + if n == 0: + raise RuntimeError("boom") + yield {"region": "fresh"}, "receipt-fresh" + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + self._trigger_cls.advanced.append(broker_payload) + + class _AckWindowTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _AckWindowProducer(cls) + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + _AckWindowTrigger.advanced.clear() + trigger = _AckWindowTrigger() + manager = SharedStreamManager() + key = trigger.shared_stream_key() + + stream1 = manager.subscribe(trigger_id=1, trigger=trigger, key=key) + old_poll_task = manager._groups[key]._poll_task + assert old_poll_task is not None + + # One tick: the poll task runs, the upstream raises immediately, and the finally + # block executes. Cancelling the ack-timeout task must not make the poll yield + # before it has evicted the key and broadcast the sentinel. + await asyncio.sleep(0) + assert key not in manager._groups, ( + "the failing ack-mode poll must evict its group synchronously — awaiting the " + "cancelled ack-timeout task must not reopen the late-subscriber window" + ) + + # A subscriber arriving inside that same tick must get a fresh group. + stream2 = manager.subscribe(trigger_id=2, trigger=trigger, key=key) + try: + events = await asyncio.wait_for(_collect_ack_stream(trigger, stream2, n=1), timeout=1.0) + assert events[0].payload == {"region": "fresh"} + finally: + # The original subscriber still has the failure sentinel waiting for it. + with pytest.raises(RuntimeError, match="boom"): + await asyncio.wait_for( + _collect(trigger.filter_shared_stream(stream1), n=1), + timeout=1.0, + ) + await old_poll_task + await manager.unsubscribe(1, key) + await manager.unsubscribe(2, key) + + assert invocations == [0, 1] + + +@pytest.mark.asyncio +async def test_directory_file_delete_trigger_path_unchanged(): + """Triggers that do NOT override create_shared_stream_producer use the fast path. + + Subscribers receive raw events (not tuples) — identical to pre-ack-mode behavior. + """ + cls = _make_trigger_class(_events_then_block([{"filename": "flag.txt"}])) + # Confirm create_shared_stream_producer is NOT overridden (fast path). + # We check that no class in the MRO before BaseEventTrigger defines it. + assert "create_shared_stream_producer" not in cls.__dict__, ( + "create_shared_stream_producer must NOT be in the class's own __dict__ for fast-path detection" + ) + + trigger = cls() + manager = SharedStreamManager() + key = trigger.shared_stream_key() + try: + stream = manager.subscribe(trigger_id=1, trigger=trigger, key=key) + events = await _collect(trigger.filter_shared_stream(stream), n=1) + assert len(events) == 1 + # The event is a raw dict, not a tuple. + assert isinstance(events[0].payload, dict) + assert events[0].payload == {"filename": "flag.txt"} + finally: + await manager.unsubscribe(1, key) + + +async def _grab_tokens(stream, tokens: list, *, n: int) -> None: + """Pull ``(event, token)`` pairs off ``stream`` until ``tokens`` holds ``n`` items.""" + async for _raw, token in stream: + tokens.append(token) + if len(tokens) >= n: + break + + +@pytest.mark.asyncio +async def test_out_of_order_resolve_still_advances_in_fanout_order(): + """Events resolved out of order are advanced strictly in fan-out order.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": 1}, "p1"), + ({"n": 2}, "p2"), + ({"n": 3}, "p3"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=3), timeout=1.0) + + # Resolve events 2 and 3 first — nothing may advance while event 1 is pending. + await tokens[1].ack() + await tokens[2].ack() + await asyncio.sleep(0) + assert cls.advanced == [], "no advance may run while the head event is unresolved" + + await tokens[0].ack() + await asyncio.sleep(0) + assert cls.advanced == ["p1", "p2", "p3"], "advances must run in fan-out order" + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_pump_serializes_advance_calls(): + """Two events resolved together never produce overlapping advance calls.""" + release = asyncio.Event() + state = {"in_flight": 0, "max_in_flight": 0} + advanced: list = [] + + class _BlockingAdvanceProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + yield {"n": 2}, "p2" + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + state["in_flight"] += 1 + state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) + await release.wait() + advanced.append(broker_payload) + state["in_flight"] -= 1 + + class _BlockingAdvanceTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _BlockingAdvanceProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _BlockingAdvanceTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) + + # Resolve both events; the pump starts the first advance, which blocks. + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) + assert state["in_flight"] == 1, "exactly one advance may be in flight" + assert advanced == [] + + release.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(advanced) >= 2: + break + await asyncio.sleep(0) + + assert advanced == ["p1", "p2"] + assert state["max_in_flight"] == 1, "advance calls must never overlap" + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_lanes_advance_independently(): + """A lane whose events are resolved does not wait for another lane's unresolved head.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "b2"}, "b2"), + ], + lane_for=lambda payload: payload[0], + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + group = manager._groups[key] + + async with _consume_in_background(s1) as collector: + await collector.wait_for(4) + tokens = collector.tokens + + # Resolve lane "b" completely while lane "a" is fully unresolved. + await tokens[1].ack() # b1 + await tokens[3].ack() # b2 + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" + + await tokens[0].ack() # a1 + await tokens[2].ack() # a2 + for _ in range(5): + await asyncio.sleep(0) + + # Contract: full per-lane projections in fan-out order. The global + # interleaving across lanes is a pump implementation detail. + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] + assert len(cls.advanced) == 4 + # All lanes drained → the per-lane index is garbage-collected. + assert group._lane_queues == {} + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_lane_internal_fifo_with_out_of_order_resolve(): + """Out-of-order resolves within a lane still advance in fan-out order, per lane.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "a3"}, "a3"), + ], + lane_for=lambda payload: payload[0], + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + async with _consume_in_background(s1) as collector: + await collector.wait_for(4) + tokens = collector.tokens + + # Resolve a2 and a3 (head a1 stays pending) plus the noise lane. + await tokens[2].ack() # a2 + await tokens[3].ack() # a3 + await tokens[1].ack() # b1 + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" + + await tokens[0].ack() # a1 + for _ in range(5): + await asyncio.sleep(0) + + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] + assert len(cls.advanced) == 4 + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_max_in_flight_one_across_lanes(): + """Even with multiple lanes ready, at most one advance call is ever in flight.""" + release = asyncio.Event() + state = {"in_flight": 0, "max_in_flight": 0} + advanced: list = [] + + class _BlockingLaneProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": "a1"}, "a1" + yield {"n": "b1"}, "b1" + await asyncio.Event().wait() + + def get_advance_lane(self, broker_payload): + return broker_payload[0] + + async def advance(self, broker_payload, outcome): + state["in_flight"] += 1 + state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) + await release.wait() + advanced.append(broker_payload) + state["in_flight"] -= 1 + + class _BlockingLaneTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _BlockingLaneProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _BlockingLaneTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + async with _consume_in_background(s1) as collector: + await collector.wait_for(2) + tokens = collector.tokens + + # Resolve both lanes; the pump starts one advance, which blocks. + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) + assert state["in_flight"] == 1, "the other lane's advance must not start while one is in flight" + assert advanced == [] + + release.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(advanced) >= 2: + break + await asyncio.sleep(0) + + assert sorted(advanced) == ["a1", "b1"] + assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_empty_snapshot_routes_through_lane(): + """Born-resolved events (no subscribers online) keep per-lane fan-out order.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "b2"}, "b2"), + ], + lane_for=lambda payload: payload[0], + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + # Subscribe then immediately unsubscribe so the group exists but has zero + # subscribers when the events arrive. + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + manager._groups[key].unsubscribe(1) # remove without stopping the poll task + + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(cls.advanced) >= 4: + break + await asyncio.sleep(0.01) + + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] + assert len(cls.advanced) == 4 + assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * 4 + + del s1 # silence unused-variable warning + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_get_advance_lane_raise_terminates_poll(): + """A get_advance_lane that raises fails the poll like any other poll failure.""" + proceed_flag = asyncio.Event() + aclose_calls: list[int] = [] + advanced: list = [] + + class _RaisingLaneProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + await proceed_flag.wait() + yield {"n": 2}, "p2" + await asyncio.Event().wait() # pragma: no cover + + def get_advance_lane(self, broker_payload): + if broker_payload == "p2": + raise RuntimeError("lane boom") + return None + + async def advance(self, broker_payload, outcome): + advanced.append(broker_payload) + + async def aclose(self): + aclose_calls.append(1) + + class _RaisingLaneTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RaisingLaneProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _RaisingLaneTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + poll_task = manager._groups[key]._poll_task + assert poll_task is not None + + # Consume and ack the first event, then release the event whose + # get_advance_lane raises. + events1 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) + assert len(events1) == 1 + proceed_flag.set() + + async def consume(): + async for _ in s1: + pass + + with pytest.raises(RuntimeError, match="lane boom"): + await asyncio.wait_for(consume(), timeout=1.0) + + assert key not in manager._groups, "the group must be evicted when get_advance_lane raises" + await asyncio.wait_for(poll_task, timeout=1.0) + assert sum(aclose_calls) == 1 + # The first event was resolved before the failure → drained on the stop path. + assert advanced == ["p1"] + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_stop_drains_all_lanes(): + """On stop the pump keeps draining until every resolved event in every lane is advanced.""" + release = asyncio.Event() + advanced: list = [] + + class _SlowFirstAdvanceProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": "a1"}, "a1" + yield {"n": "a2"}, "a2" + yield {"n": "b1"}, "b1" + yield {"n": "b2"}, "b2" + await asyncio.Event().wait() + + def get_advance_lane(self, broker_payload): + return broker_payload[0] + + async def advance(self, broker_payload, outcome): + if broker_payload == "a1": + await release.wait() + advanced.append(broker_payload) + + class _SlowFirstAdvanceTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _SlowFirstAdvanceProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _SlowFirstAdvanceTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + + # Start a1's advance and let it block in flight. + await tokens[0].ack() + for _ in range(5): + await asyncio.sleep(0) + + # Resolve everything else while a1's advance is still in flight, then + # stop the group before the pump has a chance to dispatch them. + await tokens[1].ack() + await tokens[2].ack() + await tokens[3].ack() + stop_task = asyncio.create_task(manager.unsubscribe(1, key)) + for _ in range(5): + await asyncio.sleep(0) + release.set() + await asyncio.wait_for(stop_task, timeout=1.0) + + # Drain-on-stop: a single harvesting pass would miss a2 and b2, which + # only become lane heads after the first pass already visited their lanes. + assert [p for p in advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in advanced if p[0] == "b"] == ["b1", "b2"] + assert len(advanced) == 4 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("second_action", "expected", "expected_clean"), + [ + pytest.param("ack", AdvanceOutcome(acked=2, nacked=0, failed=0), True, id="all-ack"), + pytest.param("nack", AdvanceOutcome(acked=1, nacked=1, failed=0), False, id="ack-plus-nack"), + pytest.param( + "unsubscribe", + AdvanceOutcome(acked=2, nacked=0, failed=0), + True, + id="unsubscribe-counts-as-acked", + ), + ], +) +async def test_outcome_classification(second_action, expected, expected_clean): + """Each subscriber online at broadcast time is counted in exactly one outcome field.""" + cls = _make_ack_required_trigger_class([({"msg": "outcome"}, "receipt-outcome")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + tokens: list[AckToken] = [] + await asyncio.gather( + asyncio.wait_for(_grab_tokens(s1, tokens, n=1), timeout=1.0), + asyncio.wait_for(_grab_tokens(s2, tokens, n=2), timeout=1.0), + ) + + await tokens[0].ack() + if second_action == "ack": + await tokens[1].ack() + elif second_action == "nack": + await tokens[1].nack() + else: + await manager.unsubscribe(2, key) + await asyncio.sleep(0) + + assert cls.advanced == ["receipt-outcome"] + assert cls.outcomes == [expected] + assert cls.outcomes[0].is_clean is expected_clean + # Invariant: every subscriber in the broadcast snapshot is counted exactly once. + outcome = cls.outcomes[0] + assert outcome.acked + outcome.nacked + outcome.failed == 2 + finally: + await manager.unsubscribe(1, key) + if second_action != "unsubscribe": + await manager.unsubscribe(2, key) + + +@pytest.mark.asyncio +async def test_outcome_counts_timeout_as_failed(): + """A subscriber force-failed by the ack timeout is counted as failed in the outcome.""" + cls = _make_ack_required_trigger_class([({"msg": "timeout-outcome"}, "receipt-to")]) + cls.advanced.clear() + cls.outcomes.clear() + + t_fast, t_slow = cls(), cls() + key = t_fast.shared_stream_key() + + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + try: + s_fast = manager.subscribe(trigger_id=1, trigger=t_fast, key=key) + s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) + + tokens: list[AckToken] = [] + await asyncio.gather( + asyncio.wait_for(_grab_tokens(s_fast, tokens, n=1), timeout=1.0), + # Pull the (event, token) pair off the slow stream but never resolve it. + asyncio.wait_for(_grab_tokens(s_slow, tokens, n=2), timeout=1.0), + ) + + await tokens[0].ack() + fake_clock[0] = 0.2 + # Let the timeout loop tick (cadence = 0.01s real) and the pump dispatch. + await asyncio.sleep(0.05) + + assert cls.advanced == ["receipt-to"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] + assert not cls.outcomes[0].is_clean + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_double_resolve_counts_once(): + """ack() then nack() on the same token counts the subscriber once, as acked.""" + cls = _make_ack_required_trigger_class([({"msg": "double"}, "receipt-double-resolve")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=1), timeout=1.0) + + await tokens[0].ack() + await tokens[0].nack() + await asyncio.sleep(0) + + assert cls.advanced == ["receipt-double-resolve"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_aclose_called_once_on_stop_path(): + """The producer is closed exactly once when the last subscriber leaves.""" + cls = _make_ack_required_trigger_class([({"msg": "bye"}, "receipt-bye")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + events = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) + assert len(events) == 1 + + await manager.unsubscribe(1, key) # last subscriber → group stop + assert cls.aclose_calls == 1 + + +@pytest.mark.asyncio +async def test_aclose_called_once_on_terminal_path(): + """The producer is closed exactly once when its open_stream raises.""" + aclose_calls: list[int] = [] + + class _FailingOpenProducer(SharedStreamProducer): + async def open_stream(self): + raise RuntimeError("open boom") + yield # pragma: no cover + + async def advance(self, broker_payload, outcome): # pragma: no cover + pass + + async def aclose(self): + aclose_calls.append(1) + + class _FailingOpenTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _FailingOpenProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, _token in shared_stream: # pragma: no cover + yield TriggerEvent(raw) + + t1 = _FailingOpenTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + poll_task = manager._groups[key]._poll_task + assert poll_task is not None + + async def consume(): + async for _ in stream: + pass + + with pytest.raises(RuntimeError, match="open boom"): + await asyncio.wait_for(consume(), timeout=1.0) + + assert key not in manager._groups + # The poll swallows the terminal exception after broadcasting it; once the + # task completes, the producer must have been closed exactly once. + await asyncio.wait_for(poll_task, timeout=1.0) + assert sum(aclose_calls) == 1 + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_pump_continues_after_advance_raises(): + """An advance that raises is logged; the pump still advances the next event.""" + attempted: list = [] + + class _FirstRaiseProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + yield {"n": 2}, "p2" + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + attempted.append(broker_payload) + if broker_payload == "p1": + raise RuntimeError("p1 boom") + + class _FirstRaiseTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _FirstRaiseProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _FirstRaiseTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + group = manager._groups[key] + group.log = MagicMock() + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) + + assert attempted == ["p1", "p2"], "the pump must move on to event 2 after event 1's advance raised" + error_calls = group.log.error.mock_calls + assert len(error_calls) == 1 + assert "broker advance failed" in error_calls[0].args[0] + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_force_fail_clears_subscriber_from_all_outstanding_entries(): + """A force-failed subscriber stops head-blocking earlier events' advances.""" + gate = asyncio.Event() + + class _GatedProducer(SharedStreamProducer): + def __init__(self, trigger_cls): + self._trigger_cls = trigger_cls + + async def open_stream(self): + yield {"n": 1}, "p1" + yield {"n": 2}, "p2" + await gate.wait() + yield {"n": 3}, "p3" + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + self._trigger_cls.advanced.append(broker_payload) + self._trigger_cls.outcomes.append(outcome) + + class _GatedTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + outcomes: list[AdvanceOutcome] = [] + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _GatedProducer(cls) + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t_dead, t_live = _GatedTrigger(), _GatedTrigger() + key = t_dead.shared_stream_key() + # maxsize=2: the dead subscriber's queue fills with events 1-2 and overflows + # at event 3's fan-out. The default 300s ack timeout would stall the test + # if force-fail did not clear the dead subscriber from all entries. + manager = SharedStreamManager(max_subscriber_queue=2) + try: + manager.subscribe(trigger_id=1, trigger=t_dead, key=key) + s_live = manager.subscribe(trigger_id=2, trigger=t_live, key=key) + + tokens: list[AckToken] = [] + # Live subscriber acks events 1 and 2; the dead subscriber never consumes. + await asyncio.wait_for(_grab_tokens(s_live, tokens, n=2), timeout=1.0) + await tokens[0].ack() + await tokens[1].ack() + await asyncio.sleep(0) + assert _GatedTrigger.advanced == [], "events 1-2 still wait on the dead subscriber" + + # Event 3 overflows the dead subscriber's queue → force-fail clears it + # from ALL outstanding entries, so events 1-2 advance without waiting + # for their ack timeouts. + gate.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(_GatedTrigger.advanced) >= 2: + break + await asyncio.sleep(0) + assert _GatedTrigger.advanced == ["p1", "p2"] + assert _GatedTrigger.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] * 2 + + # The live subscriber resolves event 3 and it advances too. + await asyncio.wait_for(_grab_tokens(s_live, tokens, n=3), timeout=1.0) + await tokens[2].ack() + await asyncio.sleep(0) + assert _GatedTrigger.advanced == ["p1", "p2", "p3"] + assert _GatedTrigger.outcomes[2] == AdvanceOutcome(acked=1, nacked=0, failed=1) + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_producer_factory_failure_is_terminal(): + """create_shared_stream_producer raising fails subscribers and evicts the group.""" + + class _FactoryBoomTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + raise RuntimeError("factory boom") + + async def filter_shared_stream(self, shared_stream): + async for raw, _token in shared_stream: # pragma: no cover + yield TriggerEvent(raw) + + t1 = _FactoryBoomTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + async def consume(): + async for _ in stream: + pass + + with pytest.raises(RuntimeError, match="factory boom"): + await asyncio.wait_for(consume(), timeout=1.0) + + assert key not in manager._groups + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_stop_drains_resolved_prefix_and_abandons_unresolved(): + """stop() waits for the resolved prefix's advances; unresolved events are abandoned.""" + advance_started = asyncio.Event() + advance_release = asyncio.Event() + advanced: list = [] + aclose_calls: list[int] = [] + + class _PrefixProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + yield {"n": 2}, "p2" + await asyncio.Event().wait() + + async def advance(self, broker_payload, outcome): + advance_started.set() + await advance_release.wait() + advanced.append(broker_payload) + + async def aclose(self): + aclose_calls.append(1) + + class _PrefixTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _PrefixProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _PrefixTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) + + # Resolve only event 1; its advance starts and blocks. + await tokens[0].ack() + await asyncio.wait_for(advance_started.wait(), timeout=1.0) + + # stop must wait for the in-flight advance of the resolved event... + stop_task = asyncio.create_task(manager.stop_all()) + await asyncio.sleep(0) + assert not stop_task.done(), "stop must wait for the in-flight advance" + + advance_release.set() + await asyncio.wait_for(stop_task, timeout=1.0) + + # ...and must not advance the unresolved event 2 (left to broker redelivery). + assert advanced == ["p1"] + assert sum(aclose_calls) == 1, "the producer must still be closed after the drain" +<<<<<<< HEAD +======= + + +async def _pull_pair(stream_iter): + """Pull one ``(raw_event, token)`` pair off an ack-mode stream iterator.""" + return await asyncio.wait_for(stream_iter.__anext__(), timeout=1.0) + + +async def _resume_past_yield(stream_iter) -> asyncio.Task: + """ + Resume the drain generator past its yield and leave it waiting. + + Resuming closes the previous token's binding window — the exact moment a + production filter loops back to pull the next raw event. The returned + task is still waiting on the queue; the test must cancel it (or expect + its failure) during cleanup. + """ + task = asyncio.create_task(stream_iter.__anext__()) + await asyncio.sleep(0) + return task + + +async def _cancel_quietly(task: asyncio.Task | None) -> None: + """Cancel a pending stream pull, swallowing its cancellation or failure.""" + if task is None: + return + task.cancel() + with suppress(asyncio.CancelledError, AckTimeout): + await task + + +@pytest.mark.asyncio +async def test_multi_event_binding_gates_advance_on_every_seq(): + """Two trigger events bound to one raw event both need confirmation before advance.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + # The subscriber derives two trigger events from the same raw event. + seq1 = manager.bind_pending_event(trigger_id=1, key=key) + seq2 = manager.bind_pending_event(trigger_id=1, key=key) + assert seq1 is not None + assert seq2 is not None + assert seq1 != seq2 + + await token.ack() + next_task = await _resume_past_yield(it) + assert cls.advanced == [], "advance must wait for both persist confirmations" + + manager.confirm_persisted([seq1]) + await asyncio.sleep(0) + assert cls.advanced == [], "one of two confirmations is not enough" + + manager.confirm_persisted([seq2]) + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_filtered_out_event_resolves_on_next_pull_without_confirmation(): + """An acked event with no bound trigger events resolves as soon as the filter pulls the next raw event.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + # Filtered out: ack, derive nothing, loop back for the next raw event. + await token.ack() + next_task = await _resume_past_yield(it) + await asyncio.sleep(0) + + assert cls.advanced == ["p1"], "a filtered-out event must resolve without any confirmation" + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_nack_with_bound_seq_resolves_immediately_and_late_confirm_is_noop(): + """nack() resolves without waiting for confirmations; a confirmation arriving later is ignored.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + + # nack resolves immediately — no window close, no confirmation needed. + await token.nack() + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=1, failed=0)] + + # The late confirmation finds nothing and no-ops. + manager.confirm_persisted([seq]) + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=1, failed=0)] + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_ack_before_yield_still_gates_advance_on_confirmation(): + """Regression: an ack arriving before the event is bound must not bypass the persist gate.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + # ack-before-yield: the ack lands while the binding window is still + # open and nothing is bound yet. + await token.ack() + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + + next_task = await _resume_past_yield(it) + assert cls.advanced == [], "the early ack must not resolve ahead of the confirmation" + + manager.confirm_persisted([seq]) + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_unsubscribe_with_unconfirmed_seq_waits_for_confirmation(): + """Regression: a subscriber leaving right after producing an event must not bypass the persist gate.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + it1 = s1.__aiter__() + it2 = s2.__aiter__() + + # Subscriber 1 fires (binds a trigger event) and immediately leaves. + _raw, _token1 = await _pull_pair(it1) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + await manager.unsubscribe(1, key) + + # Subscriber 2 resolves cleanly. + _raw, token2 = await _pull_pair(it2) + await token2.ack() + next_task = await _resume_past_yield(it2) + await asyncio.sleep(0) + + assert cls.advanced == [], "the departed subscriber's unconfirmed event must hold the advance" + assert len(manager._groups[key]._outstanding) == 1 + + manager.confirm_persisted([seq]) + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + # The departed subscriber still counts as acked (implicit ack-out). + assert cls.outcomes == [AdvanceOutcome(acked=2, nacked=0, failed=0)] + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(2, key) + + +@pytest.mark.asyncio +async def test_unconfirmed_persist_below_timeout_keeps_event_outstanding(): + """At 99 ms of a 100 ms timeout, an unconfirmed event stays outstanding and can still recover.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + await token.ack() + next_task = await _resume_past_yield(it) + + # Strictly inside the timeout window: the event must stay outstanding. + fake_clock[0] = 0.099 + await asyncio.sleep(0.05) + assert cls.advanced == [] + assert len(manager._groups[key]._outstanding) == 1 + + # The confirmation arrives inside the window — clean recovery. + manager.confirm_persisted([seq]) + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + finally: + await _cancel_quietly(next_task) + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_unconfirmed_persist_at_timeout_fails_subscriber(): + """A confirmation that never arrives fails the event at the ack timeout (elapsed >= timeout).""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + await token.ack() + next_task = await _resume_past_yield(it) + + # Exactly at the timeout: the unconfirmed event is force-failed. + fake_clock[0] = 0.1 + await asyncio.sleep(0.05) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=0, failed=1)] + + # The still-online subscriber sees the AckTimeout; the confirmation + # for the bound sequence number never arrives. + with pytest.raises(AckTimeout): + await asyncio.wait_for(next_task, timeout=1.0) + next_task = None + finally: + await _cancel_quietly(next_task) + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_late_confirm_after_timeout_is_noop(): + """A confirmation arriving after the timeout already failed the event changes nothing.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + await token.ack() + next_task = await _resume_past_yield(it) + + fake_clock[0] = 0.2 + await asyncio.sleep(0.05) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=0, failed=1)] + + # The confirmation arrives after the fact: no second advance, no + # change to the recorded outcome. + manager.confirm_persisted([seq]) + await asyncio.sleep(0) + assert cls.advanced == ["p1"] + assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=0, failed=1)] + finally: + await _cancel_quietly(next_task) + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_group_stop_abandons_unconfirmed_advance(): + """The last subscriber leaving with an unconfirmed event stops the group without advancing.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + it = stream.__aiter__() + _raw, token = await _pull_pair(it) + + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + await token.ack() + + # Last subscriber leaves while the confirmation is still outstanding: + # the group stops and the advance is abandoned — the broker redelivers. + await manager.unsubscribe(1, key) + assert manager._groups == {} + assert cls.advanced == [], "an unconfirmed advance must be abandoned at group stop" + assert cls.aclose_calls == 1 +>>>>>>> e927a603b1 (fixup! fixup! feat(triggers): add producer-side ack channel to shared-stream triggers) From 84af8fbb01bfb23e4ce873d774a4798af5eee19d Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 11 Jun 2026 16:43:37 +0800 Subject: [PATCH 02/15] feat(triggers): dispatch shared-stream broker advances in per-lane order Add `SharedStreamProducer.get_advance_lane` (default: a single global lane, preserving the previous behavior). Advances are dispatched one at a time, in fan-out order within a lane; different lanes do not wait for one another. A Kafka producer can return `(topic, partition)` as the lane so each partition commits `offset + 1` independently without waiting on other partitions. --- .../event-scheduling.rst | 52 ++- .../src/airflow/triggers/shared_stream.py | 150 +++++--- .../tests/unit/triggers/test_shared_stream.py | 338 +++++++++++++++++- 3 files changed, 477 insertions(+), 63 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index 821b2a7b0f79b..adb8d19f05fce 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -187,10 +187,15 @@ When this factory is overridden, the manager enters **ack mode**: :py:class:`~airflow.triggers.shared_stream.AdvanceOutcome` carrying per-event counts of how the subscribers resolved. -**Ordering guarantee**: advance calls are dispatched strictly in event -order — ``advance`` for event N is awaited only after event N-1's -``advance`` returned (or failed and was logged). When the poll ends, the -manager calls ``await producer.aclose()`` once, best-effort. +**Ordering guarantee**: by default every event belongs to the same lane, +so advance calls are dispatched strictly in event order — ``advance`` for +event N is awaited only after event N-1's ``advance`` returned (or failed +and was logged). A producer can override ``get_advance_lane`` to narrow +that ordering to within a lane: events whose lane values compare equal +are advanced in event order relative to each other, while events in +different lanes do not wait for one another. Either way, at most one +``advance`` call is awaited at a time. When the poll ends, the manager +calls ``await producer.aclose()`` once, best-effort. Example — SQS-like producer: @@ -257,27 +262,37 @@ Example — SQS-like producer: async def run(self): # pragma: no cover yield TriggerEvent({}) -Example — Kafka cumulative commit. A Kafka commit acknowledges every offset -up to the committed one, so it is only safe if no later event can be -committed while an earlier one is still pending. The ordering guarantee -above provides exactly that, which makes ``commit(offset + 1)`` correct: +Example — Kafka cumulative commit across partitions. A Kafka commit +acknowledges every offset up to the committed one within a partition, so +it is only safe if no later event from the same partition can be committed +while an earlier one is still pending — events on other partitions do not +matter. Returning ``(topic, partition)`` from ``get_advance_lane`` narrows +the ordering guarantee to exactly that granularity: each partition's +commits stay in order, and a slow partition no longer delays commits on +the other partitions: .. code-block:: python class KafkaSharedStreamProducer(SharedStreamProducer): - def __init__(self, topic: str): - self.topic = topic + def __init__(self, topics: list[str]): + self.topics = topics self.consumer = None async def open_stream(self): - self.consumer = await create_kafka_consumer(self.topic) + self.consumer = await create_kafka_consumer(self.topics) async for message in self.consumer: - yield message.value, message.offset + yield message.value, (message.topic, message.partition, message.offset) + + def get_advance_lane(self, broker_payload): + topic, partition, _offset = broker_payload + return topic, partition async def advance(self, broker_payload, outcome): - # Advance calls arrive in event order, so this cumulative commit - # can never skip past an event that is still pending. - await self.consumer.commit(broker_payload + 1) + topic, partition, offset = broker_payload + # Within one lane — here, one partition — advance calls arrive in + # event order, so this cumulative commit can never skip past an + # event that is still pending. + await self.consumer.commit(topic, partition, offset + 1) async def aclose(self): if self.consumer is not None: @@ -321,9 +336,10 @@ still governs unprocessed raw events per subscriber. The manager does **not** wait for outstanding acks before pulling the next upstream event; back-pressure is queue-bound — a subscriber whose queue is full is force-failed. The queue primarily guards against burst delivery before a -subscriber's filter runs. Broker advances, by contrast, are dispatched -strictly in event order: an event whose acks are still pending delays the -advance of every later event, with the ack timeout bounding that wait. +subscriber's filter runs. Broker advances, by contrast, are dispatched in +order within each lane: an event whose acks are still pending delays the +advance of every later event in the same lane, with the ack timeout +bounding that wait. Verifying that sharing is active ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index f558669c3f751..6f4974e661f2c 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -47,10 +47,14 @@ ``await producer.advance(broker_payload, outcome)`` exactly once — this is where the producer commits / deletes / acks on the broker. The :class:`AdvanceOutcome` carries per-event counts of how the subscribers -resolved. Advance calls are dispatched by a single pump task strictly in -fan-out order: event N's ``advance`` is awaited only after event N-1's -``advance`` returned (or raised and was logged). When the poll ends, the -manager awaits ``producer.aclose()`` once, best-effort. +resolved. Advance calls are dispatched by a single pump task: the +producer's :meth:`~SharedStreamProducer.get_advance_lane` assigns each +event to a lane, events in the same lane are advanced strictly in fan-out +order, and events in different lanes do not wait for one another. At any +moment at most one ``advance`` call is awaited globally; the default lane +assignment (every event in the same lane) preserves the original single +global order. When the poll ends, the manager awaits ``producer.aclose()`` +once, best-effort. **Snapshot-at-fan-out**: the set of subscribers that must ack an event is frozen at broadcast time. A subscriber that joins after the event was @@ -73,9 +77,9 @@ full is force-failed via :class:`_SubscriberOverflow`. The queue mainly protects against burst delivery before a subscriber's filter has had a chance to run. Broker advances, however, are dispatched by a single pump -task strictly in event order: while the head event's subscribers are still -pending, every later advance waits (the ack timeout is the backstop that -bounds this head-of-line wait). +task in per-lane order: while a lane's head event still has pending +subscribers, every later advance in that same lane waits (the ack timeout +is the backstop that bounds this per-lane head-of-line wait). Triggers that do **not** override ``create_shared_stream_producer`` run the **fast path**: no event IDs, no ack table, no AckToken — subscribers @@ -114,6 +118,7 @@ import time import weakref from abc import ABC, abstractmethod +from collections import deque from collections.abc import AsyncGenerator, AsyncIterator, Callable, Hashable from contextlib import suppress from dataclasses import dataclass @@ -247,14 +252,39 @@ async def advance(self, broker_payload: Any, outcome: AdvanceOutcome) -> None: """ Advance the broker for one fully resolved event. - Called with each event's ``broker_payload`` strictly in fan-out - order, one at a time: the call for event N is awaited only after the - call for event N-1 returned (or raised and was logged). This makes - cumulative schemes such as a Kafka offset commit safe. ``outcome`` - carries the per-event resolution counts; use it to decide whether to - commit, skip, or trigger a broker-side redeliver. + Within a lane — events for which :meth:`get_advance_lane` returned + equal values — calls arrive strictly in fan-out order: the call for + event N is awaited only after the call for event N-1 returned (or + raised and was logged). Across lanes the relative order is not + guaranteed, but at any moment at most one ``advance`` call is + awaited globally. This makes cumulative schemes such as a Kafka + offset commit safe within a lane. ``outcome`` carries the per-event + resolution counts; use it to decide whether to commit, skip, or + trigger a broker-side redeliver. """ + def get_advance_lane(self, broker_payload: Any) -> Hashable: + """ + Return the advance lane for one event. + + Events whose lane values compare equal are advanced strictly in + fan-out order relative to each other; events in different lanes do + not wait for one another. At any moment at most one ``advance`` call + is awaited globally, regardless of how many lanes exist. The default + implementation puts every event in the same lane, which preserves the + single global fan-out order. + + Called synchronously once per event before fan-out, so it must be + cheap (O(1)) and must not block. If it raises, the whole poll is + treated as failed: the group terminates and the error propagates to + every subscriber. + + Example: a Kafka producer can return ``(topic, partition)`` here — + a cumulative offset commit only needs ordering within a partition, + so a slow partition no longer delays commits on the others. + """ + return None + async def aclose(self) -> None: """ Release broker resources; called once when the poll ends. @@ -271,6 +301,7 @@ class _OutstandingEntry: pending: set[int] # trigger_ids still awaiting ack created_at: float # monotonic clock at broadcast (via injectable _now) broker_payload: Any + lane: Hashable # producer.get_advance_lane(broker_payload), taken at fan-out # Resolution counts; together they form the event's AdvanceOutcome. acked: int = 0 nacked: int = 0 @@ -381,9 +412,12 @@ def __init__( # Ack mode state — populated only when create_shared_stream_producer # is overridden. self._outstanding: dict[int, _OutstandingEntry] = {} + # Per-lane FIFO index over _outstanding: event ids in fan-out order, + # keyed by the lane each event's producer assigned at fan-out. + self._lane_queues: dict[Hashable, deque[int]] = {} self._next_event_id: int = 0 self._ack_timeout_task: asyncio.Task | None = None - # Advance pump: a single task dispatches broker advances strictly in + # Advance pump: a single task dispatches broker advances in per-lane # fan-out order, woken through this event whenever an entry resolves. self._advance_wakeup: asyncio.Event = asyncio.Event() self._pump_stopping: bool = False @@ -400,7 +434,7 @@ def start(self) -> None: def _request_pump_stop(self) -> None: """ - Ask the advance pump to drain the already-resolved prefix and exit. + Ask the advance pump to drain the already-resolved lane heads and exit. Synchronous (no await) so it can run inside ``_poll``'s terminal section without yielding. @@ -410,34 +444,54 @@ def _request_pump_stop(self) -> None: async def _run_advance_pump(self, producer: SharedStreamProducer) -> None: """ - Dispatch broker advances strictly in fan-out order, one at a time. + Dispatch broker advances in per-lane fan-out order, one at a time. - A single task harvests the resolved prefix of ``_outstanding`` (dict - insertion order is event-id order) and awaits ``producer.advance`` - for each entry in turn; an advance that raises is logged and the - pump moves on to the next entry. On stop the pump finishes the - resolved prefix, abandons unresolved entries to broker redelivery, - and exits. + A single task scans the lanes round-robin: each pass dispatches at + most one resolved head per lane, and passes repeat until one makes + no progress. An advance that raises is logged and the pump moves on. + On stop the pump keeps passing until every already-resolved + dispatchable event is drained, abandons unresolved entries to broker + redelivery, and exits. """ while True: if not self._pump_stopping: await self._advance_wakeup.wait() self._advance_wakeup.clear() - while self._outstanding: - event_id = next(iter(self._outstanding)) - entry = self._outstanding[event_id] - if entry.pending: - break - del self._outstanding[event_id] - outcome = AdvanceOutcome(acked=entry.acked, nacked=entry.nacked, failed=entry.failed) - try: - await producer.advance(entry.broker_payload, outcome) - except Exception as exc: - self.log.error( - "Producer advance raised; broker advance failed", - key=self.key, - exc_info=exc, - ) + progress = True + while progress: + progress = False + # Snapshot: lanes are added (by _poll) while we await below. + # A lane inserted during an await is invisible to this pass, + # but its appearance always comes with a wakeup.set() (born + # resolved) or a later resolve, so the next progress pass or + # the next wakeup picks it up — nothing is lost. A no-progress + # pass costs O(#lanes) and falls back to wait(). + for lane in list(self._lane_queues): + lane_queue = self._lane_queues[lane] + # The deque head is always present in _outstanding: entry + # deletion belongs to the pump alone, and it removes the + # deque slot and the dict entry together below — a + # KeyError here would be a bug. + head_id = lane_queue[0] + entry = self._outstanding[head_id] + if entry.pending: + continue + lane_queue.popleft() + del self._outstanding[head_id] + if not lane_queue: + # Sole lane GC point; _poll recreates the lane on demand. + del self._lane_queues[lane] + outcome = AdvanceOutcome(acked=entry.acked, nacked=entry.nacked, failed=entry.failed) + try: + await producer.advance(entry.broker_payload, outcome) + except Exception as exc: + self.log.error( + "Producer advance raised; broker advance failed", + key=self.key, + lane=lane, + exc_info=exc, + ) + progress = True if self._pump_stopping: return @@ -461,6 +515,9 @@ async def _poll(self) -> None: # A factory failure flows through the terminal broadcast # path below, like any other poll failure. producer = self.trigger_class.create_shared_stream_producer(self.kwargs) + # Non-Optional alias for the fan-out section below; ``producer`` + # itself stays Optional for the ``finally`` aclose. + ack_producer: SharedStreamProducer = producer self._pump_task = asyncio.create_task( self._run_advance_pump(producer), name=f"shared-stream-advance-pump[{self.key!r}]", @@ -475,6 +532,13 @@ async def _poll(self) -> None: async for item in event_source: if ack_required: raw_event, broker_payload = item + # If get_advance_lane raises — or returns an unhashable + # lane, which the setdefault below trips on — the event + # has no entry yet and the exception flows through the + # terminal broadcast path below, like any other poll + # failure. + lane = ack_producer.get_advance_lane(broker_payload) + lane_queue = self._lane_queues.setdefault(lane, deque()) # Snapshot the subscriber set at fan-out time. snapshot = set(self._subscribers.keys()) - self._failed_subscribers event_id = self._next_event_id @@ -483,11 +547,13 @@ async def _poll(self) -> None: pending=snapshot.copy(), created_at=self._now(), broker_payload=broker_payload, + lane=lane, ) + lane_queue.append(event_id) if not snapshot: # No subscribers to ack — the entry is born resolved; # route it through the pump so broker advances stay - # in fan-out order. + # in per-lane fan-out order. self._advance_wakeup.set() continue group_ref: weakref.ref[_SharedStreamGroup] = weakref.ref(self) @@ -533,7 +599,7 @@ async def _poll(self) -> None: if self._ack_timeout_task is not None and not self._ack_timeout_task.done(): self._ack_timeout_task.cancel() cancelled_ack_task = self._ack_timeout_task - # Synchronous: flags the pump to drain its resolved prefix and exit. + # Synchronous: flags the pump to drain its resolved lane heads and exit. self._request_pump_stop() if terminal_exc is not None: # Synchronous: evict from the manager and broadcast the @@ -550,8 +616,8 @@ async def _poll(self) -> None: with suppress(asyncio.CancelledError): await cancelled_ack_task if self._pump_task is not None: - # The pump exits by itself once it has dispatched the already - # resolved prefix; the suppress is defensive. + # The pump exits by itself once it has dispatched everything + # already resolved and dispatchable; the suppress is defensive. with suppress(asyncio.CancelledError): await self._pump_task if producer is not None: @@ -573,7 +639,7 @@ def _resolve_subscriber( Removes ``trigger_id`` from the pending set of ``event_id`` and adds it to the matching outcome count. If the set empties, wakes the advance pump — deleting the entry and dispatching the broker advance - are the pump's job, so advances stay in fan-out order. Duplicate + are the pump's job, so advances stay in per-lane fan-out order. Duplicate calls for the same (event_id, trigger_id) are no-ops, which keeps every subscriber counted exactly once per event. """ diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index 71d14bacdf954..ad6cdced51fbd 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -19,6 +19,7 @@ import asyncio import time import weakref +from collections.abc import Callable from contextlib import suppress from unittest.mock import MagicMock @@ -695,15 +696,21 @@ async def test_fail_overflowed_subscriber_drains_full_queue_before_putting_senti class _RecordingProducer(SharedStreamProducer): """Producer that records advance/aclose activity onto its trigger class.""" - def __init__(self, trigger_cls, events_with_payloads: list[tuple]): + def __init__(self, trigger_cls, events_with_payloads: list[tuple], *, lane_for: Callable | None = None): self._trigger_cls = trigger_cls self._events_with_payloads = events_with_payloads + self._lane_for = lane_for async def open_stream(self): for event, broker_payload in self._events_with_payloads: yield event, broker_payload await asyncio.Event().wait() + def get_advance_lane(self, broker_payload): + if self._lane_for is None: + return None + return self._lane_for(broker_payload) + async def advance(self, broker_payload, outcome): self._trigger_cls.advanced.append(broker_payload) self._trigger_cls.outcomes.append(outcome) @@ -712,7 +719,7 @@ async def aclose(self): self._trigger_cls.aclose_calls += 1 -def _make_ack_required_trigger_class(events_with_payloads: list[tuple]): +def _make_ack_required_trigger_class(events_with_payloads: list[tuple], *, lane_for: Callable | None = None): """ Return a fresh trigger class whose ``create_shared_stream_producer`` returns a :class:`_RecordingProducer` yielding ``(event, broker_payload)`` @@ -726,7 +733,7 @@ class _AckRequiredTrigger(_ProgrammableSharedStreamTrigger): @classmethod def create_shared_stream_producer(cls, kwargs): - return _RecordingProducer(cls, events_with_payloads) + return _RecordingProducer(cls, events_with_payloads, lane_for=lane_for) async def filter_shared_stream(self, shared_stream): async for raw, token in shared_stream: @@ -1783,6 +1790,331 @@ async def filter_shared_stream(self, shared_stream): await manager.unsubscribe(1, key) +# --------------------------------------------------------------------------- +# Advance lanes (get_advance_lane) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lanes_advance_independently(): + """A lane whose events are resolved does not wait for another lane's unresolved head.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "b2"}, "b2"), + ], + lane_for=lambda payload: payload[0], + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + group = manager._groups[key] + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + + # Resolve lane "b" completely while lane "a" is fully unresolved. + await tokens[1].ack() # b1 + await tokens[3].ack() # b2 + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" + + await tokens[0].ack() # a1 + await tokens[2].ack() # a2 + for _ in range(5): + await asyncio.sleep(0) + + # Contract: full per-lane projections in fan-out order. The global + # interleaving across lanes is a pump implementation detail. + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] + assert len(cls.advanced) == 4 + # All lanes drained → the per-lane index is garbage-collected. + assert group._lane_queues == {} + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_lane_internal_fifo_with_out_of_order_resolve(): + """Out-of-order resolves within a lane still advance in fan-out order, per lane.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "a3"}, "a3"), + ], + lane_for=lambda payload: payload[0], + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + + # Resolve a2 and a3 (head a1 stays pending) plus the noise lane. + await tokens[2].ack() # a2 + await tokens[3].ack() # a3 + await tokens[1].ack() # b1 + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" + + await tokens[0].ack() # a1 + for _ in range(5): + await asyncio.sleep(0) + + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] + assert len(cls.advanced) == 4 + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_max_in_flight_one_across_lanes(): + """Even with multiple lanes ready, at most one advance call is ever in flight.""" + release = asyncio.Event() + state = {"in_flight": 0, "max_in_flight": 0} + advanced: list = [] + + class _BlockingLaneProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": "a1"}, "a1" + yield {"n": "b1"}, "b1" + await asyncio.Event().wait() + + def get_advance_lane(self, broker_payload): + return broker_payload[0] + + async def advance(self, broker_payload, outcome): + state["in_flight"] += 1 + state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) + await release.wait() + advanced.append(broker_payload) + state["in_flight"] -= 1 + + class _BlockingLaneTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _BlockingLaneProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _BlockingLaneTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) + + # Resolve both lanes; the pump starts one advance, which blocks. + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) + assert state["in_flight"] == 1, "the other lane's advance must not start while one is in flight" + assert advanced == [] + + release.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(advanced) >= 2: + break + await asyncio.sleep(0) + + assert sorted(advanced) == ["a1", "b1"] + assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" + finally: + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_empty_snapshot_routes_through_lane(): + """Born-resolved events (no subscribers online) keep per-lane fan-out order.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "b2"}, "b2"), + ], + lane_for=lambda payload: payload[0], + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + # Subscribe then immediately unsubscribe so the group exists but has zero + # subscribers when the events arrive. + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + manager._groups[key].unsubscribe(1) # remove without stopping the poll task + + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(cls.advanced) >= 4: + break + await asyncio.sleep(0.01) + + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] + assert len(cls.advanced) == 4 + assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * 4 + + del s1 # silence unused-variable warning + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_get_advance_lane_raise_terminates_poll(): + """A get_advance_lane that raises fails the poll like any other poll failure.""" + proceed_flag = asyncio.Event() + aclose_calls: list[int] = [] + advanced: list = [] + + class _RaisingLaneProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + await proceed_flag.wait() + yield {"n": 2}, "p2" + await asyncio.Event().wait() # pragma: no cover + + def get_advance_lane(self, broker_payload): + if broker_payload == "p2": + raise RuntimeError("lane boom") + return None + + async def advance(self, broker_payload, outcome): + advanced.append(broker_payload) + + async def aclose(self): + aclose_calls.append(1) + + class _RaisingLaneTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RaisingLaneProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _RaisingLaneTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + poll_task = manager._groups[key]._poll_task + assert poll_task is not None + + # Consume and ack the first event, then release the event whose + # get_advance_lane raises. + events1 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) + assert len(events1) == 1 + proceed_flag.set() + + async def consume(): + async for _ in s1: + pass + + with pytest.raises(RuntimeError, match="lane boom"): + await asyncio.wait_for(consume(), timeout=1.0) + + assert key not in manager._groups, "the group must be evicted when get_advance_lane raises" + await asyncio.wait_for(poll_task, timeout=1.0) + assert sum(aclose_calls) == 1 + # The first event was resolved before the failure → drained on the stop path. + assert advanced == ["p1"] + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_stop_drains_all_lanes(): + """On stop the pump keeps draining until every resolved event in every lane is advanced.""" + release = asyncio.Event() + advanced: list = [] + + class _SlowFirstAdvanceProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": "a1"}, "a1" + yield {"n": "a2"}, "a2" + yield {"n": "b1"}, "b1" + yield {"n": "b2"}, "b2" + await asyncio.Event().wait() + + def get_advance_lane(self, broker_payload): + return broker_payload[0] + + async def advance(self, broker_payload, outcome): + if broker_payload == "a1": + await release.wait() + advanced.append(broker_payload) + + class _SlowFirstAdvanceTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _SlowFirstAdvanceProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) + + t1 = _SlowFirstAdvanceTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + tokens: list[AckToken] = [] + await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + + # Start a1's advance and let it block in flight. + await tokens[0].ack() + for _ in range(5): + await asyncio.sleep(0) + + # Resolve everything else while a1's advance is still in flight, then + # stop the group before the pump has a chance to dispatch them. + await tokens[1].ack() + await tokens[2].ack() + await tokens[3].ack() + stop_task = asyncio.create_task(manager.unsubscribe(1, key)) + for _ in range(5): + await asyncio.sleep(0) + release.set() + await asyncio.wait_for(stop_task, timeout=1.0) + + # Drain-on-stop: a single harvesting pass would miss a2 and b2, which + # only become lane heads after the first pass already visited their lanes. + assert [p for p in advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in advanced if p[0] == "b"] == ["b1", "b2"] + assert len(advanced) == 4 + + @pytest.mark.asyncio async def test_lanes_advance_independently(): """A lane whose events are resolved does not wait for another lane's unresolved head.""" From f61c14fbc67daa1f7fa3f24f93e30aeba2ca456a Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 11 Jun 2026 15:54:19 +0800 Subject: [PATCH 03/15] docs(triggers): drop redundant :py prefixes from Sphinx roles in event-scheduling doc --- .../authoring-and-scheduling/event-scheduling.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index adb8d19f05fce..ee45bea075bc4 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -79,7 +79,7 @@ cadence. See "Suitable upstreams" below for the precise scope. trigger keeps its own DB row, its own ``run_trigger`` task, and its own per-instance filtering. To participate, a subclass overrides three hooks: -* :py:meth:`~airflow.triggers.base.BaseEventTrigger.shared_stream_key` — return a key identifying the shared +* :meth:`~airflow.triggers.base.BaseEventTrigger.shared_stream_key` — return a key identifying the shared upstream (typically a tuple of strings). Triggers whose key compares equal will share one poll. Returning ``None`` (the default) opts out — the trigger runs its own independent ``run()`` loop, exactly as before. The return value is read **once** when the triggerer starts this trigger; changing it mid-lifetime has no effect on group @@ -87,11 +87,11 @@ subclass overrides three hooks: The key must be deterministic — derive it from configuration fields, never from per-call values such as ``time.time()`` or ``uuid.uuid4()``, because the comparison must be stable across the lifetime of the group. -* :py:meth:`~airflow.triggers.base.BaseEventTrigger.open_shared_stream` — a ``@classmethod`` coroutine the triggerer +* :meth:`~airflow.triggers.base.BaseEventTrigger.open_shared_stream` — a ``@classmethod`` coroutine the triggerer drives **once per shared-stream group** to yield raw events from the upstream. Because the triggerer reuses one trigger's kwargs to drive the shared poll, only rely on fields whose values participate in ``shared_stream_key``. -* :py:meth:`~airflow.triggers.base.BaseEventTrigger.filter_shared_stream` — an instance method that consumes the +* :meth:`~airflow.triggers.base.BaseEventTrigger.filter_shared_stream` — an instance method that consumes the broadcast raw stream and yields the ``TriggerEvent`` instances this trigger should fire. Per-trigger filtering (e.g. only events matching this instance's ``filename``) lives here. @@ -165,8 +165,8 @@ Producer-side ack channel For upstreams where the producer must advance (commit, delete, or ack) only after all subscribers have processed an event, override -:py:meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer` -to return a :py:class:`~airflow.triggers.shared_stream.SharedStreamProducer`. +:meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer` +to return a :class:`~airflow.triggers.shared_stream.SharedStreamProducer`. When this factory is overridden, the manager enters **ack mode**: 1. The manager calls ``create_shared_stream_producer(kwargs)`` once per @@ -184,7 +184,7 @@ When this factory is overridden, the manager enters **ack mode**: queue), the manager calls ``await producer.advance(broker_payload, outcome)`` — commit the offset, delete the SQS message, etc. The ``outcome`` is an - :py:class:`~airflow.triggers.shared_stream.AdvanceOutcome` carrying + :class:`~airflow.triggers.shared_stream.AdvanceOutcome` carrying per-event counts of how the subscribers resolved. **Ordering guarantee**: by default every event belongs to the same lane, @@ -370,7 +370,7 @@ If subscribers repeatedly overflow, there are two ways to address this: * Raise ``[triggerer] shared_stream_subscriber_queue_size`` to give the filter more slack before the overflow threshold is reached. -* Redesign :py:meth:`~airflow.triggers.base.BaseEventTrigger.shared_stream_key` so fewer +* Redesign :meth:`~airflow.triggers.base.BaseEventTrigger.shared_stream_key` so fewer sibling triggers share a single group — a narrower group reduces the rate at which any one subscriber needs to consume events. From d547cd2abc20a643d06c17e08e7218f02f1650fb Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 11 Jun 2026 15:59:22 +0800 Subject: [PATCH 04/15] refactor(triggers): drop redundant comment on TYPE_CHECKING import --- airflow-core/src/airflow/triggers/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow-core/src/airflow/triggers/base.py b/airflow-core/src/airflow/triggers/base.py index f1117fff5414a..0338af40bfa49 100644 --- a/airflow-core/src/airflow/triggers/base.py +++ b/airflow-core/src/airflow/triggers/base.py @@ -47,9 +47,6 @@ from airflow.models.taskinstance import TaskInstance from airflow.sdk.definitions.context import Context from airflow.serialization.serialized_objects import SerializedBaseOperator - - # Deferred to type-checking time: shared_stream imports this module at - # runtime, so a runtime import here would be circular. from airflow.triggers.shared_stream import SharedStreamProducer Operator: TypeAlias = MappedOperator | SerializedBaseOperator From 960a866c335e24926dd5cca6a755af2c92de5631 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 11 Jun 2026 23:33:04 +0800 Subject: [PATCH 05/15] fix(triggers): gate shared-stream broker advance on trigger event persistence A subscriber's ack now completes only after the trigger events it yielded are confirmed persisted to the metadata database, so a triggerer crash can no longer lose an event the broker already considers delivered. The runner tags outbound events with sequence numbers and the supervisor confirms them on the next state sync; every failure path (crash, lost sync, failed insert) falls back to the ack timeout, a failed outcome, and broker redelivery. Unsubscribing with unconfirmed events no longer counts as an implicit ack. --- .../event-scheduling.rst | 36 +- airflow-core/newsfragments/67523.feature.rst | 2 +- .../src/airflow/jobs/triggerer_job_runner.py | 63 +- airflow-core/src/airflow/triggers/base.py | 14 +- .../src/airflow/triggers/shared_stream.py | 280 ++++++- .../tests/unit/jobs/test_triggerer_job.py | 159 +++- .../tests/unit/triggers/test_shared_stream.py | 737 +++++++++--------- 7 files changed, 868 insertions(+), 423 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index ee45bea075bc4..59e462ba8763c 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -181,7 +181,8 @@ When this factory is overridden, the manager enters **ack mode**: the event, or ``await token.nack()`` to opt out. 4. Once every subscriber in the fan-out set has resolved an event (by ``ack()``, ``nack()``, unsubscribing, timing out, or overflowing its - queue), the manager calls + queue) **and** every ``TriggerEvent`` the subscribers derived from it + has been persisted to the metadata database, the manager calls ``await producer.advance(broker_payload, outcome)`` — commit the offset, delete the SQS message, etc. The ``outcome`` is an :class:`~airflow.triggers.shared_stream.AdvanceOutcome` carrying @@ -298,12 +299,14 @@ the other partitions: if self.consumer is not None: await self.consumer.stop() -.. warning:: - Always call ``await token.ack()`` or ``await token.nack()`` **before** - ``yield``-ing the ``TriggerEvent``. If you yield first and the caller - abandons the generator (the common single-event case), the token is - never resolved and the producer will not advance until the per-event - timeout expires. +The order of ``ack()`` relative to ``yield`` does not affect correctness: +an ack completes only after the subscriber has moved past the event and +the trigger events it derived from it were persisted, so nothing depends +on code running after the ``yield``. The one constraint on filter authors +is the binding between raw events and the trigger events derived from +them: yield every ``TriggerEvent`` derived from a raw event before pulling +the next raw event from the shared stream — which is what a +straightforward filter loop does anyway. **Snapshot-at-fan-out**: the set of subscribers that must ack a given event is frozen at the moment the event is broadcast. A subscriber that joins @@ -312,8 +315,10 @@ after the event was dispatched is not added to that event's pending set. **Per-event ack timeout**: if a subscriber has not called ``ack()`` or ``nack()`` within the ack timeout (default 5 minutes, configurable via the ``[triggerer] shared_stream_ack_timeout`` config option), the manager force-fails that -subscriber's trigger. Other subscribers are not affected; once their acks -arrive, the producer advances normally. The ack timeout is a manager-level +subscriber's trigger. The same timeout covers an ack that is waiting on a +persistence confirmation that never arrives. Other subscribers are not +affected; once their acks arrive, the producer advances normally. The ack +timeout is a manager-level safety net and does not replace any native broker session or visibility timeout. From the subscriber's perspective the force-fail surfaces as an ``AckTimeout`` (importable from ``airflow.triggers.shared_stream``) raised by the @@ -325,6 +330,19 @@ only if the subscriber needs to run cleanup before failing. triggerer restart, the broker redelivers messages that were not yet acknowledged. Subscribers must therefore be idempotent. +**Durability**: the broker advance is gated on persistence. A subscriber's +``ack()`` completes only after every ``TriggerEvent`` it derived from the +event has been stored in the metadata database; the confirmation reaches +the trigger runner on the next state sync, typically within a second or +two. If the confirmation never arrives — the triggerer crashed, or the +event could not be persisted — the ack timeout fails the event, the +producer does not commit, and the broker redelivers. A failure can +therefore cause duplicate delivery but never a lost event; idempotent +subscribers absorb the duplicates. The same trade-off applies when a +group stops while events are still awaiting confirmation (for example, +the last subscriber unsubscribes right after producing an event): the +pending advances are abandoned and the broker redelivers those events. + **nack() semantics**: ``nack()`` removes the subscriber from the pending ack set and is counted in the ``nacked`` field of the event's ``AdvanceOutcome``. What that means for the broker is the producer's diff --git a/airflow-core/newsfragments/67523.feature.rst b/airflow-core/newsfragments/67523.feature.rst index fc83bf3b75171..f5320d97bfd9e 100644 --- a/airflow-core/newsfragments/67523.feature.rst +++ b/airflow-core/newsfragments/67523.feature.rst @@ -1 +1 @@ -Shared-stream triggers now support a producer-side ack channel so message-broker upstreams (Kafka, SQS, Pub/Sub, Service Bus) can commit, delete, or ack on the broker handle after all subscribers have accepted an event. +Shared-stream triggers now support a producer-side ack channel so message-broker upstreams (Kafka, SQS, Pub/Sub, Service Bus) can commit, delete, or ack on the broker handle after all subscribers have accepted an event and the resulting trigger events have been persisted to the metadata database. diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py b/airflow-core/src/airflow/jobs/triggerer_job_runner.py index 5e9a7c728dd23..78a79390ac707 100644 --- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py +++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py @@ -294,8 +294,10 @@ class TriggerStateChanges(BaseModel): """ type: Literal["TriggerStateChanges"] = "TriggerStateChanges" + # The third tuple element is the shared-stream persist-confirmation + # seq (None for events that do not gate a broker advance). events: Annotated[ - list[tuple[int, DiscrimatedTriggerEvent]] | None, + list[tuple[int, DiscrimatedTriggerEvent, int | None]] | None, # We have to specify a default here, as otherwise Pydantic struggles to deal with the discriminated # union :shrug: Field(default=None), @@ -309,6 +311,10 @@ class TriggerStateSync(BaseModel): to_create: list[workloads.RunTrigger] to_cancel: set[int] + # Seqs of shared-stream trigger events the supervisor has persisted + # since the previous sync; the runner releases the matching broker + # advances on receipt. + events_persisted: list[int] | None = None class HITLDetailResponseResult(HITLDetailResponse): @@ -476,8 +482,13 @@ class TriggerRunnerSupervisor(WatchedSubprocess): # directly as all comms has to be initiated by the subprocess creating_triggers: deque[workloads.RunTrigger] = attrs.field(factory=deque, init=False) - # Outbound queue of events - events: deque[tuple[int, TriggerEvent]] = attrs.field(factory=deque, init=False) + # Outbound queue of events; the third element is the shared-stream + # persist-confirmation seq (None for non-shared-stream events). + events: deque[tuple[int, TriggerEvent, int | None]] = attrs.field(factory=deque, init=False) + + # Seqs of shared-stream trigger events persisted since the last state + # sync; drained into TriggerStateSync.events_persisted. + persisted_event_seqs: deque[int] = attrs.field(factory=deque, init=False) # Outbound queue of failed triggers failed_triggers: deque[tuple[int, list[str] | None]] = attrs.field(factory=deque, init=False) @@ -546,9 +557,15 @@ def _handle_request(self, msg: ToTriggerSupervisor, log: FilteringBoundLogger, r # handle leaks for every failed upload. factory.close() + # Drain the persist confirmations accumulated since the last sync. + events_persisted: list[int] = [] + while self.persisted_event_seqs: + events_persisted.append(self.persisted_event_seqs.popleft()) + response = messages.TriggerStateSync( to_create=[], to_cancel=self.cancelling_triggers, + events_persisted=events_persisted or None, ) # Pull out of these dequeues in a thread-safe manner @@ -679,10 +696,15 @@ def load_triggers(self) -> None: def handle_events(self): """Dispatch outbound events to the Trigger model which pushes them to the relevant task instances.""" while self.events: - # Get the event and its trigger ID - trigger_id, event = self.events.popleft() + # Get the event, its trigger ID, and its persist-confirmation seq + trigger_id, event, seq = self.events.popleft() # Tell the model to wake up its tasks self.on_trigger_event(trigger_id=trigger_id, event=event) + # Only reached when on_trigger_event returned, i.e. the event was + # persisted; a raise above leaves the seq unconfirmed so the + # bound shared-stream advance fails out and the broker redelivers. + if seq is not None: + self.persisted_event_seqs.append(seq) # Emit stat event stats.incr("triggers.succeeded") @@ -1072,8 +1094,9 @@ class TriggerRunner: # Inbound queue of deleted triggers to_cancel: deque[int] - # Outbound queue of events - events: deque[tuple[int, TriggerEvent]] + # Outbound queue of events; the third element is the shared-stream + # persist-confirmation seq (None for non-shared-stream events). + events: deque[tuple[int, TriggerEvent, int | None]] # Outbound queue of failed triggers failed_triggers: deque[tuple[int, BaseException | None]] @@ -1363,12 +1386,12 @@ async def cleanup_finished_triggers(self) -> list[int]: def process_trigger_events(self, finished_ids: list[int]) -> messages.TriggerStateChanges: # Copy out of our dequeues in threadsafe manner to sync state with parent - events_to_send: list[tuple[int, DiscrimatedTriggerEvent]] = [] + events_to_send: list[tuple[int, DiscrimatedTriggerEvent, int | None]] = [] failures_to_send: list[tuple[int, list[str] | None]] = [] while self.events: - trigger_id, trigger_event = self.events.popleft() - events_to_send.append((trigger_id, trigger_event)) + trigger_id, trigger_event, seq = self.events.popleft() + events_to_send.append((trigger_id, trigger_event, seq)) while self.failed_triggers: trigger_id, exc = self.failed_triggers.popleft() @@ -1383,10 +1406,10 @@ def process_trigger_events(self, finished_ids: list[int]) -> messages.TriggerSta def sanitize_trigger_events(self, msg: messages.TriggerStateChanges) -> messages.TriggerStateChanges: req_encoder = _new_encoder() - events_to_send: list[tuple[int, DiscrimatedTriggerEvent]] = [] + events_to_send: list[tuple[int, DiscrimatedTriggerEvent, int | None]] = [] if msg.events: - for trigger_id, trigger_event in msg.events: + for trigger_id, trigger_event, seq in msg.events: try: req_encoder.encode(trigger_event) except Exception as e: @@ -1395,9 +1418,13 @@ def sanitize_trigger_events(self, msg: messages.TriggerStateChanges) -> messages trigger_id, trigger_event, ) + # The dropped event's seq (if any) is never confirmed, so + # the bound shared-stream advance times out as failed and + # the broker redelivers — the safe direction for an event + # that was never persisted. self.failed_triggers.append((trigger_id, e)) else: - events_to_send.append((trigger_id, trigger_event)) + events_to_send.append((trigger_id, trigger_event, seq)) return messages.TriggerStateChanges( events=events_to_send if events_to_send else None, @@ -1418,6 +1445,8 @@ async def sync_state_to_supervisor(self, finished_ids: list[int]) -> None: if resp: self.to_create.extend(resp.to_create) self.to_cancel.extend(resp.to_cancel) + if resp.events_persisted: + self._shared_streams.confirm_persisted(resp.events_persisted) async def asend(self, msg: messages.TriggerStateChanges) -> messages.TriggerStateSync | None: try: @@ -1527,7 +1556,13 @@ async def run_trigger( "Trigger fired event", name=self.triggers[trigger_id]["name"], result=event ) self.triggers[trigger_id]["events"] += 1 - self.events.append((trigger_id, event)) + seq: int | None = None + if shared_key is not None: + # Bind and append back to back with no await in + # between, so no other coroutine can observe the + # event bound but not yet queued (or vice versa). + seq = self._shared_streams.bind_pending_event(trigger_id=trigger_id, key=shared_key) + self.events.append((trigger_id, event, seq)) span.set_status(Status(StatusCode.OK)) except asyncio.CancelledError as e: # A trigger can be cancelled for two reasons: diff --git a/airflow-core/src/airflow/triggers/base.py b/airflow-core/src/airflow/triggers/base.py index 0338af40bfa49..0b5f8e02417e9 100644 --- a/airflow-core/src/airflow/triggers/base.py +++ b/airflow-core/src/airflow/triggers/base.py @@ -285,9 +285,10 @@ class BaseEventTrigger(BaseTrigger): broker connection, supplies events through its ``open_stream`` method (instead of :meth:`open_shared_stream`), and is told to advance the broker — commit, delete, or ack the message — once all subscribers that - were online at broadcast time have resolved an event. When this factory - is not overridden, the fast path is taken: no ack tokens are issued and - subscribers receive raw events exactly as before. + were online at broadcast time have resolved an event and the trigger + events they derived from it were persisted to the metadata database. + When this factory is not overridden, the fast path is taken: no ack + tokens are issued and subscribers receive raw events exactly as before. See :mod:`airflow.triggers.shared_stream` for the full ack-mode design, including snapshot-at-fan-out semantics, per-event timeout behavior, and @@ -384,9 +385,10 @@ def create_shared_stream_producer(cls, kwargs: dict[str, Any]) -> SharedStreamPr :class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns the broker connection for the lifetime of one poll — it supplies events through ``open_stream``, is told to advance the broker once - all subscribers have resolved an event, and is closed when the poll - ends. Do not open the broker connection here; open it lazily inside - the producer's ``open_stream``. + all subscribers have resolved an event and the trigger events they + derived from it were persisted, and is closed when the poll ends. + Do not open the broker connection here; open it lazily inside the + producer's ``open_stream``. Triggers that do not override this method run the fast path: no ack tokens are issued and subscribers receive raw events from diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index 6f4974e661f2c..21e7fc3e73293 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -41,9 +41,13 @@ ``(raw_event, token)`` pairs — where ``token`` is an :class:`AckToken` — to each subscriber's queue instead of the raw event alone. A subscriber calls ``await token.ack()`` once it has accepted the event, or -``await token.nack()`` to opt out. When every subscriber that was online at -broadcast time has resolved the event (by ``ack()``, ``nack()``, -unsubscribing, or being force-failed), the manager calls +``await token.nack()`` to opt out. An ack completes the subscriber's +resolution only after the subscriber has moved past the event (pulled the +next raw event, or unsubscribed) and every +:class:`~airflow.triggers.base.TriggerEvent` it derived from the event has +been confirmed persisted to the metadata database; ``nack()`` and +force-failure resolve immediately. When every subscriber that was online +at broadcast time has resolved the event, the manager calls ``await producer.advance(broker_payload, outcome)`` exactly once — this is where the producer commits / deletes / acks on the broker. The :class:`AdvanceOutcome` carries per-event counts of how the subscribers @@ -60,15 +64,32 @@ frozen at broadcast time. A subscriber that joins after the event was broadcast is not added to that event's pending set. +**Persistence-gated advance**: the trigger events a subscriber derives +from a raw event are assigned sequence numbers as they leave the runner; +the supervisor confirms each one after the event is stored in the +metadata database, and the confirmation reaches the runner on the next +state sync (typically one sync round, a second or two — well within the +ack timeout). The subscriber's ack only completes once all of its +sequence numbers for the event are confirmed. If a confirmation never arrives — the persist +failed, or the triggerer crashed in between — the ack timeout fails the +event, the producer does not commit, and the broker redelivers. The +binding between a trigger event and the raw event it came from relies on +the filter yielding each derived event before pulling the next raw event +from the shared stream (the natural way to write a filter). + **Per-event ack timeout**: a background task scans outstanding events. Any subscriber that has not acknowledged within ``ack_timeout`` seconds is force-failed via the existing :class:`_PollFailure` path (exception type -:class:`AckTimeout`). Other subscribers are not affected; once the +:class:`AckTimeout`). The same timeout backstops persist confirmations +that never arrive. Other subscribers are not affected; once the remaining acks arrive the producer advances normally. **Triggerer restart**: outstanding acks are in-memory only. After a triggerer restart, the broker will redeliver unacknowledged events. -Subscribers must therefore be idempotent. +Subscribers must therefore be idempotent. The same applies when a group +stops while events are still awaiting persist confirmation (for example, +the last subscriber unsubscribes right after producing an event): the +pending advances are abandoned and the broker redelivers those events. **``shared_stream_subscriber_queue_size`` in ack mode**: the bound is still "unprocessed raw events per subscriber". The manager does **not** @@ -115,13 +136,14 @@ from __future__ import annotations import asyncio +import itertools import time import weakref from abc import ABC, abstractmethod from collections import deque -from collections.abc import AsyncGenerator, AsyncIterator, Callable, Hashable +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Hashable, Iterable, Iterator from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal import structlog @@ -204,10 +226,12 @@ class AdvanceOutcome: Every subscriber that was online when the event was broadcast is counted in exactly one field: - * ``acked`` — called ``token.ack()``, or unsubscribed while the event was - outstanding (implicit ack-out). + * ``acked`` — called ``token.ack()`` (or unsubscribed while the event + was outstanding — implicit ack-out) and every trigger event derived + from the event was confirmed persisted. * ``nacked`` — called ``token.nack()``. - * ``failed`` — force-failed by the manager (ack timeout or queue overflow). + * ``failed`` — force-failed by the manager (ack timeout — including a + persist confirmation that never arrived — or queue overflow). An event broadcast while no subscribers were online carries all-zero counts and is clean. @@ -294,6 +318,22 @@ async def aclose(self) -> None: """ +@dataclass(slots=True) +class _TokenBinding: + """ + Per-(event, subscriber) resolution bookkeeping for one :class:`AckToken`. + + A subscriber resolves as ``acked`` only when all three hold: ``ack()`` + was called (explicitly or implicitly on unsubscribe), the binding window + is closed (the subscriber moved past the event), and every trigger event + seq the subscriber derived from the event has been confirmed persisted. + """ + + acked: bool = False + window_closed: bool = False + unconfirmed_seqs: set[int] = field(default_factory=set) + + @dataclass class _OutstandingEntry: """State for one outstanding (broadcast-but-not-yet-advanced) event.""" @@ -302,6 +342,9 @@ class _OutstandingEntry: created_at: float # monotonic clock at broadcast (via injectable _now) broker_payload: Any lane: Hashable # producer.get_advance_lane(broker_payload), taken at fan-out + # Per-subscriber ack bookkeeping, created at fan-out alongside ``pending`` + # and removed by ``_resolve_subscriber`` when the subscriber resolves. + bindings: dict[int, _TokenBinding] = field(default_factory=dict) # Resolution counts; together they form the event's AdvanceOutcome. acked: int = 0 nacked: int = 0 @@ -316,6 +359,12 @@ class AckToken: ``await token.nack()`` to opt out — the resolution is reported to the producer through the event's :class:`AdvanceOutcome`. Repeated calls are no-ops. After the group stops, calls silently do nothing. + + An ``ack()`` does not complete the subscriber's resolution on its own: + the event counts as ``acked`` only after the subscriber has moved past it + (pulled the next raw event or unsubscribed) and every trigger event the + subscriber derived from it has been confirmed persisted to the metadata + database. ``nack()`` resolves immediately. """ __slots__ = ("_event_id", "_trigger_id", "_group_ref", "_resolved") @@ -338,17 +387,21 @@ def _resolve(self, resolution: Literal["acked", "nacked"]) -> None: return self._resolved = True group = self._group_ref() - if group is not None: - group._resolve_subscriber( - event_id=self._event_id, trigger_id=self._trigger_id, resolution=resolution - ) + if group is None: + return + if resolution == "acked": + group._record_ack(event_id=self._event_id, trigger_id=self._trigger_id) + else: + group._record_nack(event_id=self._event_id, trigger_id=self._trigger_id) async def ack(self) -> None: """ Notify the producer that this subscriber has accepted the event. Declared ``async`` so the token API leaves room for awaitable - implementations; the in-process implementation resolves synchronously. + implementations. The broker advance the ack feeds into may happen + later: it waits until the trigger events derived from this event are + persisted to the metadata database. """ self._resolve("acked") @@ -359,8 +412,8 @@ async def nack(self) -> None: How the broker reacts is the producer's decision inside :meth:`SharedStreamProducer.advance` — commit anyway, skip, or trigger a redeliver. Declared ``async`` so the token API leaves room - for awaitable implementations; the in-process implementation - resolves synchronously. + for awaitable implementations. A ``nack()`` resolves the subscriber + immediately, without waiting for any persistence confirmation. """ self._resolve("nacked") @@ -415,6 +468,13 @@ def __init__( # Per-lane FIFO index over _outstanding: event ids in fan-out order, # keyed by the lane each event's producer assigned at fan-out. self._lane_queues: dict[Hashable, deque[int]] = {} + # seq -> (event_id, trigger_id) for trigger events awaiting persist + # confirmation; entries are removed on confirmation or when the + # owning binding resolves (late confirmations then no-op). + self._seq_index: dict[int, tuple[int, int]] = {} + # Per subscriber: the token whose binding window is currently open — + # trigger events the subscriber emits now bind to this token. + self._current_token: dict[int, AckToken] = {} self._next_event_id: int = 0 self._ack_timeout_task: asyncio.Task | None = None # Advance pump: a single task dispatches broker advances in per-lane @@ -548,6 +608,7 @@ async def _poll(self) -> None: created_at=self._now(), broker_payload=broker_payload, lane=lane, + bindings={trigger_id: _TokenBinding() for trigger_id in snapshot}, ) lane_queue.append(event_id) if not snapshot: @@ -642,11 +703,21 @@ def _resolve_subscriber( are the pump's job, so advances stay in per-lane fan-out order. Duplicate calls for the same (event_id, trigger_id) are no-ops, which keeps every subscriber counted exactly once per event. + + This is the single cleanup point for the subscriber's ack + bookkeeping: the binding is popped and its unconfirmed seqs leave + ``_seq_index``, so a persist confirmation arriving after the fact + (e.g. after an ack timeout already failed the subscriber) finds + nothing and no-ops. """ entry = self._outstanding.get(event_id) if entry is None or trigger_id not in entry.pending: return # already resolved, already advanced, or never existed entry.pending.discard(trigger_id) + binding = entry.bindings.pop(trigger_id, None) + if binding is not None: + for seq in binding.unconfirmed_seqs: + self._seq_index.pop(seq, None) if resolution == "acked": entry.acked += 1 elif resolution == "nacked": @@ -656,6 +727,105 @@ def _resolve_subscriber( if not entry.pending: self._advance_wakeup.set() + def _record_ack(self, *, event_id: int, trigger_id: int) -> None: + """ + Record that the subscriber called ``ack()`` for one event. + + The subscriber resolves as acked only once the binding window is + closed and every bound seq is confirmed persisted; until then the + event stays outstanding (the ack timeout is the backstop). + """ + entry = self._outstanding.get(event_id) + if entry is None: + return + binding = entry.bindings.get(trigger_id) + if binding is None: + return # already resolved (e.g. force-failed before the ack landed) + binding.acked = True + self._maybe_complete(event_id=event_id, trigger_id=trigger_id) + + def _record_nack(self, *, event_id: int, trigger_id: int) -> None: + """Resolve the subscriber as nacked immediately; no persistence gating applies.""" + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="nacked") + + def _maybe_complete(self, *, event_id: int, trigger_id: int) -> None: + """Resolve the subscriber as acked once ack, window close, and all persist confirmations are in.""" + entry = self._outstanding.get(event_id) + if entry is None: + return + binding = entry.bindings.get(trigger_id) + if binding is None: + return + if binding.acked and binding.window_closed and not binding.unconfirmed_seqs: + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") + + def _close_binding_window(self, token: AckToken) -> None: + """ + Close the binding window of one yielded token. + + Called by ``_ack_drain`` the moment the subscriber pulls the next + item — trigger events the subscriber emits from here on belong to + the next token, and an ack recorded for this one can now complete + (subject to persist confirmations). + """ + trigger_id = token._trigger_id + if self._current_token.get(trigger_id) is token: + del self._current_token[trigger_id] + entry = self._outstanding.get(token._event_id) + if entry is None: + return + binding = entry.bindings.get(trigger_id) + if binding is None: + return + binding.window_closed = True + self._maybe_complete(event_id=token._event_id, trigger_id=trigger_id) + + def bind_pending_event(self, *, trigger_id: int, seq_counter: Iterator[int]) -> int | None: + """ + Bind one just-emitted trigger event to the subscriber's open binding window. + + Returns the seq the broker advance must wait on, or ``None`` when + there is nothing to bind to — no window open (not in ack mode, or + the event was emitted outside any raw event's window) or the + binding already resolved (nacked or force-failed). A seq is only + drawn from ``seq_counter`` when the event actually binds. + """ + token = self._current_token.get(trigger_id) + if token is None: + return None + entry = self._outstanding.get(token._event_id) + if entry is None: + return None + binding = entry.bindings.get(trigger_id) + if binding is None: + return None + seq = next(seq_counter) + binding.unconfirmed_seqs.add(seq) + self._seq_index[seq] = (token._event_id, trigger_id) + return seq + + def confirm_persisted(self, seqs: Iterable[int]) -> None: + """ + Record persist confirmations for trigger event seqs bound in this group. + + Seqs that are not (or no longer) in ``_seq_index`` — another group's + seqs, or bindings that already resolved through nack / timeout — + are ignored. + """ + for seq in seqs: + bound = self._seq_index.pop(seq, None) + if bound is None: + continue + event_id, trigger_id = bound + entry = self._outstanding.get(event_id) + if entry is None: + continue + binding = entry.bindings.get(trigger_id) + if binding is None: + continue + binding.unconfirmed_seqs.discard(seq) + self._maybe_complete(event_id=event_id, trigger_id=trigger_id) + def _fail_subscriber_in_outstanding(self, trigger_id: int) -> None: """ Resolve ``trigger_id`` as failed in every outstanding entry. @@ -708,12 +878,35 @@ async def _run_ack_timeout_loop(self) -> None: # owns entry deletion and the ordered broker advance. self._fail_subscriber_in_outstanding(trigger_id) + async def _ack_drain(self, trigger_id: int, queue: asyncio.Queue) -> AsyncGenerator[Any, None]: + """ + Ack-mode counterpart of :func:`_drain`, tracking the binding window. + + Between yielding ``(raw_event, token)`` and the subscriber pulling + the next item, the token's binding window is open: trigger events + the subscriber emits bind to it via :meth:`bind_pending_event`. The + window closes the moment the subscriber resumes this generator — + before we wait on the queue again — so a filtered-out event (acked, + nothing yielded) resolves as soon as the filter loops back for the + next raw event. + """ + while True: + item = await queue.get() + if isinstance(item, _PollFailure): + raise item.exc + _, token = item + self._current_token[trigger_id] = token + yield item + self._close_binding_window(token) + def subscribe(self, trigger_id: int) -> AsyncIterator[Any]: """Register ``trigger_id`` as a subscriber and return its raw event stream.""" if trigger_id in self._subscribers: raise RuntimeError(f"Trigger {trigger_id} already subscribed to shared stream {self.key!r}") queue: asyncio.Queue = asyncio.Queue(maxsize=self._max_subscriber_queue) self._subscribers[trigger_id] = queue + if self._is_ack_required(): + return self._ack_drain(trigger_id, queue) return _drain(queue) def unsubscribe(self, trigger_id: int) -> None: @@ -721,11 +914,22 @@ def unsubscribe(self, trigger_id: int) -> None: # (Airflow's standard idiom); dropping the queue is enough here. self._subscribers.pop(trigger_id, None) self._failed_subscribers.discard(trigger_id) - # Resolve in all outstanding entries — implicit ack-out (counted as - # acked) to prevent the producer waiting forever for a subscriber - # that has left. - for event_id in list(self._outstanding): - self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") + self._current_token.pop(trigger_id, None) + # Implicit ack-out: leaving counts as accepting every outstanding + # event, so the producer never waits forever for a subscriber that + # has left. The broker advance still waits for persist confirmation + # of any trigger events this subscriber derived from the event; with + # nothing unconfirmed the subscriber resolves immediately. + for event_id, entry in list(self._outstanding.items()): + binding = entry.bindings.get(trigger_id) + if binding is None: + # No live binding (already resolved, or pre-ack-mode entry); + # _resolve_subscriber no-ops unless still pending. + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") + continue + binding.acked = True + binding.window_closed = True + self._maybe_complete(event_id=event_id, trigger_id=trigger_id) def _fail_overflowed_subscriber(self, trigger_id: int, queue: asyncio.Queue) -> None: """ @@ -826,6 +1030,9 @@ def __init__( self._ack_timeout = ack_timeout self._now = _now self._groups: dict[Hashable, _SharedStreamGroup] = {} + # Allocator for trigger-event persist-confirmation seqs; unique per + # manager (= per runner process), shared across groups. Gaps are fine. + self._seq_counter: Iterator[int] = itertools.count() def subscribe( self, @@ -860,6 +1067,35 @@ def subscribe( self.log.debug("Shared stream group started", key=key) return group.subscribe(trigger_id) + def bind_pending_event(self, *, trigger_id: int, key: Hashable) -> int | None: + """ + Bind a trigger event the subscriber just emitted to its shared-stream ack state. + + Returns the persist-confirmation seq the runner must report through + :meth:`confirm_persisted` once the event is stored, or ``None`` when + there is nothing to gate — the group is gone, the stream is not in + ack mode, or the subscriber has no open binding window. Synchronous + and O(1); call it between taking the event off the trigger and + queueing it outbound, with no ``await`` in between. + """ + if (group := self._groups.get(key)) is None: + return None + return group.bind_pending_event(trigger_id=trigger_id, seq_counter=self._seq_counter) + + def confirm_persisted(self, seqs: Iterable[int]) -> None: + """ + Record that the trigger events behind ``seqs`` were persisted. + + Broadcast to every live group; each group resolves the sequence + numbers it owns and ignores the rest. Sequence numbers whose binding + already resolved (``nack()``, timeout) or whose group has stopped + are ignored. + """ + # Materialize so a one-shot iterator is not exhausted by the first group. + seq_list = list(seqs) + for group in self._groups.values(): + group.confirm_persisted(seq_list) + async def unsubscribe(self, trigger_id: int, key: Hashable) -> None: """ Remove a subscriber. diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py b/airflow-core/tests/unit/jobs/test_triggerer_job.py index 04a8700388cbb..7a0650693fada 100644 --- a/airflow-core/tests/unit/jobs/test_triggerer_job.py +++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py @@ -45,6 +45,7 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from pydantic import TypeAdapter from structlog.typing import FilteringBoundLogger from airflow._shared.timezones import timezone @@ -709,6 +710,12 @@ def test_blocked_main_thread_warning_threshold_decode(self) -> None: assert trigger_runner.blocked_main_thread_warning_threshold == 0.5 + @conf_vars({("triggerer", "shared_stream_ack_timeout"): "60.0"}) + def test_shared_stream_ack_timeout_config_wiring(self) -> None: + """[triggerer] shared_stream_ack_timeout is wired into SharedStreamManager.ack_timeout.""" + trigger_runner = TriggerRunner() + assert trigger_runner._shared_streams._ack_timeout == 60.0 + @pytest.mark.asyncio async def test_block_watchdog_does_not_log_when_threshold_is_not_exceeded(self) -> None: with conf_vars({("triggerer", "blocked_main_thread_warning_threshold"): "0.5"}): @@ -901,14 +908,14 @@ async def _drive(): events = list(trigger_runner.events) assert len(events) == 1 - trigger_id, event = events[0] + trigger_id, event, _seq = events[0] assert trigger_id == 1 assert event.payload == {"region": "us"} # Group is torn down on unsubscribe. assert trigger_runner._shared_streams._groups == {} def test_shared_stream_ack_mode_integration(self, session) -> None: - """A BaseEventTrigger whose producer factory is overridden advances after all acks.""" + """A BaseEventTrigger whose producer factory is overridden advances only after the acked event's persistence is confirmed.""" from airflow.triggers.base import BaseEventTrigger, TriggerEvent from airflow.triggers.shared_stream import SharedStreamProducer @@ -970,15 +977,24 @@ async def _drive(): state["trigger_runner"] = trigger_runner run_task = asyncio.create_task(trigger_runner.run_trigger(1, trigger)) - # Wait until advance has fired — unambiguous signal that ack round-trip completed. - await asyncio.wait_for(advance_called.wait(), timeout=2.0) - # Advance is dispatched from the pump task, so it can run while run_trigger is - # still suspended between yielding the event and appending it to the outbound - # queue. Wait for the event to land before cancelling, like the non-ack test above. + # Wait until the event lands on the outbound queue with its + # persist-confirmation seq. for _ in range(100): if trigger_runner.events: break await asyncio.sleep(0.01) + assert trigger_runner.events, "the trigger event must reach the outbound queue" + _trigger_id, _event, seq = trigger_runner.events[0] + assert seq is not None, "a shared ack trigger's event must carry a seq" + + # The subscriber acked, but the advance is gated on persistence: + # nothing may advance before the confirmation arrives. + assert advanced == [], "advance must not run before the persist confirmation" + + # Simulate the supervisor confirming the persist on the next sync. + trigger_runner._shared_streams.confirm_persisted([seq]) + await asyncio.wait_for(advance_called.wait(), timeout=2.0) + run_task.cancel() with contextlib.suppress(asyncio.CancelledError): await run_task @@ -988,12 +1004,14 @@ async def _drive(): events = list(trigger_runner.events) assert len(events) == 1 - trigger_id, event = events[0] + trigger_id, event, _seq = events[0] assert trigger_id == 1 assert event.payload == {"region": "us"} # The producer's advance must have been called with the broker_payload. - assert advanced == ["receipt-handle-1"], "producer.advance must be called after the subscriber acks" + assert advanced == ["receipt-handle-1"], ( + "producer.advance must be called once the acked event's persistence is confirmed" + ) # Group is torn down on unsubscribe. assert trigger_runner._shared_streams._groups == {} @@ -1120,9 +1138,9 @@ async def test_trigger_kwargs_serialization_cleanup(self, session): async def test_sync_state_to_supervisor(self, supervisor_builder): trigger_runner = TriggerRunner() trigger_runner.comms_decoder = AsyncMock(spec=TriggerCommsDecoder) - trigger_runner.events.append((1, TriggerEvent(payload={"status": "SUCCESS"}))) - trigger_runner.events.append((2, TriggerEvent(payload={"status": "FAILED"}))) - trigger_runner.events.append((3, TriggerEvent(payload={"status": "SUCCESS", "data": object()}))) + trigger_runner.events.append((1, TriggerEvent(payload={"status": "SUCCESS"}), None)) + trigger_runner.events.append((2, TriggerEvent(payload={"status": "FAILED"}), None)) + trigger_runner.events.append((3, TriggerEvent(payload={"status": "SUCCESS", "data": object()}), None)) async def asend_side_effect(msg): if msg.events and len(msg.events) == 3: @@ -1215,7 +1233,7 @@ async def test_trigger_create_race_condition_38599(session, supervisor_builder): # This calls Trigger.submit_event, which will unlink the trigger from the task instance # Simulate this call: supervisor1._service_subprocess() - supervisor1.events.append((trigger_orm.id, TriggerEvent(True))) + supervisor1.events.append((trigger_orm.id, TriggerEvent(True), None)) supervisor1.handle_events() trigger_orm = session.get(Trigger, trigger_orm.id) # This is the "pre"-condition we need to assert to test the race condition @@ -1285,7 +1303,7 @@ async def test_trigger_firing(): await asyncio.sleep(0.1) finished = await runner.cleanup_finished_triggers() if runner.events: - assert list(runner.events) == [(1, TriggerEvent(True))] + assert list(runner.events) == [(1, TriggerEvent(True), None)] assert finished == [1] break await asyncio.sleep(0.1) @@ -2388,3 +2406,116 @@ async def test_unknown_frame_id_doesnt_crash_reader(decoder_pair): assert decoder._reader_task is not None assert not decoder._reader_task.done(), "reader loop crashed unexpectedly" + + +# --------------------------------------------------------------------------- +# Shared-stream persist confirmations (runner <-> supervisor) +# --------------------------------------------------------------------------- + + +def test_trigger_state_messages_round_trip_with_seqs(): + """Event triples and persist confirmations survive the wire encode/decode.""" + changes_adapter = TypeAdapter(ToTriggerSupervisor) + changes = messages.TriggerStateChanges( + events=[(1, TriggerEvent(True), 3), (2, TriggerEvent("x"), None)], + failures=None, + finished=None, + ) + decoded = changes_adapter.validate_json(changes_adapter.dump_json(changes)) + assert isinstance(decoded, messages.TriggerStateChanges) + assert decoded.events == [(1, TriggerEvent(True), 3), (2, TriggerEvent("x"), None)] + + sync_adapter = TypeAdapter(ToTriggerRunner) + sync = messages.TriggerStateSync(to_create=[], to_cancel={5}, events_persisted=[3, 9]) + decoded_sync = sync_adapter.validate_json(sync_adapter.dump_json(sync)) + assert isinstance(decoded_sync, messages.TriggerStateSync) + assert decoded_sync.events_persisted == [3, 9] + assert decoded_sync.to_cancel == {5} + + +def test_handle_events_records_persist_confirmations(jobless_supervisor): + """handle_events confirms the seq of a persisted event; events without a seq are not confirmed.""" + event_with_seq = TriggerEvent(True) + event_without_seq = TriggerEvent("x") + jobless_supervisor.events.append((1, event_with_seq, 7)) + jobless_supervisor.events.append((2, event_without_seq, None)) + + with mock.patch.object(TriggerRunnerSupervisor, "on_trigger_event", autospec=True) as mock_event: + jobless_supervisor.handle_events() + + assert mock_event.mock_calls == [ + mock.call(jobless_supervisor, trigger_id=1, event=event_with_seq), + mock.call(jobless_supervisor, trigger_id=2, event=event_without_seq), + ] + assert list(jobless_supervisor.persisted_event_seqs) == [7] + assert len(jobless_supervisor.events) == 0 + + +def test_handle_events_does_not_confirm_seq_when_persist_fails(jobless_supervisor): + """A seq whose event failed to persist is never confirmed, so the broker advance fails out.""" + jobless_supervisor.events.append((1, TriggerEvent(True), 7)) + + with mock.patch.object( + TriggerRunnerSupervisor, + "on_trigger_event", + autospec=True, + side_effect=RuntimeError("db down"), + ): + with pytest.raises(RuntimeError, match="db down"): + jobless_supervisor.handle_events() + + assert list(jobless_supervisor.persisted_event_seqs) == [] + + +def test_state_sync_carries_and_drains_persist_confirmations(jobless_supervisor): + """The state-sync response carries pending confirmations once, then None when there are none.""" + jobless_supervisor.persisted_event_seqs.extend([3, 9]) + + with mock.patch.object(TriggerRunnerSupervisor, "send_msg", autospec=True) as mock_send: + jobless_supervisor._handle_request( + messages.TriggerStateChanges(events=None, failures=None, finished=None), + log=MagicMock(spec=FilteringBoundLogger), + req_id=1, + ) + + assert len(mock_send.mock_calls) == 1 + response = mock_send.call_args.args[1] + assert isinstance(response, messages.TriggerStateSync) + assert response.events_persisted == [3, 9] + assert len(jobless_supervisor.persisted_event_seqs) == 0 + + # The next sync has nothing pending and carries None. + with mock.patch.object(TriggerRunnerSupervisor, "send_msg", autospec=True) as mock_send: + jobless_supervisor._handle_request( + messages.TriggerStateChanges(events=None, failures=None, finished=None), + log=MagicMock(spec=FilteringBoundLogger), + req_id=2, + ) + + response = mock_send.call_args.args[1] + assert isinstance(response, messages.TriggerStateSync) + assert response.events_persisted is None + + +def test_run_trigger_appends_none_seq_for_non_shared_trigger(): + """An event from a trigger outside any shared stream carries no persist-confirmation seq.""" + trigger_runner = TriggerRunner() + trigger_runner.triggers = { + 1: {"task": MagicMock(spec=asyncio.Task), "is_watcher": False, "name": "t", "events": 0} + } + trigger = SuccessTrigger() + trigger.task_instance = MagicMock() + trigger.task_instance.map_index = -1 + + async def _drive(): + # greenback.ensure_portal() inside run_trigger needs a real asyncio + # task, like the shared-stream tests above. + await asyncio.create_task(trigger_runner.run_trigger(1, trigger)) + + asyncio.run(_drive()) + + events = list(trigger_runner.events) + assert len(events) == 1 + trigger_id, _event, seq = events[0] + assert trigger_id == 1 + assert seq is None diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index ad6cdced51fbd..99cfb50668377 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -20,7 +20,7 @@ import time import weakref from collections.abc import Callable -from contextlib import suppress +from contextlib import asynccontextmanager, suppress from unittest.mock import MagicMock import pytest @@ -753,6 +753,72 @@ async def _collect_ack_stream(trigger, stream, *, n: int) -> list: return out +class _BackgroundCollector: + """ + Items collected from a stream that is consumed like a real filter loop. + + Use via :func:`_consume_in_background`. The consuming task never breaks + out of its ``async for``, so after each item the stream generator is + resumed and the item's binding window closes — exactly what a production + ``filter_shared_stream`` loop does when it goes back for the next raw + event. Breaking out of the loop instead would leave the generator + suspended at its yield with the binding window open, so an ack could + never complete. + """ + + def __init__(self) -> None: + self.items: list = [] + # The exception (e.g. an injected AckTimeout) that ended the stream, + # if any; the consuming task records it here instead of propagating. + self.error: BaseException | None = None + + @property + def tokens(self) -> list[AckToken]: + """The AckTokens collected so far (ack-mode raw streams only).""" + return [token for _raw, token in self.items] + + async def wait_for(self, n: int, *, timeout: float = 1.0) -> list: + """Wait until ``n`` items have been collected and return them.""" + deadline = asyncio.get_event_loop().time() + timeout + while len(self.items) < n: + if self.error is not None: + raise self.error + assert asyncio.get_event_loop().time() < deadline, ( + f"collector got {len(self.items)} of {n} items within {timeout}s" + ) + await asyncio.sleep(0) + return self.items[:n] + + +@asynccontextmanager +async def _consume_in_background(stream): + """ + Consume ``stream`` in a background task, the way a real subscriber would. + + Yields a :class:`_BackgroundCollector`; the consuming task is cancelled + on exit. A stream failure is recorded on ``collector.error`` rather than + propagating out of the task. + """ + collector = _BackgroundCollector() + + async def consume() -> None: + try: + async for item in stream: + collector.items.append(item) + except asyncio.CancelledError: + raise + except BaseException as exc: + collector.error = exc + + task = asyncio.create_task(consume()) + try: + yield collector + finally: + task.cancel() + with suppress(asyncio.CancelledError): + await task + + @pytest.mark.parametrize( ("override_level", "expected"), [ @@ -816,27 +882,20 @@ async def test_ack_required_producer_advances_after_all_subscribers_ack(): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - tokens: list[AckToken] = [] - - async def collect_token(stream, trigger): - async for _raw, token in stream: - tokens.append(token) - break - # Collect the (event, token) tuples without acking yet. - await asyncio.gather( - asyncio.wait_for(collect_token(s1, t1), timeout=1.0), - asyncio.wait_for(collect_token(s2, t2), timeout=1.0), - ) + async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: + await c1.wait_for(1) + await c2.wait_for(1) + tokens: list[AckToken] = [c1.tokens[0], c2.tokens[0]] - assert len(cls.advanced) == 0, "advance must not fire before all acks" + assert len(cls.advanced) == 0, "advance must not fire before all acks" - await tokens[0].ack() - assert len(cls.advanced) == 0, "advance must not fire after only one ack" + await tokens[0].ack() + assert len(cls.advanced) == 0, "advance must not fire after only one ack" - await tokens[1].ack() - await asyncio.sleep(0) # let the advance pump run - assert cls.advanced == ["receipt-1"], "advance must fire exactly once after all acks" + await tokens[1].ack() + await asyncio.sleep(0) # let the advance pump run + assert cls.advanced == ["receipt-1"], "advance must fire exactly once after all acks" finally: await manager.unsubscribe(1, key) await manager.unsubscribe(2, key) @@ -914,46 +973,44 @@ def now() -> float: s_fast = manager.subscribe(trigger_id=1, trigger=t_fast, key=key) s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) - token_fast = None - - async def grab_fast(stream): - nonlocal token_fast - async for _raw, token in stream: - token_fast = token - break - async def drain_slow(stream): - # Collect from slow stream — we expect AckTimeout to appear. + # Pull the (event, token) pair off the slow stream but never ack — + # the deliberately stalled subscriber the timeout must catch. async for _ in stream: break - await asyncio.gather( - asyncio.wait_for(grab_fast(s_fast), timeout=1.0), - asyncio.wait_for(drain_slow(s_slow), timeout=1.0), - ) - - # fast acks; slow doesn't. - await token_fast.ack() - - # Advance fake clock past ack_timeout; the timeout loop sees now() - created_at >= 0.1. - fake_clock[0] = 0.2 - slow_queue = manager._groups[key]._subscribers.get(2) - # Let _run_ack_timeout_loop tick through a few cadence cycles (cadence=0.01s real). - await asyncio.sleep(0.05) - - assert slow_queue is not None - sentinel = slow_queue.get_nowait() - assert isinstance(sentinel, _PollFailure) - assert isinstance(sentinel.exc, AckTimeout) - - # Fast subscriber (already acked) must not have received a spurious AckTimeout. - fast_queue = manager._groups[key]._subscribers.get(1) - assert fast_queue is not None - assert fast_queue.empty(), "fast (already-acked) subscriber must not receive a spurious AckTimeout" + async with _consume_in_background(s_fast) as fast_collector: + await fast_collector.wait_for(1) + await asyncio.wait_for(drain_slow(s_slow), timeout=1.0) + token_fast = fast_collector.tokens[0] + + # fast acks; slow doesn't. + await token_fast.ack() + + # Advance fake clock past ack_timeout; the timeout loop sees now() - created_at >= 0.1. + fake_clock[0] = 0.2 + slow_queue = manager._groups[key]._subscribers.get(2) + # Let _run_ack_timeout_loop tick through a few cadence cycles (cadence=0.01s real). + await asyncio.sleep(0.05) + + assert slow_queue is not None + sentinel = slow_queue.get_nowait() + assert isinstance(sentinel, _PollFailure) + assert isinstance(sentinel.exc, AckTimeout) + + # Fast subscriber (already acked) must not have received a spurious AckTimeout. + fast_queue = manager._groups[key]._subscribers.get(1) + assert fast_queue is not None + assert fast_queue.empty(), ( + "fast (already-acked) subscriber must not receive a spurious AckTimeout" + ) + assert fast_collector.error is None, ( + "fast (already-acked) subscriber must not receive a spurious AckTimeout" + ) - # After timeout the advance should have fired (pending emptied). - await asyncio.sleep(0) - assert cls.advanced == ["receipt-timeout"] + # After timeout the advance should have fired (pending emptied). + await asyncio.sleep(0) + assert cls.advanced == ["receipt-timeout"] finally: await manager.stop_all() @@ -982,30 +1039,25 @@ def now() -> float: try: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) - token_ref = None - - async def grab_token(s): - nonlocal token_ref - async for _raw, token in s: - token_ref = token - break + async with _consume_in_background(stream) as collector: + await collector.wait_for(1) + token_ref = collector.tokens[0] - await asyncio.wait_for(grab_token(stream), timeout=1.0) - - # Advance fake clock to 99 ms — strictly inside the 100 ms ack_timeout window. - # now() - created_at = 0.099 < 0.1, so the timeout loop must NOT fire. - fake_clock[0] = 0.099 - # Let the timeout loop tick a few cadence cycles (cadence = max(0.01, 0.1/10) = 0.01s real). - await asyncio.sleep(0.05) + # Advance fake clock to 99 ms — strictly inside the 100 ms ack_timeout window. + # now() - created_at = 0.099 < 0.1, so the timeout loop must NOT fire. + fake_clock[0] = 0.099 + # Let the timeout loop tick a few cadence cycles (cadence = max(0.01, 0.1/10) = 0.01s real). + await asyncio.sleep(0.05) - subscriber_queue = manager._groups[key]._subscribers.get(1) - assert subscriber_queue is not None - assert subscriber_queue.empty(), "subscriber must not have been force-failed at t=99ms" + subscriber_queue = manager._groups[key]._subscribers.get(1) + assert subscriber_queue is not None + assert subscriber_queue.empty(), "subscriber must not have been force-failed at t=99ms" + assert collector.error is None, "subscriber must not have been force-failed at t=99ms" - # Now ack — advance should fire. - await token_ref.ack() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-boundary"] + # Now ack — advance should fire. + await token_ref.ack() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-boundary"] finally: await manager.stop_all() @@ -1167,12 +1219,13 @@ async def filter_shared_stream(self, shared_stream): assert 2 not in entry.pending, "late subscriber must not be in the pending set of earlier event" # Collect from early subscriber (filter acks immediately). - events = await asyncio.wait_for(_collect_ack_stream(t_early, s_early, n=1), timeout=1.0) - assert len(events) == 1 + async with _consume_in_background(t_early.filter_shared_stream(s_early)) as collector: + events = await collector.wait_for(1) + assert len(events) == 1 - await asyncio.sleep(0) - # The late subscriber did not participate in ack set; advance must have fired. - assert _LateTrigger.advanced == ["receipt-late"] + await asyncio.sleep(0) + # The late subscriber did not participate in ack set; advance must have fired. + assert _LateTrigger.advanced == ["receipt-late"] finally: await manager.unsubscribe(1, key) await manager.unsubscribe(2, key) @@ -1196,29 +1249,21 @@ async def test_nack_removes_subscriber_from_ack_set_without_redeliver(): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - tokens: list[AckToken] = [] + async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: + await c1.wait_for(1) + await c2.wait_for(1) - async def grab_token(stream): - async for _raw, token in stream: - tokens.append(token) - break - - await asyncio.gather( - asyncio.wait_for(grab_token(s1), timeout=1.0), - asyncio.wait_for(grab_token(s2), timeout=1.0), - ) - - # t1 acks, t2 nacks. - await tokens[0].ack() - await tokens[1].nack() - await asyncio.sleep(0) + # t1 acks, t2 nacks. + await c1.tokens[0].ack() + await c2.tokens[0].nack() + await asyncio.sleep(0) - assert cls.advanced == ["receipt-nack"], "advance must fire after ack + nack" + assert cls.advanced == ["receipt-nack"], "advance must fire after ack + nack" - # No redelivery — the manager has no redeliver mechanism. - group = manager._groups.get(key) - if group: - assert len(group._outstanding) == 0 + # No redelivery — the manager has no redeliver mechanism. + group = manager._groups.get(key) + if group: + assert len(group._outstanding) == 0 finally: await manager.unsubscribe(1, key) await manager.unsubscribe(2, key) @@ -1241,25 +1286,19 @@ async def test_double_ack_is_noop(): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - token = None - - async def grab_token(stream): - nonlocal token - async for _raw, tk in stream: - token = tk - break - - await asyncio.wait_for(grab_token(s1), timeout=1.0) - assert token is not None + async with _consume_in_background(s1) as collector: + await collector.wait_for(1) + token = collector.tokens[0] + assert token is not None - await token.ack() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-double"] + await token.ack() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-double"] - # Second ack — must be a no-op, not raise, not double-advance. - await token.ack() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-double"], "second ack must not trigger a second advance" + # Second ack — must be a no-op, not raise, not double-advance. + await token.ack() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-double"], "second ack must not trigger a second advance" finally: await manager.unsubscribe(1, key) @@ -1288,28 +1327,22 @@ async def test_cross_order_double_call_is_noop(first, second): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - token = None - - async def grab_token(stream): - nonlocal token - async for _raw, tk in stream: - token = tk - break - - await asyncio.wait_for(grab_token(s1), timeout=1.0) - assert token is not None + async with _consume_in_background(s1) as collector: + await collector.wait_for(1) + token = collector.tokens[0] + assert token is not None - # First call — should resolve the token and advance. - await getattr(token, first)() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-cross"] + # First call — should resolve the token and advance. + await getattr(token, first)() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-cross"] - # Second call (opposite) — must be a no-op; advance must not fire again. - await getattr(token, second)() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-cross"], ( - f"{second}() after {first}() must not trigger a second advance" - ) + # Second call (opposite) — must be a no-op; advance must not fire again. + await getattr(token, second)() + await asyncio.sleep(0) + assert cls.advanced == ["receipt-cross"], ( + f"{second}() after {first}() must not trigger a second advance" + ) finally: await manager.unsubscribe(1, key) @@ -1332,27 +1365,18 @@ async def test_subscriber_unsubscribe_during_outstanding_ack(): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - tokens: list[AckToken] = [] + async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: + await c1.wait_for(1) + await c2.wait_for(1) - async def grab_token(stream): - async for _raw, token in stream: - tokens.append(token) - break - - await asyncio.gather( - asyncio.wait_for(grab_token(s1), timeout=1.0), - asyncio.wait_for(grab_token(s2), timeout=1.0), - ) - - assert len(tokens) == 2 - # t1 acks; t2 leaves without acking (implicit ack-out via unsubscribe). - await tokens[0].ack() - await manager.unsubscribe(2, key) - await asyncio.sleep(0) + # t1 acks; t2 leaves without acking (implicit ack-out via unsubscribe). + await c1.tokens[0].ack() + await manager.unsubscribe(2, key) + await asyncio.sleep(0) - assert cls.advanced == ["receipt-unsub"], ( - "unsubscribe must trigger implicit ack-out so producer can advance" - ) + assert cls.advanced == ["receipt-unsub"], ( + "unsubscribe must trigger implicit ack-out so producer can advance" + ) finally: await manager.unsubscribe(1, key) @@ -1378,13 +1402,6 @@ async def test_ack_mode_queue_full_during_fanout_does_not_break_iteration(): ("filler", AckToken(event_id=-1, trigger_id=1, group_ref=weakref.ref(group))) ) - # t_ok: drive its ack stream and collect the real event. - ok_result: list = [] - - async def collect_ok(): - events = await _collect_ack_stream(t_ok, s_ok, n=1) - ok_result.extend(events) - # t_full: its queue will be drained + replaced by _PollFailure(_SubscriberOverflow). # _drain() raises the inner exception, so we expect _SubscriberOverflow here. full_exc: list[BaseException] = [] @@ -1396,22 +1413,22 @@ async def collect_full(): except Exception as exc: full_exc.append(exc) - await asyncio.gather( - asyncio.wait_for(collect_full(), timeout=2.0), - asyncio.wait_for(collect_ok(), timeout=2.0), - ) + # t_ok: drive its ack stream and collect the real event. + async with _consume_in_background(t_ok.filter_shared_stream(s_ok)) as ok_collector: + await asyncio.wait_for(collect_full(), timeout=2.0) + ok_result = await ok_collector.wait_for(1) - # Give the advance pump a tick to run. - await asyncio.sleep(0) + # Give the advance pump a tick to run. + await asyncio.sleep(0) - # t_full got an overflow failure (not a RuntimeError from set mutation). - assert len(full_exc) == 1 - assert isinstance(full_exc[0], _SubscriberOverflow) + # t_full got an overflow failure (not a RuntimeError from set mutation). + assert len(full_exc) == 1 + assert isinstance(full_exc[0], _SubscriberOverflow) - # t_ok got the real event and acknowledged it → advance was triggered. - assert len(ok_result) == 1 - assert ok_result[0].payload == {"msg": "burst"} - assert cls.advanced == ["receipt-burst"] + # t_ok got the real event and acknowledged it → advance was triggered. + assert len(ok_result) == 1 + assert ok_result[0].payload == {"msg": "burst"} + assert cls.advanced == ["receipt-burst"] finally: await manager.stop_all() @@ -1497,27 +1514,29 @@ async def filter_shared_stream(self, shared_stream): group.log = MagicMock() group.log.error = mock_log_error - # Consume the first event — this triggers an ack and schedules advance. - events1 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) - assert len(events1) == 1 + async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: + # Consume the first event — this triggers an ack and schedules advance. + events1 = await collector.wait_for(1) + assert len(events1) == 1 - # Give the advance pump a tick to run and raise. - await asyncio.sleep(0.05) + # Give the advance pump a tick to run and raise. + await asyncio.sleep(0.05) - # log.error must have been called with a message about the failed advance. - assert mock_log_error.called, "log.error must be called when producer.advance raises" - call_args = mock_log_error.call_args - assert "broker advance failed" in call_args[0][0], ( - f"log.error message must mention 'broker advance failed', got: {call_args[0][0]}" - ) + # log.error must have been called with a message about the failed advance. + assert mock_log_error.called, "log.error must be called when producer.advance raises" + call_args = mock_log_error.call_args + assert "broker advance failed" in call_args[0][0], ( + f"log.error message must mention 'broker advance failed', got: {call_args[0][0]}" + ) - # The poll group is still alive — the exception must not have killed it. - assert key in manager._groups, "group must survive a producer.advance exception" + # The poll group is still alive — the exception must not have killed it. + assert key in manager._groups, "group must survive a producer.advance exception" - # A second event can still be produced and consumed. - proceed_flag.set() - events2 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) - assert len(events2) == 1 + # A second event can still be produced and consumed (the collector + # count is cumulative across both events). + proceed_flag.set() + events2 = await collector.wait_for(2) + assert len(events2) == 2 finally: await manager.stop_all() @@ -1564,18 +1583,19 @@ async def filter_shared_stream(self, shared_stream): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - # Drive the one event through so the advance is dispatched. - events = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) - assert len(events) == 1 + async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: + # Drive the one event through so the advance is dispatched. + events = await collector.wait_for(1) + assert len(events) == 1 - # Wait for producer.advance to actually start (confirms the advance is in-flight). - await asyncio.wait_for(advance_started.wait(), timeout=1.0) + # Wait for producer.advance to actually start (confirms the advance is in-flight). + await asyncio.wait_for(advance_started.wait(), timeout=1.0) - # Unblock the advance hook and stop; advance still has a 50 ms sleep so stop() - # must genuinely await the task rather than racing past it. - advance_may_finish.set() - await manager.stop_all() - stop_returned_at = time.monotonic() + # Unblock the advance hook and stop; advance still has a 50 ms sleep so stop() + # must genuinely await the task rather than racing past it. + advance_may_finish.set() + await manager.stop_all() + stop_returned_at = time.monotonic() # stop_all() must have waited for the in-flight advance to complete. assert _SlowAdvanceTrigger.advanced == ["receipt-slow"], ( @@ -1714,18 +1734,19 @@ async def test_out_of_order_resolve_still_advances_in_fanout_order(): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=3), timeout=1.0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(3) + tokens = collector.tokens - # Resolve events 2 and 3 first — nothing may advance while event 1 is pending. - await tokens[1].ack() - await tokens[2].ack() - await asyncio.sleep(0) - assert cls.advanced == [], "no advance may run while the head event is unresolved" + # Resolve events 2 and 3 first — nothing may advance while event 1 is pending. + await tokens[1].ack() + await tokens[2].ack() + await asyncio.sleep(0) + assert cls.advanced == [], "no advance may run while the head event is unresolved" - await tokens[0].ack() - await asyncio.sleep(0) - assert cls.advanced == ["p1", "p2", "p3"], "advances must run in fan-out order" + await tokens[0].ack() + await asyncio.sleep(0) + assert cls.advanced == ["p1", "p2", "p3"], "advances must run in fan-out order" finally: await manager.unsubscribe(1, key) @@ -1766,26 +1787,27 @@ async def filter_shared_stream(self, shared_stream): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(2) + tokens = collector.tokens - # Resolve both events; the pump starts the first advance, which blocks. - await tokens[0].ack() - await tokens[1].ack() - for _ in range(5): - await asyncio.sleep(0) - assert state["in_flight"] == 1, "exactly one advance may be in flight" - assert advanced == [] + # Resolve both events; the pump starts the first advance, which blocks. + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) + assert state["in_flight"] == 1, "exactly one advance may be in flight" + assert advanced == [] - release.set() - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(advanced) >= 2: - break - await asyncio.sleep(0) + release.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(advanced) >= 2: + break + await asyncio.sleep(0) - assert advanced == ["p1", "p2"] - assert state["max_in_flight"] == 1, "advance calls must never overlap" + assert advanced == ["p1", "p2"] + assert state["max_in_flight"] == 1, "advance calls must never overlap" finally: await manager.unsubscribe(1, key) @@ -1817,28 +1839,29 @@ async def test_lanes_advance_independently(): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) group = manager._groups[key] - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(4) + tokens = collector.tokens - # Resolve lane "b" completely while lane "a" is fully unresolved. - await tokens[1].ack() # b1 - await tokens[3].ack() # b2 - for _ in range(5): - await asyncio.sleep(0) - assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" + # Resolve lane "b" completely while lane "a" is fully unresolved. + await tokens[1].ack() # b1 + await tokens[3].ack() # b2 + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" - await tokens[0].ack() # a1 - await tokens[2].ack() # a2 - for _ in range(5): - await asyncio.sleep(0) + await tokens[0].ack() # a1 + await tokens[2].ack() # a2 + for _ in range(5): + await asyncio.sleep(0) - # Contract: full per-lane projections in fan-out order. The global - # interleaving across lanes is a pump implementation detail. - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] - assert len(cls.advanced) == 4 - # All lanes drained → the per-lane index is garbage-collected. - assert group._lane_queues == {} + # Contract: full per-lane projections in fan-out order. The global + # interleaving across lanes is a pump implementation detail. + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] + assert len(cls.advanced) == 4 + # All lanes drained → the per-lane index is garbage-collected. + assert group._lane_queues == {} finally: await manager.unsubscribe(1, key) @@ -1864,24 +1887,25 @@ async def test_lane_internal_fifo_with_out_of_order_resolve(): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(4) + tokens = collector.tokens - # Resolve a2 and a3 (head a1 stays pending) plus the noise lane. - await tokens[2].ack() # a2 - await tokens[3].ack() # a3 - await tokens[1].ack() # b1 - for _ in range(5): - await asyncio.sleep(0) - assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" + # Resolve a2 and a3 (head a1 stays pending) plus the noise lane. + await tokens[2].ack() # a2 + await tokens[3].ack() # a3 + await tokens[1].ack() # b1 + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" - await tokens[0].ack() # a1 - for _ in range(5): - await asyncio.sleep(0) + await tokens[0].ack() # a1 + for _ in range(5): + await asyncio.sleep(0) - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] - assert len(cls.advanced) == 4 + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] + assert len(cls.advanced) == 4 finally: await manager.unsubscribe(1, key) @@ -1925,26 +1949,27 @@ async def filter_shared_stream(self, shared_stream): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(2) + tokens = collector.tokens - # Resolve both lanes; the pump starts one advance, which blocks. - await tokens[0].ack() - await tokens[1].ack() - for _ in range(5): - await asyncio.sleep(0) - assert state["in_flight"] == 1, "the other lane's advance must not start while one is in flight" - assert advanced == [] + # Resolve both lanes; the pump starts one advance, which blocks. + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) + assert state["in_flight"] == 1, "the other lane's advance must not start while one is in flight" + assert advanced == [] - release.set() - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(advanced) >= 2: - break - await asyncio.sleep(0) + release.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(advanced) >= 2: + break + await asyncio.sleep(0) - assert sorted(advanced) == ["a1", "b1"] - assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" + assert sorted(advanced) == ["a1", "b1"] + assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" finally: await manager.unsubscribe(1, key) @@ -2465,27 +2490,25 @@ async def test_outcome_classification(second_action, expected, expected_clean): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - tokens: list[AckToken] = [] - await asyncio.gather( - asyncio.wait_for(_grab_tokens(s1, tokens, n=1), timeout=1.0), - asyncio.wait_for(_grab_tokens(s2, tokens, n=2), timeout=1.0), - ) - - await tokens[0].ack() - if second_action == "ack": - await tokens[1].ack() - elif second_action == "nack": - await tokens[1].nack() - else: - await manager.unsubscribe(2, key) - await asyncio.sleep(0) + async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: + await c1.wait_for(1) + await c2.wait_for(1) + + await c1.tokens[0].ack() + if second_action == "ack": + await c2.tokens[0].ack() + elif second_action == "nack": + await c2.tokens[0].nack() + else: + await manager.unsubscribe(2, key) + await asyncio.sleep(0) - assert cls.advanced == ["receipt-outcome"] - assert cls.outcomes == [expected] - assert cls.outcomes[0].is_clean is expected_clean - # Invariant: every subscriber in the broadcast snapshot is counted exactly once. - outcome = cls.outcomes[0] - assert outcome.acked + outcome.nacked + outcome.failed == 2 + assert cls.advanced == ["receipt-outcome"] + assert cls.outcomes == [expected] + assert cls.outcomes[0].is_clean is expected_clean + # Invariant: every subscriber in the broadcast snapshot is counted exactly once. + outcome = cls.outcomes[0] + assert outcome.acked + outcome.nacked + outcome.failed == 2 finally: await manager.unsubscribe(1, key) if second_action != "unsubscribe": @@ -2512,21 +2535,19 @@ def now() -> float: s_fast = manager.subscribe(trigger_id=1, trigger=t_fast, key=key) s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) - tokens: list[AckToken] = [] - await asyncio.gather( - asyncio.wait_for(_grab_tokens(s_fast, tokens, n=1), timeout=1.0), - # Pull the (event, token) pair off the slow stream but never resolve it. - asyncio.wait_for(_grab_tokens(s_slow, tokens, n=2), timeout=1.0), - ) + # The slow collector pulls its (event, token) pair but never resolves it. + async with _consume_in_background(s_fast) as c_fast, _consume_in_background(s_slow) as c_slow: + await c_fast.wait_for(1) + await c_slow.wait_for(1) - await tokens[0].ack() - fake_clock[0] = 0.2 - # Let the timeout loop tick (cadence = 0.01s real) and the pump dispatch. - await asyncio.sleep(0.05) + await c_fast.tokens[0].ack() + fake_clock[0] = 0.2 + # Let the timeout loop tick (cadence = 0.01s real) and the pump dispatch. + await asyncio.sleep(0.05) - assert cls.advanced == ["receipt-to"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] - assert not cls.outcomes[0].is_clean + assert cls.advanced == ["receipt-to"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] + assert not cls.outcomes[0].is_clean finally: await manager.stop_all() @@ -2544,15 +2565,16 @@ async def test_double_resolve_counts_once(): try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=1), timeout=1.0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(1) + token = collector.tokens[0] - await tokens[0].ack() - await tokens[0].nack() - await asyncio.sleep(0) + await token.ack() + await token.nack() + await asyncio.sleep(0) - assert cls.advanced == ["receipt-double-resolve"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + assert cls.advanced == ["receipt-double-resolve"] + assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] finally: await manager.unsubscribe(1, key) @@ -2658,17 +2680,20 @@ async def filter_shared_stream(self, shared_stream): group = manager._groups[key] group.log = MagicMock() - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) - await tokens[0].ack() - await tokens[1].ack() - for _ in range(5): - await asyncio.sleep(0) + async with _consume_in_background(s1) as collector: + await collector.wait_for(2) + tokens = collector.tokens + await tokens[0].ack() + await tokens[1].ack() + for _ in range(5): + await asyncio.sleep(0) - assert attempted == ["p1", "p2"], "the pump must move on to event 2 after event 1's advance raised" - error_calls = group.log.error.mock_calls - assert len(error_calls) == 1 - assert "broker advance failed" in error_calls[0].args[0] + assert attempted == ["p1", "p2"], ( + "the pump must move on to event 2 after event 1's advance raised" + ) + error_calls = group.log.error.mock_calls + assert len(error_calls) == 1 + assert "broker advance failed" in error_calls[0].args[0] finally: await manager.unsubscribe(1, key) @@ -2716,32 +2741,33 @@ async def filter_shared_stream(self, shared_stream): manager.subscribe(trigger_id=1, trigger=t_dead, key=key) s_live = manager.subscribe(trigger_id=2, trigger=t_live, key=key) - tokens: list[AckToken] = [] # Live subscriber acks events 1 and 2; the dead subscriber never consumes. - await asyncio.wait_for(_grab_tokens(s_live, tokens, n=2), timeout=1.0) - await tokens[0].ack() - await tokens[1].ack() - await asyncio.sleep(0) - assert _GatedTrigger.advanced == [], "events 1-2 still wait on the dead subscriber" - - # Event 3 overflows the dead subscriber's queue → force-fail clears it - # from ALL outstanding entries, so events 1-2 advance without waiting - # for their ack timeouts. - gate.set() - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(_GatedTrigger.advanced) >= 2: - break + async with _consume_in_background(s_live) as collector: + await collector.wait_for(2) + tokens = collector.tokens + await tokens[0].ack() + await tokens[1].ack() await asyncio.sleep(0) - assert _GatedTrigger.advanced == ["p1", "p2"] - assert _GatedTrigger.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] * 2 + assert _GatedTrigger.advanced == [], "events 1-2 still wait on the dead subscriber" - # The live subscriber resolves event 3 and it advances too. - await asyncio.wait_for(_grab_tokens(s_live, tokens, n=3), timeout=1.0) - await tokens[2].ack() - await asyncio.sleep(0) - assert _GatedTrigger.advanced == ["p1", "p2", "p3"] - assert _GatedTrigger.outcomes[2] == AdvanceOutcome(acked=1, nacked=0, failed=1) + # Event 3 overflows the dead subscriber's queue → force-fail clears it + # from ALL outstanding entries, so events 1-2 advance without waiting + # for their ack timeouts. + gate.set() + deadline = asyncio.get_event_loop().time() + 1.0 + while asyncio.get_event_loop().time() < deadline: + if len(_GatedTrigger.advanced) >= 2: + break + await asyncio.sleep(0) + assert _GatedTrigger.advanced == ["p1", "p2"] + assert _GatedTrigger.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] * 2 + + # The live subscriber resolves event 3 and it advances too. + await collector.wait_for(3) + await collector.tokens[2].ack() + await asyncio.sleep(0) + assert _GatedTrigger.advanced == ["p1", "p2", "p3"] + assert _GatedTrigger.outcomes[2] == AdvanceOutcome(acked=1, nacked=0, failed=1) finally: await manager.stop_all() @@ -2832,8 +2858,6 @@ async def filter_shared_stream(self, shared_stream): # ...and must not advance the unresolved event 2 (left to broker redelivery). assert advanced == ["p1"] assert sum(aclose_calls) == 1, "the producer must still be closed after the drain" -<<<<<<< HEAD -======= async def _pull_pair(stream_iter): @@ -3001,7 +3025,7 @@ async def test_ack_before_yield_still_gates_advance_on_confirmation(): @pytest.mark.asyncio async def test_unsubscribe_with_unconfirmed_seq_waits_for_confirmation(): - """Regression: a subscriber leaving right after producing an event must not bypass the persist gate.""" + """Guard: a subscriber leaving right after producing an event must not bypass the persist gate.""" cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) cls.advanced.clear() cls.outcomes.clear() @@ -3195,4 +3219,3 @@ async def test_group_stop_abandons_unconfirmed_advance(): assert manager._groups == {} assert cls.advanced == [], "an unconfirmed advance must be abandoned at group stop" assert cls.aclose_calls == 1 ->>>>>>> e927a603b1 (fixup! fixup! feat(triggers): add producer-side ack channel to shared-stream triggers) From f1b35bb3d341542966023ca90224932959c919e4 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Thu, 11 Jun 2026 23:51:44 +0800 Subject: [PATCH 06/15] docs(triggers): note disabling Kafka auto-commit in the shared-stream ack example --- .../docs/authoring-and-scheduling/event-scheduling.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index 59e462ba8763c..a413c3d02415f 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -260,7 +260,7 @@ Example — SQS-like producer: else: await token.nack() - async def run(self): # pragma: no cover + async def run(self): yield TriggerEvent({}) Example — Kafka cumulative commit across partitions. A Kafka commit @@ -280,7 +280,10 @@ the other partitions: self.consumer = None async def open_stream(self): - self.consumer = await create_kafka_consumer(self.topics) + # Auto-commit must be off (the Kafka default is on), or the + # consumer commits on its own schedule and the ack channel + # no longer controls what the broker considers delivered. + self.consumer = await create_kafka_consumer(self.topics, enable_auto_commit=False) async for message in self.consumer: yield message.value, (message.topic, message.partition, message.offset) From c20c53e34917d167a73d4d99c1b97e57297d5a44 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 12 Jun 2026 15:02:23 +0800 Subject: [PATCH 07/15] refactor(triggers): replace triggerer event tuples with a named tuple TriggerEventEntry(trigger_id, event, persist_seq) replaces the bare 3-tuples in the runner/supervisor queues and the state-sync wire message; serialized bytes are unchanged. --- .../event-scheduling.rst | 4 +- .../src/airflow/jobs/triggerer_job_runner.py | 54 ++++++++++--------- .../src/airflow/triggers/shared_stream.py | 8 +-- .../tests/unit/jobs/test_triggerer_job.py | 28 +++++----- 4 files changed, 46 insertions(+), 48 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index a413c3d02415f..874a6effba58f 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -321,8 +321,8 @@ after the event was dispatched is not added to that event's pending set. subscriber's trigger. The same timeout covers an ack that is waiting on a persistence confirmation that never arrives. Other subscribers are not affected; once their acks arrive, the producer advances normally. The ack -timeout is a manager-level -safety net and does not replace any native broker session or visibility timeout. +timeout is a manager-level safety net and does not replace any native broker +session or visibility timeout. From the subscriber's perspective the force-fail surfaces as an ``AckTimeout`` (importable from ``airflow.triggers.shared_stream``) raised by the ``shared_stream`` iterator inside ``filter_shared_stream``. Letting it propagate diff --git a/airflow-core/src/airflow/jobs/triggerer_job_runner.py b/airflow-core/src/airflow/jobs/triggerer_job_runner.py index 78a79390ac707..97fd669204456 100644 --- a/airflow-core/src/airflow/jobs/triggerer_job_runner.py +++ b/airflow-core/src/airflow/jobs/triggerer_job_runner.py @@ -32,7 +32,7 @@ from datetime import datetime from socket import socket from traceback import format_exception -from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, ClassVar, Literal, TextIO, TypedDict +from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, ClassVar, Literal, NamedTuple, TextIO, TypedDict from uuid import uuid4 import anyio @@ -279,6 +279,14 @@ def _execute(self) -> int | None: log: FilteringBoundLogger = structlog.get_logger(logger_name=__name__) +class TriggerEventEntry(NamedTuple): + """A fired trigger event queued for dispatch to the Trigger model.""" + + trigger_id: int + event: DiscrimatedTriggerEvent + persist_seq: int | None # None when the event does not gate a broker advance + + # Using this as a simple namespace class messages: class StartTriggerer(BaseModel): @@ -294,10 +302,8 @@ class TriggerStateChanges(BaseModel): """ type: Literal["TriggerStateChanges"] = "TriggerStateChanges" - # The third tuple element is the shared-stream persist-confirmation - # seq (None for events that do not gate a broker advance). events: Annotated[ - list[tuple[int, DiscrimatedTriggerEvent, int | None]] | None, + list[TriggerEventEntry] | None, # We have to specify a default here, as otherwise Pydantic struggles to deal with the discriminated # union :shrug: Field(default=None), @@ -482,9 +488,8 @@ class TriggerRunnerSupervisor(WatchedSubprocess): # directly as all comms has to be initiated by the subprocess creating_triggers: deque[workloads.RunTrigger] = attrs.field(factory=deque, init=False) - # Outbound queue of events; the third element is the shared-stream - # persist-confirmation seq (None for non-shared-stream events). - events: deque[tuple[int, TriggerEvent, int | None]] = attrs.field(factory=deque, init=False) + # Outbound queue of events + events: deque[TriggerEventEntry] = attrs.field(factory=deque, init=False) # Seqs of shared-stream trigger events persisted since the last state # sync; drained into TriggerStateSync.events_persisted. @@ -696,15 +701,14 @@ def load_triggers(self) -> None: def handle_events(self): """Dispatch outbound events to the Trigger model which pushes them to the relevant task instances.""" while self.events: - # Get the event, its trigger ID, and its persist-confirmation seq - trigger_id, event, seq = self.events.popleft() + entry = self.events.popleft() # Tell the model to wake up its tasks - self.on_trigger_event(trigger_id=trigger_id, event=event) + self.on_trigger_event(trigger_id=entry.trigger_id, event=entry.event) # Only reached when on_trigger_event returned, i.e. the event was # persisted; a raise above leaves the seq unconfirmed so the # bound shared-stream advance fails out and the broker redelivers. - if seq is not None: - self.persisted_event_seqs.append(seq) + if entry.persist_seq is not None: + self.persisted_event_seqs.append(entry.persist_seq) # Emit stat event stats.incr("triggers.succeeded") @@ -1094,9 +1098,8 @@ class TriggerRunner: # Inbound queue of deleted triggers to_cancel: deque[int] - # Outbound queue of events; the third element is the shared-stream - # persist-confirmation seq (None for non-shared-stream events). - events: deque[tuple[int, TriggerEvent, int | None]] + # Outbound queue of events + events: deque[TriggerEventEntry] # Outbound queue of failed triggers failed_triggers: deque[tuple[int, BaseException | None]] @@ -1386,12 +1389,11 @@ async def cleanup_finished_triggers(self) -> list[int]: def process_trigger_events(self, finished_ids: list[int]) -> messages.TriggerStateChanges: # Copy out of our dequeues in threadsafe manner to sync state with parent - events_to_send: list[tuple[int, DiscrimatedTriggerEvent, int | None]] = [] + events_to_send: list[TriggerEventEntry] = [] failures_to_send: list[tuple[int, list[str] | None]] = [] while self.events: - trigger_id, trigger_event, seq = self.events.popleft() - events_to_send.append((trigger_id, trigger_event, seq)) + events_to_send.append(self.events.popleft()) while self.failed_triggers: trigger_id, exc = self.failed_triggers.popleft() @@ -1406,25 +1408,25 @@ def process_trigger_events(self, finished_ids: list[int]) -> messages.TriggerSta def sanitize_trigger_events(self, msg: messages.TriggerStateChanges) -> messages.TriggerStateChanges: req_encoder = _new_encoder() - events_to_send: list[tuple[int, DiscrimatedTriggerEvent, int | None]] = [] + events_to_send: list[TriggerEventEntry] = [] if msg.events: - for trigger_id, trigger_event, seq in msg.events: + for entry in msg.events: try: - req_encoder.encode(trigger_event) + req_encoder.encode(entry.event) except Exception as e: logger.error( "Trigger %s returned non-serializable result %r. Cancelling trigger.", - trigger_id, - trigger_event, + entry.trigger_id, + entry.event, ) # The dropped event's seq (if any) is never confirmed, so # the bound shared-stream advance times out as failed and # the broker redelivers — the safe direction for an event # that was never persisted. - self.failed_triggers.append((trigger_id, e)) + self.failed_triggers.append((entry.trigger_id, e)) else: - events_to_send.append((trigger_id, trigger_event, seq)) + events_to_send.append(entry) return messages.TriggerStateChanges( events=events_to_send if events_to_send else None, @@ -1562,7 +1564,7 @@ async def run_trigger( # between, so no other coroutine can observe the # event bound but not yet queued (or vice versa). seq = self._shared_streams.bind_pending_event(trigger_id=trigger_id, key=shared_key) - self.events.append((trigger_id, event, seq)) + self.events.append(TriggerEventEntry(trigger_id=trigger_id, event=event, persist_seq=seq)) span.set_status(Status(StatusCode.OK)) except asyncio.CancelledError as e: # A trigger can be cancelled for two reasons: diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index 21e7fc3e73293..447733f84d3cc 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -323,10 +323,10 @@ class _TokenBinding: """ Per-(event, subscriber) resolution bookkeeping for one :class:`AckToken`. - A subscriber resolves as ``acked`` only when all three hold: ``ack()`` - was called (explicitly or implicitly on unsubscribe), the binding window - is closed (the subscriber moved past the event), and every trigger event - seq the subscriber derived from the event has been confirmed persisted. + A subscriber resolves as ``acked`` only when all three hold: + 1. ``ack()`` was called (explicitly or implicitly on unsubscribe) + 2. the binding window is closed (the subscriber moved past the event) + 3. every trigger event seq the subscriber derived from the event has been confirmed persisted """ acked: bool = False diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py b/airflow-core/tests/unit/jobs/test_triggerer_job.py index 7a0650693fada..d93e56763334d 100644 --- a/airflow-core/tests/unit/jobs/test_triggerer_job.py +++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py @@ -58,6 +58,7 @@ ToTriggerSupervisor, TriggerCommsDecoder, TriggererJobRunner, + TriggerEventEntry, TriggerLoggingFactory, TriggerRunner, TriggerRunnerSupervisor, @@ -76,7 +77,8 @@ from airflow.sdk import DAG, BaseHook, BaseOperator from airflow.sdk.execution_time.comms import ToSupervisor, ToTask, _RequestFrame, _ResponseFrame from airflow.serialization.serialized_objects import LazyDeserializedDAG -from airflow.triggers.base import BaseTrigger, TriggerEvent +from airflow.triggers.base import BaseEventTrigger, BaseTrigger, TriggerEvent +from airflow.triggers.shared_stream import SharedStreamProducer from airflow.triggers.testing import FailureTrigger, SuccessTrigger from airflow.utils.state import State, TaskInstanceState from airflow.utils.types import DagRunType @@ -916,9 +918,6 @@ async def _drive(): def test_shared_stream_ack_mode_integration(self, session) -> None: """A BaseEventTrigger whose producer factory is overridden advances only after the acked event's persistence is confirmed.""" - from airflow.triggers.base import BaseEventTrigger, TriggerEvent - from airflow.triggers.shared_stream import SharedStreamProducer - advanced: list[Any] = [] # Container lets _drive() write trigger_runner back for post-run assertions. state: dict[str, Any] = {} @@ -1138,9 +1137,11 @@ async def test_trigger_kwargs_serialization_cleanup(self, session): async def test_sync_state_to_supervisor(self, supervisor_builder): trigger_runner = TriggerRunner() trigger_runner.comms_decoder = AsyncMock(spec=TriggerCommsDecoder) - trigger_runner.events.append((1, TriggerEvent(payload={"status": "SUCCESS"}), None)) - trigger_runner.events.append((2, TriggerEvent(payload={"status": "FAILED"}), None)) - trigger_runner.events.append((3, TriggerEvent(payload={"status": "SUCCESS", "data": object()}), None)) + trigger_runner.events.append(TriggerEventEntry(1, TriggerEvent(payload={"status": "SUCCESS"}), None)) + trigger_runner.events.append(TriggerEventEntry(2, TriggerEvent(payload={"status": "FAILED"}), None)) + trigger_runner.events.append( + TriggerEventEntry(3, TriggerEvent(payload={"status": "SUCCESS", "data": object()}), None) + ) async def asend_side_effect(msg): if msg.events and len(msg.events) == 3: @@ -1233,7 +1234,7 @@ async def test_trigger_create_race_condition_38599(session, supervisor_builder): # This calls Trigger.submit_event, which will unlink the trigger from the task instance # Simulate this call: supervisor1._service_subprocess() - supervisor1.events.append((trigger_orm.id, TriggerEvent(True), None)) + supervisor1.events.append(TriggerEventEntry(trigger_orm.id, TriggerEvent(True), None)) supervisor1.handle_events() trigger_orm = session.get(Trigger, trigger_orm.id) # This is the "pre"-condition we need to assert to test the race condition @@ -2408,11 +2409,6 @@ async def test_unknown_frame_id_doesnt_crash_reader(decoder_pair): assert not decoder._reader_task.done(), "reader loop crashed unexpectedly" -# --------------------------------------------------------------------------- -# Shared-stream persist confirmations (runner <-> supervisor) -# --------------------------------------------------------------------------- - - def test_trigger_state_messages_round_trip_with_seqs(): """Event triples and persist confirmations survive the wire encode/decode.""" changes_adapter = TypeAdapter(ToTriggerSupervisor) @@ -2437,8 +2433,8 @@ def test_handle_events_records_persist_confirmations(jobless_supervisor): """handle_events confirms the seq of a persisted event; events without a seq are not confirmed.""" event_with_seq = TriggerEvent(True) event_without_seq = TriggerEvent("x") - jobless_supervisor.events.append((1, event_with_seq, 7)) - jobless_supervisor.events.append((2, event_without_seq, None)) + jobless_supervisor.events.append(TriggerEventEntry(1, event_with_seq, 7)) + jobless_supervisor.events.append(TriggerEventEntry(2, event_without_seq, None)) with mock.patch.object(TriggerRunnerSupervisor, "on_trigger_event", autospec=True) as mock_event: jobless_supervisor.handle_events() @@ -2453,7 +2449,7 @@ def test_handle_events_records_persist_confirmations(jobless_supervisor): def test_handle_events_does_not_confirm_seq_when_persist_fails(jobless_supervisor): """A seq whose event failed to persist is never confirmed, so the broker advance fails out.""" - jobless_supervisor.events.append((1, TriggerEvent(True), 7)) + jobless_supervisor.events.append(TriggerEventEntry(1, TriggerEvent(True), 7)) with mock.patch.object( TriggerRunnerSupervisor, From be51a7656f18b199218bb59ba9a0b198e04d2460 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 12 Jun 2026 16:50:38 +0800 Subject: [PATCH 08/15] refactor(triggers): compute shared-stream ack mode once at group construction --- airflow-core/src/airflow/triggers/shared_stream.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index 447733f84d3cc..98c2279d3b224 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -462,6 +462,8 @@ def __init__( # excluded from subsequent broadcasts until they unsubscribe. self._failed_subscribers: set[int] = set() self._poll_task: asyncio.Task | None = None + # Constant for the group's lifetime: trigger_class never changes. + self._ack_required: bool = self._is_ack_required() # Ack mode state — populated only when create_shared_stream_producer # is overridden. self._outstanding: dict[int, _OutstandingEntry] = {} @@ -567,7 +569,7 @@ def _is_ack_required(self) -> bool: return False async def _poll(self) -> None: - ack_required = self._is_ack_required() + ack_required = self._ack_required producer: SharedStreamProducer | None = None terminal_exc: BaseException | None = None try: @@ -905,7 +907,7 @@ def subscribe(self, trigger_id: int) -> AsyncIterator[Any]: raise RuntimeError(f"Trigger {trigger_id} already subscribed to shared stream {self.key!r}") queue: asyncio.Queue = asyncio.Queue(maxsize=self._max_subscriber_queue) self._subscribers[trigger_id] = queue - if self._is_ack_required(): + if self._ack_required: return self._ack_drain(trigger_id, queue) return _drain(queue) From 0df647e453e1753a4173e14144150dc16fe4bdbf Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 12 Jun 2026 18:29:31 +0800 Subject: [PATCH 09/15] refactor(triggers): consolidate shared-stream ack tests with parametrize Nine sibling tests fold into four parametrized ones (same assertions, plus a previously untested nack-then-nack case); net -112 lines. --- .../tests/unit/triggers/test_shared_stream.py | 408 +++++++----------- 1 file changed, 148 insertions(+), 260 deletions(-) diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index 99cfb50668377..2d835820320c2 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -1269,50 +1269,22 @@ async def test_nack_removes_subscriber_from_ack_set_without_redeliver(): await manager.unsubscribe(2, key) -@pytest.mark.asyncio -async def test_double_ack_is_noop(): - """Calling ack() twice on the same token must not raise or advance twice.""" - cls = _make_ack_required_trigger_class( - [ - ({"msg": "double-ack"}, "receipt-double"), - ] - ) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - - async with _consume_in_background(s1) as collector: - await collector.wait_for(1) - token = collector.tokens[0] - assert token is not None - - await token.ack() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-double"] - - # Second ack — must be a no-op, not raise, not double-advance. - await token.ack() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-double"], "second ack must not trigger a second advance" - finally: - await manager.unsubscribe(1, key) - - @pytest.mark.asyncio @pytest.mark.parametrize( - ("first", "second"), + ("first", "second", "expected_outcome"), [ - ("ack", "nack"), - ("nack", "ack"), + pytest.param("ack", "nack", AdvanceOutcome(acked=1, nacked=0, failed=0), id="ack-then-nack"), + pytest.param("nack", "ack", AdvanceOutcome(acked=0, nacked=1, failed=0), id="nack-then-ack"), + pytest.param("ack", "ack", AdvanceOutcome(acked=1, nacked=0, failed=0), id="ack-then-ack"), + pytest.param("nack", "nack", AdvanceOutcome(acked=0, nacked=1, failed=0), id="nack-then-nack"), ], ) -async def test_cross_order_double_call_is_noop(first, second): - """ack-then-nack and nack-then-ack are both no-ops — _resolved blocks the second call.""" +async def test_double_resolution_is_noop_and_counts_once(first, second, expected_outcome): + """Any second resolution call on a token is a no-op — _resolved blocks it. + + advance fires exactly once, and the subscriber is counted once, by the + first call's resolution. + """ cls = _make_ack_required_trigger_class( [ ({"msg": "cross-order"}, "receipt-cross"), @@ -1337,12 +1309,13 @@ async def test_cross_order_double_call_is_noop(first, second): await asyncio.sleep(0) assert cls.advanced == ["receipt-cross"] - # Second call (opposite) — must be a no-op; advance must not fire again. + # Second call — must be a no-op, not raise; advance must not fire again. await getattr(token, second)() await asyncio.sleep(0) assert cls.advanced == ["receipt-cross"], ( f"{second}() after {first}() must not trigger a second advance" ) + assert cls.outcomes == [expected_outcome] finally: await manager.unsubscribe(1, key) @@ -1434,19 +1407,41 @@ async def collect_full(): @pytest.mark.asyncio -async def test_no_subscriber_snapshot_advances_in_order(): +@pytest.mark.parametrize( + ("lane_for", "events", "expected_per_lane"), + [ + pytest.param( + None, + [ + ({"msg": "no-sub-1"}, "p1"), + ({"msg": "no-sub-2"}, "p2"), + ({"msg": "no-sub-3"}, "p3"), + ], + {None: ["p1", "p2", "p3"]}, + id="no-lane-global-order", + ), + pytest.param( + lambda payload: payload[0], + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "b2"}, "b2"), + ], + {"a": ["a1", "a2"], "b": ["b1", "b2"]}, + id="lanes-per-lane-order", + ), + ], +) +async def test_empty_snapshot_advances_in_fanout_order(lane_for, events, expected_per_lane): """Events broadcast while no subscribers are online still advance, in fan-out order. Empty-snapshot entries are born resolved and go through the same ordered - pump as everything else; their outcomes carry all-zero counts. + pump as everything else; their outcomes carry all-zero counts. Without + lanes the full global order is preserved; with lanes the per-lane + projections keep fan-out order. """ - cls = _make_ack_required_trigger_class( - [ - ({"msg": "no-sub-1"}, "p1"), - ({"msg": "no-sub-2"}, "p2"), - ({"msg": "no-sub-3"}, "p3"), - ] - ) + cls = _make_ack_required_trigger_class(events, lane_for=lane_for) cls.advanced.clear() cls.outcomes.clear() @@ -1462,14 +1457,19 @@ async def test_no_subscriber_snapshot_advances_in_order(): # The poll task is still running; give it a few ticks to broadcast. deadline = asyncio.get_event_loop().time() + 1.0 while asyncio.get_event_loop().time() < deadline: - if len(cls.advanced) >= 3: + if len(cls.advanced) >= len(events): break await asyncio.sleep(0.01) - assert cls.advanced == ["p1", "p2", "p3"], ( - "advance must be called for every event, in fan-out order, even with no subscribers online" - ) - assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * 3 + # Per-lane projection of the advance calls; with no lane_for the + # projection is the full global sequence. + for lane, expected in expected_per_lane.items(): + projected = cls.advanced if lane_for is None else [p for p in cls.advanced if lane_for(p) == lane] + assert projected == expected, ( + "advance must be called for every event, in fan-out order, even with no subscribers online" + ) + assert len(cls.advanced) == len(events) + assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * len(events) assert all(outcome.is_clean for outcome in cls.outcomes) del s1 # silence unused-variable warning @@ -1752,36 +1752,56 @@ async def test_out_of_order_resolve_still_advances_in_fanout_order(): @pytest.mark.asyncio -async def test_pump_serializes_advance_calls(): - """Two events resolved together never produce overlapping advance calls.""" +@pytest.mark.parametrize( + ("lane_for", "payloads", "expected_advanced", "ordered"), + [ + pytest.param(None, ["p1", "p2"], ["p1", "p2"], True, id="single-lane-full-order"), + pytest.param(lambda payload: payload[0], ["a1", "b1"], ["a1", "b1"], False, id="across-lanes"), + ], +) +async def test_pump_serializes_advance_calls(lane_for, payloads, expected_advanced, ordered): + """Events resolved together never produce overlapping advance calls. + + This holds within a single lane (where the full fan-out order is also + preserved) and across lanes (where the relative order is not guaranteed, + but at most one advance call is ever in flight globally). + """ release = asyncio.Event() state = {"in_flight": 0, "max_in_flight": 0} advanced: list = [] - class _BlockingAdvanceProducer(SharedStreamProducer): - async def open_stream(self): - yield {"n": 1}, "p1" - yield {"n": 2}, "p2" - await asyncio.Event().wait() + def make_blocking_trigger(): + class _BlockingAdvanceProducer(SharedStreamProducer): + async def open_stream(self): + for payload in payloads: + yield {"n": payload}, payload + await asyncio.Event().wait() + + def get_advance_lane(self, broker_payload): + if lane_for is None: + return super().get_advance_lane(broker_payload) + return lane_for(broker_payload) + + async def advance(self, broker_payload, outcome): + state["in_flight"] += 1 + state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) + await release.wait() + advanced.append(broker_payload) + state["in_flight"] -= 1 - async def advance(self, broker_payload, outcome): - state["in_flight"] += 1 - state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) - await release.wait() - advanced.append(broker_payload) - state["in_flight"] -= 1 + class _BlockingAdvanceTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _BlockingAdvanceProducer() - class _BlockingAdvanceTrigger(_ProgrammableSharedStreamTrigger): - @classmethod - def create_shared_stream_producer(cls, kwargs): - return _BlockingAdvanceProducer() + async def filter_shared_stream(self, shared_stream): + async for raw, token in shared_stream: + await token.ack() + yield TriggerEvent(raw) - async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() - yield TriggerEvent(raw) + return _BlockingAdvanceTrigger() - t1 = _BlockingAdvanceTrigger() + t1 = make_blocking_trigger() key = t1.shared_stream_key() manager = SharedStreamManager() try: @@ -1791,7 +1811,8 @@ async def filter_shared_stream(self, shared_stream): await collector.wait_for(2) tokens = collector.tokens - # Resolve both events; the pump starts the first advance, which blocks. + # Resolve both events; the pump starts the first advance, which + # blocks. The second must not start while one is in flight. await tokens[0].ack() await tokens[1].ack() for _ in range(5): @@ -1806,8 +1827,11 @@ async def filter_shared_stream(self, shared_stream): break await asyncio.sleep(0) - assert advanced == ["p1", "p2"] - assert state["max_in_flight"] == 1, "advance calls must never overlap" + if ordered: + assert advanced == expected_advanced + else: + assert sorted(advanced) == expected_advanced + assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" finally: await manager.unsubscribe(1, key) @@ -2233,109 +2257,6 @@ async def test_lane_internal_fifo_with_out_of_order_resolve(): await manager.unsubscribe(1, key) -@pytest.mark.asyncio -async def test_max_in_flight_one_across_lanes(): - """Even with multiple lanes ready, at most one advance call is ever in flight.""" - release = asyncio.Event() - state = {"in_flight": 0, "max_in_flight": 0} - advanced: list = [] - - class _BlockingLaneProducer(SharedStreamProducer): - async def open_stream(self): - yield {"n": "a1"}, "a1" - yield {"n": "b1"}, "b1" - await asyncio.Event().wait() - - def get_advance_lane(self, broker_payload): - return broker_payload[0] - - async def advance(self, broker_payload, outcome): - state["in_flight"] += 1 - state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) - await release.wait() - advanced.append(broker_payload) - state["in_flight"] -= 1 - - class _BlockingLaneTrigger(_ProgrammableSharedStreamTrigger): - @classmethod - def create_shared_stream_producer(cls, kwargs): - return _BlockingLaneProducer() - - async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() - yield TriggerEvent(raw) - - t1 = _BlockingLaneTrigger() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - - async with _consume_in_background(s1) as collector: - await collector.wait_for(2) - tokens = collector.tokens - - # Resolve both lanes; the pump starts one advance, which blocks. - await tokens[0].ack() - await tokens[1].ack() - for _ in range(5): - await asyncio.sleep(0) - assert state["in_flight"] == 1, "the other lane's advance must not start while one is in flight" - assert advanced == [] - - release.set() - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(advanced) >= 2: - break - await asyncio.sleep(0) - - assert sorted(advanced) == ["a1", "b1"] - assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" - finally: - await manager.unsubscribe(1, key) - - -@pytest.mark.asyncio -async def test_empty_snapshot_routes_through_lane(): - """Born-resolved events (no subscribers online) keep per-lane fan-out order.""" - cls = _make_ack_required_trigger_class( - [ - ({"n": "a1"}, "a1"), - ({"n": "b1"}, "b1"), - ({"n": "a2"}, "a2"), - ({"n": "b2"}, "b2"), - ], - lane_for=lambda payload: payload[0], - ) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - - # Subscribe then immediately unsubscribe so the group exists but has zero - # subscribers when the events arrive. - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - manager._groups[key].unsubscribe(1) # remove without stopping the poll task - - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(cls.advanced) >= 4: - break - await asyncio.sleep(0.01) - - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] - assert len(cls.advanced) == 4 - assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * 4 - - del s1 # silence unused-variable warning - await manager.stop_all() - - @pytest.mark.asyncio async def test_get_advance_lane_raise_terminates_poll(): """A get_advance_lane that raises fails the poll like any other poll failure.""" @@ -2552,33 +2473,6 @@ def now() -> float: await manager.stop_all() -@pytest.mark.asyncio -async def test_double_resolve_counts_once(): - """ack() then nack() on the same token counts the subscriber once, as acked.""" - cls = _make_ack_required_trigger_class([({"msg": "double"}, "receipt-double-resolve")]) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - - async with _consume_in_background(s1) as collector: - await collector.wait_for(1) - token = collector.tokens[0] - - await token.ack() - await token.nack() - await asyncio.sleep(0) - - assert cls.advanced == ["receipt-double-resolve"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] - finally: - await manager.unsubscribe(1, key) - - @pytest.mark.asyncio async def test_aclose_called_once_on_stop_path(): """The producer is closed exactly once when the last subscriber leaves.""" @@ -3066,8 +2960,46 @@ async def test_unsubscribe_with_unconfirmed_seq_waits_for_confirmation(): @pytest.mark.asyncio -async def test_unconfirmed_persist_below_timeout_keeps_event_outstanding(): - """At 99 ms of a 100 ms timeout, an unconfirmed event stays outstanding and can still recover.""" +@pytest.mark.parametrize( + ( + "clock_value", + "advanced_before_confirm", + "outcomes_before_confirm", + "outstanding_before_confirm", + "outcomes_after_confirm", + ), + [ + pytest.param( + 0.099, + [], + [], + 1, + [AdvanceOutcome(acked=1, nacked=0, failed=0)], + id="below-timeout-confirm-recovers", + ), + pytest.param( + 0.2, + ["p1"], + [AdvanceOutcome(acked=0, nacked=0, failed=1)], + 0, + [AdvanceOutcome(acked=0, nacked=0, failed=1)], + id="late-confirm-after-timeout-is-noop", + ), + ], +) +async def test_unconfirmed_persist_confirm_timing_around_timeout( + clock_value, + advanced_before_confirm, + outcomes_before_confirm, + outstanding_before_confirm, + outcomes_after_confirm, +): + """Confirmation timing against a 100 ms ack timeout. + + Strictly inside the window the unconfirmed event stays outstanding and a + confirmation recovers it cleanly; once the timeout has already failed the + event, a late confirmation changes nothing. + """ cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) cls.advanced.clear() cls.outcomes.clear() @@ -3092,17 +3024,16 @@ def now() -> float: await token.ack() next_task = await _resume_past_yield(it) - # Strictly inside the timeout window: the event must stay outstanding. - fake_clock[0] = 0.099 + fake_clock[0] = clock_value await asyncio.sleep(0.05) - assert cls.advanced == [] - assert len(manager._groups[key]._outstanding) == 1 + assert cls.advanced == advanced_before_confirm + assert cls.outcomes == outcomes_before_confirm + assert len(manager._groups[key]._outstanding) == outstanding_before_confirm - # The confirmation arrives inside the window — clean recovery. manager.confirm_persisted([seq]) await asyncio.sleep(0) assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + assert cls.outcomes == outcomes_after_confirm finally: await _cancel_quietly(next_task) await manager.stop_all() @@ -3151,49 +3082,6 @@ def now() -> float: await manager.stop_all() -@pytest.mark.asyncio -async def test_late_confirm_after_timeout_is_noop(): - """A confirmation arriving after the timeout already failed the event changes nothing.""" - cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - - fake_clock: list[float] = [0.0] - - def now() -> float: - return fake_clock[0] - - manager = SharedStreamManager(ack_timeout=0.1, _now=now) - next_task = None - try: - stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) - it = stream.__aiter__() - _raw, token = await _pull_pair(it) - - seq = manager.bind_pending_event(trigger_id=1, key=key) - assert seq is not None - await token.ack() - next_task = await _resume_past_yield(it) - - fake_clock[0] = 0.2 - await asyncio.sleep(0.05) - assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=0, failed=1)] - - # The confirmation arrives after the fact: no second advance, no - # change to the recorded outcome. - manager.confirm_persisted([seq]) - await asyncio.sleep(0) - assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=0, failed=1)] - finally: - await _cancel_quietly(next_task) - await manager.stop_all() - - @pytest.mark.asyncio async def test_group_stop_abandons_unconfirmed_advance(): """The last subscriber leaving with an unconfirmed event stops the group without advancing.""" From 3a30e9e942b98303e6bd8af30067948a631ff8c0 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Sat, 13 Jun 2026 23:08:58 +0800 Subject: [PATCH 10/15] refactor(triggers): drop public ack tokens from shared-stream ack mode Subscribers no longer receive an AckToken. The manager derives an event's resolution from consumption progress (the filter moving past it) plus persist confirmation, so the same filter code runs unchanged on the fast path and in ack mode. The producer's advance hook now takes a per-lane batch of AdvanceItem (broker_payload + AdvanceOutcome) instead of one event at a time, so cumulative brokers commit once per batch. A filter can call reject_shared_stream_event() to terminally refuse an event, counted separately in AdvanceOutcome.rejected so a producer can dead-letter rejects while redelivering involuntary failures. --- .../event-scheduling.rst | 196 ++- .../src/airflow/config_templates/config.yml | 7 +- airflow-core/src/airflow/triggers/base.py | 40 +- .../src/airflow/triggers/shared_stream.py | 544 +++--- .../tests/unit/jobs/test_triggerer_job.py | 17 +- .../tests/unit/triggers/test_base_trigger.py | 6 +- .../tests/unit/triggers/test_shared_stream.py | 1518 ++++++++--------- 7 files changed, 1225 insertions(+), 1103 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index 874a6effba58f..5bc2f11717b87 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -167,7 +167,12 @@ For upstreams where the producer must advance (commit, delete, or ack) only after all subscribers have processed an event, override :meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer` to return a :class:`~airflow.triggers.shared_stream.SharedStreamProducer`. -When this factory is overridden, the manager enters **ack mode**: +When this factory is overridden, the manager enters **ack mode**. The +subscriber side does not change: ``filter_shared_stream`` receives raw +events exactly as on the fast path, so the same filter code works in both +modes — the framework infers when the broker may advance from each +subscriber's consumption progress and from the persistence of the trigger +events it derived. 1. The manager calls ``create_shared_stream_producer(kwargs)`` once per shared-stream group. The returned producer owns the broker connection for @@ -176,37 +181,92 @@ When this factory is overridden, the manager enters **ack mode**: 2. The producer's ``open_stream`` yields ``(event, broker_payload)`` tuples, where ``broker_payload`` is whatever the producer needs later (e.g. an SQS receipt handle, a Kafka offset, a Pub/Sub ack ID). -3. Each subscriber's ``filter_shared_stream`` receives ``(event, token)`` - pairs. The subscriber calls ``await token.ack()`` once it has accepted - the event, or ``await token.nack()`` to opt out. +3. Each subscriber's ``filter_shared_stream`` receives raw events exactly + as in the fast path. A subscriber resolves an event once it has moved + past it (pulled the next raw event, or unsubscribed) and every + ``TriggerEvent`` it derived from the event has been persisted to the + metadata database. 4. Once every subscriber in the fan-out set has resolved an event (by - ``ack()``, ``nack()``, unsubscribing, timing out, or overflowing its - queue) **and** every ``TriggerEvent`` the subscribers derived from it - has been persisted to the metadata database, the manager calls - ``await producer.advance(broker_payload, outcome)`` — commit the offset, - delete the SQS message, etc. The ``outcome`` is an - :class:`~airflow.triggers.shared_stream.AdvanceOutcome` carrying - per-event counts of how the subscribers resolved. - -**Ordering guarantee**: by default every event belongs to the same lane, -so advance calls are dispatched strictly in event order — ``advance`` for -event N is awaited only after event N-1's ``advance`` returned (or failed -and was logged). A producer can override ``get_advance_lane`` to narrow -that ordering to within a lane: events whose lane values compare equal -are advanced in event order relative to each other, while events in -different lanes do not wait for one another. Either way, at most one -``advance`` call is awaited at a time. When the poll ends, the manager -calls ``await producer.aclose()`` once, best-effort. + moving past it, unsubscribing, timing out, or overflowing its queue), + the manager calls ``await producer.advance(batch)`` with the contiguous + prefix of fully resolved events in the event's lane — commit the + offsets, delete the SQS messages, etc. Each item in the batch is an + :class:`~airflow.triggers.shared_stream.AdvanceItem` carrying the + event's ``broker_payload`` and an + :class:`~airflow.triggers.shared_stream.AdvanceOutcome` with per-event + counts of how the subscribers resolved. + +**Rejecting an event**: a subscriber's filter can actively refuse a raw +event instead of yielding a trigger event from it, by calling +:func:`~airflow.triggers.shared_stream.reject_shared_stream_event` while +processing that event. This is distinct from an involuntary failure. A +``failed`` count means a subscriber did not finish in time (ack timeout) or +fell behind (queue overflow) — the right response is usually redelivery. A +``rejected`` count means a subscriber decided the event must not produce a +trigger event and should be terminally discarded — the right response is to +dead-letter or ``nack`` it, not redeliver. The +:class:`~airflow.triggers.shared_stream.AdvanceOutcome` reports both +counts separately so the producer can apply the right per-broker action in +``advance``: + +* Azure Service Bus: dead-letter the message when ``rejected`` is non-zero, + abandon it (so the broker redelivers) when only ``failed`` is non-zero, + and complete it when every subscriber accepted the event. +* Pub/Sub: ``nack`` the message on a reject, otherwise ``ack`` it. + +The framework only reports the counts — it never dead-letters, ``nack`` s, +or redelivers on its own; that broker-specific decision lives entirely in +the producer's ``advance``. + +``reject_shared_stream_event`` is meaningful only while the filter is +processing a raw event in ack mode (the binding window of that event is +open). Called on the fast path, from a standalone ``run()``, or between two +raw events, it logs a warning and does nothing, because there is no broker +advance to influence. Because it resolves the event immediately, there is +nothing to persist and the reject does not wait on the persistence gate. + +``is_clean`` is ``True`` only when an event has no rejects and no failures — +every subscriber that was online at broadcast accepted it. A single reject +or a single failure makes ``is_clean`` ``False``. + +Example — reject inside a filter: + +.. code-block:: python + + from airflow.triggers.shared_stream import reject_shared_stream_event + + + async def filter_shared_stream(self, shared_stream): + async for raw in shared_stream: + if raw.get("malformed"): + # Never produce a trigger event from this; have the broker + # dead-letter it rather than redeliver it. + reject_shared_stream_event() + continue + yield TriggerEvent(raw) + +**Ordering guarantee**: by default every event belongs to the same lane. +The items of a batch are in event order and form their lane's contiguous +resolved prefix; within a lane, batches arrive strictly in order — the +next ``advance`` is awaited only after the previous call returned (or +failed and was logged). A producer can override ``get_advance_lane`` to +narrow that ordering to within a lane: events whose lane values compare +equal are batched and advanced in event order relative to each other, +while events in different lanes do not wait for one another. Either way, +at most one ``advance`` call is awaited at a time, and cumulative schemes +such as a Kafka offset commit only need to commit the batch's last item. +When the poll ends, the manager calls ``await producer.aclose()`` once, +best-effort. Example — SQS-like producer: .. code-block:: python - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Sequence from typing import Any from airflow.triggers.base import BaseEventTrigger, TriggerEvent - from airflow.triggers.shared_stream import AdvanceOutcome, SharedStreamProducer + from airflow.triggers.shared_stream import AdvanceItem, SharedStreamProducer class SqsSharedStreamProducer(SharedStreamProducer): @@ -222,11 +282,12 @@ Example — SQS-like producer: for msg in messages: yield msg["Body"], msg["ReceiptHandle"] - async def advance(self, broker_payload: Any, outcome: AdvanceOutcome) -> None: - # Called once all subscribers have resolved this message. - if outcome.is_clean: - await delete_sqs_message(self.client, self.queue_url, broker_payload) - # Otherwise leave the message for the visibility timeout to redeliver. + async def advance(self, batch: Sequence[AdvanceItem]) -> None: + # Called with one lane's batch of fully resolved messages. + for receipt_handle, outcome in batch: + if outcome.is_clean: + await delete_sqs_message(self.client, self.queue_url, receipt_handle) + # Otherwise leave the message for the visibility timeout to redeliver. async def aclose(self) -> None: if self.client is not None: @@ -253,12 +314,9 @@ Example — SQS-like producer: return SqsSharedStreamProducer(kwargs["queue_url"]) async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: + async for raw in shared_stream: if self.region is None or raw.get("region") == self.region: - await token.ack() yield TriggerEvent(raw) - else: - await token.nack() async def run(self): yield TriggerEvent({}) @@ -291,50 +349,48 @@ the other partitions: topic, partition, _offset = broker_payload return topic, partition - async def advance(self, broker_payload, outcome): - topic, partition, offset = broker_payload - # Within one lane — here, one partition — advance calls arrive in - # event order, so this cumulative commit can never skip past an - # event that is still pending. + async def advance(self, batch): + # The batch is one lane's — here, one partition's — contiguous + # resolved prefix, in event order, so committing the offset of + # its last item covers the whole batch and can never skip past + # an event that is still pending. + topic, partition, offset = batch[-1].broker_payload await self.consumer.commit(topic, partition, offset + 1) async def aclose(self): if self.consumer is not None: await self.consumer.stop() -The order of ``ack()`` relative to ``yield`` does not affect correctness: -an ack completes only after the subscriber has moved past the event and -the trigger events it derived from it were persisted, so nothing depends -on code running after the ``yield``. The one constraint on filter authors -is the binding between raw events and the trigger events derived from -them: yield every ``TriggerEvent`` derived from a raw event before pulling -the next raw event from the shared stream — which is what a -straightforward filter loop does anyway. - -**Snapshot-at-fan-out**: the set of subscribers that must ack a given event -is frozen at the moment the event is broadcast. A subscriber that joins -after the event was dispatched is not added to that event's pending set. - -**Per-event ack timeout**: if a subscriber has not called ``ack()`` or -``nack()`` within the ack timeout (default 5 minutes, configurable via the -``[triggerer] shared_stream_ack_timeout`` config option), the manager force-fails that -subscriber's trigger. The same timeout covers an ack that is waiting on a -persistence confirmation that never arrives. Other subscribers are not -affected; once their acks arrive, the producer advances normally. The ack -timeout is a manager-level safety net and does not replace any native broker -session or visibility timeout. +The one constraint on filter authors is the binding between raw events and +the trigger events derived from them: yield every ``TriggerEvent`` derived +from a raw event before pulling the next raw event from the shared +stream — which is what a straightforward filter loop does anyway. + +**Snapshot-at-fan-out**: the set of subscribers that must resolve a given +event is frozen at the moment the event is broadcast. A subscriber that +joins after the event was dispatched is not added to that event's pending +set. + +**Per-event ack timeout**: if a subscriber has not finished processing an +event within the ack timeout (default 5 minutes, configurable via the +``[triggerer] shared_stream_ack_timeout`` config option) — it is still on +the event, or some of its derived trigger events were never confirmed +persisted — the manager force-fails that subscriber's trigger. Other +subscribers are not affected; once they resolve, the producer advances +normally. The ack timeout is a manager-level safety net and does not +replace any native broker session or visibility timeout. From the subscriber's perspective the force-fail surfaces as an ``AckTimeout`` (importable from ``airflow.triggers.shared_stream``) raised by the ``shared_stream`` iterator inside ``filter_shared_stream``. Letting it propagate is fine — the trigger fails through the standard trigger-failure path; catch it only if the subscriber needs to run cleanup before failing. -**Triggerer restart**: outstanding acks live in memory only. After a -triggerer restart, the broker redelivers messages that were not yet -acknowledged. Subscribers must therefore be idempotent. +**Triggerer restart**: resolution state lives in memory only. After a +triggerer restart, the broker redelivers messages that were never +advanced. Subscribers must therefore be idempotent. **Durability**: the broker advance is gated on persistence. A subscriber's -``ack()`` completes only after every ``TriggerEvent`` it derived from the +resolution completes only after every ``TriggerEvent`` it derived from the event has been stored in the metadata database; the confirmation reaches the trigger runner on the next state sync, typically within a second or two. If the confirmation never arrives — the triggerer crashed, or the @@ -346,21 +402,15 @@ group stops while events are still awaiting confirmation (for example, the last subscriber unsubscribes right after producing an event): the pending advances are abandoned and the broker redelivers those events. -**nack() semantics**: ``nack()`` removes the subscriber from the pending -ack set and is counted in the ``nacked`` field of the event's -``AdvanceOutcome``. What that means for the broker is the producer's -decision inside ``advance`` — commit anyway, skip the commit, or trigger an -immediate redeliver (e.g. ``ChangeMessageVisibility(0)`` for SQS). - **``shared_stream_subscriber_queue_size`` in ack mode**: the config bound still governs unprocessed raw events per subscriber. The manager does -**not** wait for outstanding acks before pulling the next upstream event; -back-pressure is queue-bound — a subscriber whose queue is full is +**not** wait for outstanding resolutions before pulling the next upstream +event; back-pressure is queue-bound — a subscriber whose queue is full is force-failed. The queue primarily guards against burst delivery before a subscriber's filter runs. Broker advances, by contrast, are dispatched in -order within each lane: an event whose acks are still pending delays the -advance of every later event in the same lane, with the ack timeout -bounding that wait. +order within each lane: an event with pending subscribers delays the +advance of every later event in the same lane — resolved events behind it +accumulate into the next batch — with the ack timeout bounding that wait. Verifying that sharing is active ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index c000906b8b90f..a288c4fe6ed6d 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -2922,9 +2922,10 @@ triggerer: shared_stream_ack_timeout: description: | Per-event ack timeout in seconds for shared-stream triggers running in ack mode (triggers that - override ``BaseEventTrigger.create_shared_stream_producer``). A subscriber that has not called - ``token.ack()`` or ``token.nack()`` within this window is force-failed. Other subscribers in the - same group are unaffected; once their acks arrive the producer advances normally. + override ``BaseEventTrigger.create_shared_stream_producer``). A subscriber that has not finished + processing an event (moved past it and had its derived trigger events confirmed persisted) within + this window is force-failed. Other subscribers in the same group are unaffected; once they + resolve, the producer advances normally. version_added: 3.3.0 type: float example: ~ diff --git a/airflow-core/src/airflow/triggers/base.py b/airflow-core/src/airflow/triggers/base.py index 0b5f8e02417e9..6f6061e3366f8 100644 --- a/airflow-core/src/airflow/triggers/base.py +++ b/airflow-core/src/airflow/triggers/base.py @@ -268,10 +268,9 @@ class BaseEventTrigger(BaseTrigger): events. Called once per group in the triggerer. * :meth:`filter_shared_stream` — convert the shared raw stream into this trigger's own ``TriggerEvent`` instances, applying any per-trigger - filtering or transformation. In ack mode, the stream yields - ``(event, token)`` pairs, where ``token`` is an - :class:`~airflow.triggers.shared_stream.AckToken`; call - ``await token.ack()`` once the event is accepted. + filtering or transformation. The stream has the same shape in ack + mode as on the fast path, so the filter needs no mode-specific + branches. Triggers whose ``shared_stream_key`` returns ``None`` (the default) keep the existing behavior: each trigger gets its own poll loop via @@ -284,11 +283,11 @@ class BaseEventTrigger(BaseTrigger): :class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns the broker connection, supplies events through its ``open_stream`` method (instead of :meth:`open_shared_stream`), and is told to advance the - broker — commit, delete, or ack the message — once all subscribers that - were online at broadcast time have resolved an event and the trigger - events they derived from it were persisted to the metadata database. - When this factory is not overridden, the fast path is taken: no ack - tokens are issued and subscribers receive raw events exactly as before. + broker — commit, delete, or ack the messages — through per-lane batches + of events whose subscribers have all moved past them and whose derived + trigger events were persisted to the metadata database. Subscribers + receive raw events in both modes; ack mode changes only the bookkeeping + behind the stream, never the filter code. See :mod:`airflow.triggers.shared_stream` for the full ack-mode design, including snapshot-at-fan-out semantics, per-event timeout behavior, and @@ -384,15 +383,15 @@ def create_shared_stream_producer(cls, kwargs: dict[str, Any]) -> SharedStreamPr The manager calls it once per shared-stream group; the returned :class:`~airflow.triggers.shared_stream.SharedStreamProducer` owns the broker connection for the lifetime of one poll — it supplies - events through ``open_stream``, is told to advance the broker once - all subscribers have resolved an event and the trigger events they - derived from it were persisted, and is closed when the poll ends. - Do not open the broker connection here; open it lazily inside the - producer's ``open_stream``. - - Triggers that do not override this method run the fast path: no ack - tokens are issued and subscribers receive raw events from - :meth:`open_shared_stream` exactly as before. + events through ``open_stream``, is told to advance the broker in + per-lane batches of events whose subscribers have all moved past + them and had their derived trigger events confirmed persisted, and + is closed when the poll ends. Do not open the broker connection + here; open it lazily inside the producer's ``open_stream``. + + Triggers that do not override this method run the fast path: + subscribers receive raw events from :meth:`open_shared_stream` + exactly as before. The stream shape is the same in both modes. """ raise NotImplementedError(f"{cls.__name__} does not implement create_shared_stream_producer") @@ -406,6 +405,11 @@ async def filter_shared_stream(self, shared_stream: AsyncIterator[Any]) -> Async ``yield`` a :class:`TriggerEvent` for each one that should fire this trigger. + In ack mode, to have the broker terminally drop a raw event rather + than producing a trigger event from it, call + :func:`~airflow.triggers.shared_stream.reject_shared_stream_event` + while handling that raw event instead of yielding. + Required only when :meth:`shared_stream_key` returns non-``None``. """ raise NotImplementedError( diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index 98c2279d3b224..c2f5a2d609c1a 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -34,34 +34,50 @@ :meth:`~airflow.triggers.base.BaseEventTrigger.create_shared_stream_producer`, the manager switches to **ack mode** for that stream. +Subscribers receive the same stream of raw events in both modes: the same +``filter_shared_stream`` code works unchanged on the fast path and in ack +mode. The difference is resolution bookkeeping and the broker advance, +both of which the manager derives from consumption progress — no extra +subscriber API. + The factory is called once per group and returns a :class:`SharedStreamProducer` that owns the broker connection for the lifetime of one poll. The manager drives the producer's ``open_stream``, -which yields ``(raw_event, broker_payload)`` tuples, and hands -``(raw_event, token)`` pairs — where ``token`` is an :class:`AckToken` — to -each subscriber's queue instead of the raw event alone. A subscriber calls -``await token.ack()`` once it has accepted the event, or -``await token.nack()`` to opt out. An ack completes the subscriber's -resolution only after the subscriber has moved past the event (pulled the -next raw event, or unsubscribed) and every -:class:`~airflow.triggers.base.TriggerEvent` it derived from the event has -been confirmed persisted to the metadata database; ``nack()`` and -force-failure resolve immediately. When every subscriber that was online -at broadcast time has resolved the event, the manager calls -``await producer.advance(broker_payload, outcome)`` exactly once — this is -where the producer commits / deletes / acks on the broker. The -:class:`AdvanceOutcome` carries per-event counts of how the subscribers -resolved. Advance calls are dispatched by a single pump task: the -producer's :meth:`~SharedStreamProducer.get_advance_lane` assigns each -event to a lane, events in the same lane are advanced strictly in fan-out -order, and events in different lanes do not wait for one another. At any -moment at most one ``advance`` call is awaited globally; the default lane +which yields ``(raw_event, broker_payload)`` tuples, and broadcasts each +raw event to every subscriber's queue. A subscriber resolves an event +once it has moved past it (pulled the next raw event, or unsubscribed) +and every :class:`~airflow.triggers.base.TriggerEvent` it derived from +the event has been confirmed persisted to the metadata database; an event +the filter skips without yielding anything resolves cleanly as soon as +the filter loops back for the next raw event. Force-failure resolves the +subscriber immediately. When every subscriber that was online at +broadcast time has resolved an event, the event is fully resolved and +eligible for the broker advance: the manager calls +``await producer.advance(batch)`` with one lane's contiguous prefix of +fully resolved events — this is where the producer commits / deletes / +acks on the broker. Each :class:`AdvanceItem` in the batch carries the +event's ``broker_payload`` and an :class:`AdvanceOutcome` with per-event +counts of how the subscribers resolved. Advances are dispatched by a +single pump task: the producer's +:meth:`~SharedStreamProducer.get_advance_lane` assigns each event to a +lane; within a lane, batch items arrive in fan-out order and the next +batch is awaited only after the call for the previous one returned (or +raised and was logged); lanes do not wait for one another, but at any +moment at most one ``advance`` call is awaited globally. The default lane assignment (every event in the same lane) preserves the original single global order. When the poll ends, the manager awaits ``producer.aclose()`` once, best-effort. -**Snapshot-at-fan-out**: the set of subscribers that must ack an event is -frozen at broadcast time. A subscriber that joins after the event was +**Subscriber reject**: instead of yielding a trigger event, a subscriber's +filter can call :func:`reject_shared_stream_event` to terminally refuse the +raw event it is currently processing. A reject resolves immediately (there +is nothing to persist) and is counted in :attr:`AdvanceOutcome.rejected`, +separate from an involuntary ``failed``: the producer is expected to +dead-letter / ``nack`` rejects but redeliver failures. The framework only +reports the counts — the per-broker decision lives in ``advance``. + +**Snapshot-at-fan-out**: the set of subscribers that must resolve an event +is frozen at broadcast time. A subscriber that joins after the event was broadcast is not added to that event's pending set. **Persistence-gated advance**: the trigger events a subscriber derives @@ -69,7 +85,7 @@ the supervisor confirms each one after the event is stored in the metadata database, and the confirmation reaches the runner on the next state sync (typically one sync round, a second or two — well within the -ack timeout). The subscriber's ack only completes once all of its +ack timeout). The subscriber's resolution only completes once all of its sequence numbers for the event are confirmed. If a confirmation never arrives — the persist failed, or the triggerer crashed in between — the ack timeout fails the event, the producer does not commit, and the broker redelivers. The @@ -78,33 +94,37 @@ from the shared stream (the natural way to write a filter). **Per-event ack timeout**: a background task scans outstanding events. -Any subscriber that has not acknowledged within ``ack_timeout`` seconds is -force-failed via the existing :class:`_PollFailure` path (exception type -:class:`AckTimeout`). The same timeout backstops persist confirmations -that never arrive. Other subscribers are not affected; once the -remaining acks arrive the producer advances normally. - -**Triggerer restart**: outstanding acks are in-memory only. After a -triggerer restart, the broker will redeliver unacknowledged events. -Subscribers must therefore be idempotent. The same applies when a group -stops while events are still awaiting persist confirmation (for example, -the last subscriber unsubscribes right after producing an event): the -pending advances are abandoned and the broker redelivers those events. +Any subscriber that has not finished processing an event within +``ack_timeout`` seconds — still on the event, or its derived trigger +events not yet confirmed persisted — is force-failed via the existing +:class:`_PollFailure` path (exception type :class:`AckTimeout`). Other +subscribers are not affected; once they resolve, the producer advances +normally. + +**Triggerer restart**: resolution state is in-memory only. After a +triggerer restart, the broker will redeliver events that were never +advanced. Subscribers must therefore be idempotent. The same applies +when a group stops while events are still awaiting persist confirmation +(for example, the last subscriber unsubscribes right after producing an +event): the pending advances are abandoned and the broker redelivers +those events. **``shared_stream_subscriber_queue_size`` in ack mode**: the bound is still "unprocessed raw events per subscriber". The manager does **not** -wait for outstanding acks before pulling the next event from the producer's -stream; back-pressure is purely queue-bound — a subscriber whose queue is -full is force-failed via :class:`_SubscriberOverflow`. The queue mainly -protects against burst delivery before a subscriber's filter has had a -chance to run. Broker advances, however, are dispatched by a single pump -task in per-lane order: while a lane's head event still has pending -subscribers, every later advance in that same lane waits (the ack timeout -is the backstop that bounds this per-lane head-of-line wait). +wait for outstanding resolutions before pulling the next event from the +producer's stream; back-pressure is purely queue-bound — a subscriber +whose queue is full is force-failed via :class:`_SubscriberOverflow`. The +queue mainly protects against burst delivery before a subscriber's filter +has had a chance to run. Broker advances, however, are dispatched by a +single pump task in per-lane order: while a lane's head event still has +pending subscribers, every later event in that same lane waits — resolved +events behind the head accumulate into the next batch (the ack timeout is +the backstop that bounds this per-lane head-of-line wait). Triggers that do **not** override ``create_shared_stream_producer`` run the -**fast path**: no event IDs, no ack table, no AckToken — subscribers -receive raw events as before (backward-compatible). +**fast path**: no event IDs and no resolution bookkeeping. Subscribers +see the exact same stream shape in both modes, so opting a trigger into +ack mode never changes its filter code (backward-compatible). Lifecycle invariants -------------------- @@ -138,13 +158,13 @@ import asyncio import itertools import time -import weakref from abc import ABC, abstractmethod from collections import deque -from collections.abc import AsyncGenerator, AsyncIterator, Callable, Hashable, Iterable, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Hashable, Iterable, Iterator, Sequence from contextlib import suppress +from contextvars import ContextVar from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, NamedTuple import structlog @@ -155,7 +175,72 @@ log = structlog.get_logger(__name__) -__all__ = ["AckTimeout", "AckToken", "AdvanceOutcome", "SharedStreamManager", "SharedStreamProducer"] +__all__ = [ + "AckTimeout", + "AdvanceItem", + "AdvanceOutcome", + "SharedStreamManager", + "SharedStreamProducer", + "reject_shared_stream_event", +] + + +class _RejectContext(NamedTuple): + """ + The open binding window in scope while a subscriber's filter runs in ack mode. + + Set by :meth:`_SharedStreamGroup._ack_drain` for the duration of one raw + event's yield and read by :func:`reject_shared_stream_event`. + """ + + group: _SharedStreamGroup + trigger_id: int + event_id: int + + +_reject_context: ContextVar[_RejectContext | None] = ContextVar("shared_stream_reject_context", default=None) +"""Task-local handle to the binding window of the raw event currently being filtered. + +Only set while a subscriber's :meth:`~airflow.triggers.base.BaseEventTrigger.filter_shared_stream` +is processing a raw event in ack mode; ``None`` otherwise (fast path, standalone +``run()``, or between raw events). See :func:`reject_shared_stream_event`. +""" + + +def reject_shared_stream_event() -> None: + """ + Reject the shared-stream raw event the calling filter is currently processing. + + Call this from inside + :meth:`~airflow.triggers.base.BaseEventTrigger.filter_shared_stream` when, + instead of yielding a :class:`~airflow.triggers.base.TriggerEvent`, you + want the broker to treat the raw event as terminally refused — dead-letter + it (Azure Service Bus), ``nack`` it (Pub/Sub), and so on. A reject is + distinct from an involuntary failure: it counts toward + :attr:`AdvanceOutcome.rejected` rather than ``failed``, so a producer can + dead-letter rejects while still redelivering failures. + + The reject resolves the event immediately; there is no derived trigger + event to persist first. + + Only meaningful while a raw event's binding window is open — that is, + inside ``filter_shared_stream`` of an ack-mode stream, right after the + filter received a raw event. Called anywhere else (the fast path, + standalone ``run()``, or between two raw events) it logs a warning and is a + no-op, because there is no broker advance to influence. + + Invariant: this relies on the filter running in the same asyncio task as + the one driving the raw-event binding window (a task-local context variable + carries the open window). Driving the filter iteration from a different task + (for example via :func:`asyncio.to_thread` or a freshly created task) would + make the open window invisible here and turn every reject into a no-op. + """ + ctx = _reject_context.get() + if ctx is None: + log.warning("reject_shared_stream_event called outside an ack-mode binding window; ignored") + return + ctx.group._reject_pending_event(trigger_id=ctx.trigger_id, event_id=ctx.event_id) + DEFAULT_SUBSCRIBER_QUEUE_MAX = 1024 """Default per-subscriber queue size for shared streams. @@ -173,9 +258,11 @@ DEFAULT_ACK_TIMEOUT = 300.0 """Default per-event ack timeout in seconds (5 minutes). -When ack mode is active, a subscriber that has not called ``token.ack()`` or -``token.nack()`` within this window is force-failed via :class:`AckTimeout`. -Override per-manager with ``SharedStreamManager(ack_timeout=...)``. +When ack mode is active, a subscriber that has not finished processing an +event within this window — still on the event, or its derived trigger +events not yet confirmed persisted — is force-failed via +:class:`AckTimeout`. Override per-manager with +``SharedStreamManager(ack_timeout=...)``. """ @@ -210,11 +297,13 @@ def __init__(self, exc: BaseException) -> None: class AckTimeout(Exception): """ - Raised in a subscriber whose ack did not arrive within the per-event timeout. + Raised in a subscriber that did not finish processing an event within the per-event timeout. - Treated the same as :class:`_SubscriberOverflow` — the subscriber's trigger - fails through the standard trigger-failure path. Other subscribers in the same - group are unaffected; their acks still advance the producer normally. + The subscriber is either still on the event, or some of its derived + trigger events were never confirmed persisted. Treated the same as + :class:`_SubscriberOverflow` — the subscriber's trigger fails through + the standard trigger-failure path. Other subscribers in the same group + are unaffected; once they resolve, the producer advances normally. """ @@ -226,25 +315,45 @@ class AdvanceOutcome: Every subscriber that was online when the event was broadcast is counted in exactly one field: - * ``acked`` — called ``token.ack()`` (or unsubscribed while the event - was outstanding — implicit ack-out) and every trigger event derived - from the event was confirmed persisted. - * ``nacked`` — called ``token.nack()``. + * ``acked`` — the subscriber moved past the event (pulled the next raw + event, or unsubscribed while the event was outstanding) and every + trigger event it derived from the event was confirmed persisted. * ``failed`` — force-failed by the manager (ack timeout — including a persist confirmation that never arrived — or queue overflow). + * ``rejected`` — the subscriber actively refused the event by calling + :func:`reject_shared_stream_event` from its filter. Terminal: the + broker should dead-letter / ``nack`` such events rather than redeliver + them, which is what distinguishes a reject from an involuntary + ``failed`` (where redelivery is the right response). + + A producer reconciles these per-broker in :meth:`SharedStreamProducer.advance` + — the framework only reports the counts. For example a Service Bus producer + dead-letters when ``rejected`` is non-zero, abandons (redelivers) when only + ``failed`` is non-zero, and completes when every subscriber accepted the + event; a Pub/Sub producer ``nack`` s on a reject and ``ack`` s otherwise. An event broadcast while no subscribers were online carries all-zero counts and is clean. """ acked: int - nacked: int failed: int + rejected: int = 0 @property def is_clean(self) -> bool: - """Whether every subscriber accepted the event (``nacked`` and ``failed`` are both zero).""" - return self.nacked == 0 and self.failed == 0 + """Whether every subscriber accepted the event — no active reject and no involuntary failure.""" + return self.failed == 0 and self.rejected == 0 + + +class AdvanceItem(NamedTuple): + """One resolved event in the batch handed to :meth:`SharedStreamProducer.advance`.""" + + broker_payload: Any + """The opaque object the producer yielded with the raw event from ``open_stream``.""" + + outcome: AdvanceOutcome + """How the subscribers resolved the event.""" class SharedStreamProducer(ABC): @@ -272,19 +381,26 @@ def open_stream(self) -> AsyncIterator[tuple[Any, Any]]: """ @abstractmethod - async def advance(self, broker_payload: Any, outcome: AdvanceOutcome) -> None: + async def advance(self, batch: Sequence[AdvanceItem]) -> None: """ - Advance the broker for one fully resolved event. - - Within a lane — events for which :meth:`get_advance_lane` returned - equal values — calls arrive strictly in fan-out order: the call for - event N is awaited only after the call for event N-1 returned (or - raised and was logged). Across lanes the relative order is not - guaranteed, but at any moment at most one ``advance`` call is - awaited globally. This makes cumulative schemes such as a Kafka - offset commit safe within a lane. ``outcome`` carries the per-event - resolution counts; use it to decide whether to commit, skip, or - trigger a broker-side redeliver. + Advance the broker for one lane's batch of fully resolved events. + + ``batch`` is never empty. Its items are in fan-out order and form a + contiguous resolved prefix of one lane — events for which + :meth:`get_advance_lane` returned equal values. Within a lane, + batches arrive strictly in order: the next batch is awaited only + after the call for the previous one returned (or raised and was + logged). Across lanes the relative order is not guaranteed, but at + any moment at most one ``advance`` call is awaited globally. This + makes cumulative schemes such as a Kafka offset commit safe within + a lane — committing the offset of the batch's last item covers the + whole batch. Each item's :class:`AdvanceOutcome` carries the + per-event resolution counts; use them to decide whether to commit, + skip, or trigger a broker-side redeliver. + + If this method raises, the error is logged and the whole batch is + abandoned — the events are not dispatched again; the broker is + expected to redeliver whatever was not committed. """ def get_advance_lane(self, broker_payload: Any) -> Hashable: @@ -319,17 +435,15 @@ async def aclose(self) -> None: @dataclass(slots=True) -class _TokenBinding: +class _SubscriberBinding: """ - Per-(event, subscriber) resolution bookkeeping for one :class:`AckToken`. + Per-(event, subscriber) resolution bookkeeping. - A subscriber resolves as ``acked`` only when all three hold: - 1. ``ack()`` was called (explicitly or implicitly on unsubscribe) - 2. the binding window is closed (the subscriber moved past the event) - 3. every trigger event seq the subscriber derived from the event has been confirmed persisted + A subscriber resolves as ``acked`` only when both hold: + 1. the binding window is closed (the subscriber moved past the event) + 2. every trigger event seq the subscriber derived from the event has been confirmed persisted """ - acked: bool = False window_closed: bool = False unconfirmed_seqs: set[int] = field(default_factory=set) @@ -338,84 +452,18 @@ class _TokenBinding: class _OutstandingEntry: """State for one outstanding (broadcast-but-not-yet-advanced) event.""" - pending: set[int] # trigger_ids still awaiting ack + pending: set[int] # trigger_ids still awaiting resolution created_at: float # monotonic clock at broadcast (via injectable _now) broker_payload: Any lane: Hashable # producer.get_advance_lane(broker_payload), taken at fan-out - # Per-subscriber ack bookkeeping, created at fan-out alongside ``pending`` - # and removed by ``_resolve_subscriber`` when the subscriber resolves. - bindings: dict[int, _TokenBinding] = field(default_factory=dict) + # Per-subscriber resolution bookkeeping, created at fan-out alongside + # ``pending`` and removed by ``_resolve_subscriber`` when the subscriber + # resolves. + bindings: dict[int, _SubscriberBinding] = field(default_factory=dict) # Resolution counts; together they form the event's AdvanceOutcome. acked: int = 0 - nacked: int = 0 failed: int = 0 - - -class AckToken: - """ - Handed to subscribers alongside each shared-stream event in ack mode. - - Call ``await token.ack()`` once the event has been accepted, or - ``await token.nack()`` to opt out — the resolution is reported to the - producer through the event's :class:`AdvanceOutcome`. Repeated calls are - no-ops. After the group stops, calls silently do nothing. - - An ``ack()`` does not complete the subscriber's resolution on its own: - the event counts as ``acked`` only after the subscriber has moved past it - (pulled the next raw event or unsubscribed) and every trigger event the - subscriber derived from it has been confirmed persisted to the metadata - database. ``nack()`` resolves immediately. - """ - - __slots__ = ("_event_id", "_trigger_id", "_group_ref", "_resolved") - - def __init__( - self, - *, - event_id: int, - trigger_id: int, - group_ref: weakref.ref[_SharedStreamGroup], - ) -> None: - self._event_id = event_id - self._trigger_id = trigger_id - self._group_ref = group_ref - self._resolved = False - - def _resolve(self, resolution: Literal["acked", "nacked"]) -> None: - """Mark the token resolved and report it to the owning group, once.""" - if self._resolved: - return - self._resolved = True - group = self._group_ref() - if group is None: - return - if resolution == "acked": - group._record_ack(event_id=self._event_id, trigger_id=self._trigger_id) - else: - group._record_nack(event_id=self._event_id, trigger_id=self._trigger_id) - - async def ack(self) -> None: - """ - Notify the producer that this subscriber has accepted the event. - - Declared ``async`` so the token API leaves room for awaitable - implementations. The broker advance the ack feeds into may happen - later: it waits until the trigger events derived from this event are - persisted to the metadata database. - """ - self._resolve("acked") - - async def nack(self) -> None: - """ - Opt out of this event; counted as ``nacked`` in the event's :class:`AdvanceOutcome`. - - How the broker reacts is the producer's decision inside - :meth:`SharedStreamProducer.advance` — commit anyway, skip, or - trigger a redeliver. Declared ``async`` so the token API leaves room - for awaitable implementations. A ``nack()`` resolves the subscriber - immediately, without waiting for any persistence confirmation. - """ - self._resolve("nacked") + rejected: int = 0 async def _drain(queue: asyncio.Queue) -> AsyncGenerator[Any, None]: @@ -474,9 +522,9 @@ def __init__( # confirmation; entries are removed on confirmation or when the # owning binding resolves (late confirmations then no-op). self._seq_index: dict[int, tuple[int, int]] = {} - # Per subscriber: the token whose binding window is currently open — - # trigger events the subscriber emits now bind to this token. - self._current_token: dict[int, AckToken] = {} + # Per subscriber: the event id whose binding window is currently + # open — trigger events the subscriber emits now bind to this event. + self._open_windows: dict[int, int] = {} self._next_event_id: int = 0 self._ack_timeout_task: asyncio.Task | None = None # Advance pump: a single task dispatches broker advances in per-lane @@ -506,14 +554,15 @@ def _request_pump_stop(self) -> None: async def _run_advance_pump(self, producer: SharedStreamProducer) -> None: """ - Dispatch broker advances in per-lane fan-out order, one at a time. + Dispatch broker advances in per-lane fan-out order, one batch at a time. A single task scans the lanes round-robin: each pass dispatches at - most one resolved head per lane, and passes repeat until one makes - no progress. An advance that raises is logged and the pump moves on. - On stop the pump keeps passing until every already-resolved - dispatchable event is drained, abandons unresolved entries to broker - redelivery, and exits. + most one batch per lane — the lane's contiguous resolved prefix — + and passes repeat until one makes no progress. An advance that + raises is logged, its batch is abandoned to broker redelivery, and + the pump moves on. On stop the pump keeps passing until every + already-resolved dispatchable event is drained, abandons unresolved + entries to broker redelivery, and exits. """ while True: if not self._pump_stopping: @@ -533,24 +582,35 @@ async def _run_advance_pump(self, producer: SharedStreamProducer) -> None: # The deque head is always present in _outstanding: entry # deletion belongs to the pump alone, and it removes the # deque slot and the dict entry together below — a - # KeyError here would be a bug. - head_id = lane_queue[0] - entry = self._outstanding[head_id] - if entry.pending: - continue - lane_queue.popleft() - del self._outstanding[head_id] + # KeyError here would be a bug. Harvest the lane's + # contiguous resolved prefix synchronously, before the + # await, so an advance that raises abandons the whole + # batch to broker redelivery. + batch: list[AdvanceItem] = [] + while lane_queue and not self._outstanding[lane_queue[0]].pending: + head_id = lane_queue.popleft() + entry = self._outstanding.pop(head_id) + batch.append( + AdvanceItem( + entry.broker_payload, + AdvanceOutcome( + acked=entry.acked, failed=entry.failed, rejected=entry.rejected + ), + ) + ) if not lane_queue: # Sole lane GC point; _poll recreates the lane on demand. del self._lane_queues[lane] - outcome = AdvanceOutcome(acked=entry.acked, nacked=entry.nacked, failed=entry.failed) + if not batch: + continue try: - await producer.advance(entry.broker_payload, outcome) + await producer.advance(batch) except Exception as exc: self.log.error( "Producer advance raised; broker advance failed", key=self.key, lane=lane, + batch_size=len(batch), exc_info=exc, ) progress = True @@ -610,7 +670,7 @@ async def _poll(self) -> None: created_at=self._now(), broker_payload=broker_payload, lane=lane, - bindings={trigger_id: _TokenBinding() for trigger_id in snapshot}, + bindings={trigger_id: _SubscriberBinding() for trigger_id in snapshot}, ) lane_queue.append(event_id) if not snapshot: @@ -619,12 +679,10 @@ async def _poll(self) -> None: # in per-lane fan-out order. self._advance_wakeup.set() continue - group_ref: weakref.ref[_SharedStreamGroup] = weakref.ref(self) for trigger_id in snapshot: queue = self._subscribers[trigger_id] - token = AckToken(event_id=event_id, trigger_id=trigger_id, group_ref=group_ref) try: - queue.put_nowait((raw_event, token)) + queue.put_nowait((raw_event, event_id)) except asyncio.QueueFull: self._fail_overflowed_subscriber(trigger_id, queue) # The dead subscriber will never resolve anything @@ -694,7 +752,7 @@ def _resolve_subscriber( *, event_id: int, trigger_id: int, - resolution: Literal["acked", "nacked", "failed"], + resolution: Literal["acked", "failed", "rejected"], ) -> None: """ Record one subscriber's resolution of one event. @@ -706,7 +764,7 @@ def _resolve_subscriber( calls for the same (event_id, trigger_id) are no-ops, which keeps every subscriber counted exactly once per event. - This is the single cleanup point for the subscriber's ack + This is the single cleanup point for the subscriber's resolution bookkeeping: the binding is popped and its unconfirmed seqs leave ``_seq_index``, so a persist confirmation arriving after the fact (e.g. after an ack timeout already failed the subscriber) finds @@ -722,65 +780,61 @@ def _resolve_subscriber( self._seq_index.pop(seq, None) if resolution == "acked": entry.acked += 1 - elif resolution == "nacked": - entry.nacked += 1 + elif resolution == "rejected": + entry.rejected += 1 else: entry.failed += 1 if not entry.pending: self._advance_wakeup.set() - def _record_ack(self, *, event_id: int, trigger_id: int) -> None: + def _reject_pending_event(self, *, trigger_id: int, event_id: int) -> None: """ - Record that the subscriber called ``ack()`` for one event. - - The subscriber resolves as acked only once the binding window is - closed and every bound seq is confirmed persisted; until then the - event stays outstanding (the ack timeout is the backstop). + Resolve the subscriber's current open-window event as rejected. + + Called by :func:`reject_shared_stream_event` while the subscriber's + binding window for ``event_id`` is open. Unlike a normal acceptance, + a reject resolves immediately — there is no derived trigger event, so + nothing has to be persisted first and the persist gate is skipped. + + A subscriber that rejects an event and then still yields a real + trigger event for it is a subscriber-side logic error: the reject has + already popped the binding, so :meth:`bind_pending_event` returns + ``None`` and the late event simply does not bind to this raw event's + ack accounting (it still fires). The framework absorbs this safely + rather than raising. """ - entry = self._outstanding.get(event_id) - if entry is None: - return - binding = entry.bindings.get(trigger_id) - if binding is None: - return # already resolved (e.g. force-failed before the ack landed) - binding.acked = True - self._maybe_complete(event_id=event_id, trigger_id=trigger_id) - - def _record_nack(self, *, event_id: int, trigger_id: int) -> None: - """Resolve the subscriber as nacked immediately; no persistence gating applies.""" - self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="nacked") + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="rejected") def _maybe_complete(self, *, event_id: int, trigger_id: int) -> None: - """Resolve the subscriber as acked once ack, window close, and all persist confirmations are in.""" + """Resolve the subscriber as acked once the window is closed and all persist confirmations are in.""" entry = self._outstanding.get(event_id) if entry is None: return binding = entry.bindings.get(trigger_id) if binding is None: return - if binding.acked and binding.window_closed and not binding.unconfirmed_seqs: + if binding.window_closed and not binding.unconfirmed_seqs: self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") - def _close_binding_window(self, token: AckToken) -> None: + def _close_binding_window(self, *, event_id: int, trigger_id: int) -> None: """ - Close the binding window of one yielded token. + Close the subscriber's binding window for one event. Called by ``_ack_drain`` the moment the subscriber pulls the next item — trigger events the subscriber emits from here on belong to - the next token, and an ack recorded for this one can now complete + the next event, and this one can now resolve for the subscriber (subject to persist confirmations). """ - trigger_id = token._trigger_id - if self._current_token.get(trigger_id) is token: - del self._current_token[trigger_id] - entry = self._outstanding.get(token._event_id) + if self._open_windows.get(trigger_id) == event_id: + del self._open_windows[trigger_id] + entry = self._outstanding.get(event_id) if entry is None: return binding = entry.bindings.get(trigger_id) if binding is None: return binding.window_closed = True - self._maybe_complete(event_id=token._event_id, trigger_id=trigger_id) + self._maybe_complete(event_id=event_id, trigger_id=trigger_id) def bind_pending_event(self, *, trigger_id: int, seq_counter: Iterator[int]) -> int | None: """ @@ -789,13 +843,13 @@ def bind_pending_event(self, *, trigger_id: int, seq_counter: Iterator[int]) -> Returns the seq the broker advance must wait on, or ``None`` when there is nothing to bind to — no window open (not in ack mode, or the event was emitted outside any raw event's window) or the - binding already resolved (nacked or force-failed). A seq is only - drawn from ``seq_counter`` when the event actually binds. + binding already resolved (force-failed). A seq is only drawn from + ``seq_counter`` when the event actually binds. """ - token = self._current_token.get(trigger_id) - if token is None: + event_id = self._open_windows.get(trigger_id) + if event_id is None: return None - entry = self._outstanding.get(token._event_id) + entry = self._outstanding.get(event_id) if entry is None: return None binding = entry.bindings.get(trigger_id) @@ -803,7 +857,7 @@ def bind_pending_event(self, *, trigger_id: int, seq_counter: Iterator[int]) -> return None seq = next(seq_counter) binding.unconfirmed_seqs.add(seq) - self._seq_index[seq] = (token._event_id, trigger_id) + self._seq_index[seq] = (event_id, trigger_id) return seq def confirm_persisted(self, seqs: Iterable[int]) -> None: @@ -811,7 +865,7 @@ def confirm_persisted(self, seqs: Iterable[int]) -> None: Record persist confirmations for trigger event seqs bound in this group. Seqs that are not (or no longer) in ``_seq_index`` — another group's - seqs, or bindings that already resolved through nack / timeout — + seqs, or bindings that already resolved through timeout / overflow — are ignored. """ for seq in seqs: @@ -843,7 +897,7 @@ def _fail_subscriber_in_outstanding(self, trigger_id: int) -> None: async def _run_ack_timeout_loop(self) -> None: """ - Background task: force-fail subscribers whose ack is overdue. + Background task: force-fail subscribers whose resolution is overdue. Cadence is ``max(0.01, ack_timeout / 10)`` — runs ten times per timeout window, floored at 10 ms to avoid burning CPU when ``ack_timeout`` is small. @@ -870,7 +924,9 @@ async def _run_ack_timeout_loop(self) -> None: _PollFailure( AckTimeout( f"shared stream {self.key!r} trigger {trigger_id} " - f"did not ack event {event_id} within {self._ack_timeout}s" + f"did not finish processing event {event_id} within " + f"{self._ack_timeout}s (still on the event, or its " + "trigger events not yet confirmed persisted)" ) ), ) @@ -884,22 +940,40 @@ async def _ack_drain(self, trigger_id: int, queue: asyncio.Queue) -> AsyncGenera """ Ack-mode counterpart of :func:`_drain`, tracking the binding window. - Between yielding ``(raw_event, token)`` and the subscriber pulling - the next item, the token's binding window is open: trigger events - the subscriber emits bind to it via :meth:`bind_pending_event`. The - window closes the moment the subscriber resumes this generator — - before we wait on the queue again — so a filtered-out event (acked, - nothing yielded) resolves as soon as the filter loops back for the - next raw event. + Unwraps the internal ``(raw_event, event_id)`` queue items and + yields the bare raw event, so the subscriber sees the same stream + shape as on the fast path. Between the yield and the subscriber + pulling the next item, the event's binding window is open: trigger + events the subscriber emits bind to it via + :meth:`bind_pending_event`. The window closes the moment the + subscriber resumes this generator — before we wait on the queue + again — so a filtered-out event (nothing yielded) resolves as soon + as the filter loops back for the next raw event. Closing the window + is the subscriber's acceptance of the event; no explicit call is + involved. """ while True: item = await queue.get() if isinstance(item, _PollFailure): raise item.exc - _, token = item - self._current_token[trigger_id] = token - yield item - self._close_binding_window(token) + raw_event, event_id = item + self._open_windows[trigger_id] = event_id + # Expose the open window to reject_shared_stream_event(), which the + # subscriber's filter may call instead of yielding. Clear it in + # ``finally`` so cancellation / exceptions also clear it and no + # window leaks across raw events. We assign ``None`` rather than + # ``reset(token)`` on purpose: each ``__anext__`` of this generator + # runs in its caller's context, which may be a different asyncio + # task than the one that set the value (the filter is resumed from + # a fresh task in some flows), and ``ContextVar.reset`` rejects a + # token created in another context. Windows never nest, so clearing + # to ``None`` is the correct restore. + _reject_context.set(_RejectContext(group=self, trigger_id=trigger_id, event_id=event_id)) + try: + yield raw_event + finally: + _reject_context.set(None) + self._close_binding_window(event_id=event_id, trigger_id=trigger_id) def subscribe(self, trigger_id: int) -> AsyncIterator[Any]: """Register ``trigger_id`` as a subscriber and return its raw event stream.""" @@ -916,12 +990,13 @@ def unsubscribe(self, trigger_id: int) -> None: # (Airflow's standard idiom); dropping the queue is enough here. self._subscribers.pop(trigger_id, None) self._failed_subscribers.discard(trigger_id) - self._current_token.pop(trigger_id, None) - # Implicit ack-out: leaving counts as accepting every outstanding - # event, so the producer never waits forever for a subscriber that - # has left. The broker advance still waits for persist confirmation - # of any trigger events this subscriber derived from the event; with - # nothing unconfirmed the subscriber resolves immediately. + self._open_windows.pop(trigger_id, None) + # Implicit resolution: leaving closes the subscriber's window on + # every outstanding event, so the producer never waits forever for a + # subscriber that has left. The broker advance still waits for + # persist confirmation of any trigger events this subscriber derived + # from the event; with nothing unconfirmed the subscriber resolves + # immediately. for event_id, entry in list(self._outstanding.items()): binding = entry.bindings.get(trigger_id) if binding is None: @@ -929,7 +1004,6 @@ def unsubscribe(self, trigger_id: int) -> None: # _resolve_subscriber no-ops unless still pending. self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") continue - binding.acked = True binding.window_closed = True self._maybe_complete(event_id=event_id, trigger_id=trigger_id) @@ -1090,7 +1164,7 @@ def confirm_persisted(self, seqs: Iterable[int]) -> None: Broadcast to every live group; each group resolves the sequence numbers it owns and ignores the rest. Sequence numbers whose binding - already resolved (``nack()``, timeout) or whose group has stopped + already resolved (timeout, overflow) or whose group has stopped are ignored. """ # Materialize so a one-shot iterator is not exhausted by the first group. diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py b/airflow-core/tests/unit/jobs/test_triggerer_job.py index d93e56763334d..21e0d25de872e 100644 --- a/airflow-core/tests/unit/jobs/test_triggerer_job.py +++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py @@ -917,7 +917,7 @@ async def _drive(): assert trigger_runner._shared_streams._groups == {} def test_shared_stream_ack_mode_integration(self, session) -> None: - """A BaseEventTrigger whose producer factory is overridden advances only after the acked event's persistence is confirmed.""" + """A BaseEventTrigger whose producer factory is overridden advances only after the event's derived trigger event is confirmed persisted.""" advanced: list[Any] = [] # Container lets _drive() write trigger_runner back for post-run assertions. state: dict[str, Any] = {} @@ -928,8 +928,8 @@ async def open_stream(self): # Stay alive so the manager can tear us down on unsubscribe. await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): - advanced.append(broker_payload) + async def advance(self, batch): + advanced.extend(item.broker_payload for item in batch) state["advance_called"].set() class _AckSharedTrigger(BaseEventTrigger): @@ -952,12 +952,9 @@ def create_shared_stream_producer(cls, kwargs): return _AckSharedProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: + async for raw in shared_stream: if self.region is None or raw["region"] == self.region: - await token.ack() yield TriggerEvent(raw) - else: - await token.nack() async def run(self): # pragma: no cover - replaced by filter_shared_stream yield TriggerEvent({}) @@ -986,8 +983,8 @@ async def _drive(): _trigger_id, _event, seq = trigger_runner.events[0] assert seq is not None, "a shared ack trigger's event must carry a seq" - # The subscriber acked, but the advance is gated on persistence: - # nothing may advance before the confirmation arrives. + # The subscriber moved past the event, but the advance is gated + # on persistence: nothing may advance before the confirmation arrives. assert advanced == [], "advance must not run before the persist confirmation" # Simulate the supervisor confirming the persist on the next sync. @@ -1009,7 +1006,7 @@ async def _drive(): # The producer's advance must have been called with the broker_payload. assert advanced == ["receipt-handle-1"], ( - "producer.advance must be called once the acked event's persistence is confirmed" + "producer.advance must be called once the event's persistence is confirmed" ) # Group is torn down on unsubscribe. assert trigger_runner._shared_streams._groups == {} diff --git a/airflow-core/tests/unit/triggers/test_base_trigger.py b/airflow-core/tests/unit/triggers/test_base_trigger.py index 338168e7c022e..5f32cb23bfe41 100644 --- a/airflow-core/tests/unit/triggers/test_base_trigger.py +++ b/airflow-core/tests/unit/triggers/test_base_trigger.py @@ -237,9 +237,9 @@ async def stream(): def test_create_shared_stream_producer_raises_by_default(): """A subclass that does not override create_shared_stream_producer gets NotImplementedError. - The manager detects this via MRO inspection and takes the fast path (no ack - tokens issued). But calling the method directly must still raise, confirming it - is not accidentally a no-op on the base class. + The manager detects this via MRO inspection and takes the fast path (no + resolution bookkeeping). But calling the method directly must still raise, + confirming it is not accidentally a no-op on the base class. """ with pytest.raises(NotImplementedError, match="create_shared_stream_producer"): _PlainEventTrigger.create_shared_stream_producer({}) diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index 2d835820320c2..9506506e6af09 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -18,9 +18,9 @@ import asyncio import time -import weakref from collections.abc import Callable from contextlib import asynccontextmanager, suppress +from unittest import mock from unittest.mock import MagicMock import pytest @@ -29,13 +29,13 @@ from airflow.triggers.base import BaseEventTrigger, TriggerEvent from airflow.triggers.shared_stream import ( AckTimeout, - AckToken, AdvanceOutcome, SharedStreamManager, SharedStreamProducer, _PollFailure, _SharedStreamGroup, _SubscriberOverflow, + reject_shared_stream_event, ) @@ -711,9 +711,10 @@ def get_advance_lane(self, broker_payload): return None return self._lane_for(broker_payload) - async def advance(self, broker_payload, outcome): - self._trigger_cls.advanced.append(broker_payload) - self._trigger_cls.outcomes.append(outcome) + async def advance(self, batch): + self._trigger_cls.batches.append([item.broker_payload for item in batch]) + self._trigger_cls.advanced.extend(item.broker_payload for item in batch) + self._trigger_cls.outcomes.extend(item.outcome for item in batch) async def aclose(self): self._trigger_cls.aclose_calls += 1 @@ -723,12 +724,15 @@ def _make_ack_required_trigger_class(events_with_payloads: list[tuple], *, lane_ """ Return a fresh trigger class whose ``create_shared_stream_producer`` returns a :class:`_RecordingProducer` yielding ``(event, broker_payload)`` - tuples and recording each advance call into class-level lists. + tuples and recording each advance call into class-level lists — + ``batches`` keeps one list per call, ``advanced`` / ``outcomes`` are the + flattened projections. """ class _AckRequiredTrigger(_ProgrammableSharedStreamTrigger): advanced: list = [] outcomes: list[AdvanceOutcome] = [] + batches: list[list] = [] aclose_calls: int = 0 @classmethod @@ -736,13 +740,40 @@ def create_shared_stream_producer(cls, kwargs): return _RecordingProducer(cls, events_with_payloads, lane_for=lane_for) async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) return _AckRequiredTrigger +def _make_reject_trigger_class(events_with_payloads: list[tuple], *, lane_for: Callable | None = None): + """ + Like :func:`_make_ack_required_trigger_class`, but the filter rejects any + raw event whose payload is truthy on ``"reject"`` (calling + :func:`reject_shared_stream_event` instead of yielding) and yields a + ``TriggerEvent`` for the rest. + """ + + class _RejectTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + outcomes: list[AdvanceOutcome] = [] + batches: list[list] = [] + aclose_calls: int = 0 + + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RecordingProducer(cls, events_with_payloads, lane_for=lane_for) + + async def filter_shared_stream(self, shared_stream): + async for raw in shared_stream: + if raw.get("reject"): + reject_shared_stream_event() + continue + yield TriggerEvent(raw) + + return _RejectTrigger + + async def _collect_ack_stream(trigger, stream, *, n: int) -> list: """Collect ``n`` TriggerEvent items from an ack-aware filter_shared_stream.""" out = [] @@ -762,8 +793,8 @@ class _BackgroundCollector: resumed and the item's binding window closes — exactly what a production ``filter_shared_stream`` loop does when it goes back for the next raw event. Breaking out of the loop instead would leave the generator - suspended at its yield with the binding window open, so an ack could - never complete. + suspended at its yield with the binding window open, so the event could + never resolve. """ def __init__(self) -> None: @@ -772,11 +803,6 @@ def __init__(self) -> None: # if any; the consuming task records it here instead of propagating. self.error: BaseException | None = None - @property - def tokens(self) -> list[AckToken]: - """The AckTokens collected so far (ack-mode raw streams only).""" - return [token for _raw, token in self.items] - async def wait_for(self, n: int, *, timeout: float = 1.0) -> list: """Wait until ``n`` items have been collected and return them.""" deadline = asyncio.get_event_loop().time() + timeout @@ -865,8 +891,8 @@ def create_shared_stream_producer(cls, kwargs): @pytest.mark.asyncio -async def test_ack_required_producer_advances_after_all_subscribers_ack(): - """Producer's advance hook fires exactly once after both subscribers ack.""" +async def test_ack_required_producer_advances_after_all_subscribers_resolve(): + """Producer's advance hook fires exactly once after both subscribers move past the event.""" cls = _make_ack_required_trigger_class( [ ({"msg": "hello"}, "receipt-1"), @@ -878,32 +904,34 @@ async def test_ack_required_producer_advances_after_all_subscribers_ack(): t1, t2 = cls(), cls() key = t1.shared_stream_key() manager = SharedStreamManager() + next_task = None try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - # Collect the (event, token) tuples without acking yet. - async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: - await c1.wait_for(1) + # Subscriber 1 pulls the event and stays on it; subscriber 2 consumes + # it like a real filter loop (and thereby resolves it). + it1 = s1.__aiter__() + await _pull_raw(it1) + async with _consume_in_background(s2) as c2: await c2.wait_for(1) - tokens: list[AckToken] = [c1.tokens[0], c2.tokens[0]] - - assert len(cls.advanced) == 0, "advance must not fire before all acks" - - await tokens[0].ack() - assert len(cls.advanced) == 0, "advance must not fire after only one ack" + await asyncio.sleep(0) + assert len(cls.advanced) == 0, "advance must not fire while one subscriber is still on the event" - await tokens[1].ack() + next_task = await _resume_past_yield(it1) await asyncio.sleep(0) # let the advance pump run - assert cls.advanced == ["receipt-1"], "advance must fire exactly once after all acks" + assert cls.advanced == ["receipt-1"], ( + "advance must fire exactly once after all subscribers resolve" + ) finally: + await _cancel_quietly(next_task) await manager.unsubscribe(1, key) await manager.unsubscribe(2, key) @pytest.mark.asyncio -async def test_ack_required_producer_does_not_advance_on_partial_ack(): - """Producer does not advance when only one of two subscribers has acknowledged.""" +async def test_ack_required_producer_does_not_advance_on_partial_resolution(): + """Producer does not advance while one of two subscribers is still on the event.""" cls = _make_ack_required_trigger_class( [ ({"msg": "partial"}, "receipt-partial"), @@ -919,28 +947,14 @@ async def test_ack_required_producer_does_not_advance_on_partial_ack(): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - token1 = None - got_s2 = asyncio.Event() - - async def grab_token1(stream): - nonlocal token1 - async for _raw, token in stream: - token1 = token - break - - async def drain_s2(stream): - async for _ in stream: # pragma: no branch - got_s2.set() - break - - await asyncio.gather( - asyncio.wait_for(grab_token1(s1), timeout=1.0), - asyncio.wait_for(drain_s2(s2), timeout=1.0), - ) - - await token1.ack() - await asyncio.sleep(0) - assert cls.advanced == [], "producer must not advance with only one of two acks" + # Subscriber 1 pulls the event but never resumes past it. + it1 = s1.__aiter__() + await _pull_raw(it1) + # Subscriber 2 consumes and resolves it. + async with _consume_in_background(s2) as c2: + await c2.wait_for(1) + await asyncio.sleep(0) + assert cls.advanced == [], "producer must not advance while a subscriber is still on the event" finally: await manager.unsubscribe(1, key) await manager.unsubscribe(2, key) @@ -974,18 +988,16 @@ def now() -> float: s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) async def drain_slow(stream): - # Pull the (event, token) pair off the slow stream but never ack — - # the deliberately stalled subscriber the timeout must catch. + # Pull the raw event off the slow stream and stay on it — the + # deliberately stalled subscriber the timeout must catch. async for _ in stream: break async with _consume_in_background(s_fast) as fast_collector: + # The fast subscriber consumes the event and resolves; the slow + # one stays on it. await fast_collector.wait_for(1) await asyncio.wait_for(drain_slow(s_slow), timeout=1.0) - token_fast = fast_collector.tokens[0] - - # fast acks; slow doesn't. - await token_fast.ack() # Advance fake clock past ack_timeout; the timeout loop sees now() - created_at >= 0.1. fake_clock[0] = 0.2 @@ -998,14 +1010,14 @@ async def drain_slow(stream): assert isinstance(sentinel, _PollFailure) assert isinstance(sentinel.exc, AckTimeout) - # Fast subscriber (already acked) must not have received a spurious AckTimeout. + # Fast subscriber (already resolved) must not have received a spurious AckTimeout. fast_queue = manager._groups[key]._subscribers.get(1) assert fast_queue is not None assert fast_queue.empty(), ( - "fast (already-acked) subscriber must not receive a spurious AckTimeout" + "fast (already-resolved) subscriber must not receive a spurious AckTimeout" ) assert fast_collector.error is None, ( - "fast (already-acked) subscriber must not receive a spurious AckTimeout" + "fast (already-resolved) subscriber must not receive a spurious AckTimeout" ) # After timeout the advance should have fired (pending emptied). @@ -1017,7 +1029,7 @@ async def drain_slow(stream): @pytest.mark.asyncio async def test_ack_timeout_at_cap_boundary_does_not_timeout(): - """Subscriber acking inside the window (< ack_timeout) must NOT be force-failed.""" + """Subscriber resolving inside the window (< ack_timeout) must NOT be force-failed.""" cls = _make_ack_required_trigger_class( [ ({"msg": "boundary-test"}, "receipt-boundary"), @@ -1036,35 +1048,36 @@ def now() -> float: return fake_clock[0] manager = SharedStreamManager(ack_timeout=0.1, _now=now) + next_task = None try: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) - async with _consume_in_background(stream) as collector: - await collector.wait_for(1) - token_ref = collector.tokens[0] + # Pull the event and stay on it so it remains outstanding. + it = stream.__aiter__() + await _pull_raw(it) - # Advance fake clock to 99 ms — strictly inside the 100 ms ack_timeout window. - # now() - created_at = 0.099 < 0.1, so the timeout loop must NOT fire. - fake_clock[0] = 0.099 - # Let the timeout loop tick a few cadence cycles (cadence = max(0.01, 0.1/10) = 0.01s real). - await asyncio.sleep(0.05) + # Advance fake clock to 99 ms — strictly inside the 100 ms ack_timeout window. + # now() - created_at = 0.099 < 0.1, so the timeout loop must NOT fire. + fake_clock[0] = 0.099 + # Let the timeout loop tick a few cadence cycles (cadence = max(0.01, 0.1/10) = 0.01s real). + await asyncio.sleep(0.05) - subscriber_queue = manager._groups[key]._subscribers.get(1) - assert subscriber_queue is not None - assert subscriber_queue.empty(), "subscriber must not have been force-failed at t=99ms" - assert collector.error is None, "subscriber must not have been force-failed at t=99ms" + subscriber_queue = manager._groups[key]._subscribers.get(1) + assert subscriber_queue is not None + assert subscriber_queue.empty(), "subscriber must not have been force-failed at t=99ms" - # Now ack — advance should fire. - await token_ref.ack() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-boundary"] + # Now move past the event — advance should fire. + next_task = await _resume_past_yield(it) + await asyncio.sleep(0) + assert cls.advanced == ["receipt-boundary"] finally: + await _cancel_quietly(next_task) await manager.stop_all() @pytest.mark.asyncio async def test_ack_timeout_exactly_at_timeout_force_fails(): - """Subscriber whose ack is exactly ``ack_timeout`` old IS force-failed (elapsed >= timeout).""" + """Subscriber still on an event exactly ``ack_timeout`` old IS force-failed (elapsed >= timeout).""" cls = _make_ack_required_trigger_class( [ ({"msg": "exact-boundary"}, "receipt-exact"), @@ -1087,7 +1100,7 @@ def now() -> float: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) async def drain_one(s): - # Pull the (event, token) tuple but never ack. + # Pull the raw event and stay on it — never resume past the yield. async for _ in s: break @@ -1173,8 +1186,8 @@ async def open_stream(self): yield {"msg": "late-test"}, "receipt-late" await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): - self._trigger_cls.advanced.append(broker_payload) + async def advance(self, batch): + self._trigger_cls.advanced.extend(item.broker_payload for item in batch) class _LateTrigger(_ProgrammableSharedStreamTrigger): advanced: list = [] @@ -1184,8 +1197,7 @@ def create_shared_stream_producer(cls, kwargs): return _LateProducer(cls) async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) _LateTrigger.advanced.clear() @@ -1202,7 +1214,7 @@ async def filter_shared_stream(self, shared_stream): proceed.set() # Give the poll task a few ticks to run past the yield and put the - # (event, token) tuple into the early subscriber's queue. + # raw event into the early subscriber's queue. deadline = asyncio.get_event_loop().time() + 1.0 while asyncio.get_event_loop().time() < deadline: if manager._groups[key]._outstanding: @@ -1218,13 +1230,13 @@ async def filter_shared_stream(self, shared_stream): assert entry is not None assert 2 not in entry.pending, "late subscriber must not be in the pending set of earlier event" - # Collect from early subscriber (filter acks immediately). + # Collect from early subscriber (the filter loop resolves the event). async with _consume_in_background(t_early.filter_shared_stream(s_early)) as collector: events = await collector.wait_for(1) assert len(events) == 1 await asyncio.sleep(0) - # The late subscriber did not participate in ack set; advance must have fired. + # The late subscriber did not participate in the pending set; advance must have fired. assert _LateTrigger.advanced == ["receipt-late"] finally: await manager.unsubscribe(1, key) @@ -1232,97 +1244,8 @@ async def filter_shared_stream(self, shared_stream): @pytest.mark.asyncio -async def test_nack_removes_subscriber_from_ack_set_without_redeliver(): - """nack() removes the subscriber from the pending set; no redeliver happens.""" - cls = _make_ack_required_trigger_class( - [ - ({"msg": "nack-test"}, "receipt-nack"), - ] - ) - cls.advanced.clear() - cls.outcomes.clear() - - t1, t2 = cls(), cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - - async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: - await c1.wait_for(1) - await c2.wait_for(1) - - # t1 acks, t2 nacks. - await c1.tokens[0].ack() - await c2.tokens[0].nack() - await asyncio.sleep(0) - - assert cls.advanced == ["receipt-nack"], "advance must fire after ack + nack" - - # No redelivery — the manager has no redeliver mechanism. - group = manager._groups.get(key) - if group: - assert len(group._outstanding) == 0 - finally: - await manager.unsubscribe(1, key) - await manager.unsubscribe(2, key) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("first", "second", "expected_outcome"), - [ - pytest.param("ack", "nack", AdvanceOutcome(acked=1, nacked=0, failed=0), id="ack-then-nack"), - pytest.param("nack", "ack", AdvanceOutcome(acked=0, nacked=1, failed=0), id="nack-then-ack"), - pytest.param("ack", "ack", AdvanceOutcome(acked=1, nacked=0, failed=0), id="ack-then-ack"), - pytest.param("nack", "nack", AdvanceOutcome(acked=0, nacked=1, failed=0), id="nack-then-nack"), - ], -) -async def test_double_resolution_is_noop_and_counts_once(first, second, expected_outcome): - """Any second resolution call on a token is a no-op — _resolved blocks it. - - advance fires exactly once, and the subscriber is counted once, by the - first call's resolution. - """ - cls = _make_ack_required_trigger_class( - [ - ({"msg": "cross-order"}, "receipt-cross"), - ] - ) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - - async with _consume_in_background(s1) as collector: - await collector.wait_for(1) - token = collector.tokens[0] - assert token is not None - - # First call — should resolve the token and advance. - await getattr(token, first)() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-cross"] - - # Second call — must be a no-op, not raise; advance must not fire again. - await getattr(token, second)() - await asyncio.sleep(0) - assert cls.advanced == ["receipt-cross"], ( - f"{second}() after {first}() must not trigger a second advance" - ) - assert cls.outcomes == [expected_outcome] - finally: - await manager.unsubscribe(1, key) - - -@pytest.mark.asyncio -async def test_subscriber_unsubscribe_during_outstanding_ack(): - """When a subscriber leaves while its ack is pending, the group advances without it.""" +async def test_subscriber_unsubscribe_during_outstanding_event(): + """When a subscriber leaves while still on an event, the group advances without it.""" cls = _make_ack_required_trigger_class( [ ({"msg": "unsub-during-ack"}, "receipt-unsub"), @@ -1338,17 +1261,19 @@ async def test_subscriber_unsubscribe_during_outstanding_ack(): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: + # t1 consumes and resolves; t2 pulls the event, stays on it, then leaves. + it2 = s2.__aiter__() + async with _consume_in_background(s1) as c1: await c1.wait_for(1) - await c2.wait_for(1) + await _pull_raw(it2) + await asyncio.sleep(0) + assert cls.advanced == [], "the event still waits on subscriber 2" - # t1 acks; t2 leaves without acking (implicit ack-out via unsubscribe). - await c1.tokens[0].ack() await manager.unsubscribe(2, key) await asyncio.sleep(0) assert cls.advanced == ["receipt-unsub"], ( - "unsubscribe must trigger implicit ack-out so producer can advance" + "unsubscribe must resolve the departing subscriber so the producer can advance" ) finally: await manager.unsubscribe(1, key) @@ -1370,10 +1295,9 @@ async def test_ack_mode_queue_full_during_fanout_does_not_break_iteration(): s_ok = manager.subscribe(trigger_id=2, trigger=t_ok, key=key) # Pre-fill trigger 1's queue so put_nowait raises QueueFull at fan-out. + # The filler mirrors the internal (raw_event, event_id) queue shape. group = manager._groups[key] - group._subscribers[1].put_nowait( - ("filler", AckToken(event_id=-1, trigger_id=1, group_ref=weakref.ref(group))) - ) + group._subscribers[1].put_nowait(("filler", -1)) # t_full: its queue will be drained + replaced by _PollFailure(_SubscriberOverflow). # _drain() raises the inner exception, so we expect _SubscriberOverflow here. @@ -1398,7 +1322,7 @@ async def collect_full(): assert len(full_exc) == 1 assert isinstance(full_exc[0], _SubscriberOverflow) - # t_ok got the real event and acknowledged it → advance was triggered. + # t_ok got the real event and resolved it → advance was triggered. assert len(ok_result) == 1 assert ok_result[0].payload == {"msg": "burst"} assert cls.advanced == ["receipt-burst"] @@ -1469,7 +1393,7 @@ async def test_empty_snapshot_advances_in_fanout_order(lane_for, events, expecte "advance must be called for every event, in fan-out order, even with no subscribers online" ) assert len(cls.advanced) == len(events) - assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * len(events) + assert cls.outcomes == [AdvanceOutcome(0, 0)] * len(events) assert all(outcome.is_clean for outcome in cls.outcomes) del s1 # silence unused-variable warning @@ -1488,7 +1412,7 @@ async def open_stream(self): yield ({"msg": "event-2"}, "payload-2") await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): + async def advance(self, batch): raise RuntimeError("simulated broker failure") class _RaisingAdvanceTrigger(_ProgrammableSharedStreamTrigger): @@ -1497,8 +1421,7 @@ def create_shared_stream_producer(cls, kwargs): return _RaisingAdvanceProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) t1 = _RaisingAdvanceTrigger() @@ -1515,7 +1438,7 @@ async def filter_shared_stream(self, shared_stream): group.log.error = mock_log_error async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: - # Consume the first event — this triggers an ack and schedules advance. + # Consume the first event — the filter loop resolves it and schedules advance. events1 = await collector.wait_for(1) assert len(events1) == 1 @@ -1556,11 +1479,11 @@ async def open_stream(self): yield ({"msg": "slow"}, "receipt-slow") await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): + async def advance(self, batch): advance_started.set() await advance_may_finish.wait() await asyncio.sleep(0.05) # ensure stop() must actually wait for us - self._trigger_cls.advanced.append(broker_payload) + self._trigger_cls.advanced.extend(item.broker_payload for item in batch) advance_done_at.append(time.monotonic()) class _SlowAdvanceTrigger(_ProgrammableSharedStreamTrigger): @@ -1571,8 +1494,7 @@ def create_shared_stream_producer(cls, kwargs): return _SlowAdvanceProducer(cls) async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) _SlowAdvanceTrigger.advanced.clear() @@ -1628,8 +1550,8 @@ async def open_stream(self): yield {"region": "fresh"}, "receipt-fresh" await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): - self._trigger_cls.advanced.append(broker_payload) + async def advance(self, batch): + self._trigger_cls.advanced.extend(item.broker_payload for item in batch) class _AckWindowTrigger(_ProgrammableSharedStreamTrigger): advanced: list = [] @@ -1639,8 +1561,7 @@ def create_shared_stream_producer(cls, kwargs): return _AckWindowProducer(cls) async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) _AckWindowTrigger.advanced.clear() @@ -1707,14 +1628,6 @@ async def test_directory_file_delete_trigger_path_unchanged(): await manager.unsubscribe(1, key) -async def _grab_tokens(stream, tokens: list, *, n: int) -> None: - """Pull ``(event, token)`` pairs off ``stream`` until ``tokens`` holds ``n`` items.""" - async for _raw, token in stream: - tokens.append(token) - if len(tokens) >= n: - break - - @pytest.mark.asyncio async def test_out_of_order_resolve_still_advances_in_fanout_order(): """Events resolved out of order are advanced strictly in fan-out order.""" @@ -1731,23 +1644,32 @@ async def test_out_of_order_resolve_still_advances_in_fanout_order(): t1 = cls() key = t1.shared_stream_key() manager = SharedStreamManager() + next_task = None try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - async with _consume_in_background(s1) as collector: - await collector.wait_for(3) - tokens = collector.tokens + # Consume all three events, binding one persist seq to each so the + # confirmation order — not the consumption order — decides when each + # event resolves. + it = s1.__aiter__() + seqs: list[int] = [] + for _ in range(3): + await _pull_raw(it) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs.append(seq) + next_task = await _resume_past_yield(it) - # Resolve events 2 and 3 first — nothing may advance while event 1 is pending. - await tokens[1].ack() - await tokens[2].ack() - await asyncio.sleep(0) - assert cls.advanced == [], "no advance may run while the head event is unresolved" + # Resolve events 2 and 3 first — nothing may advance while event 1 is pending. + manager.confirm_persisted([seqs[1], seqs[2]]) + await asyncio.sleep(0) + assert cls.advanced == [], "no advance may run while the head event is unresolved" - await tokens[0].ack() - await asyncio.sleep(0) - assert cls.advanced == ["p1", "p2", "p3"], "advances must run in fan-out order" + manager.confirm_persisted([seqs[0]]) + await asyncio.sleep(0) + assert cls.advanced == ["p1", "p2", "p3"], "advances must run in fan-out order" finally: + await _cancel_quietly(next_task) await manager.unsubscribe(1, key) @@ -1782,11 +1704,11 @@ def get_advance_lane(self, broker_payload): return super().get_advance_lane(broker_payload) return lane_for(broker_payload) - async def advance(self, broker_payload, outcome): + async def advance(self, batch): state["in_flight"] += 1 state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) await release.wait() - advanced.append(broker_payload) + advanced.extend(item.broker_payload for item in batch) state["in_flight"] -= 1 class _BlockingAdvanceTrigger(_ProgrammableSharedStreamTrigger): @@ -1795,8 +1717,7 @@ def create_shared_stream_producer(cls, kwargs): return _BlockingAdvanceProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) return _BlockingAdvanceTrigger() @@ -1808,13 +1729,10 @@ async def filter_shared_stream(self, shared_stream): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) async with _consume_in_background(s1) as collector: + # Consuming both events resolves them; the pump starts the first + # advance, which blocks. No second call may start while one is in + # flight. await collector.wait_for(2) - tokens = collector.tokens - - # Resolve both events; the pump starts the first advance, which - # blocks. The second must not start while one is in flight. - await tokens[0].ack() - await tokens[1].ack() for _ in range(5): await asyncio.sleep(0) assert state["in_flight"] == 1, "exactly one advance may be in flight" @@ -1836,11 +1754,6 @@ async def filter_shared_stream(self, shared_stream): await manager.unsubscribe(1, key) -# --------------------------------------------------------------------------- -# Advance lanes (get_advance_lane) -# --------------------------------------------------------------------------- - - @pytest.mark.asyncio async def test_lanes_advance_independently(): """A lane whose events are resolved does not wait for another lane's unresolved head.""" @@ -1859,34 +1772,42 @@ async def test_lanes_advance_independently(): t1 = cls() key = t1.shared_stream_key() manager = SharedStreamManager() + next_task = None try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) group = manager._groups[key] - async with _consume_in_background(s1) as collector: - await collector.wait_for(4) - tokens = collector.tokens + # Consume all four events, binding a persist seq to each lane-"a" + # event so lane "a" stays unresolved while lane "b" resolves from + # consumption alone. + it = s1.__aiter__() + seqs: dict[str, int] = {} + for _ in range(4): + raw = await _pull_raw(it) + if raw["n"].startswith("a"): + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs[raw["n"]] = seq + next_task = await _resume_past_yield(it) - # Resolve lane "b" completely while lane "a" is fully unresolved. - await tokens[1].ack() # b1 - await tokens[3].ack() # b2 - for _ in range(5): - await asyncio.sleep(0) - assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" - await tokens[0].ack() # a1 - await tokens[2].ack() # a2 - for _ in range(5): - await asyncio.sleep(0) + manager.confirm_persisted([seqs["a1"]]) + manager.confirm_persisted([seqs["a2"]]) + for _ in range(5): + await asyncio.sleep(0) - # Contract: full per-lane projections in fan-out order. The global - # interleaving across lanes is a pump implementation detail. - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] - assert len(cls.advanced) == 4 - # All lanes drained → the per-lane index is garbage-collected. - assert group._lane_queues == {} + # Contract: full per-lane projections in fan-out order. The global + # interleaving across lanes is a pump implementation detail. + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] + assert len(cls.advanced) == 4 + # All lanes drained → the per-lane index is garbage-collected. + assert group._lane_queues == {} finally: + await _cancel_quietly(next_task) await manager.unsubscribe(1, key) @@ -1908,135 +1829,37 @@ async def test_lane_internal_fifo_with_out_of_order_resolve(): t1 = cls() key = t1.shared_stream_key() manager = SharedStreamManager() + next_task = None try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - async with _consume_in_background(s1) as collector: - await collector.wait_for(4) - tokens = collector.tokens - - # Resolve a2 and a3 (head a1 stays pending) plus the noise lane. - await tokens[2].ack() # a2 - await tokens[3].ack() # a3 - await tokens[1].ack() # b1 - for _ in range(5): - await asyncio.sleep(0) - assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" - - await tokens[0].ack() # a1 - for _ in range(5): - await asyncio.sleep(0) - - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] - assert len(cls.advanced) == 4 - finally: - await manager.unsubscribe(1, key) - - -@pytest.mark.asyncio -async def test_max_in_flight_one_across_lanes(): - """Even with multiple lanes ready, at most one advance call is ever in flight.""" - release = asyncio.Event() - state = {"in_flight": 0, "max_in_flight": 0} - advanced: list = [] - - class _BlockingLaneProducer(SharedStreamProducer): - async def open_stream(self): - yield {"n": "a1"}, "a1" - yield {"n": "b1"}, "b1" - await asyncio.Event().wait() - - def get_advance_lane(self, broker_payload): - return broker_payload[0] - - async def advance(self, broker_payload, outcome): - state["in_flight"] += 1 - state["max_in_flight"] = max(state["max_in_flight"], state["in_flight"]) - await release.wait() - advanced.append(broker_payload) - state["in_flight"] -= 1 - - class _BlockingLaneTrigger(_ProgrammableSharedStreamTrigger): - @classmethod - def create_shared_stream_producer(cls, kwargs): - return _BlockingLaneProducer() - - async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() - yield TriggerEvent(raw) - - t1 = _BlockingLaneTrigger() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - - async with _consume_in_background(s1) as collector: - await collector.wait_for(2) - tokens = collector.tokens + # Consume all four events; only a1 gets a persist seq, so a2, a3, and + # b1 resolve from consumption while head a1 stays pending. + it = s1.__aiter__() + a1_seq: int | None = None + for _ in range(4): + raw = await _pull_raw(it) + if raw["n"] == "a1": + a1_seq = manager.bind_pending_event(trigger_id=1, key=key) + next_task = await _resume_past_yield(it) - # Resolve both lanes; the pump starts one advance, which blocks. - await tokens[0].ack() - await tokens[1].ack() - for _ in range(5): - await asyncio.sleep(0) - assert state["in_flight"] == 1, "the other lane's advance must not start while one is in flight" - assert advanced == [] + for _ in range(5): + await asyncio.sleep(0) + assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" - release.set() - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(advanced) >= 2: - break - await asyncio.sleep(0) + assert a1_seq is not None + manager.confirm_persisted([a1_seq]) + for _ in range(5): + await asyncio.sleep(0) - assert sorted(advanced) == ["a1", "b1"] - assert state["max_in_flight"] == 1, "advance calls must never overlap, even across lanes" + assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] + assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] + assert len(cls.advanced) == 4 finally: + await _cancel_quietly(next_task) await manager.unsubscribe(1, key) -@pytest.mark.asyncio -async def test_empty_snapshot_routes_through_lane(): - """Born-resolved events (no subscribers online) keep per-lane fan-out order.""" - cls = _make_ack_required_trigger_class( - [ - ({"n": "a1"}, "a1"), - ({"n": "b1"}, "b1"), - ({"n": "a2"}, "a2"), - ({"n": "b2"}, "b2"), - ], - lane_for=lambda payload: payload[0], - ) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - - # Subscribe then immediately unsubscribe so the group exists but has zero - # subscribers when the events arrive. - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - manager._groups[key].unsubscribe(1) # remove without stopping the poll task - - deadline = asyncio.get_event_loop().time() + 1.0 - while asyncio.get_event_loop().time() < deadline: - if len(cls.advanced) >= 4: - break - await asyncio.sleep(0.01) - - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] - assert len(cls.advanced) == 4 - assert cls.outcomes == [AdvanceOutcome(0, 0, 0)] * 4 - - del s1 # silence unused-variable warning - await manager.stop_all() - - @pytest.mark.asyncio async def test_get_advance_lane_raise_terminates_poll(): """A get_advance_lane that raises fails the poll like any other poll failure.""" @@ -2056,8 +1879,8 @@ def get_advance_lane(self, broker_payload): raise RuntimeError("lane boom") return None - async def advance(self, broker_payload, outcome): - advanced.append(broker_payload) + async def advance(self, batch): + advanced.extend(item.broker_payload for item in batch) async def aclose(self): aclose_calls.append(1) @@ -2068,8 +1891,7 @@ def create_shared_stream_producer(cls, kwargs): return _RaisingLaneProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) t1 = _RaisingLaneTrigger() @@ -2080,7 +1902,7 @@ async def filter_shared_stream(self, shared_stream): poll_task = manager._groups[key]._poll_task assert poll_task is not None - # Consume and ack the first event, then release the event whose + # Consume the first event, then release the event whose # get_advance_lane raises. events1 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) assert len(events1) == 1 @@ -2118,10 +1940,10 @@ async def open_stream(self): def get_advance_lane(self, broker_payload): return broker_payload[0] - async def advance(self, broker_payload, outcome): - if broker_payload == "a1": + async def advance(self, batch): + if batch[0].broker_payload == "a1": await release.wait() - advanced.append(broker_payload) + advanced.extend(item.broker_payload for item in batch) class _SlowFirstAdvanceTrigger(_ProgrammableSharedStreamTrigger): @classmethod @@ -2129,8 +1951,7 @@ def create_shared_stream_producer(cls, kwargs): return _SlowFirstAdvanceProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) t1 = _SlowFirstAdvanceTrigger() @@ -2138,19 +1959,26 @@ async def filter_shared_stream(self, shared_stream): manager = SharedStreamManager() s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) + + # Consume all four events; every event except a1 gets a persist seq, so + # only a1 resolves from consumption alone. + it = s1.__aiter__() + seqs: dict[str, int] = {} + for _ in range(4): + raw = await _pull_raw(it) + if raw["n"] != "a1": + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs[raw["n"]] = seq # Start a1's advance and let it block in flight. - await tokens[0].ack() for _ in range(5): await asyncio.sleep(0) - # Resolve everything else while a1's advance is still in flight, then - # stop the group before the pump has a chance to dispatch them. - await tokens[1].ack() - await tokens[2].ack() - await tokens[3].ack() + # Resolve everything else while a1's advance is still in flight (b2's + # window closes on unsubscribe), then stop the group before the pump has + # a chance to dispatch them. + manager.confirm_persisted(seqs.values()) stop_task = asyncio.create_task(manager.unsubscribe(1, key)) for _ in range(5): await asyncio.sleep(0) @@ -2158,73 +1986,104 @@ async def filter_shared_stream(self, shared_stream): await asyncio.wait_for(stop_task, timeout=1.0) # Drain-on-stop: a single harvesting pass would miss a2 and b2, which - # only become lane heads after the first pass already visited their lanes. + # only become dispatchable after the first pass already visited their lanes. assert [p for p in advanced if p[0] == "a"] == ["a1", "a2"] assert [p for p in advanced if p[0] == "b"] == ["b1", "b2"] assert len(advanced) == 4 @pytest.mark.asyncio -async def test_lanes_advance_independently(): - """A lane whose events are resolved does not wait for another lane's unresolved head.""" - cls = _make_ack_required_trigger_class( - [ - ({"n": "a1"}, "a1"), - ({"n": "b1"}, "b1"), - ({"n": "a2"}, "a2"), - ({"n": "b2"}, "b2"), - ], - lane_for=lambda payload: payload[0], - ) +@pytest.mark.parametrize( + ("second_action", "expected", "expected_clean"), + [ + pytest.param("consume", AdvanceOutcome(acked=2, failed=0), True, id="all-consume"), + pytest.param( + "unsubscribe", + AdvanceOutcome(acked=2, failed=0), + True, + id="unsubscribe-counts-as-acked", + ), + ], +) +async def test_outcome_classification(second_action, expected, expected_clean): + """Each subscriber online at broadcast time is counted in exactly one outcome field.""" + cls = _make_ack_required_trigger_class([({"msg": "outcome"}, "receipt-outcome")]) cls.advanced.clear() cls.outcomes.clear() - t1 = cls() + t1, t2 = cls(), cls() key = t1.shared_stream_key() manager = SharedStreamManager() + next_task = None try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - group = manager._groups[key] + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - async with _consume_in_background(s1) as collector: - await collector.wait_for(4) - tokens = collector.tokens + it2 = s2.__aiter__() + async with _consume_in_background(s1) as c1: + await c1.wait_for(1) + await _pull_raw(it2) + if second_action == "consume": + next_task = await _resume_past_yield(it2) + else: + await manager.unsubscribe(2, key) + await asyncio.sleep(0) - # Resolve lane "b" completely while lane "a" is fully unresolved. - await tokens[1].ack() # b1 - await tokens[3].ack() # b2 - for _ in range(5): - await asyncio.sleep(0) - assert cls.advanced == ["b1", "b2"], "lane b must advance without waiting for lane a's head" + assert cls.advanced == ["receipt-outcome"] + assert cls.outcomes == [expected] + assert cls.outcomes[0].is_clean is expected_clean + # Invariant: every subscriber in the broadcast snapshot is counted exactly once. + outcome = cls.outcomes[0] + assert outcome.acked + outcome.failed == 2 + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + if second_action != "unsubscribe": + await manager.unsubscribe(2, key) - await tokens[0].ack() # a1 - await tokens[2].ack() # a2 - for _ in range(5): - await asyncio.sleep(0) - # Contract: full per-lane projections in fan-out order. The global - # interleaving across lanes is a pump implementation detail. - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1", "b2"] - assert len(cls.advanced) == 4 - # All lanes drained → the per-lane index is garbage-collected. - assert group._lane_queues == {} +@pytest.mark.asyncio +async def test_outcome_counts_timeout_as_failed(): + """A subscriber force-failed by the ack timeout is counted as failed in the outcome.""" + cls = _make_ack_required_trigger_class([({"msg": "timeout-outcome"}, "receipt-to")]) + cls.advanced.clear() + cls.outcomes.clear() + + t_fast, t_slow = cls(), cls() + key = t_fast.shared_stream_key() + + fake_clock: list[float] = [0.0] + + def now() -> float: + return fake_clock[0] + + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + try: + s_fast = manager.subscribe(trigger_id=1, trigger=t_fast, key=key) + s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) + + # The fast subscriber consumes and resolves; the slow one pulls the + # raw event and stays on it. + it_slow = s_slow.__aiter__() + async with _consume_in_background(s_fast) as c_fast: + await c_fast.wait_for(1) + await _pull_raw(it_slow) + + fake_clock[0] = 0.2 + # Let the timeout loop tick (cadence = 0.01s real) and the pump dispatch. + await asyncio.sleep(0.05) + + assert cls.advanced == ["receipt-to"] + assert cls.outcomes == [AdvanceOutcome(acked=1, failed=1)] + assert not cls.outcomes[0].is_clean finally: - await manager.unsubscribe(1, key) + await manager.stop_all() @pytest.mark.asyncio -async def test_lane_internal_fifo_with_out_of_order_resolve(): - """Out-of-order resolves within a lane still advance in fan-out order, per lane.""" - cls = _make_ack_required_trigger_class( - [ - ({"n": "a1"}, "a1"), - ({"n": "b1"}, "b1"), - ({"n": "a2"}, "a2"), - ({"n": "a3"}, "a3"), - ], - lane_for=lambda payload: payload[0], - ) +async def test_reject_counts_as_rejected_not_acked_or_failed(): + """A single subscriber rejecting an event resolves it as rejected, not acked or failed.""" + cls = _make_reject_trigger_class([({"reject": True}, "receipt-reject")]) cls.advanced.clear() cls.outcomes.clear() @@ -2233,242 +2092,233 @@ async def test_lane_internal_fifo_with_out_of_order_resolve(): manager = SharedStreamManager() try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: + # The filter rejects the only event, so nothing is ever yielded; + # give the drain loop and the advance pump a tick to resolve and + # dispatch. + await asyncio.sleep(0) + await asyncio.sleep(0) + assert cls.advanced == ["receipt-reject"] + assert cls.outcomes == [AdvanceOutcome(acked=0, failed=0, rejected=1)] + assert collector.items == [] + finally: + await manager.stop_all() - async with _consume_in_background(s1) as collector: - await collector.wait_for(4) - tokens = collector.tokens - # Resolve a2 and a3 (head a1 stays pending) plus the noise lane. - await tokens[2].ack() # a2 - await tokens[3].ack() # a3 - await tokens[1].ack() # b1 - for _ in range(5): - await asyncio.sleep(0) - assert cls.advanced == ["b1"], "lane a may not advance while its own head is unresolved" +@pytest.mark.asyncio +async def test_reject_resolves_immediately_without_persist_gate(): + """A reject advances at once, without waiting on any persist confirmation.""" + cls = _make_reject_trigger_class([({"reject": True}, "receipt-now")]) + cls.advanced.clear() + cls.outcomes.clear() - await tokens[0].ack() # a1 - for _ in range(5): - await asyncio.sleep(0) + # A fake clock pinned at 0 proves the advance does not come from the ack + # timeout path: no time elapses, yet the reject still advances. + fake_clock: list[float] = [0.0] - assert [p for p in cls.advanced if p[0] == "a"] == ["a1", "a2", "a3"] - assert [p for p in cls.advanced if p[0] == "b"] == ["b1"] - assert len(cls.advanced) == 4 + def now() -> float: + return fake_clock[0] + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + async with _consume_in_background(t1.filter_shared_stream(s1)): + await asyncio.sleep(0) + await asyncio.sleep(0) + # No persist confirmation sent and the clock never moved, yet the + # rejected event is already advanced. + assert cls.advanced == ["receipt-now"] + assert cls.outcomes == [AdvanceOutcome(acked=0, failed=0, rejected=1)] finally: - await manager.unsubscribe(1, key) + await manager.stop_all() @pytest.mark.asyncio -async def test_get_advance_lane_raise_terminates_poll(): - """A get_advance_lane that raises fails the poll like any other poll failure.""" - proceed_flag = asyncio.Event() - aclose_calls: list[int] = [] - advanced: list = [] +async def test_fan_out_mixed_ack_reject_fail_outcome(): + """Three subscribers ack / reject / fail the same event; the outcome counts each once.""" + fake_clock: list[float] = [0.0] - class _RaisingLaneProducer(SharedStreamProducer): - async def open_stream(self): - yield {"n": 1}, "p1" - await proceed_flag.wait() - yield {"n": 2}, "p2" - await asyncio.Event().wait() # pragma: no cover + def now() -> float: + return fake_clock[0] - def get_advance_lane(self, broker_payload): - if broker_payload == "p2": - raise RuntimeError("lane boom") - return None + # One producer broadcasts a single event to three subscribers; subscriber + # behaviour is decided per trigger class. + ack_cls = _make_ack_required_trigger_class([({"msg": "mixed"}, "receipt-mixed")]) + ack_cls.advanced.clear() + ack_cls.outcomes.clear() - async def advance(self, broker_payload, outcome): - advanced.append(broker_payload) + t_ack, t_reject, t_slow = ack_cls(), ack_cls(), ack_cls() + key = t_ack.shared_stream_key() + manager = SharedStreamManager(ack_timeout=0.1, _now=now) + next_task = None + try: + s_ack = manager.subscribe(trigger_id=1, trigger=t_ack, key=key) + s_reject = manager.subscribe(trigger_id=2, trigger=t_reject, key=key) + s_slow = manager.subscribe(trigger_id=3, trigger=t_slow, key=key) + + it_reject = s_reject.__aiter__() + it_slow = s_slow.__aiter__() + async with _consume_in_background(s_ack) as c_ack: + # Subscriber 1 consumes the event normally (acked). + await c_ack.wait_for(1) + # Subscriber 2 pulls the event and rejects it directly through the + # group, mirroring what reject_shared_stream_event would do while + # this subscriber's window is open. + await _pull_raw(it_reject) + manager._groups[key]._reject_pending_event(trigger_id=2, event_id=0) + # Subscriber 3 pulls the event and stalls, to be failed by the timeout. + await _pull_raw(it_slow) - async def aclose(self): - aclose_calls.append(1) + fake_clock[0] = 0.2 + await asyncio.sleep(0.05) - class _RaisingLaneTrigger(_ProgrammableSharedStreamTrigger): - @classmethod - def create_shared_stream_producer(cls, kwargs): - return _RaisingLaneProducer() + assert ack_cls.advanced == ["receipt-mixed"] + assert ack_cls.outcomes == [AdvanceOutcome(acked=1, failed=1, rejected=1)] + outcome = ack_cls.outcomes[0] + assert outcome.acked + outcome.rejected + outcome.failed == 3 + finally: + await _cancel_quietly(next_task) + await manager.stop_all() - async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() - yield TriggerEvent(raw) - t1 = _RaisingLaneTrigger() +@pytest.mark.asyncio +async def test_reject_does_not_block_later_event_in_same_lane(): + """A rejected event and a fully acked event both advance, in fan-out order.""" + cls = _make_reject_trigger_class( + [ + ({"reject": True}, "receipt-a"), + ({"reject": False, "msg": "b"}, "receipt-b"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1 = cls() key = t1.shared_stream_key() manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: + # Event A is rejected (no TriggerEvent), event B yields one. + await collector.wait_for(1) + await asyncio.sleep(0) + await asyncio.sleep(0) - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - poll_task = manager._groups[key]._poll_task - assert poll_task is not None - - # Consume and ack the first event, then release the event whose - # get_advance_lane raises. - events1 = await asyncio.wait_for(_collect_ack_stream(t1, s1, n=1), timeout=1.0) - assert len(events1) == 1 - proceed_flag.set() - - async def consume(): - async for _ in s1: - pass - - with pytest.raises(RuntimeError, match="lane boom"): - await asyncio.wait_for(consume(), timeout=1.0) - - assert key not in manager._groups, "the group must be evicted when get_advance_lane raises" - await asyncio.wait_for(poll_task, timeout=1.0) - assert sum(aclose_calls) == 1 - # The first event was resolved before the failure → drained on the stop path. - assert advanced == ["p1"] - await manager.unsubscribe(1, key) + assert cls.advanced == ["receipt-a", "receipt-b"] + assert cls.outcomes == [ + AdvanceOutcome(acked=0, failed=0, rejected=1), + AdvanceOutcome(acked=1, failed=0, rejected=0), + ] + finally: + await manager.stop_all() @pytest.mark.asyncio -async def test_stop_drains_all_lanes(): - """On stop the pump keeps draining until every resolved event in every lane is advanced.""" - release = asyncio.Event() - advanced: list = [] +async def test_reject_on_fast_path_is_no_op_and_event_still_fires(): + """On the fast path the reject signal is ignored and the event fires normally.""" - class _SlowFirstAdvanceProducer(SharedStreamProducer): - async def open_stream(self): - yield {"n": "a1"}, "a1" - yield {"n": "a2"}, "a2" - yield {"n": "b1"}, "b1" - yield {"n": "b2"}, "b2" - await asyncio.Event().wait() - - def get_advance_lane(self, broker_payload): - return broker_payload[0] + # The fast path has no producer: build a fast-mode trigger whose filter + # still calls reject_shared_stream_event for one event. + class _FastRejectTrigger(_ProgrammableSharedStreamTrigger): + async def filter_shared_stream(self, shared_stream): + async for raw in shared_stream: + if raw.get("reject"): + reject_shared_stream_event() + yield TriggerEvent(raw) - async def advance(self, broker_payload, outcome): - if broker_payload == "a1": - await release.wait() - advanced.append(broker_payload) + events = [{"reject": True, "msg": "still-fires"}] - class _SlowFirstAdvanceTrigger(_ProgrammableSharedStreamTrigger): - @classmethod - def create_shared_stream_producer(cls, kwargs): - return _SlowFirstAdvanceProducer() + async def _open(cls_, kwargs): + for event in events: + yield event + await asyncio.Event().wait() - async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() - yield TriggerEvent(raw) + _FastRejectTrigger.open_shared_stream = classmethod(_open) - t1 = _SlowFirstAdvanceTrigger() + t1 = _FastRejectTrigger() key = t1.shared_stream_key() manager = SharedStreamManager() + mock_log = MagicMock() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + with mock.patch("airflow.triggers.shared_stream.log", mock_log): + collected = await asyncio.wait_for(_collect(t1.filter_shared_stream(s1), n=1), timeout=1.0) + # The event fired despite the reject call, and the warning was logged. + assert [e.payload for e in collected] == [{"reject": True, "msg": "still-fires"}] + assert mock_log.warning.mock_calls == [ + mock.call("reject_shared_stream_event called outside an ack-mode binding window; ignored") + ] + finally: + await manager.unsubscribe(1, key) - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=4), timeout=1.0) - - # Start a1's advance and let it block in flight. - await tokens[0].ack() - for _ in range(5): - await asyncio.sleep(0) - - # Resolve everything else while a1's advance is still in flight, then - # stop the group before the pump has a chance to dispatch them. - await tokens[1].ack() - await tokens[2].ack() - await tokens[3].ack() - stop_task = asyncio.create_task(manager.unsubscribe(1, key)) - for _ in range(5): - await asyncio.sleep(0) - release.set() - await asyncio.wait_for(stop_task, timeout=1.0) - # Drain-on-stop: a single harvesting pass would miss a2 and b2, which - # only become lane heads after the first pass already visited their lanes. - assert [p for p in advanced if p[0] == "a"] == ["a1", "a2"] - assert [p for p in advanced if p[0] == "b"] == ["b1", "b2"] - assert len(advanced) == 4 +@pytest.mark.asyncio +async def test_reject_outside_any_window_is_no_op(): + """Calling the free function with no open window logs a warning and does not raise.""" + mock_log = MagicMock() + with mock.patch("airflow.triggers.shared_stream.log", mock_log): + reject_shared_stream_event() # must not raise + assert mock_log.warning.mock_calls == [ + mock.call("reject_shared_stream_event called outside an ack-mode binding window; ignored") + ] -@pytest.mark.asyncio @pytest.mark.parametrize( - ("second_action", "expected", "expected_clean"), + ("outcome", "expected_clean"), [ - pytest.param("ack", AdvanceOutcome(acked=2, nacked=0, failed=0), True, id="all-ack"), - pytest.param("nack", AdvanceOutcome(acked=1, nacked=1, failed=0), False, id="ack-plus-nack"), - pytest.param( - "unsubscribe", - AdvanceOutcome(acked=2, nacked=0, failed=0), - True, - id="unsubscribe-counts-as-acked", - ), + pytest.param(AdvanceOutcome(acked=2, failed=0, rejected=0), True, id="no-reject-no-fail"), + pytest.param(AdvanceOutcome(acked=2, failed=0, rejected=1), False, id="reject-makes-unclean"), + pytest.param(AdvanceOutcome(acked=2, failed=1, rejected=0), False, id="fail-makes-unclean"), ], ) -async def test_outcome_classification(second_action, expected, expected_clean): - """Each subscriber online at broadcast time is counted in exactly one outcome field.""" - cls = _make_ack_required_trigger_class([({"msg": "outcome"}, "receipt-outcome")]) - cls.advanced.clear() - cls.outcomes.clear() +def test_is_clean_requires_no_reject_and_no_fail(outcome, expected_clean): + """``is_clean`` is True only when both ``failed`` and ``rejected`` are zero.""" + assert outcome.is_clean is expected_clean - t1, t2 = cls(), cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) - async with _consume_in_background(s1) as c1, _consume_in_background(s2) as c2: - await c1.wait_for(1) - await c2.wait_for(1) +@pytest.mark.asyncio +async def test_reject_then_yield_real_event_fires_but_does_not_bind(): + """A subscriber that rejects an event and then yields one anyway: the late event fires but binds nothing.""" - await c1.tokens[0].ack() - if second_action == "ack": - await c2.tokens[0].ack() - elif second_action == "nack": - await c2.tokens[0].nack() - else: - await manager.unsubscribe(2, key) - await asyncio.sleep(0) + class _RejectThenYieldTrigger(_ProgrammableSharedStreamTrigger): + advanced: list = [] + outcomes: list[AdvanceOutcome] = [] + batches: list[list] = [] + aclose_calls: int = 0 - assert cls.advanced == ["receipt-outcome"] - assert cls.outcomes == [expected] - assert cls.outcomes[0].is_clean is expected_clean - # Invariant: every subscriber in the broadcast snapshot is counted exactly once. - outcome = cls.outcomes[0] - assert outcome.acked + outcome.nacked + outcome.failed == 2 - finally: - await manager.unsubscribe(1, key) - if second_action != "unsubscribe": - await manager.unsubscribe(2, key) + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RecordingProducer(cls, [({"reject": True}, "receipt-x")], lane_for=None) + async def filter_shared_stream(self, shared_stream): + async for raw in shared_stream: + reject_shared_stream_event() + # Subscriber-side logic error: still yield after rejecting. + yield TriggerEvent(raw) -@pytest.mark.asyncio -async def test_outcome_counts_timeout_as_failed(): - """A subscriber force-failed by the ack timeout is counted as failed in the outcome.""" - cls = _make_ack_required_trigger_class([({"msg": "timeout-outcome"}, "receipt-to")]) + cls = _RejectThenYieldTrigger cls.advanced.clear() cls.outcomes.clear() - t_fast, t_slow = cls(), cls() - key = t_fast.shared_stream_key() - - fake_clock: list[float] = [0.0] - - def now() -> float: - return fake_clock[0] - - manager = SharedStreamManager(ack_timeout=0.1, _now=now) + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() try: - s_fast = manager.subscribe(trigger_id=1, trigger=t_fast, key=key) - s_slow = manager.subscribe(trigger_id=2, trigger=t_slow, key=key) - - # The slow collector pulls its (event, token) pair but never resolves it. - async with _consume_in_background(s_fast) as c_fast, _consume_in_background(s_slow) as c_slow: - await c_fast.wait_for(1) - await c_slow.wait_for(1) + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: + # The trigger event still fires despite the preceding reject. + fired = await collector.wait_for(1) + assert [e.payload for e in fired] == [{"reject": True}] - await c_fast.tokens[0].ack() - fake_clock[0] = 0.2 - # Let the timeout loop tick (cadence = 0.01s real) and the pump dispatch. - await asyncio.sleep(0.05) + # The reject already popped the binding, so the late event binds + # nothing — bind_pending_event returns None and draws no seq. + assert manager.bind_pending_event(trigger_id=1, key=key) is None - assert cls.advanced == ["receipt-to"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] - assert not cls.outcomes[0].is_clean + await asyncio.sleep(0) + assert cls.advanced == ["receipt-x"] + assert cls.outcomes == [AdvanceOutcome(acked=0, failed=0, rejected=1)] finally: await manager.stop_all() @@ -2502,7 +2352,7 @@ async def open_stream(self): raise RuntimeError("open boom") yield # pragma: no cover - async def advance(self, broker_payload, outcome): # pragma: no cover + async def advance(self, batch): # pragma: no cover pass async def aclose(self): @@ -2514,7 +2364,7 @@ def create_shared_stream_producer(cls, kwargs): return _FailingOpenProducer() async def filter_shared_stream(self, shared_stream): - async for raw, _token in shared_stream: # pragma: no cover + async for raw in shared_stream: # pragma: no cover yield TriggerEvent(raw) t1 = _FailingOpenTrigger() @@ -2551,9 +2401,9 @@ async def open_stream(self): yield {"n": 2}, "p2" await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): - attempted.append(broker_payload) - if broker_payload == "p1": + async def advance(self, batch): + attempted.extend(item.broker_payload for item in batch) + if any(item.broker_payload == "p1" for item in batch): raise RuntimeError("p1 boom") class _FirstRaiseTrigger(_ProgrammableSharedStreamTrigger): @@ -2562,33 +2412,38 @@ def create_shared_stream_producer(cls, kwargs): return _FirstRaiseProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) t1 = _FirstRaiseTrigger() key = t1.shared_stream_key() manager = SharedStreamManager() + next_task = None try: s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) group = manager._groups[key] group.log = MagicMock() - async with _consume_in_background(s1) as collector: - await collector.wait_for(2) - tokens = collector.tokens - await tokens[0].ack() - await tokens[1].ack() - for _ in range(5): - await asyncio.sleep(0) + # Move past event 1; its advance dispatches alone and raises. + it = s1.__aiter__() + await _pull_raw(it) + pull2 = await _resume_past_yield(it) + for _ in range(5): + await asyncio.sleep(0) + assert attempted == ["p1"] - assert attempted == ["p1", "p2"], ( - "the pump must move on to event 2 after event 1's advance raised" - ) - error_calls = group.log.error.mock_calls - assert len(error_calls) == 1 - assert "broker advance failed" in error_calls[0].args[0] + # Move past event 2; the pump must still dispatch it. + await asyncio.wait_for(pull2, timeout=1.0) + next_task = await _resume_past_yield(it) + for _ in range(5): + await asyncio.sleep(0) + + assert attempted == ["p1", "p2"], "the pump must move on to event 2 after event 1's advance raised" + error_calls = group.log.error.mock_calls + assert len(error_calls) == 1 + assert "broker advance failed" in error_calls[0].args[0] finally: + await _cancel_quietly(next_task) await manager.unsubscribe(1, key) @@ -2608,9 +2463,9 @@ async def open_stream(self): yield {"n": 3}, "p3" await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): - self._trigger_cls.advanced.append(broker_payload) - self._trigger_cls.outcomes.append(outcome) + async def advance(self, batch): + self._trigger_cls.advanced.extend(item.broker_payload for item in batch) + self._trigger_cls.outcomes.extend(item.outcome for item in batch) class _GatedTrigger(_ProgrammableSharedStreamTrigger): advanced: list = [] @@ -2621,8 +2476,7 @@ def create_shared_stream_producer(cls, kwargs): return _GatedProducer(cls) async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) t_dead, t_live = _GatedTrigger(), _GatedTrigger() @@ -2635,12 +2489,9 @@ async def filter_shared_stream(self, shared_stream): manager.subscribe(trigger_id=1, trigger=t_dead, key=key) s_live = manager.subscribe(trigger_id=2, trigger=t_live, key=key) - # Live subscriber acks events 1 and 2; the dead subscriber never consumes. + # Live subscriber consumes events 1 and 2; the dead subscriber never consumes. async with _consume_in_background(s_live) as collector: await collector.wait_for(2) - tokens = collector.tokens - await tokens[0].ack() - await tokens[1].ack() await asyncio.sleep(0) assert _GatedTrigger.advanced == [], "events 1-2 still wait on the dead subscriber" @@ -2654,14 +2505,13 @@ async def filter_shared_stream(self, shared_stream): break await asyncio.sleep(0) assert _GatedTrigger.advanced == ["p1", "p2"] - assert _GatedTrigger.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=1)] * 2 + assert _GatedTrigger.outcomes == [AdvanceOutcome(acked=1, failed=1)] * 2 - # The live subscriber resolves event 3 and it advances too. + # The live subscriber consumes event 3 and it advances too. await collector.wait_for(3) - await collector.tokens[2].ack() await asyncio.sleep(0) assert _GatedTrigger.advanced == ["p1", "p2", "p3"] - assert _GatedTrigger.outcomes[2] == AdvanceOutcome(acked=1, nacked=0, failed=1) + assert _GatedTrigger.outcomes[2] == AdvanceOutcome(acked=1, failed=1) finally: await manager.stop_all() @@ -2676,7 +2526,7 @@ def create_shared_stream_producer(cls, kwargs): raise RuntimeError("factory boom") async def filter_shared_stream(self, shared_stream): - async for raw, _token in shared_stream: # pragma: no cover + async for raw in shared_stream: # pragma: no cover yield TriggerEvent(raw) t1 = _FactoryBoomTrigger() @@ -2710,10 +2560,10 @@ async def open_stream(self): yield {"n": 2}, "p2" await asyncio.Event().wait() - async def advance(self, broker_payload, outcome): + async def advance(self, batch): advance_started.set() await advance_release.wait() - advanced.append(broker_payload) + advanced.extend(item.broker_payload for item in batch) async def aclose(self): aclose_calls.append(1) @@ -2724,8 +2574,7 @@ def create_shared_stream_producer(cls, kwargs): return _PrefixProducer() async def filter_shared_stream(self, shared_stream): - async for raw, token in shared_stream: - await token.ack() + async for raw in shared_stream: yield TriggerEvent(raw) t1 = _PrefixTrigger() @@ -2734,12 +2583,13 @@ async def filter_shared_stream(self, shared_stream): s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - tokens: list[AckToken] = [] - await asyncio.wait_for(_grab_tokens(s1, tokens, n=2), timeout=1.0) - - # Resolve only event 1; its advance starts and blocks. - await tokens[0].ack() + # Move past event 1 (resolving it) onto event 2, which stays unresolved + # because the subscriber never resumes past its yield. + it = s1.__aiter__() + await _pull_raw(it) + pull2 = await _resume_past_yield(it) await asyncio.wait_for(advance_started.wait(), timeout=1.0) + await asyncio.wait_for(pull2, timeout=1.0) # stop must wait for the in-flight advance of the resolved event... stop_task = asyncio.create_task(manager.stop_all()) @@ -2754,8 +2604,8 @@ async def filter_shared_stream(self, shared_stream): assert sum(aclose_calls) == 1, "the producer must still be closed after the drain" -async def _pull_pair(stream_iter): - """Pull one ``(raw_event, token)`` pair off an ack-mode stream iterator.""" +async def _pull_raw(stream_iter): + """Pull one raw event off an ack-mode stream iterator.""" return await asyncio.wait_for(stream_iter.__anext__(), timeout=1.0) @@ -2763,7 +2613,7 @@ async def _resume_past_yield(stream_iter) -> asyncio.Task: """ Resume the drain generator past its yield and leave it waiting. - Resuming closes the previous token's binding window — the exact moment a + Resuming closes the previous event's binding window — the exact moment a production filter loops back to pull the next raw event. The returned task is still waiting on the queue; the test must cancel it (or expect its failure) during cleanup. @@ -2796,7 +2646,7 @@ async def test_multi_event_binding_gates_advance_on_every_seq(): try: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) it = stream.__aiter__() - _raw, token = await _pull_pair(it) + await _pull_raw(it) # The subscriber derives two trigger events from the same raw event. seq1 = manager.bind_pending_event(trigger_id=1, key=key) @@ -2805,7 +2655,6 @@ async def test_multi_event_binding_gates_advance_on_every_seq(): assert seq2 is not None assert seq1 != seq2 - await token.ack() next_task = await _resume_past_yield(it) assert cls.advanced == [], "advance must wait for both persist confirmations" @@ -2816,7 +2665,7 @@ async def test_multi_event_binding_gates_advance_on_every_seq(): manager.confirm_persisted([seq2]) await asyncio.sleep(0) assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + assert cls.outcomes == [AdvanceOutcome(acked=1, failed=0)] finally: await _cancel_quietly(next_task) await manager.unsubscribe(1, key) @@ -2824,7 +2673,7 @@ async def test_multi_event_binding_gates_advance_on_every_seq(): @pytest.mark.asyncio async def test_filtered_out_event_resolves_on_next_pull_without_confirmation(): - """An acked event with no bound trigger events resolves as soon as the filter pulls the next raw event.""" + """An event with no bound trigger events resolves as soon as the filter pulls the next raw event.""" cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) cls.advanced.clear() cls.outcomes.clear() @@ -2836,82 +2685,14 @@ async def test_filtered_out_event_resolves_on_next_pull_without_confirmation(): try: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) it = stream.__aiter__() - _raw, token = await _pull_pair(it) + await _pull_raw(it) - # Filtered out: ack, derive nothing, loop back for the next raw event. - await token.ack() + # Filtered out: derive nothing, loop back for the next raw event. next_task = await _resume_past_yield(it) await asyncio.sleep(0) assert cls.advanced == ["p1"], "a filtered-out event must resolve without any confirmation" - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] - finally: - await _cancel_quietly(next_task) - await manager.unsubscribe(1, key) - - -@pytest.mark.asyncio -async def test_nack_with_bound_seq_resolves_immediately_and_late_confirm_is_noop(): - """nack() resolves without waiting for confirmations; a confirmation arriving later is ignored.""" - cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - try: - stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) - it = stream.__aiter__() - _raw, token = await _pull_pair(it) - - seq = manager.bind_pending_event(trigger_id=1, key=key) - assert seq is not None - - # nack resolves immediately — no window close, no confirmation needed. - await token.nack() - await asyncio.sleep(0) - assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=1, failed=0)] - - # The late confirmation finds nothing and no-ops. - manager.confirm_persisted([seq]) - await asyncio.sleep(0) - assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=1, failed=0)] - finally: - await manager.unsubscribe(1, key) - - -@pytest.mark.asyncio -async def test_ack_before_yield_still_gates_advance_on_confirmation(): - """Regression: an ack arriving before the event is bound must not bypass the persist gate.""" - cls = _make_ack_required_trigger_class([({"n": 1}, "p1")]) - cls.advanced.clear() - cls.outcomes.clear() - - t1 = cls() - key = t1.shared_stream_key() - manager = SharedStreamManager() - next_task = None - try: - stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) - it = stream.__aiter__() - _raw, token = await _pull_pair(it) - - # ack-before-yield: the ack lands while the binding window is still - # open and nothing is bound yet. - await token.ack() - seq = manager.bind_pending_event(trigger_id=1, key=key) - assert seq is not None - - next_task = await _resume_past_yield(it) - assert cls.advanced == [], "the early ack must not resolve ahead of the confirmation" - - manager.confirm_persisted([seq]) - await asyncio.sleep(0) - assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=1, nacked=0, failed=0)] + assert cls.outcomes == [AdvanceOutcome(acked=1, failed=0)] finally: await _cancel_quietly(next_task) await manager.unsubscribe(1, key) @@ -2935,14 +2716,13 @@ async def test_unsubscribe_with_unconfirmed_seq_waits_for_confirmation(): it2 = s2.__aiter__() # Subscriber 1 fires (binds a trigger event) and immediately leaves. - _raw, _token1 = await _pull_pair(it1) + await _pull_raw(it1) seq = manager.bind_pending_event(trigger_id=1, key=key) assert seq is not None await manager.unsubscribe(1, key) # Subscriber 2 resolves cleanly. - _raw, token2 = await _pull_pair(it2) - await token2.ack() + await _pull_raw(it2) next_task = await _resume_past_yield(it2) await asyncio.sleep(0) @@ -2952,8 +2732,8 @@ async def test_unsubscribe_with_unconfirmed_seq_waits_for_confirmation(): manager.confirm_persisted([seq]) await asyncio.sleep(0) assert cls.advanced == ["p1"] - # The departed subscriber still counts as acked (implicit ack-out). - assert cls.outcomes == [AdvanceOutcome(acked=2, nacked=0, failed=0)] + # The departed subscriber still counts as acked (implicit resolution). + assert cls.outcomes == [AdvanceOutcome(acked=2, failed=0)] finally: await _cancel_quietly(next_task) await manager.unsubscribe(2, key) @@ -2974,15 +2754,15 @@ async def test_unsubscribe_with_unconfirmed_seq_waits_for_confirmation(): [], [], 1, - [AdvanceOutcome(acked=1, nacked=0, failed=0)], + [AdvanceOutcome(acked=1, failed=0)], id="below-timeout-confirm-recovers", ), pytest.param( 0.2, ["p1"], - [AdvanceOutcome(acked=0, nacked=0, failed=1)], + [AdvanceOutcome(acked=0, failed=1)], 0, - [AdvanceOutcome(acked=0, nacked=0, failed=1)], + [AdvanceOutcome(acked=0, failed=1)], id="late-confirm-after-timeout-is-noop", ), ], @@ -3017,11 +2797,10 @@ def now() -> float: try: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) it = stream.__aiter__() - _raw, token = await _pull_pair(it) + await _pull_raw(it) seq = manager.bind_pending_event(trigger_id=1, key=key) assert seq is not None - await token.ack() next_task = await _resume_past_yield(it) fake_clock[0] = clock_value @@ -3059,18 +2838,17 @@ def now() -> float: try: stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) it = stream.__aiter__() - _raw, token = await _pull_pair(it) + await _pull_raw(it) seq = manager.bind_pending_event(trigger_id=1, key=key) assert seq is not None - await token.ack() next_task = await _resume_past_yield(it) # Exactly at the timeout: the unconfirmed event is force-failed. fake_clock[0] = 0.1 await asyncio.sleep(0.05) assert cls.advanced == ["p1"] - assert cls.outcomes == [AdvanceOutcome(acked=0, nacked=0, failed=1)] + assert cls.outcomes == [AdvanceOutcome(acked=0, failed=1)] # The still-online subscriber sees the AckTimeout; the confirmation # for the bound sequence number never arrives. @@ -3095,11 +2873,10 @@ async def test_group_stop_abandons_unconfirmed_advance(): stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) it = stream.__aiter__() - _raw, token = await _pull_pair(it) + await _pull_raw(it) seq = manager.bind_pending_event(trigger_id=1, key=key) assert seq is not None - await token.ack() # Last subscriber leaves while the confirmation is still outstanding: # the group stops and the advance is abandoned — the broker redelivers. @@ -3107,3 +2884,222 @@ async def test_group_stop_abandons_unconfirmed_advance(): assert manager._groups == {} assert cls.advanced == [], "an unconfirmed advance must be abandoned at group stop" assert cls.aclose_calls == 1 + + +@pytest.mark.asyncio +async def test_pump_batches_contiguous_resolved_prefix(): + """Events that resolve together are dispatched as one batch, in fan-out order.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1"), ({"n": 2}, "p2"), ({"n": 3}, "p3")]) + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + # Consume all three events, binding one persist seq to each so + # nothing resolves until the confirmations land. + it = stream.__aiter__() + seqs: list[int] = [] + for _ in range(3): + await _pull_raw(it) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs.append(seq) + next_task = await _resume_past_yield(it) + for _ in range(5): + await asyncio.sleep(0) + assert cls.batches == [], "nothing may advance while every event is unconfirmed" + + # All three confirmations land in one synchronous call; the pump sees + # the whole lane resolved at once and dispatches a single batch. + manager.confirm_persisted(seqs) + for _ in range(5): + await asyncio.sleep(0) + assert cls.batches == [["p1", "p2", "p3"]], ( + "one call must carry the lane's resolved prefix in fan-out order" + ) + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_pump_batches_per_lane_independently(): + """A batch never crosses a lane boundary; each lane's resolved prefix is its own call.""" + cls = _make_ack_required_trigger_class( + [ + ({"n": "a1"}, "a1"), + ({"n": "b1"}, "b1"), + ({"n": "a2"}, "a2"), + ({"n": "b2"}, "b2"), + ], + lane_for=lambda payload: payload[0], + ) + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + it = stream.__aiter__() + seqs: list[int] = [] + for _ in range(4): + await _pull_raw(it) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs.append(seq) + next_task = await _resume_past_yield(it) + for _ in range(5): + await asyncio.sleep(0) + assert cls.batches == [] + + # Everything resolves at once, but the batches stay per lane. + manager.confirm_persisted(seqs) + for _ in range(5): + await asyncio.sleep(0) + assert sorted(cls.batches) == [["a1", "a2"], ["b1", "b2"]], ( + "each lane's resolved prefix must be dispatched as its own batch" + ) + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_partial_prefix_waits_for_lane_head(): + """Resolved events behind an unresolved head wait, then ship in one batch with it.""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1"), ({"n": 2}, "p2"), ({"n": 3}, "p3")]) + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + it = stream.__aiter__() + seqs: list[int] = [] + for _ in range(3): + await _pull_raw(it) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs.append(seq) + next_task = await _resume_past_yield(it) + + # Events 2 and 3 resolve out of order; the lane head is still pending. + manager.confirm_persisted([seqs[1], seqs[2]]) + for _ in range(5): + await asyncio.sleep(0) + assert cls.batches == [], "no batch may ship while the lane head is unresolved" + + # The head resolves; the whole prefix ships as one batch in fan-out order. + manager.confirm_persisted([seqs[0]]) + for _ in range(5): + await asyncio.sleep(0) + assert cls.batches == [["p1", "p2", "p3"]] + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_advance_raise_abandons_whole_batch(): + """A batch whose advance raises is logged and abandoned; later events form fresh batches.""" + calls: list[list] = [] + proceed = asyncio.Event() + + class _RaiseOnFirstBatchProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + yield {"n": 2}, "p2" + await proceed.wait() + yield {"n": 3}, "p3" + await asyncio.Event().wait() + + async def advance(self, batch): + calls.append([item.broker_payload for item in batch]) + if len(calls) == 1: + raise RuntimeError("batch boom") + + class _RaiseOnFirstBatchTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RaiseOnFirstBatchProducer() + + t1 = _RaiseOnFirstBatchTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + group = manager._groups[key] + group.log = MagicMock() + + # Resolve events 1 and 2 together so they form one two-event batch. + it = s1.__aiter__() + seqs: list[int] = [] + for _ in range(2): + await _pull_raw(it) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs.append(seq) + next_task = await _resume_past_yield(it) + manager.confirm_persisted(seqs) + for _ in range(5): + await asyncio.sleep(0) + + # The batch raised: it is logged, already off the books, and never + # dispatched again — the broker is left to redeliver. + assert calls == [["p1", "p2"]] + assert group._outstanding == {} + assert len(group.log.error.mock_calls) == 1 + + # The pump keeps running; a later event forms a fresh batch. + proceed.set() + await asyncio.wait_for(next_task, timeout=1.0) + next_task = await _resume_past_yield(it) + for _ in range(5): + await asyncio.sleep(0) + assert calls == [["p1", "p2"], ["p3"]] + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) + + +@pytest.mark.asyncio +async def test_trickle_resolution_yields_singleton_batches(): + """Events that resolve one at a time degenerate to singleton batches (the old behavior).""" + cls = _make_ack_required_trigger_class([({"n": 1}, "p1"), ({"n": 2}, "p2"), ({"n": 3}, "p3")]) + + t1 = cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + stream = manager.subscribe(trigger_id=1, trigger=t1, key=key) + + it = stream.__aiter__() + seqs: list[int] = [] + for _ in range(3): + await _pull_raw(it) + seq = manager.bind_pending_event(trigger_id=1, key=key) + assert seq is not None + seqs.append(seq) + next_task = await _resume_past_yield(it) + + # Resolve one event per pump pass. + for seq in seqs: + manager.confirm_persisted([seq]) + for _ in range(5): + await asyncio.sleep(0) + + assert cls.batches == [["p1"], ["p2"], ["p3"]], ( + "trickled resolutions must each ship as their own batch" + ) + finally: + await _cancel_quietly(next_task) + await manager.unsubscribe(1, key) From e45b4510331f1536432af9fefc0afadddcce8e1c Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 15 Jun 2026 16:39:24 +0800 Subject: [PATCH 11/15] fix(triggers): redeliver never-pulled shared-stream events on unsubscribe When a shared-stream subscriber leaves on graceful triggerer shutdown, cancellation, or trigger reassignment, only the event it actually pulled now resolves as accepted. Events still queued that the subscriber never pulled resolve as failed so the broker redelivers them, instead of committing/deleting a broker message with no persisted TriggerEvent behind it. A zero-subscriber broadcast (all-zero counts) is no longer reported as clean, since nothing was accepted for the producer to commit on. --- .../src/airflow/triggers/shared_stream.py | 45 ++++-- .../tests/unit/triggers/test_shared_stream.py | 129 +++++++++++++++++- 2 files changed, 163 insertions(+), 11 deletions(-) diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index c2f5a2d609c1a..ab04af0946f53 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -333,7 +333,8 @@ class AdvanceOutcome: event; a Pub/Sub producer ``nack`` s on a reject and ``ack`` s otherwise. An event broadcast while no subscribers were online carries all-zero - counts and is clean. + counts and is **not** clean — nothing was accepted, so there is nothing + the producer should commit on. """ acked: int @@ -342,8 +343,13 @@ class AdvanceOutcome: @property def is_clean(self) -> bool: - """Whether every subscriber accepted the event — no active reject and no involuntary failure.""" - return self.failed == 0 and self.rejected == 0 + """ + Whether every subscriber accepted the event. + + No active reject, no involuntary failure, and at least one subscriber + acked. A zero-subscriber broadcast (all-zero counts) is not clean. + """ + return self.failed == 0 and self.rejected == 0 and self.acked > 0 class AdvanceItem(NamedTuple): @@ -990,13 +996,27 @@ def unsubscribe(self, trigger_id: int) -> None: # (Airflow's standard idiom); dropping the queue is enough here. self._subscribers.pop(trigger_id, None) self._failed_subscribers.discard(trigger_id) + # Capture the event the subscriber is currently sitting on (if any) + # before clearing it from the open-windows map. + open_event_id = self._open_windows.get(trigger_id) self._open_windows.pop(trigger_id, None) # Implicit resolution: leaving closes the subscriber's window on # every outstanding event, so the producer never waits forever for a - # subscriber that has left. The broker advance still waits for - # persist confirmation of any trigger events this subscriber derived - # from the event; with nothing unconfirmed the subscriber resolves - # immediately. + # subscriber that has left. + # + # Three cases for live bindings: + # - window_closed=True: the subscriber pulled this event and moved past + # it (closed the binding window) before unsubscribing. Persist + # confirmations may still be draining. Route to the acked path so + # any remaining unconfirmed seqs can drain before the entry resolves. + # - window_closed=False and event_id == open_event_id: the subscriber + # pulled this event and is currently sitting on it (the open window). + # Close the window now and let any unconfirmed seqs drain via the + # acked path. + # - window_closed=False and event_id != open_event_id: the event was + # fan-out-enqueued but the subscriber left before ever pulling it. + # Resolving as failed tells the broker to redeliver rather than + # committing an event no one persisted. for event_id, entry in list(self._outstanding.items()): binding = entry.bindings.get(trigger_id) if binding is None: @@ -1004,8 +1024,15 @@ def unsubscribe(self, trigger_id: int) -> None: # _resolve_subscriber no-ops unless still pending. self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="acked") continue - binding.window_closed = True - self._maybe_complete(event_id=event_id, trigger_id=trigger_id) + if binding.window_closed or event_id == open_event_id: + # Subscriber pulled this event — either already moved past it + # (window_closed=True, persist confirmations may still be draining) or + # is currently sitting on it (open window). Resolve via the acked path. + binding.window_closed = True + self._maybe_complete(event_id=event_id, trigger_id=trigger_id) + else: + # Subscriber never pulled this event; redeliver via failed. + self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="failed") def _fail_overflowed_subscriber(self, trigger_id: int, queue: asyncio.Queue) -> None: """ diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index 9506506e6af09..11cc025063490 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -1279,6 +1279,128 @@ async def test_subscriber_unsubscribe_during_outstanding_event(): await manager.unsubscribe(1, key) +@pytest.mark.asyncio +async def test_subscriber_unsubscribe_without_pulling_redelivers(): + """When a subscriber leaves without ever pulling an event, the event resolves as failed (redeliverable). + + This is the cap-boundary pair for ``test_subscriber_unsubscribe_during_outstanding_event``: + - pulled-then-left → the event counts as acked for that subscriber (existing test). + - never-pulled-left → the event counts as failed so the broker redelivers (this test). + """ + cls = _make_ack_required_trigger_class( + [ + ({"msg": "never-pulled"}, "receipt-never-pulled"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + manager.subscribe(trigger_id=2, trigger=t2, key=key) + + # t1 consumes and resolves; t2 never calls _pull_raw — event stays queued. + async with _consume_in_background(s1) as c1: + await c1.wait_for(1) + await asyncio.sleep(0) + assert cls.advanced == [], "the event still waits on subscriber 2" + + # t2 leaves without ever pulling the event. + await manager.unsubscribe(2, key) + await asyncio.sleep(0) + + assert cls.advanced == ["receipt-never-pulled"], ( + "unsubscribe must resolve the departing subscriber so the producer can advance" + ) + finally: + await manager.unsubscribe(1, key) + + assert len(cls.outcomes) == 1, "exactly one event must have been advanced" + outcome = cls.outcomes[0] + assert outcome.failed >= 1, "a subscriber that left without pulling must be counted as failed, not acked" + assert not outcome.is_clean, "an outcome with failed > 0 must not be clean" + + +@pytest.mark.asyncio +async def test_subscriber_unsubscribe_after_pulling_event_a_then_moving_to_event_b(): + """Guard: a subscriber that pulled event A (emitting a TriggerEvent) and then moved to event B + must resolve event A as *acked* on unsubscribe — not failed. + + Regression for the ``event_id == open_event_id``-only discriminant in ``unsubscribe``: + when the subscriber moves to event B, ``open_event_id`` becomes B and A's binding has + ``window_closed=True``. The old code fell into the ``else`` (failed) branch for A because + ``event_id_A != open_event_id_B``. The fix checks ``binding.window_closed`` first. + """ + cls = _make_ack_required_trigger_class( + [ + ({"msg": "event-a"}, "receipt-a"), + ({"msg": "event-b"}, "receipt-b"), + ] + ) + cls.advanced.clear() + cls.outcomes.clear() + + t1, t2 = cls(), cls() + key = t1.shared_stream_key() + manager = SharedStreamManager() + next_task = None + try: + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + # Subscriber 2 consumes both events and stays resident so the group + # remains alive while we exercise subscriber 1. + async with _consume_in_background(s2) as c2: + await c2.wait_for(2) + + # Subscriber 1: pull event A, bind a persist seq (simulating an emitted + # TriggerEvent that has not yet been confirmed persisted), then move past + # event A onto event B. At this point: + # - event A: window_closed=True, unconfirmed_seqs={seq_a} + # - event B: open window (open_event_id == event_b_id) + it1 = s1.__aiter__() + await _pull_raw(it1) # pulls event A; open window = A + seq_a = manager.bind_pending_event(trigger_id=1, key=key) + assert seq_a is not None, "seq must bind while event A's window is open" + + # Resume past event A → closes A's window, opens event B's. + pull_b = await _resume_past_yield(it1) + await asyncio.wait_for(pull_b, timeout=1.0) # now on event B + + # Subscriber 1 unsubscribes while sitting on event B with event A's + # confirmation still outstanding. + await manager.unsubscribe(1, key) + await asyncio.sleep(0) + + # Neither event may have advanced yet — event A is still waiting on + # the persist confirmation for seq_a. + assert cls.advanced == [], "event A must not advance before its persist confirmation arrives" + + # Confirm event A's seq. Both events should now advance. + manager.confirm_persisted([seq_a]) + await asyncio.sleep(0) + + assert "receipt-a" in cls.advanced, ( + "event A must resolve as acked (not failed) — subscriber pulled it " + "and emitted a TriggerEvent before moving to event B" + ) + assert len(cls.outcomes) >= 1 + # Locate event A's outcome: the one with receipt-a. + idx_a = cls.advanced.index("receipt-a") + outcome_a = cls.outcomes[idx_a] + assert outcome_a.acked >= 1, "subscriber 1 must be counted as acked for event A, not failed" + assert outcome_a.failed == 0, ( + "event A outcome must have failed=0; subscriber pulled it before leaving" + ) + finally: + await _cancel_quietly(next_task) + # s2 is consumed in the background context above; group may already be gone. + await manager.unsubscribe(2, key) + + @pytest.mark.asyncio async def test_ack_mode_queue_full_during_fanout_does_not_break_iteration(): """A full queue at fan-out time must not break the iteration of remaining subscribers.""" @@ -1394,7 +1516,9 @@ async def test_empty_snapshot_advances_in_fanout_order(lane_for, events, expecte ) assert len(cls.advanced) == len(events) assert cls.outcomes == [AdvanceOutcome(0, 0)] * len(events) - assert all(outcome.is_clean for outcome in cls.outcomes) + assert not any(outcome.is_clean for outcome in cls.outcomes), ( + "a zero-subscriber broadcast still advances in fan-out order but is no longer clean" + ) del s1 # silence unused-variable warning await manager.stop_all() @@ -2271,10 +2395,11 @@ async def test_reject_outside_any_window_is_no_op(): pytest.param(AdvanceOutcome(acked=2, failed=0, rejected=0), True, id="no-reject-no-fail"), pytest.param(AdvanceOutcome(acked=2, failed=0, rejected=1), False, id="reject-makes-unclean"), pytest.param(AdvanceOutcome(acked=2, failed=1, rejected=0), False, id="fail-makes-unclean"), + pytest.param(AdvanceOutcome(acked=0, failed=0, rejected=0), False, id="zero-subscribers-not-clean"), ], ) def test_is_clean_requires_no_reject_and_no_fail(outcome, expected_clean): - """``is_clean`` is True only when both ``failed`` and ``rejected`` are zero.""" + """``is_clean`` is True only when both ``failed`` and ``rejected`` are zero and at least one subscriber acked.""" assert outcome.is_clean is expected_clean From 92c2801f973560c3c717034ad8939a74c3d23499 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 15 Jun 2026 16:44:49 +0800 Subject: [PATCH 12/15] test(triggers): drop unused session fixture from shared-stream ack integration test --- airflow-core/tests/unit/jobs/test_triggerer_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow-core/tests/unit/jobs/test_triggerer_job.py b/airflow-core/tests/unit/jobs/test_triggerer_job.py index 21e0d25de872e..923779df0034c 100644 --- a/airflow-core/tests/unit/jobs/test_triggerer_job.py +++ b/airflow-core/tests/unit/jobs/test_triggerer_job.py @@ -916,7 +916,7 @@ async def _drive(): # Group is torn down on unsubscribe. assert trigger_runner._shared_streams._groups == {} - def test_shared_stream_ack_mode_integration(self, session) -> None: + def test_shared_stream_ack_mode_integration(self) -> None: """A BaseEventTrigger whose producer factory is overridden advances only after the event's derived trigger event is confirmed persisted.""" advanced: list[Any] = [] # Container lets _drive() write trigger_runner back for post-run assertions. From 3cf823fbe17039781d4ae8315b7610b8df0353cd Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 15 Jun 2026 16:44:49 +0800 Subject: [PATCH 13/15] docs(triggers): route non-clean outcomes to a DLQ in shared-stream advance example The Kafka producer example now inspects each AdvanceItem's outcome and sends non-clean events to a dead-letter queue before the cumulative offset commit, instead of committing every batched event as delivered. Also align the is_clean prose with the new zero-subscriber-is-not-clean semantics. --- .../event-scheduling.rst | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index 5bc2f11717b87..967a11459bda6 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -225,9 +225,11 @@ raw events, it logs a warning and does nothing, because there is no broker advance to influence. Because it resolves the event immediately, there is nothing to persist and the reject does not wait on the persistence gate. -``is_clean`` is ``True`` only when an event has no rejects and no failures — -every subscriber that was online at broadcast accepted it. A single reject -or a single failure makes ``is_clean`` ``False``. +``is_clean`` is ``True`` only when every subscriber that was online at +broadcast accepted the event: no rejects, no failures, and at least one +subscriber acked. A single reject or failure makes it ``False``, and so +does a zero-subscriber broadcast (all-zero counts) — nothing was accepted, +so there is nothing the producer should commit on. Example — reject inside a filter: @@ -354,6 +356,16 @@ the other partitions: # resolved prefix, in event order, so committing the offset of # its last item covers the whole batch and can never skip past # an event that is still pending. + # + # Inspect each item's outcome before committing: non-clean events + # (rejects, failures, or a zero-subscriber broadcast) should go to + # a dead-letter queue rather than be committed as delivered. + to_dlq = [] + for broker_payload, outcome in batch: + if not outcome.is_clean: + to_dlq.append(broker_payload) + if to_dlq: + handle_dlq(to_dlq) topic, partition, offset = batch[-1].broker_payload await self.consumer.commit(topic, partition, offset + 1) From a393fed83a76f59c9ed714a11eafccbc663fd8a1 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 15 Jun 2026 16:44:49 +0800 Subject: [PATCH 14/15] improvement(triggers): log shared-stream event rejection for observability --- .../authoring-and-scheduling/event-scheduling.rst | 2 +- .../src/airflow/triggers/shared_stream.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index 967a11459bda6..e55054616f93a 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -227,7 +227,7 @@ nothing to persist and the reject does not wait on the persistence gate. ``is_clean`` is ``True`` only when every subscriber that was online at broadcast accepted the event: no rejects, no failures, and at least one -subscriber acked. A single reject or failure makes it ``False``, and so +subscriber acknowledged it. A single reject or failure makes it ``False``, and so does a zero-subscriber broadcast (all-zero counts) — nothing was accepted, so there is nothing the producer should commit on. diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index ab04af0946f53..c433e7e68a5a6 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -239,6 +239,11 @@ def reject_shared_stream_event() -> None: if ctx is None: log.warning("reject_shared_stream_event called outside an ack-mode binding window; ignored") return + log.info( + "Rejecting shared-stream event", + trigger_id=ctx.trigger_id, + event_id=ctx.event_id, + ) ctx.group._reject_pending_event(trigger_id=ctx.trigger_id, event_id=ctx.event_id) @@ -347,7 +352,7 @@ def is_clean(self) -> bool: Whether every subscriber accepted the event. No active reject, no involuntary failure, and at least one subscriber - acked. A zero-subscriber broadcast (all-zero counts) is not clean. + acknowledged. A zero-subscriber broadcast (all-zero counts) is not clean. """ return self.failed == 0 and self.rejected == 0 and self.acked > 0 @@ -812,7 +817,7 @@ def _reject_pending_event(self, *, trigger_id: int, event_id: int) -> None: self._resolve_subscriber(event_id=event_id, trigger_id=trigger_id, resolution="rejected") def _maybe_complete(self, *, event_id: int, trigger_id: int) -> None: - """Resolve the subscriber as acked once the window is closed and all persist confirmations are in.""" + """Resolve the subscriber as acknowledged once the window is closed and all persist confirmations are in.""" entry = self._outstanding.get(event_id) if entry is None: return @@ -1007,12 +1012,12 @@ def unsubscribe(self, trigger_id: int) -> None: # Three cases for live bindings: # - window_closed=True: the subscriber pulled this event and moved past # it (closed the binding window) before unsubscribing. Persist - # confirmations may still be draining. Route to the acked path so + # confirmations may still be draining. Route to the acknowledged path so # any remaining unconfirmed seqs can drain before the entry resolves. # - window_closed=False and event_id == open_event_id: the subscriber # pulled this event and is currently sitting on it (the open window). # Close the window now and let any unconfirmed seqs drain via the - # acked path. + # acknowledged path. # - window_closed=False and event_id != open_event_id: the event was # fan-out-enqueued but the subscriber left before ever pulling it. # Resolving as failed tells the broker to redeliver rather than @@ -1027,7 +1032,7 @@ def unsubscribe(self, trigger_id: int) -> None: if binding.window_closed or event_id == open_event_id: # Subscriber pulled this event — either already moved past it # (window_closed=True, persist confirmations may still be draining) or - # is currently sitting on it (open window). Resolve via the acked path. + # is currently sitting on it (open window). Resolve via the acknowledged path. binding.window_closed = True self._maybe_complete(event_id=event_id, trigger_id=trigger_id) else: From e1c906953b7057188e63dc5b05bcc1a0508e5add Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Mon, 15 Jun 2026 22:08:09 +0800 Subject: [PATCH 15/15] fix(triggers): terminate shared-stream group when producer advance fails A producer advance that raises now terminates the whole shared-stream group and fails every subscriber, instead of logging the error and continuing. On a cumulative-commit broker (e.g. single-partition Kafka), continuing let a later batch's successful commit advance the offset past the abandoned batch, silently dropping events that were never committed. Terminating leaves the offset uncommitted so the broker redelivers from the last committed point. --- .../event-scheduling.rst | 13 +- .../src/airflow/triggers/shared_stream.py | 90 +++++--- .../tests/unit/triggers/test_shared_stream.py | 214 ++++++++++++------ 3 files changed, 226 insertions(+), 91 deletions(-) diff --git a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst index e55054616f93a..b60b39cf821f3 100644 --- a/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst +++ b/airflow-core/docs/authoring-and-scheduling/event-scheduling.rst @@ -250,8 +250,13 @@ Example — reject inside a filter: **Ordering guarantee**: by default every event belongs to the same lane. The items of a batch are in event order and form their lane's contiguous resolved prefix; within a lane, batches arrive strictly in order — the -next ``advance`` is awaited only after the previous call returned (or -failed and was logged). A producer can override ``get_advance_lane`` to +next ``advance`` is awaited only after the previous call returned. If +``advance`` raises, the error is logged and the **whole shared-stream +group is terminated**: every subscriber receives a failure sentinel and +the broker redelivers from the never-committed offset (a loud, safe +failure rather than a silent data skip). Recovering more gracefully from +transient failures would require the producer to track which offsets are +safe to recommit. A producer can override ``get_advance_lane`` to narrow that ordering to within a lane: events whose lane values compare equal are batched and advanced in event order relative to each other, while events in different lanes do not wait for one another. Either way, @@ -357,6 +362,10 @@ the other partitions: # its last item covers the whole batch and can never skip past # an event that is still pending. # + # If this method raises, the whole shared-stream group is + # terminated and the broker redelivers from the last committed + # offset — no silent data skip. + # # Inspect each item's outcome before committing: non-clean events # (rejects, failures, or a zero-subscriber broadcast) should go to # a dead-letter queue rather than be committed as delivered. diff --git a/airflow-core/src/airflow/triggers/shared_stream.py b/airflow-core/src/airflow/triggers/shared_stream.py index c433e7e68a5a6..93ccc3e80dafa 100644 --- a/airflow-core/src/airflow/triggers/shared_stream.py +++ b/airflow-core/src/airflow/triggers/shared_stream.py @@ -61,8 +61,9 @@ single pump task: the producer's :meth:`~SharedStreamProducer.get_advance_lane` assigns each event to a lane; within a lane, batch items arrive in fan-out order and the next -batch is awaited only after the call for the previous one returned (or -raised and was logged); lanes do not wait for one another, but at any +batch is awaited only after the call for the previous one returned; if it +raises, the whole group is terminated and the broker redelivers from the +never-committed offset; lanes do not wait for one another, but at any moment at most one ``advance`` call is awaited globally. The default lane assignment (every event in the same lane) preserves the original single global order. When the poll ends, the manager awaits ``producer.aclose()`` @@ -400,18 +401,20 @@ async def advance(self, batch: Sequence[AdvanceItem]) -> None: contiguous resolved prefix of one lane — events for which :meth:`get_advance_lane` returned equal values. Within a lane, batches arrive strictly in order: the next batch is awaited only - after the call for the previous one returned (or raised and was - logged). Across lanes the relative order is not guaranteed, but at - any moment at most one ``advance`` call is awaited globally. This - makes cumulative schemes such as a Kafka offset commit safe within - a lane — committing the offset of the batch's last item covers the - whole batch. Each item's :class:`AdvanceOutcome` carries the - per-event resolution counts; use them to decide whether to commit, - skip, or trigger a broker-side redeliver. - - If this method raises, the error is logged and the whole batch is - abandoned — the events are not dispatched again; the broker is - expected to redeliver whatever was not committed. + after the call for the previous one returned. Across lanes the + relative order is not guaranteed, but at any moment at most one + ``advance`` call is awaited globally. This makes cumulative schemes + such as a Kafka offset commit safe within a lane — committing the + offset of the batch's last item covers the whole batch. Each item's + :class:`AdvanceOutcome` carries the per-event resolution counts; use + them to decide whether to commit, skip, or trigger a broker-side + redeliver. + + If this method raises, the error is logged and the whole + shared-stream group is terminated: every subscriber receives a + failure sentinel and the broker redelivers from the never-committed + offset. Terminating is the safe default; finer-grained per-lane + retry would require tracking which offsets are safe to recommit. """ def get_advance_lane(self, broker_payload: Any) -> Hashable: @@ -543,6 +546,9 @@ def __init__( self._advance_wakeup: asyncio.Event = asyncio.Event() self._pump_stopping: bool = False self._pump_task: asyncio.Task | None = None + # Set to True by _fail_group; guards against double-broadcast when both + # the pump's advance failure and _poll's finally reach the terminal path. + self._terminated: bool = False def start(self) -> None: """Start the underlying poll loop. Call exactly once per group.""" @@ -563,17 +569,42 @@ def _request_pump_stop(self) -> None: self._pump_stopping = True self._advance_wakeup.set() + def _fail_group(self, exc: BaseException) -> None: + """ + Idempotent, synchronous terminal broadcast. + + Evict the group and deliver + ``_PollFailure(exc)`` to every subscriber. + + Must be called from a synchronous section (no ``await`` inside) so the + broadcast completes before any coroutine can observe the group in a + half-torn-down state (see "Lifecycle invariants" in the module docstring). + Guarded by ``_terminated`` so calling it twice is a no-op — both + ``_poll``'s finally and the advance pump can reach this path without + double-broadcasting. + """ + if self._terminated: + return + self._terminated = True + self._on_poll_terminate(self) + failure = _PollFailure(exc) + for queue in self._subscribers.values(): + self._drain_and_offer_failure(queue, failure) + async def _run_advance_pump(self, producer: SharedStreamProducer) -> None: """ Dispatch broker advances in per-lane fan-out order, one batch at a time. A single task scans the lanes round-robin: each pass dispatches at most one batch per lane — the lane's contiguous resolved prefix — - and passes repeat until one makes no progress. An advance that - raises is logged, its batch is abandoned to broker redelivery, and - the pump moves on. On stop the pump keeps passing until every - already-resolved dispatchable event is drained, abandons unresolved - entries to broker redelivery, and exits. + and passes repeat until one makes no progress. If ``advance`` raises, + the error is logged and the whole shared-stream group is terminated: + the manager evicts the key, every subscriber receives a + ``_PollFailure``, and the broker redelivers from the never-committed + offset. Terminating on any advance failure is the safe default; see + the neutral note in the ``except`` block below. On stop + the pump keeps passing until every already-resolved dispatchable event + is drained, abandons unresolved entries to broker redelivery, and exits. """ while True: if not self._pump_stopping: @@ -618,12 +649,22 @@ async def _run_advance_pump(self, producer: SharedStreamProducer) -> None: await producer.advance(batch) except Exception as exc: self.log.error( - "Producer advance raised; broker advance failed", + "Producer advance raised; terminating shared-stream group", key=self.key, lane=lane, batch_size=len(batch), exc_info=exc, ) + # Broadcast the failure synchronously before cancelling + # _poll so no late subscriber can attach to a dead group. + # Any advance failure terminates the whole group: the safe + # default is to stop and let the broker redeliver from the + # last committed offset. Per-lane retry would need extra + # state to track which offsets are safe to recommit. + self._fail_group(exc) + if self._poll_task is not None and not self._poll_task.done(): + self._poll_task.cancel() + return progress = True if self._pump_stopping: return @@ -737,12 +778,9 @@ async def _poll(self) -> None: # Synchronous: evict from the manager and broadcast the # sentinel before returning to the loop, so no coroutine can # observe ``_groups[key]`` pointing at a dead poll. - self._on_poll_terminate(self) - failure = _PollFailure(terminal_exc) - for queue in self._subscribers.values(): - # Drain stale events then put the failure sentinel so every - # subscriber wakes up even if its queue was at capacity. - self._drain_and_offer_failure(queue, failure) + # _fail_group is idempotent — if the advance pump already + # broadcast (advance failure path), this is a no-op. + self._fail_group(terminal_exc) # End of synchronous section; yields are safe from here on. if cancelled_ack_task is not None: with suppress(asyncio.CancelledError): diff --git a/airflow-core/tests/unit/triggers/test_shared_stream.py b/airflow-core/tests/unit/triggers/test_shared_stream.py index 11cc025063490..152056ed389bd 100644 --- a/airflow-core/tests/unit/triggers/test_shared_stream.py +++ b/airflow-core/tests/unit/triggers/test_shared_stream.py @@ -1525,15 +1525,12 @@ async def test_empty_snapshot_advances_in_fanout_order(lane_for, events, expecte @pytest.mark.asyncio -async def test_producer_advance_exception_is_logged(): - """When producer.advance raises, the error is logged and the poll group stays alive.""" - proceed_flag = asyncio.Event() +async def test_producer_advance_exception_terminates_group(): + """When producer.advance raises, the error is logged and the whole poll group terminates.""" class _RaisingAdvanceProducer(SharedStreamProducer): async def open_stream(self): yield ({"msg": "event-1"}, "payload-1") - await proceed_flag.wait() - yield ({"msg": "event-2"}, "payload-2") await asyncio.Event().wait() async def advance(self, batch): @@ -1557,9 +1554,7 @@ async def filter_shared_stream(self, shared_stream): # Intercept log.error on the group so we can assert it was called. group = manager._groups[key] - mock_log_error = MagicMock() group.log = MagicMock() - group.log.error = mock_log_error async with _consume_in_background(t1.filter_shared_stream(s1)) as collector: # Consume the first event — the filter loop resolves it and schedules advance. @@ -1569,21 +1564,16 @@ async def filter_shared_stream(self, shared_stream): # Give the advance pump a tick to run and raise. await asyncio.sleep(0.05) - # log.error must have been called with a message about the failed advance. - assert mock_log_error.called, "log.error must be called when producer.advance raises" - call_args = mock_log_error.call_args - assert "broker advance failed" in call_args[0][0], ( - f"log.error message must mention 'broker advance failed', got: {call_args[0][0]}" + # log.error must have been called with a message about the terminated group. + assert group.log.error.called, "log.error must be called when producer.advance raises" + call_args = group.log.error.call_args + assert "terminating shared-stream group" in call_args[0][0], ( + f"log.error message must mention 'terminating shared-stream group', got: {call_args[0][0]}" ) - # The poll group is still alive — the exception must not have killed it. - assert key in manager._groups, "group must survive a producer.advance exception" - - # A second event can still be produced and consumed (the collector - # count is cumulative across both events). - proceed_flag.set() - events2 = await collector.wait_for(2) - assert len(events2) == 2 + # The poll group must be evicted — the advance exception terminates the group. + await asyncio.sleep(0.05) + assert key not in manager._groups, "group must be evicted after producer.advance exception" finally: await manager.stop_all() @@ -2516,18 +2506,23 @@ async def consume(): @pytest.mark.asyncio -async def test_pump_continues_after_advance_raises(): - """An advance that raises is logged; the pump still advances the next event.""" +async def test_pump_terminates_group_after_advance_raises(): + """An advance that raises terminates the group; no subsequent advance is attempted.""" attempted: list = [] + # Gate p2 so it only enters the stream after p1 has been advanced; this + # ensures p1 resolves alone (batch = ["p1"]) and p2 never reaches advance. + p2_gate = asyncio.Event() class _FirstRaiseProducer(SharedStreamProducer): async def open_stream(self): yield {"n": 1}, "p1" + await p2_gate.wait() yield {"n": 2}, "p2" await asyncio.Event().wait() async def advance(self, batch): attempted.extend(item.broker_payload for item in batch) + p2_gate.set() # signal: p1 has been advanced (or batched) if any(item.broker_payload == "p1" for item in batch): raise RuntimeError("p1 boom") @@ -2543,33 +2538,33 @@ async def filter_shared_stream(self, shared_stream): t1 = _FirstRaiseTrigger() key = t1.shared_stream_key() manager = SharedStreamManager() - next_task = None - try: - s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) - group = manager._groups[key] - group.log = MagicMock() - # Move past event 1; its advance dispatches alone and raises. - it = s1.__aiter__() - await _pull_raw(it) - pull2 = await _resume_past_yield(it) - for _ in range(5): - await asyncio.sleep(0) - assert attempted == ["p1"] + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + group = manager._groups[key] + group.log = MagicMock() - # Move past event 2; the pump must still dispatch it. - await asyncio.wait_for(pull2, timeout=1.0) - next_task = await _resume_past_yield(it) - for _ in range(5): - await asyncio.sleep(0) + async with _consume_in_background(t1.filter_shared_stream(s1)) as c1: + try: + # Drive event 1 through the filter; the loop-back closes its binding + # window, triggering the pump to dispatch the advance that raises. + await c1.wait_for(1) - assert attempted == ["p1", "p2"], "the pump must move on to event 2 after event 1's advance raised" - error_calls = group.log.error.mock_calls - assert len(error_calls) == 1 - assert "broker advance failed" in error_calls[0].args[0] - finally: - await _cancel_quietly(next_task) - await manager.unsubscribe(1, key) + # Give the pump time to run advance() (which sets p2_gate) and + # call _fail_group; also gives the producer time to yield p2 (which + # should be ignored since the group is already terminated). + await asyncio.sleep(0.05) + + # The group must now be terminated — event 2 must never be advanced. + assert attempted == ["p1"], "the pump must not advance event 2 after event 1's advance raised" + assert key not in manager._groups, "group must be evicted after advance raises" + error_calls = group.log.error.mock_calls + assert len(error_calls) == 1 + assert "terminating shared-stream group" in error_calls[0].args[0] + # The subscriber must have received the advance exception via _PollFailure. + assert isinstance(c1.error, RuntimeError), f"subscriber must fail; got: {c1.error!r}" + assert str(c1.error) == "p1 boom" + finally: + await manager.stop_all() @pytest.mark.asyncio @@ -3132,17 +3127,14 @@ async def test_partial_prefix_waits_for_lane_head(): @pytest.mark.asyncio -async def test_advance_raise_abandons_whole_batch(): - """A batch whose advance raises is logged and abandoned; later events form fresh batches.""" +async def test_advance_raise_terminates_group_and_no_later_batch(): + """A batch whose advance raises terminates the whole group; no later event is advanced.""" calls: list[list] = [] - proceed = asyncio.Event() class _RaiseOnFirstBatchProducer(SharedStreamProducer): async def open_stream(self): yield {"n": 1}, "p1" yield {"n": 2}, "p2" - await proceed.wait() - yield {"n": 3}, "p3" await asyncio.Event().wait() async def advance(self, batch): @@ -3177,22 +3169,118 @@ def create_shared_stream_producer(cls, kwargs): for _ in range(5): await asyncio.sleep(0) - # The batch raised: it is logged, already off the books, and never - # dispatched again — the broker is left to redeliver. + # The batch raised — the group is terminated. assert calls == [["p1", "p2"]] - assert group._outstanding == {} assert len(group.log.error.mock_calls) == 1 - - # The pump keeps running; a later event forms a fresh batch. - proceed.set() - await asyncio.wait_for(next_task, timeout=1.0) - next_task = await _resume_past_yield(it) - for _ in range(5): - await asyncio.sleep(0) - assert calls == [["p1", "p2"], ["p3"]] + await asyncio.sleep(0.05) + assert key not in manager._groups, "group must be evicted after advance batch raise" + # The subscriber must have received the failure. + with pytest.raises(RuntimeError, match="batch boom"): + await asyncio.wait_for(next_task, timeout=1.0) + next_task = None finally: await _cancel_quietly(next_task) - await manager.unsubscribe(1, key) + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_advance_failure_fans_out_to_all_subscribers(): + """advance() raise → every subscriber gets a _PollFailure; group is evicted from _groups.""" + + class _RaisingProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + await asyncio.Event().wait() + + async def advance(self, batch): + raise RuntimeError("all-subscribers boom") + + class _RaisingTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _RaisingProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw in shared_stream: + yield TriggerEvent(raw) + + t1, t2 = _RaisingTrigger(), _RaisingTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + s2 = manager.subscribe(trigger_id=2, trigger=t2, key=key) + + async with ( + _consume_in_background(t1.filter_shared_stream(s1)) as c1, + _consume_in_background(t2.filter_shared_stream(s2)) as c2, + ): + try: + # Both subscribers consume the first event (resolving it, triggering advance). + await c1.wait_for(1) + await c2.wait_for(1) + + # Give the pump time to run and raise. + await asyncio.sleep(0.05) + + # Group must be evicted. + assert key not in manager._groups, "group must be evicted after advance failure" + # Both collectors must have captured the same RuntimeError. + assert isinstance(c1.error, RuntimeError), f"subscriber 1 must fail; got: {c1.error!r}" + assert isinstance(c2.error, RuntimeError), f"subscriber 2 must fail; got: {c2.error!r}" + assert str(c1.error) == "all-subscribers boom" + assert str(c2.error) == "all-subscribers boom" + finally: + await manager.stop_all() + + +@pytest.mark.asyncio +async def test_advance_failure_idempotent_with_stop(): + """advance failure followed by stop() does not double-broadcast _PollFailure.""" + advance_started = asyncio.Event() + + class _SlowRaisingProducer(SharedStreamProducer): + async def open_stream(self): + yield {"n": 1}, "p1" + await asyncio.Event().wait() + + async def advance(self, batch): + advance_started.set() + await asyncio.sleep(0.05) # slow enough for stop() to race + raise RuntimeError("idempotent boom") + + class _SlowRaisingTrigger(_ProgrammableSharedStreamTrigger): + @classmethod + def create_shared_stream_producer(cls, kwargs): + return _SlowRaisingProducer() + + async def filter_shared_stream(self, shared_stream): + async for raw in shared_stream: + yield TriggerEvent(raw) + + t1 = _SlowRaisingTrigger() + key = t1.shared_stream_key() + manager = SharedStreamManager() + + s1 = manager.subscribe(trigger_id=1, trigger=t1, key=key) + group = manager._groups[key] + + async with _consume_in_background(t1.filter_shared_stream(s1)) as c1: + await c1.wait_for(1) + # Wait until advance is in-flight. + await asyncio.wait_for(advance_started.wait(), timeout=1.0) + # Now stop the manager concurrently with the in-flight advance raise. + await manager.stop_all() + + # _terminated must be True (fail_group fired exactly once). + assert group._terminated is True + # The group must be gone (evicted by fail_group, not left behind by stop). + assert key not in manager._groups + # The failure must have reached the subscriber exactly once: one real event + # was consumed before advance raised, and the error is the expected one. + assert len(c1.items) == 1 + assert isinstance(c1.error, RuntimeError) + assert str(c1.error) == "idempotent boom" @pytest.mark.asyncio