Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions go-sdk/airflow/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,16 @@ type Registerable interface{ registerable() }
// Add every task to a Dag before registering the Dag. [DagRef.Task], [DagRef.If], [IfRef.Then]
// and [IfRef.Else] panic once the Dag is registered.
//
// Register is where a Dag's task dependencies are checked for a cycle, over the whole graph at
// once: [TaskRef.Before], [TaskRef.After] and [Inputs] each record an edge without walking the
// graph, so building a Dag stays linear in its edges however many a task has.
//
// Register panics if a task handler with the same dag_id and task_id is already registered,
// if a Dag with the same dag_id is already registered, if a task handler and a Dag have the
// same dag_id, or if [BundleRef.Serve] has already been called: registration closes when
// serving starts. A task handler runs a task of a Python Dag, so its dag_id cannot also belong
// to a Dag authored in Go. Register also panics if a Dag has a condition from [DagRef.If]
// without a task from [IfRef.Then].
// same dag_id, if the task dependencies of a Dag contain a cycle, or if [BundleRef.Serve] has
// already been called: registration closes when serving starts. A task handler runs a task of a
// Python Dag, so its dag_id cannot also belong to a Dag authored in Go. Register also panics if a
// Dag has a condition from [DagRef.If] without a task from [IfRef.Then].
func (b *BundleRef) Register(items ...Registerable) {
if b.closed.Load() {
panic(
Expand Down
54 changes: 54 additions & 0 deletions go-sdk/airflow/bundle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,60 @@ func TestRegisterRejectsATaskHandlerWithTheDagIDOfADag(t *testing.T) {
}
}

// TestRegisterRejectsACycle covers the check that reads a Dag's edges as a whole. Before and
// After record an edge without walking the graph, so a cycle between other tasks is Register's to
// find, and every edge of the call that closed it is recorded until then.
func TestRegisterRejectsACycle(t *testing.T) {
dag := Dag("etl")
extracted := orderedTask(t, dag, "extract")
loaded := orderedTask(t, dag, "load")
notified := orderedTask(t, dag, "notify")
extracted.Before(loaded)
loaded.Before(notified, extracted)

assert.PanicsWithValue(t,
`airflow.BundleRef.Register: the task dependencies of Dag "etl" contain a cycle: `+
`extract -> load -> extract`,
func() { Bundle().Register(dag) },
)
// The verbs recorded what they were given, and the Dag can still be corrected.
assertTasks(t, loaded.downstreams, notified, extracted)
assert.False(t, dag.registered)
}

// TestRegisterRejectsACycleThroughAnInputsEdge pins that the check sees the edges Inputs
// declared, which is what DagRef.Task recording them is for. It is the case ADR-0008 names:
// b := dag.Task(B, Inputs(a)) followed by b.Before(a) is a genuine cycle in accepted syntax.
func TestRegisterRejectsACycleThroughAnInputsEdge(t *testing.T) {
dag := Dag("etl")
read := dag.Task(readRows)
counted := dag.Task(countRows, Inputs(read))
notified := orderedTask(t, dag, "notify")
counted.Before(notified)
counted.Before(read)

assert.PanicsWithValue(t,
`airflow.BundleRef.Register: the task dependencies of Dag "etl" contain a cycle: `+
`readRows -> countRows -> readRows`,
func() { Bundle().Register(dag) },
)
}

// TestRegisterTakesADagWhoseTasksShareADownstream pins that the walk follows a diamond, where a
// task is reached twice without any cycle.
func TestRegisterTakesADagWhoseTasksShareADownstream(t *testing.T) {
dag := Dag("etl")
extracted := orderedTask(t, dag, "extract")
notified := orderedTask(t, dag, "notify")
cleaned := orderedTask(t, dag, "cleanup")
done := orderedTask(t, dag, "done")
extracted.Before(notified, cleaned).Before(done)

Bundle().Register(dag)

assert.True(t, dag.registered)
}

func TestRegisterRejectsNilDag(t *testing.T) {
var dag *DagRef
assert.PanicsWithValue(t, "airflow.BundleRef.Register: cannot register a nil *airflow.DagRef",
Expand Down
29 changes: 26 additions & 3 deletions go-sdk/airflow/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ type DagRef struct {
registered bool
tasks []*TaskRef
tasksByID map[string]*TaskRef
// edgeLabels holds every edge of the Dag, whether Inputs, Before or After declared it, and
// the label that Label put on it. An edge with no label maps to the empty string, so a
// lookup reports whether the edge has been declared.
edgeLabels map[edgeKey]string
}

// Dag returns an empty Dag with the given dag_id. An optional [DagSpec] holds the rest of the
Expand Down Expand Up @@ -74,7 +78,8 @@ func (*DagRef) registerable() {}

// TaskRef is a task that [DagRef.Task] added to a Dag. Pass it to [Inputs] to give its result
// to a task that DagRef.Task or [DagRef.If] adds later. Pass it to [IfRef.Then] or [IfRef.Else]
// to run it on one side of a condition.
// to run it on one side of a condition. A TaskRef is a [Node], so [TaskRef.Before] and
// [TaskRef.After] order it against another task.
type TaskRef struct {
dag *DagRef
taskID string
Expand All @@ -85,7 +90,12 @@ type TaskRef struct {
// inputs holds the tasks that Inputs passed, in the order of the parameters they fill. Each
// of them is an upstream task of this one.
inputs []*TaskRef
task bundle.Task
// upstreams and downstreams hold the edges of the task, in the order they were declared and
// without a repeat, so that an edge is recorded in both directions. Inputs, Before and After
// all record an edge here.
upstreams []*TaskRef
downstreams []*TaskRef
task bundle.Task
// triggerDagRun is the checked copy of the TriggerDagRunSpec of a task from TriggerDagRun.
// It is nil for a task that runs a Go function. A task from TriggerDagRun runs no Go
// function, so its resultType, inputs and task are nil.
Expand Down Expand Up @@ -270,11 +280,18 @@ func (d *DagRef) addTask(method string, fn any, opts []TaskOption, ifRef *IfRef)
}
d.tasksByID[taskID] = task
d.tasks = append(d.tasks, task)
// Inputs passes a task once per parameter it fills, so the same task can arrive twice. The
// edge is one either way, and the task is new, so no edge to it carries a label to settle.
for _, upstream := range upstreams {
Comment thread
henry3260 marked this conversation as resolved.
Resolved
d.addEdgeLocked(upstream, task, "")
}
return task
}

// markRegistered marks d as registered, which stops any further change to d. It panics instead
// when a condition from If has no task from Then.
// when a condition from If has no task from Then, or when the edges of d close a cycle. The Dag
// is whole by then, so one walk of the graph answers for every edge its tasks declared, and a Dag
// that fails a check stays unregistered and can still be corrected.
func (d *DagRef) markRegistered() {
d.mu.Lock()
defer d.mu.Unlock()
Expand All @@ -288,6 +305,12 @@ func (d *DagRef) markRegistered() {
))
}
}
if cycle := d.cycleLocked(); cycle != nil {
Comment thread
henry3260 marked this conversation as resolved.
Resolved
panic(fmt.Sprintf(
"airflow.BundleRef.Register: the task dependencies of Dag %q contain a cycle: %s",
d.dagID, strings.Join(cycle, " -> "),
))
}
d.registered = true
}

Expand Down
7 changes: 7 additions & 0 deletions go-sdk/airflow/if.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ type IfRef struct {
// - false skips the task from Then
// - an error fails the task, which then skips nothing
//
// A task from Then or Else runs after the condition, so naming it records an edge from the
// condition to it, as [TaskRef.Before] would. Declaring that edge as well changes nothing.
//
// The condition skips only that one task. In Python, a branch skips every task directly after it
// that it does not follow, and a short circuit skips every task after it. After the skip, the
// trigger rule of each task after the skipped task decides whether that task runs. A task that
Expand Down Expand Up @@ -172,6 +175,10 @@ func (g *IfRef) setTask(side string, task *TaskRef) {
))
}
*slot = task
// The task runs after the condition, so it is a downstream task of it. Recording the edge is
// what puts the condition in the serialized Dag as the task's upstream, and what lets
// registration see a cycle that runs through a condition.
d.addEdgeLocked(g.task, task, "")
}

// wrapCondition wraps fn as the task of g. The task skips the side of g that the result of fn
Expand Down
50 changes: 50 additions & 0 deletions go-sdk/airflow/if_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,39 @@ func TestThenAndElseNameTheTasksOfTheCondition(t *testing.T) {
assert.Same(t, reported, reversed.elseTask)
}

// TestThenAndElseRecordTheEdgeFromTheCondition pins that naming a task orders it after the
// condition, which is what puts the condition in the serialized Dag as the task's upstream.
func TestThenAndElseRecordTheEdgeFromTheCondition(t *testing.T) {
dag := Dag("etl")
loaded := dag.Task(load)
reported := dag.Task(reportEmpty)

gate := dag.If(isReady)
gate.Then(loaded)
gate.Else(reported)

assertTasks(t, gate.task.downstreams, loaded, reported)
assertTasks(t, loaded.upstreams, gate.task)
assertTasks(t, reported.upstreams, gate.task)
assertEdgeLabel(t, dag, "isReady", "load", "")
assertEdgeLabel(t, dag, "isReady", "reportEmpty", "")
}

// TestDeclaringTheEdgeOfAConditionAgainChangesNothing covers an author who also writes the edge
// that Then records, which is as idempotent as declaring any edge twice.
func TestDeclaringTheEdgeOfAConditionAgainChangesNothing(t *testing.T) {
dag := Dag("etl")
loaded := dag.Task(load)

gate := dag.If(isReady)
gate.Then(loaded)
gate.task.Before(Label(loaded, "when ready"))

assertTasks(t, gate.task.downstreams, loaded)
assertTasks(t, loaded.upstreams, gate.task)
assertEdgeLabel(t, dag, "isReady", "load", "when ready")
}

func TestThenAndElseRejectATaskOutsideTheDag(t *testing.T) {
dag := Dag("etl")
loaded := dag.Task(load)
Expand Down Expand Up @@ -485,6 +518,23 @@ func runCondition(gate *IfRef, results, earlier map[string]any) (*conditionClien
return client, gate.task.task.Execute(ctx, discardLogger(), args)
}

// TestRegisterRejectsACycleThroughACondition pins that the edge Then records is one the cycle
// check walks, so a cycle that runs through a condition is rejected like any other.
func TestRegisterRejectsACycleThroughACondition(t *testing.T) {
dag := Dag("etl")
read := dag.Task(readRows)
loaded := dag.Task(load)
dag.If(hasRows, Inputs(read)).Then(loaded)
loaded.Before(read)

assert.PanicsWithValue(t,
`airflow.BundleRef.Register: the task dependencies of Dag "etl" contain a cycle: `+
`readRows -> hasRows -> load -> readRows`,
func() { Bundle().Register(dag) },
)
assert.False(t, dag.registered)
}

func TestConditionSkipsTheSideThatItDoesNotTake(t *testing.T) {
tests := []struct {
name string
Expand Down
Loading