Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions providers/openlineage/docs/macros.rst
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@ These macros:

* ``lineage_job_namespace()`` - Returns OpenLineage namespace for a given task_instance
* ``lineage_job_name(task_instance)`` - Returns OpenLineage job name for a given task_instance
* ``lineage_job_type()`` - Returns OpenLineage job type facet fields (``processingType``, ``integration``, ``jobType``) of an Airflow task
* ``lineage_run_id(task_instance)`` - Returns the generated OpenLineage run id for a given task_instance

describe the Airflow task and should be used as **parent** information when configuring child jobs. From the child job's perspective,
Expand Down Expand Up @@ -367,6 +368,9 @@ These macros:
* ``lineage_root_job_namespace(task_instance)`` - Returns OpenLineage namespace of root job of a given task_instance
* ``lineage_root_job_name(task_instance)`` - Returns OpenLineage job name of root job of a given task_instance
* ``lineage_root_run_id(task_instance)`` - Returns OpenLineage run ID of root run of a given task_instance
* ``lineage_root_job_type(task_instance)`` - Returns OpenLineage job type facet fields (``processingType``, ``integration``,
``jobType``) of root job of a given task_instance, or ``None`` if the root comes from the DagRun conf and its job type
is missing or invalid

forward the Airflow task's root information into the child job and should be used as **root** information when configuring child jobs.
This allows the child job to maintain the complete job hierarchy, especially in scenarios where tasks are executed as part of a larger workflow.
Expand All @@ -385,6 +389,34 @@ This allows the child job to maintain the complete job hierarchy, especially in
},
)
Job type macros
^^^^^^^^^^^^^^^^

Unlike the other ``lineage_*`` macros, ``lineage_job_type()`` and ``lineage_root_job_type(task_instance)`` return a
dictionary (``lineage_root_job_type`` may also return ``None``) instead of a string. Rendered in a template as is,
the dictionary becomes its Python representation (with single quotes, and ``None`` for an unknown type), which is not
valid JSON. When the value is passed as a string, for example in a Spark conf or an HTTP payload, serialize it with
the ``tojson`` filter:

.. code-block:: python
PythonOperator(
task_id="trigger_external_job",
python_callable=call_external_api,
op_kwargs={
"parent_job_type": "{{ macros.OpenLineageProviderPlugin.lineage_job_type() | tojson }}",
"root_job_type": "{{ macros.OpenLineageProviderPlugin.lineage_root_job_type(task_instance) | tojson }}",
},
)
Alternatively, set ``render_template_as_native_obj=True`` on the Dag to get the dictionaries as they are, or call
the macros directly from Python.

When a Dag run is triggered with job types in its ``openlineage`` conf (``parentJobType`` and ``rootParentJobType``,
as injected by ``TriggerDagRunOperator``), they are added as the ``jobType`` facet of the parent and root jobs in
the ``ParentRunFacet``. Only complete job type information is supported - a value missing any of ``processingType``,
``integration`` or ``jobType`` is ignored.

Joined identifiers
^^^^^^^^^^^^^^^^^^^

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
resolve_airflow_connection_auth,
)
from airflow.providers.openlineage.utils.utils import (
_JOB_TYPE_DAG,
_JOB_TYPE_TASK,
_PRODUCER,
OpenLineageRedactor,
build_dag_run_ol_run_id,
Expand Down Expand Up @@ -72,9 +74,6 @@

set_producer(_PRODUCER)

_JOB_TYPE_DAG: Literal["DAG"] = "DAG"
_JOB_TYPE_TASK: Literal["TASK"] = "TASK"


class OpenLineageAdapter(LoggingMixin):
"""Translate Airflow metadata to OpenLineage events instead of creating them from Airflow code."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
from airflow.providers.openlineage import conf
from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
from airflow.providers.openlineage.utils.utils import (
_JOB_TYPE_DAG,
_JOB_TYPE_TASK,
get_airflow_job_type,
get_job_name,
get_parent_information_from_dagrun_conf,
get_regional_task_instance_run_id,
Expand Down Expand Up @@ -55,6 +58,21 @@ def lineage_job_name(task_instance: TaskInstance):
return get_job_name(task_instance)


def lineage_job_type() -> dict[str, str]:
Comment thread
mobuchowski marked this conversation as resolved.
Resolved
"""
Macro function which returns the OpenLineage job type facet fields of an Airflow task.
The result has the ``processingType``, ``integration`` and ``jobType`` fields of the ``JobTypeJobFacet``
and can be forwarded to a child job together with the parent information.
Note that a dictionary is returned, not a string - see the guide for how to use it in templates.
.. seealso::
For more information take a look at the guide:
:ref:`howto/macros:openlineage`
"""
return get_airflow_job_type(_JOB_TYPE_TASK)


def lineage_run_id(task_instance: TaskInstance):
"""
Macro function which returns the generated run id (UUID) for a given task.
Expand Down Expand Up @@ -133,6 +151,27 @@ def lineage_root_run_id(task_instance: TaskInstance):
)


def lineage_root_job_type(task_instance: TaskInstance) -> dict[str, str] | None:
"""
Macro function which returns the OpenLineage job type facet fields of the root job of given task.
The result has the ``processingType``, ``integration`` and ``jobType`` fields of the ``JobTypeJobFacet``.
The type is taken from the same source as the other ``lineage_root_*`` macros: DagRun conf root
information, then DagRun conf parent information used as root, then the Dag of the task.
None is returned if the root comes from the DagRun conf, but its job type is missing or invalid.
.. seealso::
For more information take a look at the guide:
:ref:`howto/macros:openlineage`
"""
dr_conf = _get_dag_run_conf(task_instance=task_instance)
if ol_root_info := get_root_information_from_dagrun_conf(dr_conf=dr_conf):
return ol_root_info.get("root_parent_job_type")
if ol_parent_info := get_parent_information_from_dagrun_conf(dr_conf=dr_conf):
return ol_parent_info.get("parent_job_type")
return get_airflow_job_type(_JOB_TYPE_DAG)


def lineage_root_job_namespace(task_instance: TaskInstance):
root_parent_job_namespace = _get_ol_root_id("root_parent_job_namespace", task_instance)
if root_parent_job_namespace:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@
from airflow.providers.openlineage.plugins.macros import (
lineage_job_name,
lineage_job_namespace,
lineage_job_type,
lineage_parent_id,
lineage_root_job_name,
lineage_root_job_namespace,
lineage_root_job_type,
lineage_root_parent_id,
lineage_root_run_id,
lineage_run_id,
Expand All @@ -48,11 +50,13 @@ class OpenLineageProviderPlugin(AirflowPlugin):
macros = [
lineage_job_namespace,
lineage_job_name,
lineage_job_type,
lineage_run_id,
lineage_parent_id,
lineage_root_run_id,
lineage_root_job_name,
lineage_root_job_namespace,
lineage_root_job_type,
lineage_root_parent_id,
]
listeners = [get_openlineage_listener()]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@
_get_logical_date,
lineage_job_name,
lineage_job_namespace,
lineage_job_type,
lineage_root_job_name,
lineage_root_job_namespace,
lineage_root_job_type,
lineage_root_run_id,
lineage_run_id,
)
Expand Down Expand Up @@ -267,6 +269,8 @@ def _create_ol_event_pair(
root_parent_run_id=lineage_root_run_id(task_instance),
root_parent_job_name=lineage_root_job_name(task_instance),
root_parent_job_namespace=lineage_root_job_namespace(task_instance),
parent_job_type=lineage_job_type(),
root_parent_job_type=lineage_root_job_type(task_instance),
)
job_type_facet: dict = {
"jobType": job_type_job.JobTypeJobFacet(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from contextlib import suppress
from functools import wraps
from importlib import metadata
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Literal
from uuid import UUID

import attrs
Expand Down Expand Up @@ -135,6 +135,8 @@
_NOMINAL_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
_MAX_DOC_BYTES = 64 * 1024 # 64 kilobytes
_PRODUCER = f"https://github.com/apache/airflow/tree/providers-openlineage/{OPENLINEAGE_PROVIDER_VERSION}"
_JOB_TYPE_DAG: Literal["DAG"] = "DAG"
_JOB_TYPE_TASK: Literal["TASK"] = "TASK"


def try_import_from_string(string: str) -> Any:
Expand Down Expand Up @@ -354,34 +356,62 @@ def build_task_event_job_facets(

return {
**(additional_job_facets or {}),
"jobType": job_type_job.JobTypeJobFacet(
jobType="TASK", integration="AIRFLOW", processingType="BATCH", producer=_PRODUCER
),
**_get_job_type_facets(get_airflow_job_type(_JOB_TYPE_TASK)),
**documentation_facet,
**ownership_facet,
**tags_facet,
}


def get_airflow_job_type(job_type: Literal["DAG", "TASK"]) -> dict[str, str]:
"""Return the job type facet fields of an Airflow Dag or task."""
return {"processingType": "BATCH", "integration": "AIRFLOW", "jobType": job_type}


def _get_job_type_facets(job_type: dict[str, str] | None) -> dict[str, JobFacet]:
if not job_type:
return {}
return {
"jobType": job_type_job.JobTypeJobFacet(
processingType=job_type["processingType"],
integration=job_type["integration"],
jobType=job_type["jobType"],
producer=_PRODUCER,
)
}


def _get_parent_run_facet(
parent_run_id: str,
parent_job_name: str,
parent_job_namespace: str = conf.namespace(),
root_parent_run_id: str | None = None,
root_parent_job_name: str | None = None,
root_parent_job_namespace: str | None = None,
parent_job_type: dict[str, str] | None = None,
root_parent_job_type: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Create the parent run facet from identifiers."""
if not any((root_parent_run_id, root_parent_job_name, root_parent_job_namespace)):
root_parent_job_type = parent_job_type
root_parent_run_id = root_parent_run_id or parent_run_id
root_parent_job_name = root_parent_job_name or parent_job_name
root_parent_job_namespace = root_parent_job_namespace or parent_job_namespace
return {
"parent": parent_run.ParentRunFacet(
run=parent_run.Run(runId=parent_run_id),
job=parent_run.Job(namespace=parent_job_namespace, name=parent_job_name),
job=parent_run.Job(
namespace=parent_job_namespace,
name=parent_job_name,
facets=_get_job_type_facets(parent_job_type),
),
root=parent_run.Root(
run=parent_run.RootRun(runId=root_parent_run_id),
job=parent_run.RootJob(namespace=root_parent_job_namespace, name=root_parent_job_name),
job=parent_run.RootJob(
namespace=root_parent_job_namespace,
name=root_parent_job_name,
facets=_get_job_type_facets(root_parent_job_type),
),
),
)
}
Expand All @@ -397,6 +427,7 @@ def get_task_parent_run_facet(
dr_conf: dict | None = None,
) -> dict[str, Any]:
"""Retrieve the parent run facet."""
root_parent_job_type = None
all_root_info = (root_parent_run_id, root_parent_job_namespace, root_parent_job_name)
# If not all root identifiers are provided explicitly, try to get them from dagrun conf.
if not all(all_root_info):
Expand All @@ -417,11 +448,13 @@ def get_task_parent_run_facet(
root_parent_run_id = root_info["root_parent_run_id"]
root_parent_job_namespace = root_info["root_parent_job_namespace"]
root_parent_job_name = root_info["root_parent_job_name"]
root_parent_job_type = root_info.get("root_parent_job_type")
# If not present, check for parent identifiers in dagrun conf and use them as root
elif parent_info := get_parent_information_from_dagrun_conf(dr_conf):
root_parent_run_id = parent_info["parent_run_id"]
root_parent_job_namespace = parent_info["parent_job_namespace"]
root_parent_job_name = parent_info["parent_job_name"]
root_parent_job_type = parent_info.get("parent_job_type")

return _get_parent_run_facet(
parent_run_id=parent_run_id,
Expand All @@ -430,6 +463,8 @@ def get_task_parent_run_facet(
root_parent_run_id=root_parent_run_id,
root_parent_job_namespace=root_parent_job_namespace,
root_parent_job_name=root_parent_job_name,
parent_job_type=get_airflow_job_type(_JOB_TYPE_DAG),
root_parent_job_type=root_parent_job_type,
Comment thread
mobuchowski marked this conversation as resolved.
Resolved
)


Expand All @@ -441,7 +476,29 @@ def _get_openlineage_data_from_dagrun_conf(dr_conf: dict | None) -> dict:
return ol_data if isinstance(ol_data, dict) else {}


def get_root_information_from_dagrun_conf(dr_conf: dict | None) -> dict[str, str]:
def _get_job_type_from_ol_data(ol_data: dict, key: str) -> dict[str, str] | None:
"""Return the job type facet fields stored under ``key`` if they are all valid, otherwise None."""
job_type = ol_data.get(key)
if job_type is None:
return None
required_fields = ("processingType", "integration", "jobType")
# The spec allows partial job type, but only complete information is supported for now.
# Other fields (e.g. `_producer` of a serialized facet) are ignored.
if not isinstance(job_type, dict) or not all(
isinstance(job_type.get(field), str) and job_type[field] for field in required_fields
):
log.warning(
"Invalid OpenLineage %s '%s' in DAG run config - expected a dictionary with non-empty "
"string values for all of: %s.",
key,
job_type,
", ".join(required_fields),
)
return None
return {field: job_type[field] for field in required_fields}


def get_root_information_from_dagrun_conf(dr_conf: dict | None) -> dict[str, Any]:
"""Extract root parent run and job information from a DAG run config."""
ol_data = _get_openlineage_data_from_dagrun_conf(dr_conf)
if not ol_data:
Expand Down Expand Up @@ -483,14 +540,17 @@ def get_root_information_from_dagrun_conf(dr_conf: dict | None) -> dict[str, str
root_parent_job_namespace,
root_parent_job_name,
)
return {
root_info: dict[str, Any] = {
"root_parent_run_id": root_parent_run_id,
"root_parent_job_namespace": root_parent_job_namespace,
"root_parent_job_name": root_parent_job_name,
}
if root_parent_job_type := _get_job_type_from_ol_data(ol_data, "rootParentJobType"):
root_info["root_parent_job_type"] = root_parent_job_type
return root_info


def get_parent_information_from_dagrun_conf(dr_conf: dict | None) -> dict[str, str]:
def get_parent_information_from_dagrun_conf(dr_conf: dict | None) -> dict[str, Any]:
"""Extract parent run and job information from a DAG run config."""
ol_data = _get_openlineage_data_from_dagrun_conf(dr_conf)
if not ol_data:
Expand Down Expand Up @@ -533,11 +593,14 @@ def get_parent_information_from_dagrun_conf(dr_conf: dict | None) -> dict[str, s
parent_job_name,
)

return {
parent_info: dict[str, Any] = {
"parent_run_id": parent_run_id,
"parent_job_namespace": parent_job_namespace,
"parent_job_name": parent_job_name,
}
if parent_job_type := _get_job_type_from_ol_data(ol_data, "parentJobType"):
parent_info["parent_job_type"] = parent_job_type
return parent_info


def get_dag_parent_run_facet(dr_conf: dict | None) -> dict[str, parent_run.ParentRunFacet]:
Expand Down Expand Up @@ -572,10 +635,12 @@ def get_dag_parent_run_facet(dr_conf: dict | None) -> dict[str, parent_run.Paren
return {}

root_parent_run_id, root_parent_job_namespace, root_parent_job_name = None, None, None
root_parent_job_type = None
if root_info := get_root_information_from_dagrun_conf(dr_conf):
root_parent_run_id = root_info["root_parent_run_id"]
root_parent_job_namespace = root_info["root_parent_job_namespace"]
root_parent_job_name = root_info["root_parent_job_name"]
root_parent_job_type = root_info.get("root_parent_job_type")
else:
log.debug(
"Missing OpenLineage root identifiers in DAG run config, "
Expand All @@ -590,6 +655,8 @@ def get_dag_parent_run_facet(dr_conf: dict | None) -> dict[str, parent_run.Paren
root_parent_run_id=root_parent_run_id,
root_parent_job_namespace=root_parent_job_namespace,
root_parent_job_name=root_parent_job_name,
parent_job_type=parent_info.get("parent_job_type"),
root_parent_job_type=root_parent_job_type,
)


Expand Down
Loading
Loading