apache-airflow-providers-openlineage 1.8.0rc1__py3-none-any.whl → 1.9.0rc1__py3-none-any.whl

airflow/providers/openlineage/LICENSE

@@ -215,7 +215,7 @@ Third party Apache 2.0 licenses
 
 The following components are provided under the Apache 2.0 License.
 See project link for details. The text of each license is also included
-at licenses/LICENSE-[project].txt.
+at 3rd-party-licenses/LICENSE-[project].txt.
 
     (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
     (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
@@ -227,7 +227,7 @@ MIT licenses
 ========================================================================
 
 The following components are provided under the MIT License. See project link for details.
-The text of each license is also included at licenses/LICENSE-[project].txt.
+The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
 
     (MIT License) jquery v3.5.1 (https://jquery.org/license/)
     (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
@@ -243,11 +243,11 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
 BSD 3-Clause licenses
 ========================================================================
 The following components are provided under the BSD 3-Clause license. See project links for details.
-The text of each license is also included at licenses/LICENSE-[project].txt.
+The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
 
     (BSD 3 License) d3 v5.16.0 (https://d3js.org)
     (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
     (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
 
 ========================================================================
-See licenses/LICENSES-ui.txt for packages used in `/airflow/www`
+See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.8.0"
+__version__ = "1.9.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

airflow/providers/openlineage/conf.py

@@ -33,7 +33,15 @@ from __future__ import annotations
 import os
 from typing import Any
 
-from airflow.compat.functools import cache
+# Disable caching if we're inside tests - this makes config easier to mock.
+if os.getenv("PYTEST_VERSION"):
+
+    def decorator(func):
+        return func
+
+    cache = decorator
+else:
+    from airflow.compat.functools import cache
 from airflow.configuration import conf
 
 _CONFIG_SECTION = "openlineage"
@@ -130,3 +138,10 @@ def dag_state_change_process_pool_size() -> int:
     """[openlineage] dag_state_change_process_pool_size."""
     option = conf.get(_CONFIG_SECTION, "dag_state_change_process_pool_size", fallback="")
     return _safe_int_convert(str(option).strip(), default=1)
+
+
+@cache
+def execution_timeout() -> int:
+    """[openlineage] execution_timeout."""
+    option = conf.get(_CONFIG_SECTION, "execution_timeout", fallback="")
+    return _safe_int_convert(str(option).strip(), default=10)

airflow/providers/openlineage/facets/AirflowJobFacet.json

@@ -0,0 +1,40 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$defs": {
+      "AirflowJobFacet": {
+        "allOf": [
+          {
+            "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+          },
+          {
+            "type": "object",
+            "properties": {
+              "taskTree": {
+                "description": "The hierarchical structure of tasks in the DAG.",
+                "type": "object",
+                "additionalProperties": true
+              },
+              "taskGroups": {
+                "description": "Information about all task groups within the DAG.",
+                "type": "object",
+                "additionalProperties": true
+              },
+              "tasks": {
+                "description": "Details of all individual tasks within the DAG.",
+                "type": "object",
+                "additionalProperties": true
+              }
+            },
+            "required": ["taskTree", "taskGroups", "tasks"]
+          }
+        ],
+        "type": "object"
+      }
+    },
+    "type": "object",
+    "properties": {
+      "airflow": {
+        "$ref": "#/$defs/AirflowJobFacet"
+      }
+    }
+  }

airflow/providers/openlineage/facets/AirflowRunFacet.json

@@ -0,0 +1,261 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$defs": {
+    "AirflowRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "dag": {
+              "$ref": "#/$defs/DAG"
+            },
+            "dagRun": {
+              "$ref": "#/$defs/DagRun"
+            },
+            "taskInstance": {
+              "$ref": "#/$defs/TaskInstance"
+            },
+            "task": {
+              "$ref": "#/$defs/Task"
+            },
+            "taskUuid": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "dag",
+            "dagRun",
+            "taskInstance",
+            "task",
+            "taskUuid"
+          ]
+        }
+      ]
+    },
+    "Task": {
+      "type": "object",
+      "properties": {
+        "depends_on_past": {
+          "type": "boolean"
+        },
+        "downstream_task_ids": {
+          "type": "string"
+        },
+        "execution_timeout": {
+          "type": "string"
+        },
+        "executor_config": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "ignore_first_depends_on_past": {
+          "type": "boolean"
+        },
+        "is_setup": {
+          "type": "boolean"
+        },
+        "is_teardown": {
+          "type": "boolean"
+        },
+        "mapped": {
+          "type": "boolean"
+        },
+        "max_active_tis_per_dag": {
+          "type": "integer"
+        },
+        "max_active_tis_per_dagrun": {
+          "type": "integer"
+        },
+        "max_retry_delay": {
+          "type": "string"
+        },
+        "multiple_outputs": {
+          "type": "boolean"
+        },
+        "operator_class": {
+          "description": "Module + class name of the operator",
+          "type": "string"
+        },
+        "owner": {
+          "type": "string"
+        },
+        "priority_weight": {
+          "type": "integer"
+        },
+        "queue": {
+          "type": "string"
+        },
+        "retries": {
+          "type": "integer"
+        },
+        "retry_exponential_backoff": {
+          "type": "boolean"
+        },
+        "run_as_user": {
+          "type": "string"
+        },
+        "sla": {
+          "type": "number"
+        },
+        "task_id": {
+          "type": "string"
+        },
+        "trigger_rule": {
+          "type": "string"
+        },
+        "upstream_task_ids": {
+          "type": "string"
+        },
+        "wait_for_downstream": {
+          "type": "boolean"
+        },
+        "wait_for_past_depends_before_skipping": {
+          "type": "boolean"
+        },
+        "weight_rule": {
+          "type": "string"
+        },
+        "task_group": {
+          "description": "Task group related information",
+          "type": "object",
+          "properties": {
+            "group_id": {
+              "type": "string"
+            },
+            "downstream_group_ids": {
+              "type": "string"
+            },
+            "downstream_task_ids": {
+              "type": "string"
+            },
+            "prefix_group_id": {
+              "type": "boolean"
+            },
+            "tooltip": {
+              "type": "string"
+            },
+            "upstream_group_ids": {
+              "type": "string"
+            },
+            "upstream_task_ids": {
+              "type": "string"
+            }
+          },
+          "additionalProperties": true,
+          "required": ["group_id"]
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "task_id"
+      ]
+    },
+    "DAG": {
+      "type": "object",
+      "properties": {
+        "dag_id": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "owner": {
+          "type": "string"
+        },
+        "schedule_interval": {
+          "type": "string"
+        },
+        "start_date": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "tags": {
+          "type": "string"
+        },
+        "timetable": {
+          "description": "Describes timetable (successor of schedule_interval)",
+          "type": "object",
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "dag_id",
+        "start_date"
+      ]
+    },
+    "TaskInstance": {
+      "type": "object",
+      "properties": {
+        "duration": {
+          "type": "number"
+        },
+        "map_index": {
+          "type": "integer"
+        },
+        "pool": {
+          "type": "string"
+        },
+        "try_number": {
+          "type": "integer"
+        },
+        "queued_dttm": {
+          "type": "string",
+          "format": "date-time"
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "pool",
+        "try_number"
+      ]
+    },
+    "DagRun": {
+      "type": "object",
+      "properties": {
+        "conf": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "dag_id": {
+          "type": "string"
+        },
+        "data_interval_start": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "data_interval_end": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "external_trigger": {
+          "type": "boolean"
+        },
+        "run_id": {
+          "type": "string"
+        },
+        "run_type": {
+          "type": "string"
+        },
+        "start_date": {
+          "type": "string",
+          "format": "date-time"
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "dag_id",
+        "run_id"
+      ]
+    }
+  },
+  "type": "object",
+  "properties": {
+    "airflow": {
+      "$ref": "#/$defs/AirflowRunFacet"
+    }
+  }
+}

airflow/providers/openlineage/facets/AirflowStateRunFacet.json

@@ -0,0 +1,34 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$defs": {
+      "AirflowStateRunFacet": {
+        "allOf": [
+          {
+            "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+          },
+          {
+            "type": "object",
+            "properties": {
+              "dagRunState": {
+                "description": "The final status of the entire DagRun",
+                "type": "string"
+              },
+              "tasksState": {
+                "description": "Mapping of task IDs to their respective states",
+                "type": "object",
+                "additionalProperties": true
+              }
+            },
+            "required": ["dagRunState", "tasksState"]
+          }
+        ],
+        "type": "object"
+      }
+    },
+    "type": "object",
+    "properties": {
+      "airflowState": {
+        "$ref": "#/$defs/AirflowStateRunFacet"
+      }
+    }
+  }

airflow/providers/openlineage/facets/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1715684338,
+        "source-date-epoch": 1718605195,
         "versions": [
+            "1.9.0",
             "1.8.0",
             "1.7.1",
             "1.7.0",
@@ -50,8 +51,8 @@ def get_provider_info():
             "apache-airflow>=2.7.0",
             "apache-airflow-providers-common-sql>=1.6.0",
             "attrs>=22.2",
-            "openlineage-integration-common>=0.28.0",
-            "openlineage-python>=0.28.0",
+            "openlineage-integration-common>=1.16.0",
+            "openlineage-python>=1.16.0",
         ],
         "integrations": [
             {
@@ -134,6 +135,13 @@ def get_provider_info():
                         "type": "integer",
                         "version_added": "1.8.0",
                     },
+                    "execution_timeout": {
+                        "description": "Maximum amount of time (in seconds) that OpenLineage can spend executing metadata extraction.\n",
+                        "default": "10",
+                        "example": None,
+                        "type": "integer",
+                        "version_added": "1.9.0",
+                    },
                 },
             }
         },

airflow/providers/openlineage/plugins/adapter.py

@@ -17,7 +17,6 @@
 from __future__ import annotations
 
 import traceback
-import uuid
 from contextlib import ExitStack
 from typing import TYPE_CHECKING
 
@@ -36,13 +35,19 @@ from openlineage.client.facet import (
     SourceCodeLocationJobFacet,
 )
 from openlineage.client.run import Job, Run, RunEvent, RunState
+from openlineage.client.uuid import generate_static_uuid
 
 from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
-from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
+from airflow.providers.openlineage.utils.utils import (
+    OpenLineageRedactor,
+    get_airflow_state_run_facet,
+)
 from airflow.stats import Stats
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
+    from datetime import datetime
+
     from airflow.models.dagrun import DagRun
     from airflow.providers.openlineage.extractors import OperatorLineage
     from airflow.utils.log.secrets_masker import SecretsMasker
@@ -111,15 +116,25 @@ class OpenLineageAdapter(LoggingMixin):
             return yaml.safe_load(config_file)
 
     @staticmethod
-    def build_dag_run_id(dag_id, dag_run_id):
-        return str(uuid.uuid3(uuid.NAMESPACE_URL, f"{conf.namespace()}.{dag_id}.{dag_run_id}"))
+    def build_dag_run_id(dag_id: str, execution_date: datetime) -> str:
+        return str(
+            generate_static_uuid(
+                instant=execution_date,
+                data=f"{conf.namespace()}.{dag_id}".encode(),
+            )
+        )
 
     @staticmethod
-    def build_task_instance_run_id(dag_id, task_id, execution_date, try_number):
+    def build_task_instance_run_id(
+        dag_id: str,
+        task_id: str,
+        try_number: int,
+        execution_date: datetime,
+    ):
         return str(
-            uuid.uuid3(
-                uuid.NAMESPACE_URL,
-                f"{conf.namespace()}.{dag_id}.{task_id}.{execution_date}.{try_number}",
+            generate_static_uuid(
+                instant=execution_date,
+                data=f"{conf.namespace()}.{dag_id}.{task_id}.{try_number}".encode(),
             )
         )
 
@@ -264,6 +279,7 @@ class OpenLineageAdapter(LoggingMixin):
         parent_run_id: str | None,
         end_time: str,
         task: OperatorLineage,
+        error: str | BaseException | None = None,
     ) -> RunEvent:
         """
         Emit openlineage event of type FAIL.
@@ -275,7 +291,16 @@ class OpenLineageAdapter(LoggingMixin):
         :param parent_run_id: identifier of job spawning this task
         :param end_time: time of task completion
         :param task: metadata container with information extracted from operator
+        :param error: error
         """
+        error_facet = {}
+        if error:
+            if isinstance(error, BaseException):
+                import traceback
+
+                error = "\\n".join(traceback.format_exception(type(error), error, error.__traceback__))
+            error_facet = {"errorMessage": ErrorMessageRunFacet(message=error, programmingLanguage="python")}
+
         event = RunEvent(
             eventType=RunState.FAIL,
             eventTime=end_time,
@@ -284,7 +309,7 @@ class OpenLineageAdapter(LoggingMixin):
                 job_name=job_name,
                 parent_job_name=parent_job_name,
                 parent_run_id=parent_run_id,
-                run_facets=task.run_facets,
+                run_facets={**task.run_facets, **error_facet},
             ),
             job=self._build_job(job_name, job_type=_JOB_TYPE_TASK, job_facets=task.job_facets),
             inputs=task.inputs,
@@ -299,14 +324,24 @@ class OpenLineageAdapter(LoggingMixin):
         msg: str,
         nominal_start_time: str,
         nominal_end_time: str,
+        job_facets: dict[str, BaseFacet] | None = None,  # Custom job facets
     ):
         try:
             event = RunEvent(
                 eventType=RunState.START,
                 eventTime=dag_run.start_date.isoformat(),
-                job=self._build_job(job_name=dag_run.dag_id, job_type=_JOB_TYPE_DAG),
+                job=self._build_job(
+                    job_name=dag_run.dag_id,
+                    job_type=_JOB_TYPE_DAG,
+                    job_description=dag_run.dag.description if dag_run.dag else None,
+                    owners=[x.strip() for x in dag_run.dag.owner.split(",")] if dag_run.dag else None,
+                    job_facets=job_facets,
+                ),
                 run=self._build_run(
-                    run_id=self.build_dag_run_id(dag_run.dag_id, dag_run.run_id),
+                    run_id=self.build_dag_run_id(
+                        dag_id=dag_run.dag_id,
+                        execution_date=dag_run.execution_date,
+                    ),
                     job_name=dag_run.dag_id,
                     nominal_start_time=nominal_start_time,
                     nominal_end_time=nominal_end_time,
@@ -328,7 +363,13 @@ class OpenLineageAdapter(LoggingMixin):
                 eventType=RunState.COMPLETE,
                 eventTime=dag_run.end_date.isoformat(),
                 job=self._build_job(job_name=dag_run.dag_id, job_type=_JOB_TYPE_DAG),
-                run=Run(runId=self.build_dag_run_id(dag_run.dag_id, dag_run.run_id)),
+                run=Run(
+                    runId=self.build_dag_run_id(
+                        dag_id=dag_run.dag_id,
+                        execution_date=dag_run.execution_date,
+                    ),
+                    facets={**get_airflow_state_run_facet(dag_run)},
+                ),
                 inputs=[],
                 outputs=[],
                 producer=_PRODUCER,
@@ -347,8 +388,14 @@ class OpenLineageAdapter(LoggingMixin):
                 eventTime=dag_run.end_date.isoformat(),
                 job=self._build_job(job_name=dag_run.dag_id, job_type=_JOB_TYPE_DAG),
                 run=Run(
-                    runId=self.build_dag_run_id(dag_run.dag_id, dag_run.run_id),
-                    facets={"errorMessage": ErrorMessageRunFacet(message=msg, programmingLanguage="python")},
+                    runId=self.build_dag_run_id(
+                        dag_id=dag_run.dag_id,
+                        execution_date=dag_run.execution_date,
+                    ),
+                    facets={
+                        "errorMessage": ErrorMessageRunFacet(message=msg, programmingLanguage="python"),
+                        **get_airflow_state_run_facet(dag_run),
+                    },
                 ),
                 inputs=[],
                 outputs=[],

airflow/providers/openlineage/plugins/facets.py

@@ -39,15 +39,56 @@ class AirflowMappedTaskRunFacet(BaseFacet):
 
     @classmethod
     def from_task_instance(cls, task_instance):
-        task = task_instance.task
-        from airflow.providers.openlineage.utils.utils import get_operator_class
+        from airflow.providers.openlineage.utils.utils import get_fully_qualified_class_name
 
         return cls(
             mapIndex=task_instance.map_index,
-            operatorClass=f"{get_operator_class(task).__module__}.{get_operator_class(task).__name__}",
+            operatorClass=get_fully_qualified_class_name(task_instance.task),
         )
 
 
+@define(slots=False)
+class AirflowJobFacet(BaseFacet):
+    """
+    Composite Airflow job facet.
+
+    This facet encapsulates all the necessary information to re-create full scope of an Airflow DAG logic,
+    enabling reconstruction, visualization, and analysis of DAGs in a comprehensive manner.
+    It includes detailed representations of the tasks, task groups, and their hierarchical relationships,
+    making it possible to draw a graph that visually represents the entire DAG structure (like in Airflow UI).
+    It also indicates whether a task should emit an OpenLineage (OL) event, enabling consumers to anticipate
+    the number of events and identify the tasks from which they can expect these events.
+
+    Attributes:
+        taskTree: A dictionary representing the hierarchical structure of tasks in the DAG.
+        taskGroups: A dictionary that contains information about task groups within the DAG.
+        tasks: A dictionary detailing individual tasks within the DAG.
+    """
+
+    taskTree: dict
+    taskGroups: dict
+    tasks: dict
+
+
+@define(slots=False)
+class AirflowStateRunFacet(BaseFacet):
+    """
+    Airflow facet providing state information.
+
+    This facet is designed to be sent at a completion event, offering state information about
+    the DAG run and each individual task. This information is crucial for understanding
+    the execution flow and comprehensive post-run analysis and debugging, including why certain tasks
+    did not emit events, which can occur due to the use of control flow operators like the BranchOperator.
+
+    Attributes:
+        dagRunState: This indicates the final status of the entire DAG run (e.g., "success", "failed").
+        tasksState: A dictionary mapping task IDs to their respective states. (e.g., "failed", "skipped").
+    """
+
+    dagRunState: str
+    tasksState: dict[str, str]
+
+
 @define(slots=False)
 class AirflowRunFacet(BaseFacet):
     """Composite Airflow run facet."""