apache-airflow-providers-openlineage 1.11.0rc1__tar.gz → 1.12.0__tar.gz

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.11.0rc1
+Version: 1.12.0
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,15 +21,16 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-sql>=1.6.0rc0
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.2.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.6.0
+Requires-Dist: apache-airflow>=2.8.0
 Requires-Dist: attrs>=22.2
-Requires-Dist: openlineage-integration-common>=1.16.0
-Requires-Dist: openlineage-python>=1.16.0
+Requires-Dist: openlineage-integration-common>=1.22.0
+Requires-Dist: openlineage-python>=1.22.0
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +81,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.11.0.rc1``
+Release: ``1.12.0``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -93,7 +94,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/>`_.
 
 Installation
 ------------
@@ -107,15 +108,16 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.8.0``
-``apache-airflow-providers-common-sql``  ``>=1.6.0``
-``attrs``                                ``>=22.2``
-``openlineage-integration-common``       ``>=1.16.0``
-``openlineage-python``                   ``>=1.16.0``
-=======================================  ==================
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.8.0``
+``apache-airflow-providers-common-sql``     ``>=1.6.0``
+``apache-airflow-providers-common-compat``  ``>=1.2.0``
+``attrs``                                   ``>=22.2``
+``openlineage-integration-common``          ``>=1.22.0``
+``openlineage-python``                      ``>=1.22.0``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -137,4 +139,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.11.0.rc1``
+Release: ``1.12.0``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/>`_.
 
 Installation
 ------------
@@ -69,15 +69,16 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================================  ==================
-PIP package                              Version required
-=======================================  ==================
-``apache-airflow``                       ``>=2.8.0``
-``apache-airflow-providers-common-sql``  ``>=1.6.0``
-``attrs``                                ``>=22.2``
-``openlineage-integration-common``       ``>=1.16.0``
-``openlineage-python``                   ``>=1.16.0``
-=======================================  ==================
+==========================================  ==================
+PIP package                                 Version required
+==========================================  ==================
+``apache-airflow``                          ``>=2.8.0``
+``apache-airflow-providers-common-sql``     ``>=1.6.0``
+``apache-airflow-providers-common-compat``  ``>=1.2.0``
+``attrs``                                   ``>=22.2``
+``openlineage-integration-common``          ``>=1.22.0``
+``openlineage-python``                      ``>=1.22.0``
+==========================================  ==================
 
 Cross provider package dependencies
 -----------------------------------
@@ -99,4 +100,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.11.0"
+__version__ = "1.12.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/extractors/manager.py

@@ -25,6 +25,7 @@ from airflow.providers.openlineage.extractors.bash import BashExtractor
 from airflow.providers.openlineage.extractors.python import PythonExtractor
 from airflow.providers.openlineage.utils.utils import (
     get_unknown_source_attribute_run_facet,
+    translate_airflow_dataset,
     try_import_from_string,
 )
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -90,7 +91,6 @@ class ExtractorManager(LoggingMixin):
             f"task_id={task.task_id} "
             f"airflow_run_id={dagrun.run_id} "
         )
-
         if extractor:
             # Extracting advanced metadata is only possible when extractor for particular operator
             # is defined. Without it, we can't extract any input or output data.
@@ -105,14 +105,22 @@ class ExtractorManager(LoggingMixin):
                 task_metadata = self.validate_task_metadata(task_metadata)
                 if task_metadata:
                     if (not task_metadata.inputs) and (not task_metadata.outputs):
-                        self.extract_inlets_and_outlets(task_metadata, task.inlets, task.outlets)
-
+                        if (hook_lineage := self.get_hook_lineage()) is not None:
+                            inputs, outputs = hook_lineage
+                            task_metadata.inputs = inputs
+                            task_metadata.outputs = outputs
+                        else:
+                            self.extract_inlets_and_outlets(task_metadata, task.inlets, task.outlets)
                     return task_metadata
 
             except Exception as e:
                 self.log.warning(
                     "Failed to extract metadata using found extractor %s - %s %s", extractor, e, task_info
                 )
+        elif (hook_lineage := self.get_hook_lineage()) is not None:
+            inputs, outputs = hook_lineage
+            task_metadata = OperatorLineage(inputs=inputs, outputs=outputs)
+            return task_metadata
         else:
             self.log.debug("Unable to find an extractor %s", task_info)
 
@@ -168,6 +176,30 @@ class ExtractorManager(LoggingMixin):
             if d:
                 task_metadata.outputs.append(d)
 
+    def get_hook_lineage(self) -> tuple[list[Dataset], list[Dataset]] | None:
+        try:
+            from airflow.lineage.hook import get_hook_lineage_collector
+        except ImportError:
+            return None
+
+        if not get_hook_lineage_collector().has_collected:
+            return None
+
+        return (
+            [
+                dataset
+                for dataset_info in get_hook_lineage_collector().collected_datasets.inputs
+                if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
+                is not None
+            ],
+            [
+                dataset
+                for dataset_info in get_hook_lineage_collector().collected_datasets.outputs
+                if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
+                is not None
+            ],
+        )
+
     @staticmethod
     def convert_to_ol_dataset_from_object_storage_uri(uri: str) -> Dataset | None:
         from urllib.parse import urlparse

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1723970474,
+        "source-date-epoch": 1726861079,
         "versions": [
+            "1.12.0",
             "1.11.0",
             "1.10.0",
             "1.9.1",
@@ -53,9 +54,10 @@ def get_provider_info():
         "dependencies": [
             "apache-airflow>=2.8.0",
             "apache-airflow-providers-common-sql>=1.6.0",
+            "apache-airflow-providers-common-compat>=1.2.0",
             "attrs>=22.2",
-            "openlineage-integration-common>=1.16.0",
-            "openlineage-python>=1.16.0",
+            "openlineage-integration-common>=1.22.0",
+            "openlineage-python>=1.22.0",
         ],
         "integrations": [
             {

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/plugins/adapter.py

@@ -40,7 +40,6 @@ from openlineage.client.uuid import generate_static_uuid
 from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
 from airflow.providers.openlineage.utils.utils import (
     OpenLineageRedactor,
-    get_airflow_dag_run_facet,
     get_airflow_debug_facet,
     get_airflow_state_run_facet,
 )
@@ -50,9 +49,9 @@ from airflow.utils.log.logging_mixin import LoggingMixin
 if TYPE_CHECKING:
     from datetime import datetime
 
-    from airflow.models.dagrun import DagRun
     from airflow.providers.openlineage.extractors import OperatorLineage
     from airflow.utils.log.secrets_masker import SecretsMasker
+    from airflow.utils.state import DagRunState
 
 _PRODUCER = f"https://github.com/apache/airflow/tree/providers-openlineage/{OPENLINEAGE_PROVIDER_VERSION}"
 
@@ -118,10 +117,10 @@ class OpenLineageAdapter(LoggingMixin):
             return yaml.safe_load(config_file)
 
     @staticmethod
-    def build_dag_run_id(dag_id: str, execution_date: datetime) -> str:
+    def build_dag_run_id(dag_id: str, logical_date: datetime) -> str:
         return str(
             generate_static_uuid(
-                instant=execution_date,
+                instant=logical_date,
                 data=f"{conf.namespace()}.{dag_id}".encode(),
             )
         )
@@ -336,33 +335,36 @@ class OpenLineageAdapter(LoggingMixin):
 
     def dag_started(
         self,
-        dag_run: DagRun,
-        msg: str,
+        dag_id: str,
+        logical_date: datetime,
+        start_date: datetime,
         nominal_start_time: str,
         nominal_end_time: str,
+        owners: list[str],
+        run_facets: dict[str, RunFacet],
+        description: str | None = None,
         job_facets: dict[str, JobFacet] | None = None,  # Custom job facets
     ):
         try:
-            owner = [x.strip() for x in dag_run.dag.owner.split(",")] if dag_run.dag else None
             event = RunEvent(
                 eventType=RunState.START,
-                eventTime=dag_run.start_date.isoformat(),
+                eventTime=start_date.isoformat(),
                 job=self._build_job(
-                    job_name=dag_run.dag_id,
+                    job_name=dag_id,
                     job_type=_JOB_TYPE_DAG,
-                    job_description=dag_run.dag.description if dag_run.dag else None,
-                    owners=owner,
+                    job_description=description,
+                    owners=owners,
                     job_facets=job_facets,
                 ),
                 run=self._build_run(
                     run_id=self.build_dag_run_id(
-                        dag_id=dag_run.dag_id,
-                        execution_date=dag_run.execution_date,
+                        dag_id=dag_id,
+                        logical_date=logical_date,
                     ),
-                    job_name=dag_run.dag_id,
+                    job_name=dag_id,
                     nominal_start_time=nominal_start_time,
                     nominal_end_time=nominal_end_time,
-                    run_facets={**get_airflow_dag_run_facet(dag_run), **get_airflow_debug_facet()},
+                    run_facets={**run_facets, **get_airflow_debug_facet()},
                 ),
                 inputs=[],
                 outputs=[],
@@ -375,18 +377,29 @@ class OpenLineageAdapter(LoggingMixin):
             # This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
             self.log.warning("Failed to emit DAG started event: \n %s", traceback.format_exc())
 
-    def dag_success(self, dag_run: DagRun, msg: str):
+    def dag_success(
+        self,
+        dag_id: str,
+        run_id: str,
+        end_date: datetime,
+        logical_date: datetime,
+        dag_run_state: DagRunState,
+        task_ids: list[str],
+    ):
         try:
             event = RunEvent(
                 eventType=RunState.COMPLETE,
-                eventTime=dag_run.end_date.isoformat(),
-                job=self._build_job(job_name=dag_run.dag_id, job_type=_JOB_TYPE_DAG),
+                eventTime=end_date.isoformat(),
+                job=self._build_job(job_name=dag_id, job_type=_JOB_TYPE_DAG),
                 run=Run(
                     runId=self.build_dag_run_id(
-                        dag_id=dag_run.dag_id,
-                        execution_date=dag_run.execution_date,
+                        dag_id=dag_id,
+                        logical_date=logical_date,
                     ),
-                    facets={**get_airflow_state_run_facet(dag_run), **get_airflow_debug_facet()},
+                    facets={
+                        **get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
+                        **get_airflow_debug_facet(),
+                    },
                 ),
                 inputs=[],
                 outputs=[],
@@ -399,22 +412,31 @@ class OpenLineageAdapter(LoggingMixin):
             # This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
             self.log.warning("Failed to emit DAG success event: \n %s", traceback.format_exc())
 
-    def dag_failed(self, dag_run: DagRun, msg: str):
+    def dag_failed(
+        self,
+        dag_id: str,
+        run_id: str,
+        end_date: datetime,
+        logical_date: datetime,
+        dag_run_state: DagRunState,
+        task_ids: list[str],
+        msg: str,
+    ):
         try:
             event = RunEvent(
                 eventType=RunState.FAIL,
-                eventTime=dag_run.end_date.isoformat(),
-                job=self._build_job(job_name=dag_run.dag_id, job_type=_JOB_TYPE_DAG),
+                eventTime=end_date.isoformat(),
+                job=self._build_job(job_name=dag_id, job_type=_JOB_TYPE_DAG),
                 run=Run(
                     runId=self.build_dag_run_id(
-                        dag_id=dag_run.dag_id,
-                        execution_date=dag_run.execution_date,
+                        dag_id=dag_id,
+                        logical_date=logical_date,
                     ),
                     facets={
                         "errorMessage": error_message_run.ErrorMessageRunFacet(
                             message=msg, programmingLanguage="python"
                         ),
-                        **get_airflow_state_run_facet(dag_run),
+                        **get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
                         **get_airflow_debug_facet(),
                     },
                 ),

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/plugins/listener.py

@@ -27,11 +27,13 @@ from setproctitle import getproctitle, setproctitle
 
 from airflow import settings
 from airflow.listeners import hookimpl
+from airflow.models import DagRun
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors import ExtractorManager
 from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter, RunState
 from airflow.providers.openlineage.utils.utils import (
     IS_AIRFLOW_2_10_OR_HIGHER,
+    get_airflow_dag_run_facet,
     get_airflow_debug_facet,
     get_airflow_job_facet,
     get_airflow_mapped_task_facet,
@@ -51,7 +53,7 @@ from airflow.utils.timeout import timeout
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
 
-    from airflow.models import DagRun, TaskInstance
+    from airflow.models import TaskInstance
 
 _openlineage_listener: OpenLineageListener | None = None
 
@@ -134,7 +136,7 @@ class OpenLineageListener:
                 return
             parent_run_id = self.adapter.build_dag_run_id(
                 dag_id=dag.dag_id,
-                execution_date=dagrun.execution_date,
+                logical_date=dagrun.logical_date,
             )
 
             task_uuid = self.adapter.build_task_instance_run_id(
@@ -213,7 +215,7 @@ class OpenLineageListener:
         def on_success():
             parent_run_id = OpenLineageAdapter.build_dag_run_id(
                 dag_id=dag.dag_id,
-                execution_date=dagrun.execution_date,
+                logical_date=dagrun.logical_date,
             )
 
             task_uuid = OpenLineageAdapter.build_task_instance_run_id(
@@ -312,7 +314,7 @@ class OpenLineageListener:
         def on_failure():
             parent_run_id = OpenLineageAdapter.build_dag_run_id(
                 dag_id=dag.dag_id,
-                execution_date=dagrun.execution_date,
+                logical_date=dagrun.logical_date,
             )
 
             task_uuid = OpenLineageAdapter.build_task_instance_run_id(
@@ -413,65 +415,120 @@ class OpenLineageListener:
 
     @hookimpl
     def on_dag_run_running(self, dag_run: DagRun, msg: str) -> None:
-        if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
-            self.log.debug(
-                "Skipping OpenLineage event emission for DAG `%s` "
-                "due to lack of explicit lineage enablement for DAG while "
-                "[openlineage] selective_enable is on.",
-                dag_run.dag_id,
+        try:
+            if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
+                self.log.debug(
+                    "Skipping OpenLineage event emission for DAG `%s` "
+                    "due to lack of explicit lineage enablement for DAG while "
+                    "[openlineage] selective_enable is on.",
+                    dag_run.dag_id,
+                )
+                return
+
+            if not self.executor:
+                self.log.debug("Executor have not started before `on_dag_run_running`")
+                return
+
+            data_interval_start = (
+                dag_run.data_interval_start.isoformat() if dag_run.data_interval_start else None
             )
-            return
+            data_interval_end = dag_run.data_interval_end.isoformat() if dag_run.data_interval_end else None
 
-        if not self.executor:
-            self.log.debug("Executor have not started before `on_dag_run_running`")
-            return
+            run_facets = {**get_airflow_dag_run_facet(dag_run)}
 
-        data_interval_start = dag_run.data_interval_start.isoformat() if dag_run.data_interval_start else None
-        data_interval_end = dag_run.data_interval_end.isoformat() if dag_run.data_interval_end else None
-        self.executor.submit(
-            self.adapter.dag_started,
-            dag_run=dag_run,
-            msg=msg,
-            nominal_start_time=data_interval_start,
-            nominal_end_time=data_interval_end,
-            # AirflowJobFacet should be created outside ProcessPoolExecutor that pickles objects,
-            # as it causes lack of some TaskGroup attributes and crashes event emission.
-            job_facets=get_airflow_job_facet(dag_run=dag_run),
-        )
+            self.submit_callable(
+                self.adapter.dag_started,
+                dag_id=dag_run.dag_id,
+                run_id=dag_run.run_id,
+                logical_date=dag_run.logical_date,
+                start_date=dag_run.start_date,
+                nominal_start_time=data_interval_start,
+                nominal_end_time=data_interval_end,
+                run_facets=run_facets,
+                owners=[x.strip() for x in dag_run.dag.owner.split(",")] if dag_run.dag else None,
+                description=dag_run.dag.description if dag_run.dag else None,
+                # AirflowJobFacet should be created outside ProcessPoolExecutor that pickles objects,
+                # as it causes lack of some TaskGroup attributes and crashes event emission.
+                job_facets=get_airflow_job_facet(dag_run=dag_run),
+            )
+        except BaseException as e:
+            self.log.warning("OpenLineage received exception in method on_dag_run_running", exc_info=e)
 
     @hookimpl
     def on_dag_run_success(self, dag_run: DagRun, msg: str) -> None:
-        if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
-            self.log.debug(
-                "Skipping OpenLineage event emission for DAG `%s` "
-                "due to lack of explicit lineage enablement for DAG while "
-                "[openlineage] selective_enable is on.",
-                dag_run.dag_id,
-            )
-            return
+        try:
+            if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
+                self.log.debug(
+                    "Skipping OpenLineage event emission for DAG `%s` "
+                    "due to lack of explicit lineage enablement for DAG while "
+                    "[openlineage] selective_enable is on.",
+                    dag_run.dag_id,
+                )
+                return
 
-        if not self.executor:
-            self.log.debug("Executor have not started before `on_dag_run_success`")
-            return
+            if not self.executor:
+                self.log.debug("Executor have not started before `on_dag_run_success`")
+                return
 
-        self.executor.submit(self.adapter.dag_success, dag_run=dag_run, msg=msg)
+            if IS_AIRFLOW_2_10_OR_HIGHER:
+                task_ids = DagRun._get_partial_task_ids(dag_run.dag)
+            else:
+                task_ids = dag_run.dag.task_ids if dag_run.dag and dag_run.dag.partial else None
+            self.submit_callable(
+                self.adapter.dag_success,
+                dag_id=dag_run.dag_id,
+                run_id=dag_run.run_id,
+                end_date=dag_run.end_date,
+                logical_date=dag_run.logical_date,
+                task_ids=task_ids,
+                dag_run_state=dag_run.get_state(),
+            )
+        except BaseException as e:
+            self.log.warning("OpenLineage received exception in method on_dag_run_success", exc_info=e)
 
     @hookimpl
     def on_dag_run_failed(self, dag_run: DagRun, msg: str) -> None:
-        if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
-            self.log.debug(
-                "Skipping OpenLineage event emission for DAG `%s` "
-                "due to lack of explicit lineage enablement for DAG while "
-                "[openlineage] selective_enable is on.",
-                dag_run.dag_id,
+        try:
+            if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
+                self.log.debug(
+                    "Skipping OpenLineage event emission for DAG `%s` "
+                    "due to lack of explicit lineage enablement for DAG while "
+                    "[openlineage] selective_enable is on.",
+                    dag_run.dag_id,
+                )
+                return
+
+            if not self.executor:
+                self.log.debug("Executor have not started before `on_dag_run_failed`")
+                return
+
+            if IS_AIRFLOW_2_10_OR_HIGHER:
+                task_ids = DagRun._get_partial_task_ids(dag_run.dag)
+            else:
+                task_ids = dag_run.dag.task_ids if dag_run.dag and dag_run.dag.partial else None
+            self.submit_callable(
+                self.adapter.dag_failed,
+                dag_id=dag_run.dag_id,
+                run_id=dag_run.run_id,
+                end_date=dag_run.end_date,
+                logical_date=dag_run.logical_date,
+                dag_run_state=dag_run.get_state(),
+                task_ids=task_ids,
+                msg=msg,
             )
-            return
+        except BaseException as e:
+            self.log.warning("OpenLineage received exception in method on_dag_run_failed", exc_info=e)
 
-        if not self.executor:
-            self.log.debug("Executor have not started before `on_dag_run_failed`")
-            return
+    def submit_callable(self, callable, *args, **kwargs):
+        fut = self.executor.submit(callable, *args, **kwargs)
+        fut.add_done_callback(self.log_submit_error)
+        return fut
 
-        self.executor.submit(self.adapter.dag_failed, dag_run=dag_run, msg=msg)
+    def log_submit_error(self, fut):
+        if fut.exception():
+            self.log.warning("Failed to submit method to executor", exc_info=fut.exception())
+        else:
+            self.log.debug("Successfully submitted method to executor")
 
 
 def get_openlineage_listener() -> OpenLineageListener:

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/plugins/openlineage.py

@@ -25,6 +25,7 @@ from airflow.providers.openlineage.plugins.macros import (
     lineage_parent_id,
     lineage_run_id,
 )
+from airflow.providers.openlineage.utils.utils import IS_AIRFLOW_2_10_OR_HIGHER
 
 
 class OpenLineageProviderPlugin(AirflowPlugin):
@@ -39,6 +40,10 @@ class OpenLineageProviderPlugin(AirflowPlugin):
     if not conf.is_disabled():
         macros = [lineage_job_namespace, lineage_job_name, lineage_run_id, lineage_parent_id]
         listeners = [get_openlineage_listener()]
+        if IS_AIRFLOW_2_10_OR_HIGHER:
+            from airflow.lineage.hook import HookLineageReader
+
+            hook_lineage_readers = [HookLineageReader]
     else:
         macros = []
         listeners = []

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/airflow/providers/openlineage/utils/utils.py

@@ -33,7 +33,7 @@ from packaging.version import Version
 from airflow import __version__ as AIRFLOW_VERSION
 from airflow.datasets import Dataset
 from airflow.exceptions import AirflowProviderDeprecationWarning  # TODO: move this maybe to Airflow's logic?
-from airflow.models import DAG, BaseOperator, MappedOperator, Operator
+from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.plugins.facets import (
     AirflowDagRunFacet,
@@ -58,9 +58,8 @@ if TYPE_CHECKING:
     from openlineage.client.event_v2 import Dataset as OpenLineageDataset
     from openlineage.client.facet_v2 import RunFacet
 
-    from airflow.models import DagRun, TaskInstance
-    from airflow.utils.state import TaskInstanceState
-
+    from airflow.models import TaskInstance
+    from airflow.utils.state import DagRunState, TaskInstanceState
 
 log = logging.getLogger(__name__)
 _NOMINAL_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
@@ -199,7 +198,7 @@ class InfoJsonEncodable(dict):
             return value.isoformat()
         if isinstance(value, datetime.timedelta):
             return f"{value.total_seconds()} seconds"
-        if isinstance(value, (set, tuple)):
+        if isinstance(value, (set, list, tuple)):
             return str(list(value))
         return value
 
@@ -244,7 +243,16 @@ class InfoJsonEncodable(dict):
 class DagInfo(InfoJsonEncodable):
     """Defines encoding DAG object to JSON."""
 
-    includes = ["dag_id", "description", "fileloc", "owner", "schedule_interval", "start_date", "tags"]
+    includes = [
+        "dag_id",
+        "description",
+        "fileloc",
+        "owner",
+        "schedule_interval",  # For Airflow 2.
+        "timetable_summary",  # For Airflow 3.
+        "start_date",
+        "tags",
+    ]
     casts = {"timetable": lambda dag: dag.timetable.serialize() if getattr(dag, "timetable", None) else None}
     renames = {"_dag_id": "dag_id"}
 
@@ -423,59 +431,25 @@ def get_airflow_job_facet(dag_run: DagRun) -> dict[str, AirflowJobFacet]:
         return {}
     return {
         "airflow": AirflowJobFacet(
-            taskTree=_get_parsed_dag_tree(dag_run.dag),
+            taskTree={},  # caused OOM errors, to be removed, see #41587
             taskGroups=_get_task_groups_details(dag_run.dag),
             tasks=_get_tasks_details(dag_run.dag),
         )
     }
 
 
-def get_airflow_state_run_facet(dag_run: DagRun) -> dict[str, AirflowStateRunFacet]:
+def get_airflow_state_run_facet(
+    dag_id: str, run_id: str, task_ids: list[str], dag_run_state: DagRunState
+) -> dict[str, AirflowStateRunFacet]:
+    tis = DagRun.fetch_task_instances(dag_id=dag_id, run_id=run_id, task_ids=task_ids)
     return {
         "airflowState": AirflowStateRunFacet(
-            dagRunState=dag_run.get_state(),
-            tasksState={ti.task_id: ti.state for ti in dag_run.get_task_instances()},
+            dagRunState=dag_run_state,
+            tasksState={ti.task_id: ti.state for ti in tis},
         )
     }
 
 
-def _get_parsed_dag_tree(dag: DAG) -> dict:
-    """
-    Get DAG's tasks hierarchy representation.
-
-    While the task dependencies are defined as following:
-    task >> [task_2, task_4] >> task_7
-    task_3 >> task_5
-    task_6  # has no dependencies, it's a root and a leaf
-
-    The result of this function will look like:
-    {
-        "task": {
-            "task_2": {
-                "task_7": {}
-            },
-            "task_4": {
-                "task_7": {}
-            }
-        },
-        "task_3": {
-            "task_5": {}
-        },
-        "task_6": {}
-    }
-    """
-
-    def get_downstream(task: Operator, current_dict: dict):
-        current_dict[task.task_id] = {}
-        for tmp_task in sorted(task.downstream_list, key=lambda x: x.task_id):
-            get_downstream(tmp_task, current_dict[task.task_id])
-
-    task_dict: dict = {}
-    for t in sorted(dag.roots, key=lambda x: x.task_id):
-        get_downstream(t, task_dict)
-    return task_dict
-
-
 def _get_tasks_details(dag: DAG) -> dict:
     tasks = {
         single_task.task_id: {
@@ -487,8 +461,9 @@ def _get_tasks_details(dag: DAG) -> dict:
             "ui_label": single_task.label,
             "is_setup": single_task.is_setup,
             "is_teardown": single_task.is_teardown,
+            "downstream_task_ids": sorted(single_task.downstream_task_ids),
         }
-        for single_task in dag.tasks
+        for single_task in sorted(dag.tasks, key=lambda x: x.task_id)
     }
 
     return tasks

{apache_airflow_providers_openlineage-1.11.0rc1 → apache_airflow_providers_openlineage-1.12.0}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-openlineage"
-version = "1.11.0.rc1"
+version = "1.12.0"
 description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -56,16 +56,17 @@ classifiers = [
 ]
 requires-python = "~=3.8"
 dependencies = [
-    "apache-airflow-providers-common-sql>=1.6.0rc0",
-    "apache-airflow>=2.8.0rc0",
+    "apache-airflow-providers-common-compat>=1.2.0",
+    "apache-airflow-providers-common-sql>=1.6.0",
+    "apache-airflow>=2.8.0",
     "attrs>=22.2",
-    "openlineage-integration-common>=1.16.0",
-    "openlineage-python>=1.16.0",
+    "openlineage-integration-common>=1.22.0",
+    "openlineage-python>=1.22.0",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"