apache-airflow-providers-openlineage 1.14.0rc1__tar.gz → 2.0.0rc1__tar.gz

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apache-airflow-providers-openlineage
-Version: 1.14.0rc1
+Version: 2.0.0rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,23 +20,19 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-compat>=1.2.1rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.3.0rc0
 Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow>=2.9.0rc0
 Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.24.2
 Requires-Dist: openlineage-python>=1.24.2
-Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
-Requires-Dist: apache-airflow-providers-common-sql ; extra == "common-sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
-Project-URL: Twitter, https://twitter.com/ApacheAirflow
+Project-URL: Twitter, https://x.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
-Provides-Extra: common-compat
-Provides-Extra: common-sql
 
 
 .. Licensed to the Apache Software Foundation (ASF) under one
@@ -82,7 +78,7 @@ Provides-Extra: common-sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.14.0.rc1``
+Release: ``2.0.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -95,7 +91,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0/>`_.
 
 Installation
 ------------
@@ -112,9 +108,9 @@ Requirements
 ==========================================  ==================
 PIP package                                 Version required
 ==========================================  ==================
-``apache-airflow``                          ``>=2.8.0``
+``apache-airflow``                          ``>=2.9.0``
 ``apache-airflow-providers-common-sql``     ``>=1.20.0``
-``apache-airflow-providers-common-compat``  ``>=1.2.1``
+``apache-airflow-providers-common-compat``  ``>=1.3.0``
 ``attrs``                                   ``>=22.2``
 ``openlineage-integration-common``          ``>=1.24.2``
 ``openlineage-python``                      ``>=1.24.2``
@@ -141,4 +137,4 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.14.0.rc1``
+Release: ``2.0.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0/>`_.
 
 Installation
 ------------
@@ -72,9 +72,9 @@ Requirements
 ==========================================  ==================
 PIP package                                 Version required
 ==========================================  ==================
-``apache-airflow``                          ``>=2.8.0``
+``apache-airflow``                          ``>=2.9.0``
 ``apache-airflow-providers-common-sql``     ``>=1.20.0``
-``apache-airflow-providers-common-compat``  ``>=1.2.1``
+``apache-airflow-providers-common-compat``  ``>=1.3.0``
 ``attrs``                                   ``>=22.2``
 ``openlineage-integration-common``          ``>=1.24.2``
 ``openlineage-python``                      ``>=1.24.2``
@@ -101,4 +101,4 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.14.0"
+__version__ = "2.0.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.8.0"
+    "2.9.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.8.0+"
+        f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.9.0+"
     )

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/conf.py

@@ -77,6 +77,12 @@ def selective_enable() -> bool:
     return conf.getboolean(_CONFIG_SECTION, "selective_enable", fallback="False")
 
 
+@cache
+def spark_inject_parent_job_info() -> bool:
+    """[openlineage] spark_inject_parent_job_info."""
+    return conf.getboolean(_CONFIG_SECTION, "spark_inject_parent_job_info", fallback="False")
+
+
 @cache
 def custom_extractors() -> set[str]:
     """[openlineage] extractors."""

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/extractors/base.py

@@ -29,7 +29,7 @@ with warnings.catch_warnings():
     from openlineage.client.facet import BaseFacet as BaseFacet_V1
 from openlineage.client.facet_v2 import JobFacet, RunFacet
 
-from airflow.providers.openlineage.utils.utils import IS_AIRFLOW_2_10_OR_HIGHER
+from airflow.providers.openlineage.utils.utils import AIRFLOW_V_2_10_PLUS
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import TaskInstanceState
 
@@ -117,7 +117,7 @@ class DefaultExtractor(BaseExtractor):
 
     def extract_on_complete(self, task_instance) -> OperatorLineage | None:
         failed_states = [TaskInstanceState.FAILED, TaskInstanceState.UP_FOR_RETRY]
-        if not IS_AIRFLOW_2_10_OR_HIGHER:  # todo: remove when min airflow version >= 2.10.0
+        if not AIRFLOW_V_2_10_PLUS:  # todo: remove when min airflow version >= 2.10.0
             # Before fix (#41053) implemented in Airflow 2.10 TaskInstance's state was still RUNNING when
             # being passed to listener's on_failure method. Since `extract_on_complete()` is only called
             # after task completion, RUNNING state means that we are dealing with FAILED task in < 2.10

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/extractors/manager.py

@@ -16,7 +16,8 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
 
 from airflow.providers.common.compat.openlineage.utils.utils import (
     translate_airflow_asset,

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1730013356,
+        "source-date-epoch": 1734535974,
         "versions": [
+            "2.0.0",
             "1.14.0",
             "1.13.0",
             "1.12.2",
@@ -56,9 +57,9 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.8.0",
+            "apache-airflow>=2.9.0",
             "apache-airflow-providers-common-sql>=1.20.0",
-            "apache-airflow-providers-common-compat>=1.2.1",
+            "apache-airflow-providers-common-compat>=1.3.0",
             "attrs>=22.2",
             "openlineage-integration-common>=1.24.2",
             "openlineage-python>=1.24.2",
@@ -172,6 +173,13 @@ def get_provider_info():
                         "type": "boolean",
                         "version_added": "1.11.0",
                     },
+                    "spark_inject_parent_job_info": {
+                        "description": "Automatically inject OpenLineage's parent job (namespace, job name, run id) information into Spark\napplication properties for supported Operators.\n",
+                        "type": "boolean",
+                        "default": "False",
+                        "example": None,
+                        "version_added": "1.15.0",
+                    },
                 },
             }
         },

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/plugins/adapter.py

@@ -84,7 +84,7 @@ class OpenLineageAdapter(LoggingMixin):
                     "OpenLineage configuration found. Transport type: `%s`",
                     config.get("type", "no type provided"),
                 )
-                self._client = OpenLineageClient(config=config)
+                self._client = OpenLineageClient(config=config)  # type: ignore[call-arg]
             else:
                 self.log.debug(
                     "OpenLineage configuration not found directly in Airflow. "
@@ -115,11 +115,11 @@ class OpenLineageAdapter(LoggingMixin):
             return yaml.safe_load(config_file)
 
     @staticmethod
-    def build_dag_run_id(dag_id: str, logical_date: datetime) -> str:
+    def build_dag_run_id(dag_id: str, logical_date: datetime, clear_number: int) -> str:
         return str(
             generate_static_uuid(
                 instant=logical_date,
-                data=f"{conf.namespace()}.{dag_id}".encode(),
+                data=f"{conf.namespace()}.{dag_id}.{clear_number}".encode(),
             )
         )
 
@@ -128,12 +128,13 @@ class OpenLineageAdapter(LoggingMixin):
         dag_id: str,
         task_id: str,
         try_number: int,
-        execution_date: datetime,
+        logical_date: datetime,
+        map_index: int,
     ):
         return str(
             generate_static_uuid(
-                instant=execution_date,
-                data=f"{conf.namespace()}.{dag_id}.{task_id}.{try_number}".encode(),
+                instant=logical_date,
+                data=f"{conf.namespace()}.{dag_id}.{task_id}.{try_number}.{map_index}".encode(),
             )
         )
 
@@ -156,10 +157,10 @@ class OpenLineageAdapter(LoggingMixin):
                 stack.enter_context(Stats.timer("ol.emit.attempts"))
                 self._client.emit(redacted_event)
                 self.log.debug("Successfully emitted OpenLineage event of id %s", event.run.runId)
-        except Exception as e:
+        except Exception:
             Stats.incr("ol.emit.failed")
             self.log.warning("Failed to emit OpenLineage event of id %s", event.run.runId)
-            self.log.debug("OpenLineage emission failure: %s", e)
+            self.log.debug("OpenLineage emission failure: %s", exc_info=True)
 
         return redacted_event
 
@@ -332,6 +333,7 @@ class OpenLineageAdapter(LoggingMixin):
         nominal_end_time: str,
         owners: list[str],
         run_facets: dict[str, RunFacet],
+        clear_number: int,
         description: str | None = None,
         job_facets: dict[str, JobFacet] | None = None,  # Custom job facets
     ):
@@ -348,8 +350,7 @@ class OpenLineageAdapter(LoggingMixin):
                 ),
                 run=self._build_run(
                     run_id=self.build_dag_run_id(
-                        dag_id=dag_id,
-                        logical_date=logical_date,
+                        dag_id=dag_id, logical_date=logical_date, clear_number=clear_number
                     ),
                     job_name=dag_id,
                     nominal_start_time=nominal_start_time,
@@ -373,6 +374,7 @@ class OpenLineageAdapter(LoggingMixin):
         run_id: str,
         end_date: datetime,
         logical_date: datetime,
+        clear_number: int,
         dag_run_state: DagRunState,
         task_ids: list[str],
     ):
@@ -383,8 +385,7 @@ class OpenLineageAdapter(LoggingMixin):
                 job=self._build_job(job_name=dag_id, job_type=_JOB_TYPE_DAG),
                 run=Run(
                     runId=self.build_dag_run_id(
-                        dag_id=dag_id,
-                        logical_date=logical_date,
+                        dag_id=dag_id, logical_date=logical_date, clear_number=clear_number
                     ),
                     facets={
                         **get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
@@ -408,6 +409,7 @@ class OpenLineageAdapter(LoggingMixin):
         run_id: str,
         end_date: datetime,
         logical_date: datetime,
+        clear_number: int,
         dag_run_state: DagRunState,
         task_ids: list[str],
         msg: str,
@@ -421,6 +423,7 @@ class OpenLineageAdapter(LoggingMixin):
                     runId=self.build_dag_run_id(
                         dag_id=dag_id,
                         logical_date=logical_date,
+                        clear_number=clear_number,
                     ),
                     facets={
                         "errorMessage": error_message_run.ErrorMessageRunFacet(

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/plugins/listener.py

@@ -32,7 +32,7 @@ from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors import ExtractorManager
 from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter, RunState
 from airflow.providers.openlineage.utils.utils import (
-    IS_AIRFLOW_2_10_OR_HIGHER,
+    AIRFLOW_V_2_10_PLUS,
     get_airflow_dag_run_facet,
     get_airflow_debug_facet,
     get_airflow_job_facet,
@@ -42,6 +42,7 @@ from airflow.providers.openlineage.utils.utils import (
     get_user_provided_run_facets,
     is_operator_disabled,
     is_selective_lineage_enabled,
+    is_ti_rescheduled_already,
     print_warning,
 )
 from airflow.settings import configure_orm
@@ -60,7 +61,7 @@ _openlineage_listener: OpenLineageListener | None = None
 
 def _get_try_number_success(val):
     # todo: remove when min airflow version >= 2.10.0
-    if IS_AIRFLOW_2_10_OR_HIGHER:
+    if AIRFLOW_V_2_10_PLUS:
         return val.try_number
     return val.try_number - 1
 
@@ -134,16 +135,28 @@ class OpenLineageListener:
             # we return here because Airflow 2.3 needs task from deferred state
             if task_instance.next_method is not None:
                 return
+
+            if is_ti_rescheduled_already(task_instance):
+                self.log.debug("Skipping this instance of rescheduled task - START event was emitted already")
+                return
+
             parent_run_id = self.adapter.build_dag_run_id(
                 dag_id=dag.dag_id,
                 logical_date=dagrun.logical_date,
+                clear_number=dagrun.clear_number,
             )
 
+            if hasattr(task_instance, "logical_date"):
+                logical_date = task_instance.logical_date
+            else:
+                logical_date = task_instance.execution_date
+
             task_uuid = self.adapter.build_task_instance_run_id(
                 dag_id=dag.dag_id,
                 task_id=task.task_id,
                 try_number=task_instance.try_number,
-                execution_date=task_instance.execution_date,
+                logical_date=logical_date,
+                map_index=task_instance.map_index,
             )
             event_type = RunState.RUNNING.value.lower()
             operator_name = task.task_type.lower()
@@ -213,16 +226,22 @@ class OpenLineageListener:
 
         @print_warning(self.log)
         def on_success():
-            parent_run_id = OpenLineageAdapter.build_dag_run_id(
+            parent_run_id = self.adapter.build_dag_run_id(
                 dag_id=dag.dag_id,
                 logical_date=dagrun.logical_date,
+                clear_number=dagrun.clear_number,
             )
 
-            task_uuid = OpenLineageAdapter.build_task_instance_run_id(
+            if hasattr(task_instance, "logical_date"):
+                logical_date = task_instance.logical_date
+            else:
+                logical_date = task_instance.execution_date
+            task_uuid = self.adapter.build_task_instance_run_id(
                 dag_id=dag.dag_id,
                 task_id=task.task_id,
                 try_number=_get_try_number_success(task_instance),
-                execution_date=task_instance.execution_date,
+                logical_date=logical_date,
+                map_index=task_instance.map_index,
             )
             event_type = RunState.COMPLETE.value.lower()
             operator_name = task.task_type.lower()
@@ -254,7 +273,7 @@ class OpenLineageListener:
 
         self._execute(on_success, "on_success", use_fork=True)
 
-    if IS_AIRFLOW_2_10_OR_HIGHER:
+    if AIRFLOW_V_2_10_PLUS:
 
         @hookimpl
         def on_task_instance_failed(
@@ -312,16 +331,23 @@ class OpenLineageListener:
 
         @print_warning(self.log)
         def on_failure():
-            parent_run_id = OpenLineageAdapter.build_dag_run_id(
+            parent_run_id = self.adapter.build_dag_run_id(
                 dag_id=dag.dag_id,
                 logical_date=dagrun.logical_date,
+                clear_number=dagrun.clear_number,
             )
 
-            task_uuid = OpenLineageAdapter.build_task_instance_run_id(
+            if hasattr(task_instance, "logical_date"):
+                logical_date = task_instance.logical_date
+            else:
+                logical_date = task_instance.execution_date
+
+            task_uuid = self.adapter.build_task_instance_run_id(
                 dag_id=dag.dag_id,
                 task_id=task.task_id,
                 try_number=task_instance.try_number,
-                execution_date=task_instance.execution_date,
+                logical_date=logical_date,
+                map_index=task_instance.map_index,
             )
             event_type = RunState.FAIL.value.lower()
             operator_name = task.task_type.lower()
@@ -444,6 +470,7 @@ class OpenLineageListener:
                 nominal_start_time=data_interval_start,
                 nominal_end_time=data_interval_end,
                 run_facets=run_facets,
+                clear_number=dag_run.clear_number,
                 owners=[x.strip() for x in dag_run.dag.owner.split(",")] if dag_run.dag else None,
                 description=dag_run.dag.description if dag_run.dag else None,
                 # AirflowJobFacet should be created outside ProcessPoolExecutor that pickles objects,
@@ -469,7 +496,7 @@ class OpenLineageListener:
                 self.log.debug("Executor have not started before `on_dag_run_success`")
                 return
 
-            if IS_AIRFLOW_2_10_OR_HIGHER:
+            if AIRFLOW_V_2_10_PLUS:
                 task_ids = DagRun._get_partial_task_ids(dag_run.dag)
             else:
                 task_ids = dag_run.dag.task_ids if dag_run.dag and dag_run.dag.partial else None
@@ -479,6 +506,7 @@ class OpenLineageListener:
                 run_id=dag_run.run_id,
                 end_date=dag_run.end_date,
                 logical_date=dag_run.logical_date,
+                clear_number=dag_run.clear_number,
                 task_ids=task_ids,
                 dag_run_state=dag_run.get_state(),
             )
@@ -501,7 +529,7 @@ class OpenLineageListener:
                 self.log.debug("Executor have not started before `on_dag_run_failed`")
                 return
 
-            if IS_AIRFLOW_2_10_OR_HIGHER:
+            if AIRFLOW_V_2_10_PLUS:
                 task_ids = DagRun._get_partial_task_ids(dag_run.dag)
             else:
                 task_ids = dag_run.dag.task_ids if dag_run.dag and dag_run.dag.partial else None
@@ -511,6 +539,7 @@ class OpenLineageListener:
                 run_id=dag_run.run_id,
                 end_date=dag_run.end_date,
                 logical_date=dag_run.logical_date,
+                clear_number=dag_run.clear_number,
                 dag_run_state=dag_run.get_state(),
                 task_ids=task_ids,
                 msg=msg,

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/plugins/macros.py

@@ -58,11 +58,16 @@ def lineage_run_id(task_instance: TaskInstance):
         For more information take a look at the guide:
         :ref:`howto/macros:openlineage`
     """
+    if hasattr(task_instance, "logical_date"):
+        logical_date = task_instance.logical_date
+    else:
+        logical_date = task_instance.execution_date
     return OpenLineageAdapter.build_task_instance_run_id(
         dag_id=task_instance.dag_id,
         task_id=task_instance.task_id,
         try_number=task_instance.try_number,
-        execution_date=task_instance.execution_date,
+        logical_date=logical_date,
+        map_index=task_instance.map_index,
     )
 
 

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/plugins/openlineage.py

@@ -25,7 +25,7 @@ from airflow.providers.openlineage.plugins.macros import (
     lineage_parent_id,
     lineage_run_id,
 )
-from airflow.providers.openlineage.utils.utils import IS_AIRFLOW_2_10_OR_HIGHER
+from airflow.providers.openlineage.version_compat import AIRFLOW_V_2_10_PLUS
 
 
 class OpenLineageProviderPlugin(AirflowPlugin):
@@ -40,7 +40,7 @@ class OpenLineageProviderPlugin(AirflowPlugin):
     if not conf.is_disabled():
         macros = [lineage_job_namespace, lineage_job_name, lineage_run_id, lineage_parent_id]
         listeners = [get_openlineage_listener()]
-        if IS_AIRFLOW_2_10_OR_HIGHER:
+        if AIRFLOW_V_2_10_PLUS:
             from airflow.lineage.hook import HookLineageReader
 
             hook_lineage_readers = [HookLineageReader]

apache_airflow_providers_openlineage-2.0.0rc1/airflow/providers/openlineage/utils/spark.py

@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from airflow.providers.openlineage.plugins.macros import (
+    lineage_job_name,
+    lineage_job_namespace,
+    lineage_run_id,
+)
+
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
+log = logging.getLogger(__name__)
+
+
+def _get_parent_job_information_as_spark_properties(context: Context) -> dict:
+    """
+    Retrieve parent job information as Spark properties.
+
+    Args:
+        context: The context containing task instance information.
+
+    Returns:
+        Spark properties with the parent job information.
+    """
+    ti = context["ti"]
+    return {
+        "spark.openlineage.parentJobNamespace": lineage_job_namespace(),
+        "spark.openlineage.parentJobName": lineage_job_name(ti),  # type: ignore[arg-type]
+        "spark.openlineage.parentRunId": lineage_run_id(ti),  # type: ignore[arg-type]
+    }
+
+
+def _is_parent_job_information_present_in_spark_properties(properties: dict) -> bool:
+    """
+    Check if any parent job information is present in Spark properties.
+
+    Args:
+        properties: Spark properties.
+
+    Returns:
+        True if parent job information is present, False otherwise.
+    """
+    return any(str(key).startswith("spark.openlineage.parent") for key in properties)
+
+
+def inject_parent_job_information_into_spark_properties(properties: dict, context: Context) -> dict:
+    """
+    Inject parent job information into Spark properties if not already present.
+
+    Args:
+        properties: Spark properties.
+        context: The context containing task instance information.
+
+    Returns:
+        Modified Spark properties with OpenLineage parent job information properties injected, if applicable.
+    """
+    if _is_parent_job_information_present_in_spark_properties(properties):
+        log.info(
+            "Some OpenLineage properties with parent job information are already present "
+            "in Spark properties. Skipping the injection of OpenLineage "
+            "parent job information into Spark properties."
+        )
+        return properties
+
+    ol_parent_job_properties = _get_parent_job_information_as_spark_properties(context)
+    return {**properties, **ol_parent_job_properties}

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/utils/sql.py

@@ -20,7 +20,7 @@ import logging
 from collections import defaultdict
 from contextlib import closing
 from enum import IntEnum
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 from attrs import define
 from openlineage.client.event_v2 import Dataset
@@ -50,7 +50,7 @@ class ColumnIndex(IntEnum):
     DATABASE = 5
 
 
-TablesHierarchy = Dict[Optional[str], Dict[Optional[str], List[str]]]
+TablesHierarchy = dict[Optional[str], dict[Optional[str], list[str]]]
 
 
 @define

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/airflow/providers/openlineage/utils/utils.py

@@ -23,22 +23,20 @@ import logging
 from contextlib import suppress
 from functools import wraps
 from importlib import metadata
-from typing import TYPE_CHECKING, Any, Callable, Iterable
+from typing import TYPE_CHECKING, Any, Callable
 
 import attrs
-from deprecated import deprecated
 from openlineage.client.utils import RedactMixin
-from packaging.version import Version
+from sqlalchemy import exists
 
 from airflow import __version__ as AIRFLOW_VERSION
-from airflow.exceptions import (
-    AirflowProviderDeprecationWarning,
-)
 
 # TODO: move this maybe to Airflow's logic?
-from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
-from airflow.providers.common.compat.assets import Asset
-from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
+from airflow.models import DAG, BaseOperator, DagRun, MappedOperator, TaskReschedule
+from airflow.providers.openlineage import (
+    __version__ as OPENLINEAGE_PROVIDER_VERSION,
+    conf,
+)
 from airflow.providers.openlineage.plugins.facets import (
     AirflowDagRunFacet,
     AirflowDebugRunFacet,
@@ -53,6 +51,8 @@ from airflow.providers.openlineage.utils.selective_enable import (
     is_dag_lineage_enabled,
     is_task_lineage_enabled,
 )
+from airflow.providers.openlineage.version_compat import AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS
+from airflow.sensors.base import BaseSensorOperator
 from airflow.serialization.serialized_objects import SerializedBaseOperator
 from airflow.utils.context import AirflowContextDeprecationWarning
 from airflow.utils.log.secrets_masker import (
@@ -62,17 +62,27 @@ from airflow.utils.log.secrets_masker import (
     should_hide_value_for_key,
 )
 from airflow.utils.module_loading import import_string
+from airflow.utils.session import NEW_SESSION, provide_session
 
 if TYPE_CHECKING:
     from openlineage.client.event_v2 import Dataset as OpenLineageDataset
     from openlineage.client.facet_v2 import RunFacet, processing_engine_run
 
     from airflow.models import TaskInstance
+    from airflow.providers.common.compat.assets import Asset
     from airflow.utils.state import DagRunState, TaskInstanceState
+else:
+    try:
+        from airflow.providers.common.compat.assets import Asset
+    except ImportError:
+        if AIRFLOW_V_3_0_PLUS:
+            from airflow.sdk.definitions.asset import Asset
+        else:
+            # dataset is renamed to asset since Airflow 3.0
+            from airflow.datasets import Dataset as Asset
 
 log = logging.getLogger(__name__)
 _NOMINAL_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
-IS_AIRFLOW_2_10_OR_HIGHER = Version(Version(AIRFLOW_VERSION).base_version) >= Version("2.10.0")
 
 
 def try_import_from_string(string: str) -> Any:
@@ -167,6 +177,28 @@ def is_selective_lineage_enabled(obj: DAG | BaseOperator | MappedOperator) -> bo
         raise TypeError("is_selective_lineage_enabled can only be used on DAG or Operator objects")
 
 
+@provide_session
+def is_ti_rescheduled_already(ti: TaskInstance, session=NEW_SESSION):
+    if not isinstance(ti.task, BaseSensorOperator):
+        return False
+
+    if not ti.task.reschedule:
+        return False
+
+    return (
+        session.query(
+            exists().where(
+                TaskReschedule.dag_id == ti.dag_id,
+                TaskReschedule.task_id == ti.task_id,
+                TaskReschedule.run_id == ti.run_id,
+                TaskReschedule.map_index == ti.map_index,
+                TaskReschedule.try_number == ti.try_number,
+            )
+        ).scalar()
+        is True
+    )
+
+
 class InfoJsonEncodable(dict):
     """
     Airflow objects might not be json-encodable overall.
@@ -200,6 +232,7 @@ class InfoJsonEncodable(dict):
             self,
             **{field: InfoJsonEncodable._cast_basic_types(getattr(self, field)) for field in self._fields},
         )
+        del self.obj
 
     @staticmethod
     def _cast_basic_types(value):
@@ -660,11 +693,11 @@ def print_warning(log):
         def wrapper(*args, **kwargs):
             try:
                 return f(*args, **kwargs)
-            except Exception as e:
+            except Exception:
                 log.warning(
-                    "Note: exception below is being caught: it's printed for visibility. However OpenLineage events aren't being emitted. If you see that, task has completed successfully despite not getting OL events."
+                    "OpenLineage event emission failed. Exception below is being caught: it's printed for visibility. This has no impact on actual task execution status.",
+                    exc_info=True,
                 )
-                log.warning(e)
 
         return wrapper
 
@@ -676,23 +709,9 @@ def get_filtered_unknown_operator_keys(operator: BaseOperator) -> dict:
     return {attr: value for attr, value in operator.__dict__.items() if attr not in not_required_keys}
 
 
-@deprecated(
-    reason=(
-        "`airflow.providers.openlineage.utils.utils.normalize_sql` "
-        "has been deprecated and will be removed in future"
-    ),
-    category=AirflowProviderDeprecationWarning,
-)
-def normalize_sql(sql: str | Iterable[str]):
-    if isinstance(sql, str):
-        sql = [stmt for stmt in sql.split(";") if stmt != ""]
-    sql = [obj for stmt in sql for obj in stmt.split(";") if obj != ""]
-    return ";\n".join(sql)
-
-
 def should_use_external_connection(hook) -> bool:
     # If we're at Airflow 2.10, the execution is process-isolated, so we can safely run those again.
-    if not IS_AIRFLOW_2_10_OR_HIGHER:
+    if not AIRFLOW_V_2_10_PLUS:
         return hook.__class__.__name__ not in [
             "SnowflakeHook",
             "SnowflakeSqlApiHook",
@@ -708,9 +727,9 @@ def translate_airflow_asset(asset: Asset, lineage_context) -> OpenLineageDataset
     This function returns None if no URI normalizer is defined, no asset converter is found or
     some core Airflow changes are missing and ImportError is raised.
     """
-    try:
-        from airflow.assets import _get_normalized_scheme
-    except ModuleNotFoundError:
+    if AIRFLOW_V_3_0_PLUS:
+        from airflow.sdk.definitions.asset import _get_normalized_scheme
+    else:
         try:
             from airflow.datasets import _get_normalized_scheme  # type: ignore[no-redef, attr-defined]
         except ImportError:

apache_airflow_providers_openlineage-2.0.0rc1/airflow/providers/openlineage/version_compat.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
+# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
+# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
+# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
+#
+from __future__ import annotations
+
+
+def get_base_airflow_version_tuple() -> tuple[int, int, int]:
+    from packaging.version import Version
+
+    from airflow import __version__
+
+    airflow_version = Version(__version__)
+    return airflow_version.major, airflow_version.minor, airflow_version.micro
+
+
+AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
+AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)

{apache_airflow_providers_openlineage-1.14.0rc1 → apache_airflow_providers_openlineage-2.0.0rc1}/pyproject.toml

@@ -27,7 +27,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-openlineage"
-version = "1.14.0.rc1"
+version = "2.0.0.rc1"
 description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -54,34 +54,27 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 dependencies = [
-    "apache-airflow-providers-common-compat>=1.2.1rc0",
+    "apache-airflow-providers-common-compat>=1.3.0rc0",
     "apache-airflow-providers-common-sql>=1.20.0rc0",
-    "apache-airflow>=2.8.0rc0",
+    "apache-airflow>=2.9.0rc0",
     "attrs>=22.2",
     "openlineage-integration-common>=1.24.2",
     "openlineage-python>=1.24.2",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.14.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.0.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
-"Twitter" = "https://twitter.com/ApacheAirflow"
+"Twitter" = "https://x.com/ApacheAirflow"
 "YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
 
 [project.entry-points."apache_airflow_provider"]
 provider_info = "airflow.providers.openlineage.get_provider_info:get_provider_info"
 [project.entry-points."airflow.plugins"]
 openlineage = "airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin"
-[project.optional-dependencies]
-"common.compat" = [
-    "apache-airflow-providers-common-compat",
-]
-"common.sql" = [
-    "apache-airflow-providers-common-sql",
-]
 
 [tool.flit.module]
 name = "airflow.providers.openlineage"