apache-airflow-providers-openlineage 2.1.2rc1__py3-none-any.whl → 2.1.3rc1__py3-none-any.whl

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "2.1.2"
+__version__ = "2.1.3"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/openlineage/extractors/base.py

@@ -84,7 +84,7 @@ class BaseExtractor(ABC, LoggingMixin):
         return self.extract()
 
     def extract_on_failure(self, task_instance) -> OperatorLineage | None:
-        return self.extract()
+        return self.extract_on_complete(task_instance)
 
 
 class DefaultExtractor(BaseExtractor):
@@ -152,6 +152,10 @@ class DefaultExtractor(BaseExtractor):
                 "OpenLineage provider method failed to import OpenLineage integration. "
                 "This should not happen."
             )
-        except Exception:
-            self.log.warning("OpenLineage provider method failed to extract data from provider.")
+        except Exception as e:
+            self.log.warning(
+                "OpenLineage method failed to extract data from Operator with the following exception: `%s`",
+                e,
+            )
+            self.log.debug("OpenLineage extraction failure details:", exc_info=True)
         return None

airflow/providers/openlineage/extractors/manager.py

@@ -134,7 +134,7 @@ class ExtractorManager(LoggingMixin):
                             task_metadata.inputs = inputs
                             task_metadata.outputs = outputs
                         else:
-                            self.extract_inlets_and_outlets(task_metadata, task.inlets, task.outlets)
+                            self.extract_inlets_and_outlets(task_metadata, task)
                     return task_metadata
 
             except Exception as e:
@@ -144,6 +144,7 @@ class ExtractorManager(LoggingMixin):
                     e,
                     task_info,
                 )
+                self.log.debug("OpenLineage extraction failure details:", exc_info=True)
         elif (hook_lineage := self.get_hook_lineage()) is not None:
             inputs, outputs = hook_lineage
             task_metadata = OperatorLineage(inputs=inputs, outputs=outputs)
@@ -155,9 +156,7 @@ class ExtractorManager(LoggingMixin):
             task_metadata = OperatorLineage(
                 run_facets=get_unknown_source_attribute_run_facet(task=task),
             )
-            inlets = task.get_inlet_defs()
-            outlets = task.get_outlet_defs()
-            self.extract_inlets_and_outlets(task_metadata, inlets, outlets)
+            self.extract_inlets_and_outlets(task_metadata, task)
             return task_metadata
 
         return OperatorLineage()
@@ -182,29 +181,21 @@ class ExtractorManager(LoggingMixin):
             return extractor(task)
         return None
 
-    def extract_inlets_and_outlets(
-        self,
-        task_metadata: OperatorLineage,
-        inlets: list,
-        outlets: list,
-    ):
-        if inlets or outlets:
+    def extract_inlets_and_outlets(self, task_metadata: OperatorLineage, task) -> None:
+        if task.inlets or task.outlets:
             self.log.debug("Manually extracting lineage metadata from inlets and outlets")
-        for i in inlets:
+        for i in task.inlets:
             d = self.convert_to_ol_dataset(i)
             if d:
                 task_metadata.inputs.append(d)
-        for o in outlets:
+        for o in task.outlets:
             d = self.convert_to_ol_dataset(o)
             if d:
                 task_metadata.outputs.append(d)
 
-    @staticmethod
-    def get_hook_lineage() -> tuple[list[Dataset], list[Dataset]] | None:
+    def get_hook_lineage(self) -> tuple[list[Dataset], list[Dataset]] | None:
         try:
-            from airflow.providers.common.compat.lineage.hook import (
-                get_hook_lineage_collector,
-            )
+            from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
         except ImportError:
             return None
 
@@ -213,6 +204,7 @@ class ExtractorManager(LoggingMixin):
         if not get_hook_lineage_collector().has_collected:
             return None
 
+        self.log.debug("OpenLineage will extract lineage from Hook Lineage Collector.")
         return (
             [
                 asset
@@ -322,5 +314,5 @@ class ExtractorManager(LoggingMixin):
                 job_facets=task_metadata.job_facets,
             )
         except AttributeError:
-            self.log.warning("Extractor returns non-valid metadata: %s", task_metadata)
+            self.log.warning("OpenLineage extractor returns non-valid metadata: `%s`", task_metadata)
             return None

airflow/providers/openlineage/get_provider_info.py

@@ -26,38 +26,6 @@ def get_provider_info():
         "package-name": "apache-airflow-providers-openlineage",
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
-        "state": "ready",
-        "source-date-epoch": 1742478177,
-        "versions": [
-            "2.1.2",
-            "2.1.1",
-            "2.1.0",
-            "2.0.0",
-            "1.14.0",
-            "1.13.0",
-            "1.12.2",
-            "1.12.1",
-            "1.12.0",
-            "1.11.0",
-            "1.10.0",
-            "1.9.1",
-            "1.9.0",
-            "1.8.0",
-            "1.7.1",
-            "1.7.0",
-            "1.6.0",
-            "1.5.0",
-            "1.4.0",
-            "1.3.1",
-            "1.3.0",
-            "1.2.1",
-            "1.2.0",
-            "1.1.1",
-            "1.1.0",
-            "1.0.2",
-            "1.0.1",
-            "1.0.0",
-        ],
         "integrations": [
             {
                 "integration-name": "OpenLineage",
@@ -184,13 +152,4 @@ def get_provider_info():
                 },
             }
         },
-        "dependencies": [
-            "apache-airflow>=2.9.0",
-            "apache-airflow-providers-common-sql>=1.20.0",
-            "apache-airflow-providers-common-compat>=1.4.0",
-            "attrs>=22.2",
-            "openlineage-integration-common>=1.24.2",
-            "openlineage-python>=1.24.2",
-        ],
-        "devel-dependencies": ["uuid6>=2024.7.10"],
     }

airflow/providers/openlineage/plugins/adapter.py

@@ -159,11 +159,20 @@ class OpenLineageAdapter(LoggingMixin):
                 stack.enter_context(Stats.timer(f"ol.emit.attempts.{event_type}.{transport_type}"))
                 stack.enter_context(Stats.timer("ol.emit.attempts"))
                 self._client.emit(redacted_event)
-                self.log.debug("Successfully emitted OpenLineage event of id %s", event.run.runId)
-        except Exception:
+                self.log.info(
+                    "Successfully emitted OpenLineage `%s` event of id `%s`",
+                    event_type.upper(),
+                    event.run.runId,
+                )
+        except Exception as e:
             Stats.incr("ol.emit.failed")
-            self.log.warning("Failed to emit OpenLineage event of id %s", event.run.runId)
-            self.log.debug("OpenLineage emission failure: %s", exc_info=True)
+            self.log.warning(
+                "Failed to emit OpenLineage `%s` event of id `%s` with the following exception: `%s`",
+                event_type.upper(),
+                event.run.runId,
+                e,
+            )
+            self.log.debug("OpenLineage emission failure details:", exc_info=True)
 
         return redacted_event
 
@@ -371,7 +380,7 @@ class OpenLineageAdapter(LoggingMixin):
             # Catch all exceptions to prevent ProcessPoolExecutor from silently swallowing them.
             # This ensures that any unexpected exceptions are logged for debugging purposes.
             # This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
-            self.log.warning("Failed to emit DAG started event: \n %s", traceback.format_exc())
+            self.log.warning("Failed to emit OpenLineage DAG started event: \n %s", traceback.format_exc())
 
     def dag_success(
         self,
@@ -409,7 +418,7 @@ class OpenLineageAdapter(LoggingMixin):
             # Catch all exceptions to prevent ProcessPoolExecutor from silently swallowing them.
             # This ensures that any unexpected exceptions are logged for debugging purposes.
             # This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
-            self.log.warning("Failed to emit DAG success event: \n %s", traceback.format_exc())
+            self.log.warning("Failed to emit OpenLineage DAG success event: \n %s", traceback.format_exc())
 
     def dag_failed(
         self,
@@ -453,7 +462,7 @@ class OpenLineageAdapter(LoggingMixin):
             # Catch all exceptions to prevent ProcessPoolExecutor from silently swallowing them.
             # This ensures that any unexpected exceptions are logged for debugging purposes.
             # This part cannot be wrapped to deduplicate code, otherwise the method cannot be pickled in multiprocessing.
-            self.log.warning("Failed to emit DAG failed event: \n %s", traceback.format_exc())
+            self.log.warning("Failed to emit OpenLineage DAG failed event: \n %s", traceback.format_exc())
 
     @staticmethod
     def _build_run(

airflow/providers/openlineage/plugins/listener.py

@@ -482,7 +482,9 @@ class OpenLineageListener:
                 process.wait(conf.execution_timeout())
             except psutil.TimeoutExpired:
                 self.log.warning(
-                    "OpenLineage process %s expired. This should not affect process execution.", pid
+                    "OpenLineage process with pid `%s` expired and will be terminated by listener. "
+                    "This has no impact on actual task execution status.",
+                    pid,
                 )
                 self._terminate_with_wait(process)
             except BaseException:

airflow/providers/openlineage/utils/selective_enable.py

@@ -38,7 +38,7 @@ DISABLE_OL_PARAM = Param(False, const=False)
 T = TypeVar("T", bound="DAG | Operator")
 
 if TYPE_CHECKING:
-    from airflow.sdk.definitions.baseoperator import BaseOperator as SdkBaseOperator
+    from airflow.sdk.bases.operator import BaseOperator as SdkBaseOperator
 
 
 log = logging.getLogger(__name__)

airflow/providers/openlineage/utils/utils.py

@@ -210,10 +210,22 @@ def is_ti_rescheduled_already(ti: TaskInstance, session=NEW_SESSION):
 
     if not ti.task.reschedule:
         return False
-
+    if AIRFLOW_V_3_0_PLUS:
+        return (
+            session.query(
+                exists().where(TaskReschedule.ti_id == ti.id, TaskReschedule.try_number == ti.try_number)
+            ).scalar()
+            is True
+        )
     return (
         session.query(
-            exists().where(TaskReschedule.ti_id == ti.id, TaskReschedule.try_number == ti.try_number)
+            exists().where(
+                TaskReschedule.dag_id == ti.dag_id,
+                TaskReschedule.task_id == ti.task_id,
+                TaskReschedule.run_id == ti.run_id,
+                TaskReschedule.map_index == ti.map_index,
+                TaskReschedule.try_number == ti.try_number,
+            )
         ).scalar()
         is True
     )
@@ -360,6 +372,8 @@ class DagRunInfo(InfoJsonEncodable):
         "data_interval_start",
         "data_interval_end",
         "external_trigger",  # Removed in Airflow 3, use run_type instead
+        "logical_date",  # Airflow 3
+        "run_after",  # Airflow 3
         "run_id",
         "run_type",
         "start_date",
@@ -430,7 +444,6 @@ class TaskInfo(InfoJsonEncodable):
         "upstream_task_ids",
         "wait_for_downstream",
         "wait_for_past_depends_before_skipping",
-        "weight_rule",
     ]
     casts = {
         "operator_class": lambda task: task.task_type,
@@ -745,7 +758,9 @@ def print_warning(log):
                 return f(*args, **kwargs)
             except Exception:
                 log.warning(
-                    "OpenLineage event emission failed. Exception below is being caught: it's printed for visibility. This has no impact on actual task execution status.",
+                    "OpenLineage event emission failed. "
+                    "Exception below is being caught but it's printed for visibility. "
+                    "This has no impact on actual task execution status.",
                     exc_info=True,
                 )
 

{apache_airflow_providers_openlineage-2.1.2rc1.dist-info → apache_airflow_providers_openlineage-2.1.3rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-openlineage
-Version: 2.1.2rc1
+Version: 2.1.3rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -27,8 +27,8 @@ Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.24.2
 Requires-Dist: openlineage-python>=1.24.2
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.3/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.3
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -59,7 +59,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``2.1.2``
+Release: ``2.1.3``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -72,7 +72,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.3/>`_.
 
 Installation
 ------------
@@ -118,5 +118,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.3/changelog.html>`_.
 

{apache_airflow_providers_openlineage-2.1.2rc1.dist-info → apache_airflow_providers_openlineage-2.1.3rc1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 airflow/providers/openlineage/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/openlineage/__init__.py,sha256=z82Hjldc_TSS3Uwai9WOyuJKxfAG5BH4NlVuMbMSc8g,1498
+airflow/providers/openlineage/__init__.py,sha256=R6VkEEY6inOTMg3aCbbydjy-qIpvgikyWt7OGFyLts4,1498
 airflow/providers/openlineage/conf.py,sha256=aYdLU7iHBdGIU8ZAC5iUiIDgXP9gvP9r_z5hTAbXPOU,5535
-airflow/providers/openlineage/get_provider_info.py,sha256=ryuVSqcK5KIwhNtglm59artrNsQbzFjFNu-yfjOGetY,10320
+airflow/providers/openlineage/get_provider_info.py,sha256=108mAg-tdcTBmPYwJ9M3wcmhc-i1bAs5OB2MLnSBA-4,9256
 airflow/providers/openlineage/sqlparser.py,sha256=N38XhkU-lxwxnYevQpq63JOBi4rzp0q56JjxO3H24W8,20340
 airflow/providers/openlineage/version_compat.py,sha256=aHg90_DtgoSnQvILFICexMyNlHlALBdaeWqkX3dFDug,1605
 airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
-airflow/providers/openlineage/extractors/base.py,sha256=roOVjCxY2P9BN4SCEnQuO7cGi41PGWnozfRpYZzd9k0,6152
+airflow/providers/openlineage/extractors/base.py,sha256=ZXRlvMSak8kUfur-BxrgAxeylMQFG-iT-LusQguIFLc,6342
 airflow/providers/openlineage/extractors/bash.py,sha256=3aR0PXs8fzRLibRxXN1R8wMZnGzyCur7mjpy8e5GC4A,2583
-airflow/providers/openlineage/extractors/manager.py,sha256=O_5_AsVNmlSp6Hlqcvj7DRwgSSKXMgBBeaJCzktOpjc,12872
+airflow/providers/openlineage/extractors/manager.py,sha256=g3WJRBR2-XZHTG7qAR4UEviwtymvDArhlPMVD3c4q_g,12862
 airflow/providers/openlineage/extractors/python.py,sha256=hVWOplMlBimrpPKPeW6vm75a8OmAYMU1oJzqMz8Jh90,3171
 airflow/providers/openlineage/facets/AirflowDagRunFacet.json,sha256=ie6c-J3-wGgk80WDTGWePz18o6DbW--TNM7BMF4WfcU,2251
 airflow/providers/openlineage/facets/AirflowDebugRunFacet.json,sha256=_zA5gFqGje5MOH1SmdMeA5ViOHvW_pV4oijEAvkuBbY,768
@@ -16,17 +16,17 @@ airflow/providers/openlineage/facets/AirflowRunFacet.json,sha256=70mEaZShgSJp-2x
 airflow/providers/openlineage/facets/AirflowStateRunFacet.json,sha256=xhHQEKD9Jopw-oqbkCCrrwFjfXnxvuJAritsmegKjuQ,937
 airflow/providers/openlineage/facets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/openlineage/plugins/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/openlineage/plugins/adapter.py,sha256=799jV2xIyD0Eu1nRIT9CwzmQqRv-3BZXP1IOMJzSryo,20325
+airflow/providers/openlineage/plugins/adapter.py,sha256=wCGJ3rMDpiyFlPaonUQwGzs0hyNqU-4__e0Z9nAbcyI,20620
 airflow/providers/openlineage/plugins/facets.py,sha256=VvyMYR6ONkC95q5FdNmohv0scbA1Ej_B5cQ97as5GvA,4161
-airflow/providers/openlineage/plugins/listener.py,sha256=yjyLbXJGbZEFMXmj5XtTeQFxfmFl-R2vHawGeUAue-Y,25983
+airflow/providers/openlineage/plugins/listener.py,sha256=KlBKT9VkdOrZxvQHsLZWWq_g4jPhaa2GdVxmHy_EVhM,26083
 airflow/providers/openlineage/plugins/macros.py,sha256=qrHLjE95Uq8H-W9CIkQe5Y9Pu1O-GErhpDV2olGaGQM,3730
 airflow/providers/openlineage/plugins/openlineage.py,sha256=HD3mYNPfXd-buZydEpuAY-naVBXhausU2LYUNhL48QA,1906
 airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/openlineage/utils/selective_enable.py,sha256=ZJUH_iS0thup2qYAVcjOgNIru6E8bKc56_pNQHuc8Fg,3451
+airflow/providers/openlineage/utils/selective_enable.py,sha256=YyrUQ7Djv5o46XdH83N_G8AXAZ9C_aKPa534pbNVp08,3441
 airflow/providers/openlineage/utils/spark.py,sha256=-2XfUaV0WISK6vHSBmB9E78xkuPjO3fM1tDQCZG7j9I,7303
 airflow/providers/openlineage/utils/sql.py,sha256=vkKrrdENEMVG8gtzV6yuTXMa2Z9fBAEXmxDVIDaVncI,9571
-airflow/providers/openlineage/utils/utils.py,sha256=Ww7Aryq4gfWbSf2ittdNTn_ksaxnD4P_xiTlKUy2Okc,28787
-apache_airflow_providers_openlineage-2.1.2rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
-apache_airflow_providers_openlineage-2.1.2rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
-apache_airflow_providers_openlineage-2.1.2rc1.dist-info/METADATA,sha256=jy4lwp2jQarfvdobMK_lbkoklCPvanEdJe7ezzPFdZE,5697
-apache_airflow_providers_openlineage-2.1.2rc1.dist-info/RECORD,,
+airflow/providers/openlineage/utils/utils.py,sha256=_TP49gO4GeyXqqqRgYY-7bgNaM1cczLTv-3lQFGLMfM,29329
+apache_airflow_providers_openlineage-2.1.3rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
+apache_airflow_providers_openlineage-2.1.3rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_openlineage-2.1.3rc1.dist-info/METADATA,sha256=mwWQRF94TgSsKPNgzcVQ4iUWmoCiCwDc-bVBMB87sKA,5697
+apache_airflow_providers_openlineage-2.1.3rc1.dist-info/RECORD,,

{apache_airflow_providers_openlineage-2.1.2rc1.dist-info → apache_airflow_providers_openlineage-2.1.3rc1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.11.0
+Generator: flit 3.12.0
 Root-Is-Purelib: true
 Tag: py3-none-any