apache-airflow-providers-openlineage 2.7.2__py3-none-any.whl → 2.7.3rc1__py3-none-any.whl

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "2.7.2"
+__version__ = "2.7.3"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

airflow/providers/openlineage/plugins/openlineage.py

@@ -18,16 +18,20 @@ from __future__ import annotations
 
 from airflow.plugins_manager import AirflowPlugin
 from airflow.providers.openlineage import conf
-from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
-from airflow.providers.openlineage.plugins.macros import (
-    lineage_job_name,
-    lineage_job_namespace,
-    lineage_parent_id,
-    lineage_root_job_name,
-    lineage_root_parent_id,
-    lineage_root_run_id,
-    lineage_run_id,
-)
+
+# Conditional imports - only load expensive dependencies when plugin is enabled
+if not conf.is_disabled():
+    from airflow.lineage.hook import HookLineageReader
+    from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
+    from airflow.providers.openlineage.plugins.macros import (
+        lineage_job_name,
+        lineage_job_namespace,
+        lineage_parent_id,
+        lineage_root_job_name,
+        lineage_root_parent_id,
+        lineage_root_run_id,
+        lineage_run_id,
+    )
 
 
 class OpenLineageProviderPlugin(AirflowPlugin):
@@ -50,8 +54,6 @@ class OpenLineageProviderPlugin(AirflowPlugin):
             lineage_root_parent_id,
         ]
         listeners = [get_openlineage_listener()]
-        from airflow.lineage.hook import HookLineageReader
-
         hook_lineage_readers = [HookLineageReader]
     else:
         macros = []

airflow/providers/openlineage/sqlparser.py

@@ -232,8 +232,8 @@ class SQLParser(LoggingMixin):
             else None,
         )
 
+    @staticmethod
     def get_metadata_from_parser(
-        self,
         inputs: list[DbTableMeta],
         outputs: list[DbTableMeta],
         database_info: DatabaseInfo,
@@ -315,6 +315,7 @@ class SQLParser(LoggingMixin):
         :param database_info: database specific information
         :param database: when passed it takes precedence over parsed database name
         :param sqlalchemy_engine: when passed, engine's dialect is used to compile SQL queries
+        :param use_connection: if call to db should be performed to enrich datasets (e.g., with schema)
         """
         job_facets: dict[str, JobFacet] = {"sql": sql_job.SQLJobFacet(query=self.normalize_sql(sql))}
         parse_result = self.parse(sql=self.split_sql_string(sql))
@@ -338,17 +339,28 @@ class SQLParser(LoggingMixin):
             )
 
         namespace = self.create_namespace(database_info=database_info)
+        inputs: list[Dataset] = []
+        outputs: list[Dataset] = []
         if use_connection:
-            inputs, outputs = self.parse_table_schemas(
-                hook=hook,
-                inputs=parse_result.in_tables,
-                outputs=parse_result.out_tables,
-                namespace=namespace,
-                database=database,
-                database_info=database_info,
-                sqlalchemy_engine=sqlalchemy_engine,
-            )
-        else:
+            try:
+                inputs, outputs = self.parse_table_schemas(
+                    hook=hook,
+                    inputs=parse_result.in_tables,
+                    outputs=parse_result.out_tables,
+                    namespace=namespace,
+                    database=database,
+                    database_info=database_info,
+                    sqlalchemy_engine=sqlalchemy_engine,
+                )
+            except Exception as e:
+                self.log.warning(
+                    "OpenLineage method failed to enrich datasets using db metadata. Exception: `%s`",
+                    e,
+                )
+                self.log.debug("OpenLineage failure details:", exc_info=True)
+
+        # If call to db failed or was not performed, use datasets from sql parsing alone
+        if not inputs and not outputs:
             inputs, outputs = self.get_metadata_from_parser(
                 inputs=parse_result.in_tables,
                 outputs=parse_result.out_tables,

airflow/providers/openlineage/utils/utils.py

@@ -751,16 +751,20 @@ def get_airflow_state_run_facet(
     dag_id: str, run_id: str, task_ids: list[str], dag_run_state: DagRunState
 ) -> dict[str, AirflowStateRunFacet]:
     tis = DagRun.fetch_task_instances(dag_id=dag_id, run_id=run_id, task_ids=task_ids)
+
+    def get_task_duration(ti):
+        if ti.duration is not None:
+            return ti.duration
+        if ti.end_date is not None and ti.start_date is not None:
+            return (ti.end_date - ti.start_date).total_seconds()
+        # Fallback to 0.0 for tasks with missing timestamps (e.g., skipped/terminated tasks)
+        return 0.0
+
     return {
         "airflowState": AirflowStateRunFacet(
             dagRunState=dag_run_state,
             tasksState={ti.task_id: ti.state for ti in tis},
-            tasksDuration={
-                ti.task_id: ti.duration
-                if ti.duration is not None
-                else (ti.end_date - ti.start_date).total_seconds()
-                for ti in tis
-            },
+            tasksDuration={ti.task_id: get_task_duration(ti) for ti in tis},
         )
     }
 

{apache_airflow_providers_openlineage-2.7.2.dist-info → apache_airflow_providers_openlineage-2.7.3rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-openlineage
-Version: 2.7.2
+Version: 2.7.3rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,15 +20,15 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.10.0
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.4.0
+Requires-Dist: apache-airflow>=2.10.0rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.8.0rc1
 Requires-Dist: attrs>=22.2
-Requires-Dist: openlineage-integration-common>=1.36.0
-Requires-Dist: openlineage-python>=1.36.0
+Requires-Dist: openlineage-integration-common>=1.38.0
+Requires-Dist: openlineage-python>=1.38.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.7.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.7.2
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.7.3/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-openlineage/2.7.3
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -59,7 +59,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``2.7.2``
+Release: ``2.7.3``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -72,7 +72,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.7.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.7.3/>`_.
 
 Installation
 ------------
@@ -91,10 +91,10 @@ PIP package                                 Version required
 ==========================================  ==================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-sql``     ``>=1.20.0``
-``apache-airflow-providers-common-compat``  ``>=1.4.0``
+``apache-airflow-providers-common-compat``  ``>=1.8.0``
 ``attrs``                                   ``>=22.2``
-``openlineage-integration-common``          ``>=1.36.0``
-``openlineage-python``                      ``>=1.36.0``
+``openlineage-integration-common``          ``>=1.38.0``
+``openlineage-python``                      ``>=1.38.0``
 ==========================================  ==================
 
 Cross provider package dependencies
@@ -118,5 +118,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.7.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.7.3/changelog.html>`_.
 

{apache_airflow_providers_openlineage-2.7.2.dist-info → apache_airflow_providers_openlineage-2.7.3rc1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 airflow/providers/openlineage/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/openlineage/__init__.py,sha256=HJa2Y1jJqKcfTwfH_OU7tXIq5436vV_BMiQFRJva0oA,1500
+airflow/providers/openlineage/__init__.py,sha256=WVDyhuTV4ymGp3pA3G3ut6p-P3KlQPqhLVZXtt1sPco,1500
 airflow/providers/openlineage/conf.py,sha256=9v2DpQ84BBCdRxPlh8QsboTqX8HXe-qeHVcTMRL5c3o,5807
 airflow/providers/openlineage/get_provider_info.py,sha256=2Oy13q-jA-UYt-a9pYBk4PnImYshGnJCPD1Jj80ChNw,9453
-airflow/providers/openlineage/sqlparser.py,sha256=8Aq0qbUUBthKjXBV756p2aBf8RYfCuBBfgxwhGpQIg4,20360
+airflow/providers/openlineage/sqlparser.py,sha256=XtfIyBtCw61TDf0num4uD-7I264Hygsv2dTEIgylorQ,20998
 airflow/providers/openlineage/version_compat.py,sha256=weU73JlGTWEcfDLnuFGOXw9Yiagp-bU_--nRgogt-jk,2020
 airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
 airflow/providers/openlineage/extractors/base.py,sha256=0K7prvOeYjs30P87zgcOmABZOZYsw0WYoFBstS_vgmY,6449
@@ -22,13 +22,13 @@ airflow/providers/openlineage/plugins/adapter.py,sha256=moIgAoBTTVaKuU-i00xvkKyR
 airflow/providers/openlineage/plugins/facets.py,sha256=x2EPwst9MsoO53OpFV_aANO_rhiPq_2GLP4UOrqBnnQ,4279
 airflow/providers/openlineage/plugins/listener.py,sha256=IQbTH5yvbR7R85Ffuvxq3oCi_5a_VDy_FVongNSMCfI,32329
 airflow/providers/openlineage/plugins/macros.py,sha256=RfxkpNq78CHzfTAf9X7MQ_zRArMRu9sSD2j69fPSK7s,5265
-airflow/providers/openlineage/plugins/openlineage.py,sha256=dP3GOVtOGAIokeaeRx2OW_c1TKAxDvATlD9OGMyXqr0,2032
+airflow/providers/openlineage/plugins/openlineage.py,sha256=Czx9h8qDmfClhOxRuFOm7F1f9qG3jMXIvDzjXesaLTE,2175
 airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/openlineage/utils/selective_enable.py,sha256=XpozjxcWBASAQXSR2N0GkA-QZVq6EmerOzyR4-eQ64M,3521
 airflow/providers/openlineage/utils/spark.py,sha256=X5liLxVLgQcgPF_0lFtQULeMOv_9dGj-HFjtZvWFgOo,7626
 airflow/providers/openlineage/utils/sql.py,sha256=b_k2fUyGGWzR1eau7tgq7vKQJsR7wPQzDF8M-WRq6jk,9548
-airflow/providers/openlineage/utils/utils.py,sha256=PPG4YzX3pm4uviwG_ZBpiXtR-KkbaDmMHsDOLesUulI,37276
-apache_airflow_providers_openlineage-2.7.2.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
-apache_airflow_providers_openlineage-2.7.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_openlineage-2.7.2.dist-info/METADATA,sha256=KlrninmxJMynH9HCjskb4PZ48udLWmq43J9l_mzTQUM,5688
-apache_airflow_providers_openlineage-2.7.2.dist-info/RECORD,,
+airflow/providers/openlineage/utils/utils.py,sha256=VfGKEzeRxKe06-ASuA1MmSNSNw8RxaANeJpYdAw4cC0,37473
+apache_airflow_providers_openlineage-2.7.3rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
+apache_airflow_providers_openlineage-2.7.3rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_openlineage-2.7.3rc1.dist-info/METADATA,sha256=sJ32EOS-It9sWfTC8Mfq8PnZs7GXL4LEsu60u85kcS0,5714
+apache_airflow_providers_openlineage-2.7.3rc1.dist-info/RECORD,,