apache-airflow-providers-openlineage 1.12.2rc1__py3-none-any.whl → 1.13.0rc1__py3-none-any.whl

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.12.2"
+__version__ = "1.13.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

airflow/providers/openlineage/extractors/manager.py

@@ -18,7 +18,9 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterator
 
-from airflow.providers.common.compat.openlineage.utils.utils import translate_airflow_asset
+from airflow.providers.common.compat.openlineage.utils.utils import (
+    translate_airflow_asset,
+)
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors import BaseExtractor, OperatorLineage
 from airflow.providers.openlineage.extractors.base import DefaultExtractor
@@ -61,7 +63,8 @@ class ExtractorManager(LoggingMixin):
             extractor: type[BaseExtractor] | None = try_import_from_string(extractor_path)
             if not extractor:
                 self.log.warning(
-                    "OpenLineage is unable to import custom extractor `%s`; will ignore it.", extractor_path
+                    "OpenLineage is unable to import custom extractor `%s`; will ignore it.",
+                    extractor_path,
                 )
                 continue
             for operator_class in extractor.get_operator_classnames():
@@ -95,13 +98,21 @@ class ExtractorManager(LoggingMixin):
             # Extracting advanced metadata is only possible when extractor for particular operator
             # is defined. Without it, we can't extract any input or output data.
             try:
-                self.log.debug("Using extractor %s %s", extractor.__class__.__name__, str(task_info))
+                self.log.debug(
+                    "Using extractor %s %s",
+                    extractor.__class__.__name__,
+                    str(task_info),
+                )
                 if complete:
                     task_metadata = extractor.extract_on_complete(task_instance)
                 else:
                     task_metadata = extractor.extract()
 
-                self.log.debug("Found task metadata for operation %s: %s", task.task_id, str(task_metadata))
+                self.log.debug(
+                    "Found task metadata for operation %s: %s",
+                    task.task_id,
+                    str(task_metadata),
+                )
                 task_metadata = self.validate_task_metadata(task_metadata)
                 if task_metadata:
                     if (not task_metadata.inputs) and (not task_metadata.outputs):
@@ -115,7 +126,10 @@ class ExtractorManager(LoggingMixin):
 
             except Exception as e:
                 self.log.warning(
-                    "Failed to extract metadata using found extractor %s - %s %s", extractor, e, task_info
+                    "Failed to extract metadata using found extractor %s - %s %s",
+                    extractor,
+                    e,
+                    task_info,
                 )
         elif (hook_lineage := self.get_hook_lineage()) is not None:
             inputs, outputs = hook_lineage
@@ -178,16 +192,9 @@ class ExtractorManager(LoggingMixin):
 
     def get_hook_lineage(self) -> tuple[list[Dataset], list[Dataset]] | None:
         try:
-            from importlib.util import find_spec
-
-            if find_spec("airflow.assets"):
-                from airflow.lineage.hook import get_hook_lineage_collector
-            else:
-                # TODO: import from common.compat directly after common.compat providers with
-                # asset_compat_lineage_collector released
-                from airflow.providers.openlineage.utils.asset_compat_lineage_collector import (
-                    get_hook_lineage_collector,
-                )
+            from airflow.providers.common.compat.lineage.hook import (
+                get_hook_lineage_collector,
+            )
         except ImportError:
             return None
 

airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1728485291,
+        "source-date-epoch": 1730013356,
         "versions": [
+            "1.13.0",
             "1.12.2",
             "1.12.1",
             "1.12.0",
@@ -56,7 +57,7 @@ def get_provider_info():
         "dependencies": [
             "apache-airflow>=2.8.0",
             "apache-airflow-providers-common-sql>=1.6.0",
-            "apache-airflow-providers-common-compat>=1.2.0",
+            "apache-airflow-providers-common-compat>=1.2.1",
             "attrs>=22.2",
             "openlineage-integration-common>=1.22.0",
             "openlineage-python>=1.22.0",

airflow/providers/openlineage/utils/utils.py

@@ -31,8 +31,13 @@ from openlineage.client.utils import RedactMixin
 from packaging.version import Version
 
 from airflow import __version__ as AIRFLOW_VERSION
-from airflow.exceptions import AirflowProviderDeprecationWarning  # TODO: move this maybe to Airflow's logic?
+from airflow.exceptions import (
+    AirflowProviderDeprecationWarning,
+)
+
+# TODO: move this maybe to Airflow's logic?
 from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
+from airflow.providers.common.compat.assets import Asset
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.plugins.facets import (
     AirflowDagRunFacet,
@@ -50,14 +55,14 @@ from airflow.providers.openlineage.utils.selective_enable import (
 )
 from airflow.serialization.serialized_objects import SerializedBaseOperator
 from airflow.utils.context import AirflowContextDeprecationWarning
-from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key
+from airflow.utils.log.secrets_masker import (
+    Redactable,
+    Redacted,
+    SecretsMasker,
+    should_hide_value_for_key,
+)
 from airflow.utils.module_loading import import_string
 
-try:
-    from airflow.assets import Asset
-except ModuleNotFoundError:
-    from airflow.datasets import Dataset as Asset  # type: ignore[no-redef]
-
 if TYPE_CHECKING:
     from openlineage.client.event_v2 import Dataset as OpenLineageDataset
     from openlineage.client.facet_v2 import RunFacet
@@ -477,7 +482,6 @@ def _get_task_groups_details(dag: DAG) -> dict:
     return {
         tg_id: {
             "parent_group": tg.parent_group.group_id,
-            "tooltip": tg.tooltip,
             "ui_color": tg.ui_color,
             "ui_fgcolor": tg.ui_fgcolor,
             "ui_label": tg.label,
@@ -501,7 +505,11 @@ def _emits_ol_events(task: BaseOperator | MappedOperator) -> bool:
     )
 
     emits_ol_events = all(
-        (config_selective_enabled, not config_disabled_for_operators, not is_skipped_as_empty_operator)
+        (
+            config_selective_enabled,
+            not config_disabled_for_operators,
+            not is_skipped_as_empty_operator,
+        )
     )
     return emits_ol_events
 
@@ -567,7 +575,12 @@ class OpenLineageRedactor(SecretsMasker):
                             setattr(
                                 item,
                                 dict_key,
-                                self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth),
+                                self._redact(
+                                    subval,
+                                    name=dict_key,
+                                    depth=(depth + 1),
+                                    max_depth=max_depth,
+                                ),
                             )
                     return item
                 elif is_json_serializable(item) and hasattr(item, "__dict__"):
@@ -578,7 +591,12 @@ class OpenLineageRedactor(SecretsMasker):
                             setattr(
                                 item,
                                 dict_key,
-                                self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth),
+                                self._redact(
+                                    subval,
+                                    name=dict_key,
+                                    depth=(depth + 1),
+                                    max_depth=max_depth,
+                                ),
                             )
                     return item
                 else:
@@ -641,13 +659,17 @@ def normalize_sql(sql: str | Iterable[str]):
 def should_use_external_connection(hook) -> bool:
     # If we're at Airflow 2.10, the execution is process-isolated, so we can safely run those again.
     if not IS_AIRFLOW_2_10_OR_HIGHER:
-        return hook.__class__.__name__ not in ["SnowflakeHook", "SnowflakeSqlApiHook", "RedshiftSQLHook"]
+        return hook.__class__.__name__ not in [
+            "SnowflakeHook",
+            "SnowflakeSqlApiHook",
+            "RedshiftSQLHook",
+        ]
     return True
 
 
 def translate_airflow_asset(asset: Asset, lineage_context) -> OpenLineageDataset | None:
     """
-    Convert a Asset with an AIP-60 compliant URI to an OpenLineageDataset.
+    Convert an Asset with an AIP-60 compliant URI to an OpenLineageDataset.
 
     This function returns None if no URI normalizer is defined, no asset converter is found or
     some core Airflow changes are missing and ImportError is raised.
@@ -656,7 +678,7 @@ def translate_airflow_asset(asset: Asset, lineage_context) -> OpenLineageDataset
         from airflow.assets import _get_normalized_scheme
     except ModuleNotFoundError:
         try:
-            from airflow.datasets import _get_normalized_scheme  # type: ignore[no-redef]
+            from airflow.datasets import _get_normalized_scheme  # type: ignore[no-redef, attr-defined]
         except ImportError:
             return None
 

{apache_airflow_providers_openlineage-1.12.2rc1.dist-info → apache_airflow_providers_openlineage-1.13.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.12.2rc1
+Version: 1.13.0rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow-providers-common-compat>=1.2.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.2.1rc0
 Requires-Dist: apache-airflow-providers-common-sql>=1.6.0rc0
 Requires-Dist: apache-airflow>=2.8.0rc0
 Requires-Dist: attrs>=22.2
@@ -29,8 +29,8 @@ Requires-Dist: openlineage-python>=1.22.0
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.13.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.13.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -82,7 +82,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.12.2.rc1``
+Release: ``1.13.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -95,7 +95,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.13.0/>`_.
 
 Installation
 ------------
@@ -114,7 +114,7 @@ PIP package                                 Version required
 ==========================================  ==================
 ``apache-airflow``                          ``>=2.8.0``
 ``apache-airflow-providers-common-sql``     ``>=1.6.0``
-``apache-airflow-providers-common-compat``  ``>=1.2.0``
+``apache-airflow-providers-common-compat``  ``>=1.2.1``
 ``attrs``                                   ``>=22.2``
 ``openlineage-integration-common``          ``>=1.22.0``
 ``openlineage-python``                      ``>=1.22.0``
@@ -141,4 +141,4 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.13.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.12.2rc1.dist-info → apache_airflow_providers_openlineage-1.13.0rc1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 airflow/providers/openlineage/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/openlineage/__init__.py,sha256=ebc-6rZfaCgWlVuLsW6FA1fuGFXkFheh8ricGwCd3kw,1499
+airflow/providers/openlineage/__init__.py,sha256=IjDwbAoImu75D6-6urGxe4iyvRUy0FKCLdbE7XPJW6s,1499
 airflow/providers/openlineage/conf.py,sha256=mOOiF25MhsDtDxewm5X-yf4uitH1eFr-LINucSgmdE8,5136
-airflow/providers/openlineage/get_provider_info.py,sha256=t1MzS0YvFEPXgdk_-KMzITB5l2qiKW0nMgFb6HRntKs,8971
+airflow/providers/openlineage/get_provider_info.py,sha256=ZdtzQH8v6vCjKaOE-Qm50x-Wo1NjGKDed0a5kdA4RIc,8993
 airflow/providers/openlineage/sqlparser.py,sha256=c7q3VVw41S87ZFozrkrEr2oZK79N12mC3KdDs9V1IuM,15581
 airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
 airflow/providers/openlineage/extractors/base.py,sha256=olafVPBKxeGjqXerCYM0vj2q78Lm4ErWjex_R6nhjKY,6635
 airflow/providers/openlineage/extractors/bash.py,sha256=3aR0PXs8fzRLibRxXN1R8wMZnGzyCur7mjpy8e5GC4A,2583
-airflow/providers/openlineage/extractors/manager.py,sha256=sDSp_ajHiRcbFfX3YL_bx7EhYSKHhCVU0jTx04VzRvQ,12386
+airflow/providers/openlineage/extractors/manager.py,sha256=TU1flv6W3jtn5xxwpqw8K6f2Ni_mZbqqRZK2v3ZiE28,12261
 airflow/providers/openlineage/extractors/python.py,sha256=hVWOplMlBimrpPKPeW6vm75a8OmAYMU1oJzqMz8Jh90,3171
 airflow/providers/openlineage/facets/AirflowDagRunFacet.json,sha256=ie6c-J3-wGgk80WDTGWePz18o6DbW--TNM7BMF4WfcU,2251
 airflow/providers/openlineage/facets/AirflowDebugRunFacet.json,sha256=_zA5gFqGje5MOH1SmdMeA5ViOHvW_pV4oijEAvkuBbY,768
@@ -21,11 +21,10 @@ airflow/providers/openlineage/plugins/listener.py,sha256=j83NmEvdT46FTg2lko8yg-Y
 airflow/providers/openlineage/plugins/macros.py,sha256=hgFA3ZdQibyn4KXIOsKYBm4WRKDLA5q6Asscx5rvNfM,3076
 airflow/providers/openlineage/plugins/openlineage.py,sha256=T0L5Yxpyq_wzs2_hltJCMY5NKzgsYp0vuEn8LppV5PU,1915
 airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/openlineage/utils/asset_compat_lineage_collector.py,sha256=iixSY7KS4HE8X6kdJ6BxT3D0_E_ptn4OmhxVgmLrqYc,3697
 airflow/providers/openlineage/utils/selective_enable.py,sha256=dFJ7wK7J_-BFwcOKp9tqFOSrASV3lmLv7HtRkEuMk3Q,3087
 airflow/providers/openlineage/utils/sql.py,sha256=bnuU9WvjVKcWVMN3cUp0jaHtU5_ZRM5I1OP1WhIdztg,9583
-airflow/providers/openlineage/utils/utils.py,sha256=WAU65ERSk4_RKM2QXmIcrDnYcq44l9jMq7xLvkhQJe0,24210
-apache_airflow_providers_openlineage-1.12.2rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
-apache_airflow_providers_openlineage-1.12.2rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_openlineage-1.12.2rc1.dist-info/METADATA,sha256=spjsxZleiQRnm6gwbSi6x4cTQ5OLB5fuadcLbfoM-_4,6767
-apache_airflow_providers_openlineage-1.12.2rc1.dist-info/RECORD,,
+airflow/providers/openlineage/utils/utils.py,sha256=mvMq5aaUnO3ujhj3tfot9mA4gShtdOHMzECzqj_OQvk,24582
+apache_airflow_providers_openlineage-1.13.0rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
+apache_airflow_providers_openlineage-1.13.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_openlineage-1.13.0rc1.dist-info/METADATA,sha256=edOQfokrRBhJ_7n6syCvbgfRPw4oB1XIGOpGVALmIJE,6767
+apache_airflow_providers_openlineage-1.13.0rc1.dist-info/RECORD,,

airflow/providers/openlineage/utils/asset_compat_lineage_collector.py

@@ -1,108 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from importlib.util import find_spec
-
-# TODO: replace this module with common.compat provider once common.compat 1.3.0 released
-
-
-def _get_asset_compat_hook_lineage_collector():
-    from airflow.lineage.hook import get_hook_lineage_collector
-
-    collector = get_hook_lineage_collector()
-
-    if all(
-        getattr(collector, asset_method_name, None)
-        for asset_method_name in ("add_input_asset", "add_output_asset", "collected_assets")
-    ):
-        return collector
-
-    # dataset is renamed as asset in Airflow 3.0
-
-    from functools import wraps
-
-    from airflow.lineage.hook import DatasetLineageInfo, HookLineage
-
-    DatasetLineageInfo.asset = DatasetLineageInfo.dataset
-
-    def rename_dataset_kwargs_as_assets_kwargs(function):
-        @wraps(function)
-        def wrapper(*args, **kwargs):
-            if "asset_kwargs" in kwargs:
-                kwargs["dataset_kwargs"] = kwargs.pop("asset_kwargs")
-
-            if "asset_extra" in kwargs:
-                kwargs["dataset_extra"] = kwargs.pop("asset_extra")
-
-            return function(*args, **kwargs)
-
-        return wrapper
-
-    collector.create_asset = rename_dataset_kwargs_as_assets_kwargs(collector.create_dataset)
-    collector.add_input_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_input_dataset)
-    collector.add_output_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_output_dataset)
-
-    def collected_assets_compat(collector) -> HookLineage:
-        """Get the collected hook lineage information."""
-        lineage = collector.collected_datasets
-        return HookLineage(
-            [
-                DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
-                for item in lineage.inputs
-            ],
-            [
-                DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
-                for item in lineage.outputs
-            ],
-        )
-
-    setattr(
-        collector.__class__,
-        "collected_assets",
-        property(lambda collector: collected_assets_compat(collector)),
-    )
-
-    return collector
-
-
-def get_hook_lineage_collector():
-    # HookLineageCollector added in 2.10
-    try:
-        if find_spec("airflow.assets"):
-            # Dataset has been renamed as Asset in 3.0
-            from airflow.lineage.hook import get_hook_lineage_collector
-
-            return get_hook_lineage_collector()
-
-        return _get_asset_compat_hook_lineage_collector()
-    except ImportError:
-
-        class NoOpCollector:
-            """
-            NoOpCollector is a hook lineage collector that does nothing.
-
-            It is used when you want to disable lineage collection.
-            """
-
-            def add_input_asset(self, *_, **__):
-                pass
-
-            def add_output_asset(self, *_, **__):
-                pass
-
-        return NoOpCollector()