apache-airflow-providers-openlineage 1.12.1rc1__py3-none-any.whl → 1.12.2rc1__py3-none-any.whl

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.12.1"
+__version__ = "1.12.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

airflow/providers/openlineage/conf.py

@@ -35,7 +35,7 @@ if os.getenv("PYTEST_VERSION"):
 
     cache = decorator
 else:
-    from airflow.compat.functools import cache
+    from functools import cache
 from airflow.configuration import conf
 
 _CONFIG_SECTION = "openlineage"

airflow/providers/openlineage/extractors/manager.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterator
 
+from airflow.providers.common.compat.openlineage.utils.utils import translate_airflow_asset
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors import BaseExtractor, OperatorLineage
 from airflow.providers.openlineage.extractors.base import DefaultExtractor
@@ -25,7 +26,6 @@ from airflow.providers.openlineage.extractors.bash import BashExtractor
 from airflow.providers.openlineage.extractors.python import PythonExtractor
 from airflow.providers.openlineage.utils.utils import (
     get_unknown_source_attribute_run_facet,
-    translate_airflow_dataset,
     try_import_from_string,
 )
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -178,7 +178,16 @@ class ExtractorManager(LoggingMixin):
 
     def get_hook_lineage(self) -> tuple[list[Dataset], list[Dataset]] | None:
         try:
-            from airflow.lineage.hook import get_hook_lineage_collector
+            from importlib.util import find_spec
+
+            if find_spec("airflow.assets"):
+                from airflow.lineage.hook import get_hook_lineage_collector
+            else:
+                # TODO: import from common.compat directly after common.compat providers with
+                # asset_compat_lineage_collector released
+                from airflow.providers.openlineage.utils.asset_compat_lineage_collector import (
+                    get_hook_lineage_collector,
+                )
         except ImportError:
             return None
 
@@ -187,16 +196,14 @@ class ExtractorManager(LoggingMixin):
 
         return (
             [
-                dataset
-                for dataset_info in get_hook_lineage_collector().collected_datasets.inputs
-                if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
-                is not None
+                asset
+                for asset_info in get_hook_lineage_collector().collected_assets.inputs
+                if (asset := translate_airflow_asset(asset_info.asset, asset_info.context)) is not None
             ],
             [
-                dataset
-                for dataset_info in get_hook_lineage_collector().collected_datasets.outputs
-                if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
-                is not None
+                asset
+                for asset_info in get_hook_lineage_collector().collected_assets.outputs
+                if (asset := translate_airflow_asset(asset_info.asset, asset_info.context)) is not None
             ],
         )
 

airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1727372276,
+        "source-date-epoch": 1728485291,
         "versions": [
+            "1.12.2",
             "1.12.1",
             "1.12.0",
             "1.11.0",
@@ -88,7 +89,7 @@ def get_provider_info():
                     "disabled_for_operators": {
                         "description": "Exclude some Operators from emitting OpenLineage events by passing a string of semicolon separated\nfull import paths of Operators to disable.\n",
                         "type": "string",
-                        "example": "airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator",
+                        "example": "airflow.providers.standard.operators.bash.BashOperator; airflow.operators.python.PythonOperator",
                         "default": "",
                         "version_added": "1.1.0",
                     },

airflow/providers/openlineage/utils/asset_compat_lineage_collector.py

@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from importlib.util import find_spec
+
+# TODO: replace this module with common.compat provider once common.compat 1.3.0 released
+
+
+def _get_asset_compat_hook_lineage_collector():
+    from airflow.lineage.hook import get_hook_lineage_collector
+
+    collector = get_hook_lineage_collector()
+
+    if all(
+        getattr(collector, asset_method_name, None)
+        for asset_method_name in ("add_input_asset", "add_output_asset", "collected_assets")
+    ):
+        return collector
+
+    # dataset is renamed as asset in Airflow 3.0
+
+    from functools import wraps
+
+    from airflow.lineage.hook import DatasetLineageInfo, HookLineage
+
+    DatasetLineageInfo.asset = DatasetLineageInfo.dataset
+
+    def rename_dataset_kwargs_as_assets_kwargs(function):
+        @wraps(function)
+        def wrapper(*args, **kwargs):
+            if "asset_kwargs" in kwargs:
+                kwargs["dataset_kwargs"] = kwargs.pop("asset_kwargs")
+
+            if "asset_extra" in kwargs:
+                kwargs["dataset_extra"] = kwargs.pop("asset_extra")
+
+            return function(*args, **kwargs)
+
+        return wrapper
+
+    collector.create_asset = rename_dataset_kwargs_as_assets_kwargs(collector.create_dataset)
+    collector.add_input_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_input_dataset)
+    collector.add_output_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_output_dataset)
+
+    def collected_assets_compat(collector) -> HookLineage:
+        """Get the collected hook lineage information."""
+        lineage = collector.collected_datasets
+        return HookLineage(
+            [
+                DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
+                for item in lineage.inputs
+            ],
+            [
+                DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
+                for item in lineage.outputs
+            ],
+        )
+
+    setattr(
+        collector.__class__,
+        "collected_assets",
+        property(lambda collector: collected_assets_compat(collector)),
+    )
+
+    return collector
+
+
+def get_hook_lineage_collector():
+    # HookLineageCollector added in 2.10
+    try:
+        if find_spec("airflow.assets"):
+            # Dataset has been renamed as Asset in 3.0
+            from airflow.lineage.hook import get_hook_lineage_collector
+
+            return get_hook_lineage_collector()
+
+        return _get_asset_compat_hook_lineage_collector()
+    except ImportError:
+
+        class NoOpCollector:
+            """
+            NoOpCollector is a hook lineage collector that does nothing.
+
+            It is used when you want to disable lineage collection.
+            """
+
+            def add_input_asset(self, *_, **__):
+                pass
+
+            def add_output_asset(self, *_, **__):
+                pass
+
+        return NoOpCollector()

airflow/providers/openlineage/utils/utils.py

@@ -31,7 +31,6 @@ from openlineage.client.utils import RedactMixin
 from packaging.version import Version
 
 from airflow import __version__ as AIRFLOW_VERSION
-from airflow.datasets import Dataset
 from airflow.exceptions import AirflowProviderDeprecationWarning  # TODO: move this maybe to Airflow's logic?
 from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
 from airflow.providers.openlineage import conf
@@ -54,6 +53,11 @@ from airflow.utils.context import AirflowContextDeprecationWarning
 from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key
 from airflow.utils.module_loading import import_string
 
+try:
+    from airflow.assets import Asset
+except ModuleNotFoundError:
+    from airflow.datasets import Dataset as Asset  # type: ignore[no-redef]
+
 if TYPE_CHECKING:
     from openlineage.client.event_v2 import Dataset as OpenLineageDataset
     from openlineage.client.facet_v2 import RunFacet
@@ -283,8 +287,8 @@ class TaskInstanceInfo(InfoJsonEncodable):
     }
 
 
-class DatasetInfo(InfoJsonEncodable):
-    """Defines encoding Airflow Dataset object to JSON."""
+class AssetInfo(InfoJsonEncodable):
+    """Defines encoding Airflow Asset object to JSON."""
 
     includes = ["uri", "extra"]
 
@@ -335,8 +339,8 @@ class TaskInfo(InfoJsonEncodable):
             if hasattr(task, "task_group") and getattr(task.task_group, "_group_id", None)
             else None
         ),
-        "inlets": lambda task: [DatasetInfo(i) for i in task.inlets if isinstance(i, Dataset)],
-        "outlets": lambda task: [DatasetInfo(o) for o in task.outlets if isinstance(o, Dataset)],
+        "inlets": lambda task: [AssetInfo(i) for i in task.inlets if isinstance(i, Asset)],
+        "outlets": lambda task: [AssetInfo(o) for o in task.outlets if isinstance(o, Asset)],
     }
 
 
@@ -641,19 +645,29 @@ def should_use_external_connection(hook) -> bool:
     return True
 
 
-def translate_airflow_dataset(dataset: Dataset, lineage_context) -> OpenLineageDataset | None:
+def translate_airflow_asset(asset: Asset, lineage_context) -> OpenLineageDataset | None:
     """
-    Convert a Dataset with an AIP-60 compliant URI to an OpenLineageDataset.
+    Convert a Asset with an AIP-60 compliant URI to an OpenLineageDataset.
 
-    This function returns None if no URI normalizer is defined, no dataset converter is found or
+    This function returns None if no URI normalizer is defined, no asset converter is found or
     some core Airflow changes are missing and ImportError is raised.
     """
     try:
-        from airflow.datasets import _get_normalized_scheme
+        from airflow.assets import _get_normalized_scheme
+    except ModuleNotFoundError:
+        try:
+            from airflow.datasets import _get_normalized_scheme  # type: ignore[no-redef]
+        except ImportError:
+            return None
+
+    try:
         from airflow.providers_manager import ProvidersManager
 
-        ol_converters = ProvidersManager().dataset_to_openlineage_converters
-        normalized_uri = dataset.normalized_uri
+        ol_converters = getattr(ProvidersManager(), "asset_to_openlineage_converters", None)
+        if not ol_converters:
+            ol_converters = ProvidersManager().dataset_to_openlineage_converters  # type: ignore[attr-defined]
+
+        normalized_uri = asset.normalized_uri
     except (ImportError, AttributeError):
         return None
 
@@ -666,4 +680,4 @@ def translate_airflow_dataset(dataset: Dataset, lineage_context) -> OpenLineageD
     if (airflow_to_ol_converter := ol_converters.get(normalized_scheme)) is None:
         return None
 
-    return airflow_to_ol_converter(Dataset(uri=normalized_uri, extra=dataset.extra), lineage_context)
+    return airflow_to_ol_converter(Asset(uri=normalized_uri, extra=asset.extra), lineage_context)

{apache_airflow_providers_openlineage-1.12.1rc1.dist-info → apache_airflow_providers_openlineage-1.12.2rc1.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.12.1rc1
+Version: 1.12.2rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.8
+Requires-Python: ~=3.9
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,7 +15,6 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -27,14 +26,16 @@ Requires-Dist: apache-airflow>=2.8.0rc0
 Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.22.0
 Requires-Dist: openlineage-python>=1.22.0
+Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
+Provides-Extra: common.compat
 Provides-Extra: common.sql
 
 
@@ -81,7 +82,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.12.1.rc1``
+Release: ``1.12.2.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -94,7 +95,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/>`_.
 
 Installation
 ------------
@@ -103,7 +104,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-openlineage``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -129,14 +130,15 @@ You can install such cross-provider dependencies when installing from PyPI. For
 
 .. code-block:: bash
 
-    pip install apache-airflow-providers-openlineage[common.sql]
+    pip install apache-airflow-providers-openlineage[common.compat]
 
 
-============================================================================================================  ==============
-Dependent package                                                                                             Extra
-============================================================================================================  ==============
-`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_  ``common.sql``
-============================================================================================================  ==============
+==================================================================================================================  =================
+Dependent package                                                                                                   Extra
+==================================================================================================================  =================
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
+`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_        ``common.sql``
+==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html>`_.

{apache_airflow_providers_openlineage-1.12.1rc1.dist-info → apache_airflow_providers_openlineage-1.12.2rc1.dist-info}/RECORD

@@ -1,12 +1,12 @@
 airflow/providers/openlineage/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/openlineage/__init__.py,sha256=iTQAADKhEuTkxtt--WbdUTMy4EknzeKmeOkQc6FCFNo,1499
-airflow/providers/openlineage/conf.py,sha256=paV6AHxPJPdQyVMoTGBRHymT-bpbs4Tovttqy9oicMs,5151
-airflow/providers/openlineage/get_provider_info.py,sha256=Yjn7Hu7JakCb7A8a-wizfJwgx_vpcuHZavkLmkBFH-A,8929
+airflow/providers/openlineage/__init__.py,sha256=ebc-6rZfaCgWlVuLsW6FA1fuGFXkFheh8ricGwCd3kw,1499
+airflow/providers/openlineage/conf.py,sha256=mOOiF25MhsDtDxewm5X-yf4uitH1eFr-LINucSgmdE8,5136
+airflow/providers/openlineage/get_provider_info.py,sha256=t1MzS0YvFEPXgdk_-KMzITB5l2qiKW0nMgFb6HRntKs,8971
 airflow/providers/openlineage/sqlparser.py,sha256=c7q3VVw41S87ZFozrkrEr2oZK79N12mC3KdDs9V1IuM,15581
 airflow/providers/openlineage/extractors/__init__.py,sha256=I0X4f6zUniclyD9zT0DFHRImpCpJVP4MkPJT3cd7X5I,1081
 airflow/providers/openlineage/extractors/base.py,sha256=olafVPBKxeGjqXerCYM0vj2q78Lm4ErWjex_R6nhjKY,6635
 airflow/providers/openlineage/extractors/bash.py,sha256=3aR0PXs8fzRLibRxXN1R8wMZnGzyCur7mjpy8e5GC4A,2583
-airflow/providers/openlineage/extractors/manager.py,sha256=Z5fZN6dt67xfJFtlD-_Xnc-tYpxOp66o-ASm6Sn2Vpk,11958
+airflow/providers/openlineage/extractors/manager.py,sha256=sDSp_ajHiRcbFfX3YL_bx7EhYSKHhCVU0jTx04VzRvQ,12386
 airflow/providers/openlineage/extractors/python.py,sha256=hVWOplMlBimrpPKPeW6vm75a8OmAYMU1oJzqMz8Jh90,3171
 airflow/providers/openlineage/facets/AirflowDagRunFacet.json,sha256=ie6c-J3-wGgk80WDTGWePz18o6DbW--TNM7BMF4WfcU,2251
 airflow/providers/openlineage/facets/AirflowDebugRunFacet.json,sha256=_zA5gFqGje5MOH1SmdMeA5ViOHvW_pV4oijEAvkuBbY,768
@@ -21,10 +21,11 @@ airflow/providers/openlineage/plugins/listener.py,sha256=j83NmEvdT46FTg2lko8yg-Y
 airflow/providers/openlineage/plugins/macros.py,sha256=hgFA3ZdQibyn4KXIOsKYBm4WRKDLA5q6Asscx5rvNfM,3076
 airflow/providers/openlineage/plugins/openlineage.py,sha256=T0L5Yxpyq_wzs2_hltJCMY5NKzgsYp0vuEn8LppV5PU,1915
 airflow/providers/openlineage/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/openlineage/utils/asset_compat_lineage_collector.py,sha256=iixSY7KS4HE8X6kdJ6BxT3D0_E_ptn4OmhxVgmLrqYc,3697
 airflow/providers/openlineage/utils/selective_enable.py,sha256=dFJ7wK7J_-BFwcOKp9tqFOSrASV3lmLv7HtRkEuMk3Q,3087
 airflow/providers/openlineage/utils/sql.py,sha256=bnuU9WvjVKcWVMN3cUp0jaHtU5_ZRM5I1OP1WhIdztg,9583
-airflow/providers/openlineage/utils/utils.py,sha256=ps8hFtO6N8TSWdOzB0RAo9Y3lkcuGfoLS9EM7411JGI,23775
-apache_airflow_providers_openlineage-1.12.1rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
-apache_airflow_providers_openlineage-1.12.1rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_openlineage-1.12.1rc1.dist-info/METADATA,sha256=NRQFqplCsaUDXfM6QHJllzeoqtDV3mAs1imbN2CWk1c,6534
-apache_airflow_providers_openlineage-1.12.1rc1.dist-info/RECORD,,
+airflow/providers/openlineage/utils/utils.py,sha256=WAU65ERSk4_RKM2QXmIcrDnYcq44l9jMq7xLvkhQJe0,24210
+apache_airflow_providers_openlineage-1.12.2rc1.dist-info/entry_points.txt,sha256=GAx0_i2OeZzqaiiiYuA-xchICDXiCT5kVqpKSxsOjt4,214
+apache_airflow_providers_openlineage-1.12.2rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_openlineage-1.12.2rc1.dist-info/METADATA,sha256=spjsxZleiQRnm6gwbSi6x4cTQ5OLB5fuadcLbfoM-_4,6767
+apache_airflow_providers_openlineage-1.12.2rc1.dist-info/RECORD,,