apache-airflow-providers-openlineage 1.12.1__tar.gz → 1.12.2__tar.gz

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.12.1
+Version: 1.12.2
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.8
+Requires-Python: ~=3.9
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,7 +15,6 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -27,14 +26,16 @@ Requires-Dist: apache-airflow>=2.8.0
 Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.22.0
 Requires-Dist: openlineage-python>=1.22.0
+Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
+Provides-Extra: common.compat
 Provides-Extra: common.sql
 
 
@@ -81,7 +82,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.12.1``
+Release: ``1.12.2``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -94,7 +95,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/>`_.
 
 Installation
 ------------
@@ -103,7 +104,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-openlineage``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -129,14 +130,15 @@ You can install such cross-provider dependencies when installing from PyPI. For
 
 .. code-block:: bash
 
-    pip install apache-airflow-providers-openlineage[common.sql]
+    pip install apache-airflow-providers-openlineage[common.compat]
 
 
-============================================================================================================  ==============
-Dependent package                                                                                             Extra
-============================================================================================================  ==============
-`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_  ``common.sql``
-============================================================================================================  ==============
+==================================================================================================================  =================
+Dependent package                                                                                                   Extra
+==================================================================================================================  =================
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
+`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_        ``common.sql``
+==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html>`_.

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.12.1``
+Release: ``1.12.2``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/>`_.
 
 Installation
 ------------
@@ -64,7 +64,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-openlineage``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -90,14 +90,15 @@ You can install such cross-provider dependencies when installing from PyPI. For
 
 .. code-block:: bash
 
-    pip install apache-airflow-providers-openlineage[common.sql]
+    pip install apache-airflow-providers-openlineage[common.compat]
 
 
-============================================================================================================  ==============
-Dependent package                                                                                             Extra
-============================================================================================================  ==============
-`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_  ``common.sql``
-============================================================================================================  ==============
+==================================================================================================================  =================
+Dependent package                                                                                                   Extra
+==================================================================================================================  =================
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
+`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_        ``common.sql``
+==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html>`_.

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.12.1"
+__version__ = "1.12.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/airflow/providers/openlineage/conf.py

@@ -35,7 +35,7 @@ if os.getenv("PYTEST_VERSION"):
 
     cache = decorator
 else:
-    from airflow.compat.functools import cache
+    from functools import cache
 from airflow.configuration import conf
 
 _CONFIG_SECTION = "openlineage"

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/airflow/providers/openlineage/extractors/manager.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterator
 
+from airflow.providers.common.compat.openlineage.utils.utils import translate_airflow_asset
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors import BaseExtractor, OperatorLineage
 from airflow.providers.openlineage.extractors.base import DefaultExtractor
@@ -25,7 +26,6 @@ from airflow.providers.openlineage.extractors.bash import BashExtractor
 from airflow.providers.openlineage.extractors.python import PythonExtractor
 from airflow.providers.openlineage.utils.utils import (
     get_unknown_source_attribute_run_facet,
-    translate_airflow_dataset,
     try_import_from_string,
 )
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -178,7 +178,16 @@ class ExtractorManager(LoggingMixin):
 
     def get_hook_lineage(self) -> tuple[list[Dataset], list[Dataset]] | None:
         try:
-            from airflow.lineage.hook import get_hook_lineage_collector
+            from importlib.util import find_spec
+
+            if find_spec("airflow.assets"):
+                from airflow.lineage.hook import get_hook_lineage_collector
+            else:
+                # TODO: import from common.compat directly after common.compat providers with
+                # asset_compat_lineage_collector released
+                from airflow.providers.openlineage.utils.asset_compat_lineage_collector import (
+                    get_hook_lineage_collector,
+                )
         except ImportError:
             return None
 
@@ -187,16 +196,14 @@ class ExtractorManager(LoggingMixin):
 
         return (
             [
-                dataset
-                for dataset_info in get_hook_lineage_collector().collected_datasets.inputs
-                if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
-                is not None
+                asset
+                for asset_info in get_hook_lineage_collector().collected_assets.inputs
+                if (asset := translate_airflow_asset(asset_info.asset, asset_info.context)) is not None
             ],
             [
-                dataset
-                for dataset_info in get_hook_lineage_collector().collected_datasets.outputs
-                if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
-                is not None
+                asset
+                for asset_info in get_hook_lineage_collector().collected_assets.outputs
+                if (asset := translate_airflow_asset(asset_info.asset, asset_info.context)) is not None
             ],
         )
 

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1727372276,
+        "source-date-epoch": 1728485291,
         "versions": [
+            "1.12.2",
             "1.12.1",
             "1.12.0",
             "1.11.0",
@@ -88,7 +89,7 @@ def get_provider_info():
                     "disabled_for_operators": {
                         "description": "Exclude some Operators from emitting OpenLineage events by passing a string of semicolon separated\nfull import paths of Operators to disable.\n",
                         "type": "string",
-                        "example": "airflow.operators.bash.BashOperator;airflow.operators.python.PythonOperator",
+                        "example": "airflow.providers.standard.operators.bash.BashOperator; airflow.operators.python.PythonOperator",
                         "default": "",
                         "version_added": "1.1.0",
                     },

apache_airflow_providers_openlineage-1.12.2/airflow/providers/openlineage/utils/asset_compat_lineage_collector.py

@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from importlib.util import find_spec
+
+# TODO: replace this module with common.compat provider once common.compat 1.3.0 released
+
+
+def _get_asset_compat_hook_lineage_collector():
+    from airflow.lineage.hook import get_hook_lineage_collector
+
+    collector = get_hook_lineage_collector()
+
+    if all(
+        getattr(collector, asset_method_name, None)
+        for asset_method_name in ("add_input_asset", "add_output_asset", "collected_assets")
+    ):
+        return collector
+
+    # dataset is renamed as asset in Airflow 3.0
+
+    from functools import wraps
+
+    from airflow.lineage.hook import DatasetLineageInfo, HookLineage
+
+    DatasetLineageInfo.asset = DatasetLineageInfo.dataset
+
+    def rename_dataset_kwargs_as_assets_kwargs(function):
+        @wraps(function)
+        def wrapper(*args, **kwargs):
+            if "asset_kwargs" in kwargs:
+                kwargs["dataset_kwargs"] = kwargs.pop("asset_kwargs")
+
+            if "asset_extra" in kwargs:
+                kwargs["dataset_extra"] = kwargs.pop("asset_extra")
+
+            return function(*args, **kwargs)
+
+        return wrapper
+
+    collector.create_asset = rename_dataset_kwargs_as_assets_kwargs(collector.create_dataset)
+    collector.add_input_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_input_dataset)
+    collector.add_output_asset = rename_dataset_kwargs_as_assets_kwargs(collector.add_output_dataset)
+
+    def collected_assets_compat(collector) -> HookLineage:
+        """Get the collected hook lineage information."""
+        lineage = collector.collected_datasets
+        return HookLineage(
+            [
+                DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
+                for item in lineage.inputs
+            ],
+            [
+                DatasetLineageInfo(dataset=item.dataset, count=item.count, context=item.context)
+                for item in lineage.outputs
+            ],
+        )
+
+    setattr(
+        collector.__class__,
+        "collected_assets",
+        property(lambda collector: collected_assets_compat(collector)),
+    )
+
+    return collector
+
+
+def get_hook_lineage_collector():
+    # HookLineageCollector added in 2.10
+    try:
+        if find_spec("airflow.assets"):
+            # Dataset has been renamed as Asset in 3.0
+            from airflow.lineage.hook import get_hook_lineage_collector
+
+            return get_hook_lineage_collector()
+
+        return _get_asset_compat_hook_lineage_collector()
+    except ImportError:
+
+        class NoOpCollector:
+            """
+            NoOpCollector is a hook lineage collector that does nothing.
+
+            It is used when you want to disable lineage collection.
+            """
+
+            def add_input_asset(self, *_, **__):
+                pass
+
+            def add_output_asset(self, *_, **__):
+                pass
+
+        return NoOpCollector()

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/airflow/providers/openlineage/utils/utils.py

@@ -31,7 +31,6 @@ from openlineage.client.utils import RedactMixin
 from packaging.version import Version
 
 from airflow import __version__ as AIRFLOW_VERSION
-from airflow.datasets import Dataset
 from airflow.exceptions import AirflowProviderDeprecationWarning  # TODO: move this maybe to Airflow's logic?
 from airflow.models import DAG, BaseOperator, DagRun, MappedOperator
 from airflow.providers.openlineage import conf
@@ -54,6 +53,11 @@ from airflow.utils.context import AirflowContextDeprecationWarning
 from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key
 from airflow.utils.module_loading import import_string
 
+try:
+    from airflow.assets import Asset
+except ModuleNotFoundError:
+    from airflow.datasets import Dataset as Asset  # type: ignore[no-redef]
+
 if TYPE_CHECKING:
     from openlineage.client.event_v2 import Dataset as OpenLineageDataset
     from openlineage.client.facet_v2 import RunFacet
@@ -283,8 +287,8 @@ class TaskInstanceInfo(InfoJsonEncodable):
     }
 
 
-class DatasetInfo(InfoJsonEncodable):
-    """Defines encoding Airflow Dataset object to JSON."""
+class AssetInfo(InfoJsonEncodable):
+    """Defines encoding Airflow Asset object to JSON."""
 
     includes = ["uri", "extra"]
 
@@ -335,8 +339,8 @@ class TaskInfo(InfoJsonEncodable):
             if hasattr(task, "task_group") and getattr(task.task_group, "_group_id", None)
             else None
         ),
-        "inlets": lambda task: [DatasetInfo(i) for i in task.inlets if isinstance(i, Dataset)],
-        "outlets": lambda task: [DatasetInfo(o) for o in task.outlets if isinstance(o, Dataset)],
+        "inlets": lambda task: [AssetInfo(i) for i in task.inlets if isinstance(i, Asset)],
+        "outlets": lambda task: [AssetInfo(o) for o in task.outlets if isinstance(o, Asset)],
     }
 
 
@@ -641,19 +645,29 @@ def should_use_external_connection(hook) -> bool:
     return True
 
 
-def translate_airflow_dataset(dataset: Dataset, lineage_context) -> OpenLineageDataset | None:
+def translate_airflow_asset(asset: Asset, lineage_context) -> OpenLineageDataset | None:
     """
-    Convert a Dataset with an AIP-60 compliant URI to an OpenLineageDataset.
+    Convert a Asset with an AIP-60 compliant URI to an OpenLineageDataset.
 
-    This function returns None if no URI normalizer is defined, no dataset converter is found or
+    This function returns None if no URI normalizer is defined, no asset converter is found or
     some core Airflow changes are missing and ImportError is raised.
     """
     try:
-        from airflow.datasets import _get_normalized_scheme
+        from airflow.assets import _get_normalized_scheme
+    except ModuleNotFoundError:
+        try:
+            from airflow.datasets import _get_normalized_scheme  # type: ignore[no-redef]
+        except ImportError:
+            return None
+
+    try:
         from airflow.providers_manager import ProvidersManager
 
-        ol_converters = ProvidersManager().dataset_to_openlineage_converters
-        normalized_uri = dataset.normalized_uri
+        ol_converters = getattr(ProvidersManager(), "asset_to_openlineage_converters", None)
+        if not ol_converters:
+            ol_converters = ProvidersManager().dataset_to_openlineage_converters  # type: ignore[attr-defined]
+
+        normalized_uri = asset.normalized_uri
     except (ImportError, AttributeError):
         return None
 
@@ -666,4 +680,4 @@ def translate_airflow_dataset(dataset: Dataset, lineage_context) -> OpenLineageD
     if (airflow_to_ol_converter := ol_converters.get(normalized_scheme)) is None:
         return None
 
-    return airflow_to_ol_converter(Dataset(uri=normalized_uri, extra=dataset.extra), lineage_context)
+    return airflow_to_ol_converter(Asset(uri=normalized_uri, extra=asset.extra), lineage_context)

{apache_airflow_providers_openlineage-1.12.1 → apache_airflow_providers_openlineage-1.12.2}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-openlineage"
-version = "1.12.1"
+version = "1.12.2"
 description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -47,14 +47,13 @@ classifiers = [
     "Framework :: Apache Airflow",
     "Framework :: Apache Airflow :: Provider",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
 ]
-requires-python = "~=3.8"
+requires-python = "~=3.9"
 dependencies = [
     "apache-airflow-providers-common-compat>=1.2.0",
     "apache-airflow-providers-common-sql>=1.6.0",
@@ -65,8 +64,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.12.2/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -78,6 +77,9 @@ provider_info = "airflow.providers.openlineage.get_provider_info:get_provider_in
 [project.entry-points."airflow.plugins"]
 openlineage = "airflow.providers.openlineage.plugins.openlineage:OpenLineageProviderPlugin"
 [project.optional-dependencies]
+"common.compat" = [
+    "apache-airflow-providers-common-compat",
+]
 "common.sql" = [
     "apache-airflow-providers-common-sql",
 ]