apache-airflow-providers-openlineage 1.6.0__tar.gz → 1.7.0__tar.gz

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.6.0
+Version: 1.7.0
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow-providers-common-sql>=1.6.0
 Requires-Dist: apache-airflow>=2.7.0
@@ -27,8 +28,8 @@ Requires-Dist: openlineage-integration-common>=0.28.0
 Requires-Dist: openlineage-python>=0.28.0
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -79,7 +80,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.6.0``
+Release: ``1.7.0``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -92,7 +93,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/>`_.
 
 Installation
 ------------
@@ -101,7 +102,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-openlineage``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11
+The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -136,4 +137,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.6.0``
+Release: ``1.7.0``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/>`_.
 
 Installation
 ------------
@@ -64,7 +64,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-openlineage``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11
+The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -99,4 +99,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "1.6.0"
+__version__ = "1.7.0"
 
 try:
     from airflow import __version__ as airflow_version

apache_airflow_providers_openlineage-1.7.0/airflow/providers/openlineage/conf.py

@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from airflow.compat.functools import cache
+from airflow.configuration import conf
+
+_CONFIG_SECTION = "openlineage"
+
+
+@cache
+def config_path(check_legacy_env_var: bool = True) -> str:
+    """[openlineage] config_path."""
+    option = conf.get(_CONFIG_SECTION, "config_path", fallback="")
+    if check_legacy_env_var and not option:
+        option = os.getenv("OPENLINEAGE_CONFIG", "")
+    return option
+
+
+@cache
+def is_source_enabled() -> bool:
+    """[openlineage] disable_source_code."""
+    option = conf.get(_CONFIG_SECTION, "disable_source_code", fallback="")
+    if not option:
+        option = os.getenv("OPENLINEAGE_AIRFLOW_DISABLE_SOURCE_CODE", "")
+    return option.lower() not in ("true", "1", "t")
+
+
+@cache
+def disabled_operators() -> set[str]:
+    """[openlineage] disabled_for_operators."""
+    option = conf.get(_CONFIG_SECTION, "disabled_for_operators", fallback="")
+    return set(operator.strip() for operator in option.split(";") if operator.strip())
+
+
+@cache
+def selective_enable() -> bool:
+    return conf.getboolean(_CONFIG_SECTION, "selective_enable", fallback=False)
+
+
+@cache
+def custom_extractors() -> set[str]:
+    """[openlineage] extractors."""
+    option = conf.get(_CONFIG_SECTION, "extractors", fallback="")
+    if not option:
+        option = os.getenv("OPENLINEAGE_EXTRACTORS", "")
+    return set(extractor.strip() for extractor in option.split(";") if extractor.strip())
+
+
+@cache
+def namespace() -> str:
+    """[openlineage] namespace."""
+    option = conf.get(_CONFIG_SECTION, "namespace", fallback="")
+    if not option:
+        option = os.getenv("OPENLINEAGE_NAMESPACE", "default")
+    return option
+
+
+@cache
+def transport() -> dict[str, Any]:
+    """[openlineage] transport."""
+    option = conf.getjson(_CONFIG_SECTION, "transport", fallback={})
+    if not isinstance(option, dict):
+        raise ValueError(f"OpenLineage transport `{option}` is not a dict")
+    return option
+
+
+@cache
+def is_disabled() -> bool:
+    """[openlineage] disabled + some extra checks."""
+
+    def _is_true(val):
+        return str(val).lower().strip() in ("true", "1", "t")
+
+    option = conf.get(_CONFIG_SECTION, "disabled", fallback="")
+    if _is_true(option):
+        return True
+
+    option = os.getenv("OPENLINEAGE_DISABLED", "")
+    if _is_true(option):
+        return True
+
+    # Check if both 'transport' and 'config_path' are not present and also
+    # if legacy 'OPENLINEAGE_URL' environment variables is not set
+    return transport() == {} and config_path(True) == "" and os.getenv("OPENLINEAGE_URL", "") == ""

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/extractors/base.py

@@ -18,12 +18,10 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from functools import cached_property
 from typing import TYPE_CHECKING
 
 from attrs import Factory, define
 
-from airflow.configuration import conf
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import TaskInstanceState
 
@@ -64,33 +62,10 @@ class BaseExtractor(ABC, LoggingMixin):
         """
         raise NotImplementedError()
 
-    @cached_property
-    def disabled_operators(self) -> set[str]:
-        return set(
-            operator.strip() for operator in conf.get("openlineage", "disabled_for_operators").split(";")
-        )
-
-    @cached_property
-    def _is_operator_disabled(self) -> bool:
-        fully_qualified_class_name = (
-            self.operator.__class__.__module__ + "." + self.operator.__class__.__name__
-        )
-        return fully_qualified_class_name in self.disabled_operators
-
-    def validate(self):
-        assert self.operator.task_type in self.get_operator_classnames()
-
     @abstractmethod
-    def _execute_extraction(self) -> OperatorLineage | None:
-        ...
+    def _execute_extraction(self) -> OperatorLineage | None: ...
 
     def extract(self) -> OperatorLineage | None:
-        if self._is_operator_disabled:
-            self.log.debug(
-                f"Skipping extraction for operator {self.operator.task_type} "
-                "due to its presence in [openlineage] openlineage_disabled_for_operators."
-            )
-            return None
         return self._execute_extraction()
 
     def extract_on_complete(self, task_instance) -> OperatorLineage | None:
@@ -121,18 +96,12 @@ class DefaultExtractor(BaseExtractor):
             return None
         except AttributeError:
             self.log.debug(
-                f"Operator {self.operator.task_type} does not have the "
-                "get_openlineage_facets_on_start method."
+                "Operator %s does not have the get_openlineage_facets_on_start method.",
+                self.operator.task_type,
             )
             return None
 
     def extract_on_complete(self, task_instance) -> OperatorLineage | None:
-        if self._is_operator_disabled:
-            self.log.debug(
-                f"Skipping extraction for operator {self.operator.task_type} "
-                "due to its presence in [openlineage] openlineage_disabled_for_operators."
-            )
-            return None
         if task_instance.state == TaskInstanceState.FAILED:
             on_failed = getattr(self.operator, "get_openlineage_facets_on_failure", None)
             if on_failed and callable(on_failed):

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/extractors/bash.py

@@ -19,15 +19,9 @@ from __future__ import annotations
 
 from openlineage.client.facet import SourceCodeJobFacet
 
+from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
-from airflow.providers.openlineage.plugins.facets import (
-    UnknownOperatorAttributeRunFacet,
-    UnknownOperatorInstance,
-)
-from airflow.providers.openlineage.utils.utils import (
-    get_filtered_unknown_operator_keys,
-    is_source_enabled,
-)
+from airflow.providers.openlineage.utils.utils import get_unknown_source_attribute_run_facet
 
 """
 :meta private:
@@ -51,7 +45,7 @@ class BashExtractor(BaseExtractor):
 
     def _execute_extraction(self) -> OperatorLineage | None:
         job_facets: dict = {}
-        if is_source_enabled():
+        if conf.is_source_enabled():
             job_facets = {
                 "sourceCode": SourceCodeJobFacet(
                     language="bash",
@@ -62,19 +56,9 @@ class BashExtractor(BaseExtractor):
 
         return OperatorLineage(
             job_facets=job_facets,
-            run_facets={
-                # The BashOperator is recorded as an "unknownSource" even though we have an
-                # extractor, as the <i>data lineage</i> cannot be determined from the operator
-                # directly.
-                "unknownSourceAttribute": UnknownOperatorAttributeRunFacet(
-                    unknownItems=[
-                        UnknownOperatorInstance(
-                            name="BashOperator",
-                            properties=get_filtered_unknown_operator_keys(self.operator),
-                        )
-                    ]
-                )
-            },
+            # The BashOperator is recorded as an "unknownSource" even though we have an extractor,
+            # as the <i>data lineage</i> cannot be determined from the operator directly.
+            run_facets=get_unknown_source_attribute_run_facet(task=self.operator, name="BashOperator"),
         )
 
     def extract(self) -> OperatorLineage | None:

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/extractors/manager.py

@@ -16,20 +16,15 @@
 # under the License.
 from __future__ import annotations
 
-import os
 from contextlib import suppress
 from typing import TYPE_CHECKING, Iterator
 
-from airflow.configuration import conf
+from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors import BaseExtractor, OperatorLineage
 from airflow.providers.openlineage.extractors.base import DefaultExtractor
 from airflow.providers.openlineage.extractors.bash import BashExtractor
 from airflow.providers.openlineage.extractors.python import PythonExtractor
-from airflow.providers.openlineage.plugins.facets import (
-    UnknownOperatorAttributeRunFacet,
-    UnknownOperatorInstance,
-)
-from airflow.providers.openlineage.utils.utils import get_filtered_unknown_operator_keys
+from airflow.providers.openlineage.utils.utils import get_unknown_source_attribute_run_facet
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.module_loading import import_string
 
@@ -65,22 +60,17 @@ class ExtractorManager(LoggingMixin):
             for operator_class in extractor.get_operator_classnames():
                 self.extractors[operator_class] = extractor
 
-        # Semicolon-separated extractors in Airflow configuration or OPENLINEAGE_EXTRACTORS variable.
-        # Extractors should implement BaseExtractor
-        env_extractors = conf.get("openlineage", "extractors", fallback=os.getenv("OPENLINEAGE_EXTRACTORS"))
-        # skip either when it's empty string or None
-        if env_extractors:
-            for extractor in env_extractors.split(";"):
-                extractor: type[BaseExtractor] = try_import_from_string(extractor.strip())
-                for operator_class in extractor.get_operator_classnames():
-                    if operator_class in self.extractors:
-                        self.log.debug(
-                            "Duplicate extractor found for `%s`. `%s` will be used instead of `%s`",
-                            operator_class,
-                            extractor,
-                            self.extractors[operator_class],
-                        )
-                    self.extractors[operator_class] = extractor
+        for extractor_path in conf.custom_extractors():
+            extractor: type[BaseExtractor] = try_import_from_string(extractor_path)
+            for operator_class in extractor.get_operator_classnames():
+                if operator_class in self.extractors:
+                    self.log.debug(
+                        "Duplicate extractor found for `%s`. `%s` will be used instead of `%s`",
+                        operator_class,
+                        extractor_path,
+                        self.extractors[operator_class],
+                    )
+                self.extractors[operator_class] = extractor
 
     def add_extractor(self, operator_class: str, extractor: type[BaseExtractor]):
         self.extractors[operator_class] = extractor
@@ -121,16 +111,7 @@ class ExtractorManager(LoggingMixin):
 
             # Only include the unkonwnSourceAttribute facet if there is no extractor
             task_metadata = OperatorLineage(
-                run_facets={
-                    "unknownSourceAttribute": UnknownOperatorAttributeRunFacet(
-                        unknownItems=[
-                            UnknownOperatorInstance(
-                                name=task.task_type,
-                                properties=get_filtered_unknown_operator_keys(task),
-                            )
-                        ]
-                    )
-                },
+                run_facets=get_unknown_source_attribute_run_facet(task=task),
             )
             inlets = task.get_inlet_defs()
             outlets = task.get_outlet_defs()

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/extractors/python.py

@@ -22,15 +22,9 @@ from typing import Callable
 
 from openlineage.client.facet import SourceCodeJobFacet
 
+from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
-from airflow.providers.openlineage.plugins.facets import (
-    UnknownOperatorAttributeRunFacet,
-    UnknownOperatorInstance,
-)
-from airflow.providers.openlineage.utils.utils import (
-    get_filtered_unknown_operator_keys,
-    is_source_enabled,
-)
+from airflow.providers.openlineage.utils.utils import get_unknown_source_attribute_run_facet
 
 """
 :meta private:
@@ -55,7 +49,7 @@ class PythonExtractor(BaseExtractor):
     def _execute_extraction(self) -> OperatorLineage | None:
         source_code = self.get_source_code(self.operator.python_callable)
         job_facet: dict = {}
-        if is_source_enabled() and source_code:
+        if conf.is_source_enabled() and source_code:
             job_facet = {
                 "sourceCode": SourceCodeJobFacet(
                     language="python",
@@ -65,19 +59,9 @@ class PythonExtractor(BaseExtractor):
             }
         return OperatorLineage(
             job_facets=job_facet,
-            run_facets={
-                # The PythonOperator is recorded as an "unknownSource" even though we have an
-                # extractor, as the data lineage cannot be determined from the operator
-                # directly.
-                "unknownSourceAttribute": UnknownOperatorAttributeRunFacet(
-                    unknownItems=[
-                        UnknownOperatorInstance(
-                            name="PythonOperator",
-                            properties=get_filtered_unknown_operator_keys(self.operator),
-                        )
-                    ]
-                )
-            },
+            # The PythonOperator is recorded as an "unknownSource" even though we have an extractor,
+            # as the <i>data lineage</i> cannot be determined from the operator directly.
+            run_facets=get_unknown_source_attribute_run_facet(task=self.operator, name="PythonOperator"),
         )
 
     def get_source_code(self, callable: Callable) -> str | None:

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1709555960,
+        "source-date-epoch": 1712666247,
         "versions": [
+            "1.7.0",
             "1.6.0",
             "1.5.0",
             "1.4.0",
@@ -82,6 +83,13 @@ def get_provider_info():
                         "default": "",
                         "version_added": "1.1.0",
                     },
+                    "selective_enable": {
+                        "description": "If this setting is enabled, OpenLineage integration won't collect and emit metadata,\nunless you explicitly enable it per `DAG` or `Task` using  `enable_lineage` method.\n",
+                        "type": "boolean",
+                        "default": "False",
+                        "example": None,
+                        "version_added": "1.7.0",
+                    },
                     "namespace": {
                         "description": "Set namespace that the lineage data belongs to, so that if you use multiple OpenLineage producers,\nevents coming from them will be logically separated.\n",
                         "version_added": None,

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/plugins/adapter.py

@@ -16,7 +16,6 @@
 # under the License.
 from __future__ import annotations
 
-import os
 import uuid
 from contextlib import ExitStack
 from typing import TYPE_CHECKING
@@ -37,8 +36,7 @@ from openlineage.client.facet import (
 )
 from openlineage.client.run import Job, Run, RunEvent, RunState
 
-from airflow.configuration import conf
-from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION
+from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
 from airflow.providers.openlineage.utils.utils import OpenLineageRedactor
 from airflow.stats import Stats
 from airflow.utils.log.logging_mixin import LoggingMixin
@@ -48,12 +46,6 @@ if TYPE_CHECKING:
     from airflow.providers.openlineage.extractors import OperatorLineage
     from airflow.utils.log.secrets_masker import SecretsMasker
 
-_DAG_DEFAULT_NAMESPACE = "default"
-
-_DAG_NAMESPACE = conf.get(
-    "openlineage", "namespace", fallback=os.getenv("OPENLINEAGE_NAMESPACE", _DAG_DEFAULT_NAMESPACE)
-)
-
 _PRODUCER = f"https://github.com/apache/airflow/tree/providers-openlineage/{OPENLINEAGE_PROVIDER_VERSION}"
 
 set_producer(_PRODUCER)
@@ -88,18 +80,16 @@ class OpenLineageAdapter(LoggingMixin):
 
     def get_openlineage_config(self) -> dict | None:
         # First, try to read from YAML file
-        openlineage_config_path = conf.get("openlineage", "config_path")
+        openlineage_config_path = conf.config_path(check_legacy_env_var=False)
         if openlineage_config_path:
             config = self._read_yaml_config(openlineage_config_path)
             if config:
                 return config.get("transport", None)
         # Second, try to get transport config
-        transport = conf.getjson("openlineage", "transport")
-        if not transport:
+        transport_config = conf.transport()
+        if not transport_config:
             return None
-        elif not isinstance(transport, dict):
-            raise ValueError(f"{transport} is not a dict")
-        return transport
+        return transport_config
 
     def _read_yaml_config(self, path: str) -> dict | None:
         with open(path) as config_file:
@@ -107,14 +97,14 @@ class OpenLineageAdapter(LoggingMixin):
 
     @staticmethod
     def build_dag_run_id(dag_id, dag_run_id):
-        return str(uuid.uuid3(uuid.NAMESPACE_URL, f"{_DAG_NAMESPACE}.{dag_id}.{dag_run_id}"))
+        return str(uuid.uuid3(uuid.NAMESPACE_URL, f"{conf.namespace()}.{dag_id}.{dag_run_id}"))
 
     @staticmethod
     def build_task_instance_run_id(dag_id, task_id, execution_date, try_number):
         return str(
             uuid.uuid3(
                 uuid.NAMESPACE_URL,
-                f"{_DAG_NAMESPACE}.{dag_id}.{task_id}.{execution_date}.{try_number}",
+                f"{conf.namespace()}.{dag_id}.{task_id}.{execution_date}.{try_number}",
             )
         )
 
@@ -353,7 +343,7 @@ class OpenLineageAdapter(LoggingMixin):
         if parent_run_id:
             parent_run_facet = ParentRunFacet.create(
                 runId=parent_run_id,
-                namespace=_DAG_NAMESPACE,
+                namespace=conf.namespace(),
                 name=parent_job_name or job_name,
             )
             facets.update(
@@ -396,4 +386,4 @@ class OpenLineageAdapter(LoggingMixin):
 
         facets.update({"jobType": job_type})
 
-        return Job(_DAG_NAMESPACE, job_name, facets)
+        return Job(conf.namespace(), job_name, facets)

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/plugins/facets.py

@@ -17,10 +17,17 @@
 from __future__ import annotations
 
 from attrs import define
+from deprecated import deprecated
 from openlineage.client.facet import BaseFacet
 from openlineage.client.utils import RedactMixin
 
+from airflow.exceptions import AirflowProviderDeprecationWarning
 
+
+@deprecated(
+    reason="To be removed in the next release. Make sure to use information from AirflowRunFacet instead.",
+    category=AirflowProviderDeprecationWarning,
+)
 @define(slots=False)
 class AirflowMappedTaskRunFacet(BaseFacet):
     """Run facet containing information about mapped tasks."""
@@ -66,6 +73,10 @@ class UnknownOperatorInstance(RedactMixin):
     _skip_redact = ["name", "type"]
 
 
+@deprecated(
+    reason="To be removed in the next release. Make sure to use information from AirflowRunFacet instead.",
+    category=AirflowProviderDeprecationWarning,
+)
 @define(slots=False)
 class UnknownOperatorAttributeRunFacet(BaseFacet):
     """RunFacet that describes unknown operators in an Airflow DAG."""

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/plugins/listener.py

@@ -30,6 +30,8 @@ from airflow.providers.openlineage.utils.utils import (
     get_airflow_run_facet,
     get_custom_facets,
     get_job_name,
+    is_operator_disabled,
+    is_selective_lineage_enabled,
     print_warning,
 )
 from airflow.stats import Stats
@@ -59,17 +61,31 @@ class OpenLineageListener:
         task_instance: TaskInstance,
         session: Session,  # This will always be QUEUED
     ):
-        if not hasattr(task_instance, "task"):
+        if not getattr(task_instance, "task", None) is not None:
             self.log.warning(
-                f"No task set for TI object task_id: {task_instance.task_id} - "
-                f"dag_id: {task_instance.dag_id} - run_id {task_instance.run_id}"
+                "No task set for TI object task_id: %s - dag_id: %s - run_id %s",
+                task_instance.task_id,
+                task_instance.dag_id,
+                task_instance.run_id,
             )
             return
 
         self.log.debug("OpenLineage listener got notification about task instance start")
         dagrun = task_instance.dag_run
         task = task_instance.task
+        if TYPE_CHECKING:
+            assert task
         dag = task.dag
+        if is_operator_disabled(task):
+            self.log.debug(
+                "Skipping OpenLineage event emission for operator %s "
+                "due to its presence in [openlineage] disabled_for_operators.",
+                task.task_type,
+            )
+            return None
+
+        if not is_selective_lineage_enabled(task):
+            return
 
         @print_warning(self.log)
         def on_running():
@@ -127,7 +143,19 @@ class OpenLineageListener:
 
         dagrun = task_instance.dag_run
         task = task_instance.task
+        if TYPE_CHECKING:
+            assert task
         dag = task.dag
+        if is_operator_disabled(task):
+            self.log.debug(
+                "Skipping OpenLineage event emission for operator %s "
+                "due to its presence in [openlineage] disabled_for_operators.",
+                task.task_type,
+            )
+            return None
+
+        if not is_selective_lineage_enabled(task):
+            return
 
         @print_warning(self.log)
         def on_success():
@@ -170,7 +198,19 @@ class OpenLineageListener:
 
         dagrun = task_instance.dag_run
         task = task_instance.task
+        if TYPE_CHECKING:
+            assert task
         dag = task.dag
+        if is_operator_disabled(task):
+            self.log.debug(
+                "Skipping OpenLineage event emission for operator %s "
+                "due to its presence in [openlineage] disabled_for_operators.",
+                task.task_type,
+            )
+            return None
+
+        if not is_selective_lineage_enabled(task):
+            return
 
         @print_warning(self.log)
         def on_failure():
@@ -220,12 +260,13 @@ class OpenLineageListener:
     @hookimpl
     def before_stopping(self, component):
         self.log.debug("before_stopping: %s", component.__class__.__name__)
-        # TODO: configure this with Airflow config
         with timeout(30):
             self.executor.shutdown(wait=True)
 
     @hookimpl
     def on_dag_run_running(self, dag_run: DagRun, msg: str):
+        if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
+            return
         data_interval_start = dag_run.data_interval_start.isoformat() if dag_run.data_interval_start else None
         data_interval_end = dag_run.data_interval_end.isoformat() if dag_run.data_interval_end else None
         self.executor.submit(
@@ -238,6 +279,8 @@ class OpenLineageListener:
 
     @hookimpl
     def on_dag_run_success(self, dag_run: DagRun, msg: str):
+        if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
+            return
         if not self.executor:
             self.log.debug("Executor have not started before `on_dag_run_success`")
             return
@@ -245,6 +288,8 @@ class OpenLineageListener:
 
     @hookimpl
     def on_dag_run_failed(self, dag_run: DagRun, msg: str):
+        if dag_run.dag and not is_selective_lineage_enabled(dag_run.dag):
+            return
         if not self.executor:
             self.log.debug("Executor have not started before `on_dag_run_failed`")
             return

apache_airflow_providers_openlineage-1.7.0/airflow/providers/openlineage/plugins/macros.py

@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.providers.openlineage import conf
+from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
+from airflow.providers.openlineage.utils.utils import get_job_name
+
+if TYPE_CHECKING:
+    from airflow.models import TaskInstance
+
+
+def lineage_job_namespace():
+    """
+    Macro function which returns Airflow OpenLineage namespace.
+
+    .. seealso::
+        For more information take a look at the guide:
+        :ref:`howto/macros:openlineage`
+    """
+    return conf.namespace()
+
+
+def lineage_job_name(task_instance: TaskInstance):
+    """
+    Macro function which returns Airflow task name in OpenLineage format (`<dag_id>.<task_id>`).
+
+    .. seealso::
+        For more information take a look at the guide:
+        :ref:`howto/macros:openlineage`
+    """
+    return get_job_name(task_instance)
+
+
+def lineage_run_id(task_instance: TaskInstance):
+    """
+    Macro function which returns the generated run id (UUID) for a given task.
+
+    This can be used to forward the run id from a task to a child run so the job hierarchy is preserved.
+
+    .. seealso::
+        For more information take a look at the guide:
+        :ref:`howto/macros:openlineage`
+    """
+    return OpenLineageAdapter.build_task_instance_run_id(
+        dag_id=task_instance.dag_id,
+        task_id=task_instance.task_id,
+        execution_date=task_instance.execution_date,
+        try_number=task_instance.try_number,
+    )
+
+
+def lineage_parent_id(task_instance: TaskInstance):
+    """
+    Macro function which returns a unique identifier of given task that can be used to create ParentRunFacet.
+
+    This identifier is composed of the namespace, job name, and generated run id for given task, structured
+    as '{namespace}/{job_name}/{run_id}'. This can be used to forward task information from a task to a child
+    run so the job hierarchy is preserved. Child run can easily create ParentRunFacet from these information.
+
+    .. seealso::
+        For more information take a look at the guide:
+        :ref:`howto/macros:openlineage`
+    """
+    return "/".join(
+        (
+            lineage_job_namespace(),
+            lineage_job_name(task_instance),
+            lineage_run_id(task_instance),
+        )
+    )

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/plugins/openlineage.py

@@ -16,25 +16,15 @@
 # under the License.
 from __future__ import annotations
 
-import os
-
-from airflow.configuration import conf
 from airflow.plugins_manager import AirflowPlugin
+from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
-from airflow.providers.openlineage.plugins.macros import lineage_parent_id, lineage_run_id
-
-
-def _is_disabled() -> bool:
-    return (
-        conf.getboolean("openlineage", "disabled", fallback=False)
-        or os.getenv("OPENLINEAGE_DISABLED", "false").lower() == "true"
-        or (
-            conf.get("openlineage", "transport", fallback="") == ""
-            and conf.get("openlineage", "config_path", fallback="") == ""
-            and os.getenv("OPENLINEAGE_URL", "") == ""
-            and os.getenv("OPENLINEAGE_CONFIG", "") == ""
-        )
-    )
+from airflow.providers.openlineage.plugins.macros import (
+    lineage_job_name,
+    lineage_job_namespace,
+    lineage_parent_id,
+    lineage_run_id,
+)
 
 
 class OpenLineageProviderPlugin(AirflowPlugin):
@@ -46,6 +36,6 @@ class OpenLineageProviderPlugin(AirflowPlugin):
     """
 
     name = "OpenLineageProviderPlugin"
-    if not _is_disabled():
-        macros = [lineage_run_id, lineage_parent_id]
+    if not conf.is_disabled():
+        macros = [lineage_job_namespace, lineage_job_name, lineage_run_id, lineage_parent_id]
         listeners = [get_openlineage_listener()]

apache_airflow_providers_openlineage-1.7.0/airflow/providers/openlineage/utils/selective_enable.py

@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import logging
+from typing import TypeVar
+
+from airflow.models import DAG, Operator, Param
+from airflow.models.xcom_arg import XComArg
+
+ENABLE_OL_PARAM_NAME = "_selective_enable_ol"
+ENABLE_OL_PARAM = Param(True, const=True)
+DISABLE_OL_PARAM = Param(False, const=False)
+T = TypeVar("T", bound="DAG | Operator")
+
+log = logging.getLogger(__name__)
+
+
+def enable_lineage(obj: T) -> T:
+    """Set selective enable OpenLineage parameter to True.
+
+    The method also propagates param to tasks if the object is DAG.
+    """
+    if isinstance(obj, XComArg):
+        enable_lineage(obj.operator)
+        return obj
+    # propagate param to tasks
+    if isinstance(obj, DAG):
+        for task in obj.task_dict.values():
+            enable_lineage(task)
+    obj.params[ENABLE_OL_PARAM_NAME] = ENABLE_OL_PARAM
+    return obj
+
+
+def disable_lineage(obj: T) -> T:
+    """Set selective enable OpenLineage parameter to False.
+
+    The method also propagates param to tasks if the object is DAG.
+    """
+    if isinstance(obj, XComArg):
+        disable_lineage(obj.operator)
+        return obj
+    # propagate param to tasks
+    if isinstance(obj, DAG):
+        for task in obj.task_dict.values():
+            disable_lineage(task)
+    obj.params[ENABLE_OL_PARAM_NAME] = DISABLE_OL_PARAM
+    return obj
+
+
+def is_task_lineage_enabled(task: Operator) -> bool:
+    """Check if selective enable OpenLineage parameter is set to True on task level."""
+    if task.params.get(ENABLE_OL_PARAM_NAME) is False:
+        log.debug(
+            "OpenLineage event emission suppressed. Task for this functionality is selectively disabled."
+        )
+    return task.params.get(ENABLE_OL_PARAM_NAME) is True
+
+
+def is_dag_lineage_enabled(dag: DAG) -> bool:
+    """Check if DAG is selectively enabled to emit OpenLineage events.
+
+    The method also checks if selective enable parameter is set to True
+    or if any of the tasks in DAG is selectively enabled.
+    """
+    if dag.params.get(ENABLE_OL_PARAM_NAME) is False:
+        log.debug(
+            "OpenLineage event emission suppressed. DAG for this functionality is selectively disabled."
+        )
+    return dag.params.get(ENABLE_OL_PARAM_NAME) is True or any(
+        is_task_lineage_enabled(task) for task in dag.tasks
+    )

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/airflow/providers/openlineage/utils/utils.py

@@ -20,124 +20,43 @@ from __future__ import annotations
 import datetime
 import json
 import logging
-import os
 from contextlib import suppress
 from functools import wraps
 from typing import TYPE_CHECKING, Any, Iterable
-from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 
 import attrs
-from attrs import asdict
+from openlineage.client.utils import RedactMixin  # TODO: move this maybe to Airflow's logic?
 
-# TODO: move this maybe to Airflow's logic?
-from openlineage.client.utils import RedactMixin
-
-from airflow.compat.functools import cache
-from airflow.configuration import conf
+from airflow.models import DAG, BaseOperator, MappedOperator
+from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.plugins.facets import (
     AirflowMappedTaskRunFacet,
     AirflowRunFacet,
+    UnknownOperatorAttributeRunFacet,
+    UnknownOperatorInstance,
+)
+from airflow.providers.openlineage.utils.selective_enable import (
+    is_dag_lineage_enabled,
+    is_task_lineage_enabled,
 )
 from airflow.utils.context import AirflowContextDeprecationWarning
 from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key
 
 if TYPE_CHECKING:
-    from airflow.models import DAG, BaseOperator, Connection, DagRun, TaskInstance
+    from airflow.models import DagRun, TaskInstance
 
 
 log = logging.getLogger(__name__)
 _NOMINAL_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 
-def openlineage_job_name(dag_id: str, task_id: str) -> str:
-    return f"{dag_id}.{task_id}"
-
-
 def get_operator_class(task: BaseOperator) -> type:
     if task.__class__.__name__ in ("DecoratedMappedOperator", "MappedOperator"):
         return task.operator_class
     return task.__class__
 
 
-def to_json_encodable(task: BaseOperator) -> dict[str, object]:
-    def _task_encoder(obj):
-        from airflow.models import DAG
-
-        if isinstance(obj, datetime.datetime):
-            return obj.isoformat()
-        elif isinstance(obj, DAG):
-            return {
-                "dag_id": obj.dag_id,
-                "tags": obj.tags,
-                "schedule_interval": obj.schedule_interval,
-                "timetable": obj.timetable.serialize(),
-            }
-        else:
-            return str(obj)
-
-    return json.loads(json.dumps(task.__dict__, default=_task_encoder))
-
-
-def url_to_https(url) -> str | None:
-    # Ensure URL exists
-    if not url:
-        return None
-
-    base_url = None
-    if url.startswith("git@"):
-        part = url.split("git@")[1:2]
-        if part:
-            base_url = f'https://{part[0].replace(":", "/", 1)}'
-    elif url.startswith("https://"):
-        base_url = url
-
-    if not base_url:
-        raise ValueError(f"Unable to extract location from: {url}")
-
-    if base_url.endswith(".git"):
-        base_url = base_url[:-4]
-    return base_url
-
-
-def redacted_connection_uri(conn: Connection, filtered_params=None, filtered_prefixes=None):
-    """
-    Return the connection URI for the given Connection.
-
-    This method additionally filters URI by removing query parameters that are known to carry sensitive data
-    like username, password, access key.
-    """
-    if filtered_prefixes is None:
-        filtered_prefixes = []
-    if filtered_params is None:
-        filtered_params = []
-
-    def filter_key_params(k: str):
-        return k not in filtered_params and any(substr in k for substr in filtered_prefixes)
-
-    conn_uri = conn.get_uri()
-    parsed = urlparse(conn_uri)
-
-    # Remove username and password
-    netloc = f"{parsed.hostname}" + (f":{parsed.port}" if parsed.port else "")
-    parsed = parsed._replace(netloc=netloc)
-    if parsed.query:
-        query_dict = dict(parse_qsl(parsed.query))
-        if conn.EXTRA_KEY in query_dict:
-            query_dict = json.loads(query_dict[conn.EXTRA_KEY])
-        filtered_qs = {k: v for k, v in query_dict.items() if not filter_key_params(k)}
-        parsed = parsed._replace(query=urlencode(filtered_qs))
-    return urlunparse(parsed)
-
-
-def get_connection(conn_id) -> Connection | None:
-    from airflow.hooks.base import BaseHook
-
-    with suppress(Exception):
-        return BaseHook.get_connection(conn_id=conn_id)
-    return None
-
-
-def get_job_name(task):
+def get_job_name(task: TaskInstance) -> str:
     return f"{task.dag_id}.{task.task_id}"
 
 
@@ -150,6 +69,26 @@ def get_custom_facets(task_instance: TaskInstance | None = None) -> dict[str, An
     return custom_facets
 
 
+def get_fully_qualified_class_name(operator: BaseOperator | MappedOperator) -> str:
+    return operator.__class__.__module__ + "." + operator.__class__.__name__
+
+
+def is_operator_disabled(operator: BaseOperator | MappedOperator) -> bool:
+    return get_fully_qualified_class_name(operator) in conf.disabled_operators()
+
+
+def is_selective_lineage_enabled(obj: DAG | BaseOperator | MappedOperator) -> bool:
+    """If selective enable is active check if DAG or Task is enabled to emit events."""
+    if not conf.selective_enable():
+        return True
+    if isinstance(obj, DAG):
+        return is_dag_lineage_enabled(obj)
+    elif isinstance(obj, (BaseOperator, MappedOperator)):
+        return is_task_lineage_enabled(obj)
+    else:
+        raise TypeError("is_selective_lineage_enabled can only be used on DAG or Operator objects")
+
+
 class InfoJsonEncodable(dict):
     """
     Airflow objects might not be json-encodable overall.
@@ -205,7 +144,7 @@ class InfoJsonEncodable(dict):
 
     def _include_fields(self):
         if self.includes and self.excludes:
-            raise Exception("Don't use both includes and excludes.")
+            raise ValueError("Don't use both includes and excludes.")
         if self.includes:
             for field in self.includes:
                 if field not in self._fields and hasattr(self.obj, field):
@@ -256,23 +195,34 @@ class TaskInfo(InfoJsonEncodable):
     """Defines encoding BaseOperator/AbstractOperator object to JSON."""
 
     renames = {
-        "_BaseOperator__init_kwargs": "args",
         "_BaseOperator__from_mapped": "mapped",
         "_downstream_task_ids": "downstream_task_ids",
         "_upstream_task_ids": "upstream_task_ids",
+        "_is_setup": "is_setup",
+        "_is_teardown": "is_teardown",
     }
-    excludes = [
-        "_BaseOperator__instantiated",
-        "_dag",
-        "_hook",
-        "_log",
-        "_outlets",
-        "_inlets",
-        "_lock_for_execution",
-        "handler",
-        "params",
-        "python_callable",
-        "retry_delay",
+    includes = [
+        "depends_on_past",
+        "downstream_task_ids",
+        "execution_timeout",
+        "executor_config",
+        "ignore_first_depends_on_past",
+        "max_active_tis_per_dag",
+        "max_active_tis_per_dagrun",
+        "max_retry_delay",
+        "multiple_outputs",
+        "owner",
+        "priority_weight",
+        "queue",
+        "retries",
+        "retry_exponential_backoff",
+        "run_as_user",
+        "task_id",
+        "trigger_rule",
+        "upstream_task_ids",
+        "wait_for_downstream",
+        "wait_for_past_depends_before_skipping",
+        "weight_rule",
     ]
     casts = {
         "operator_class": lambda task: task.task_type,
@@ -306,18 +256,30 @@ def get_airflow_run_facet(
     task_uuid: str,
 ):
     return {
-        "airflow": json.loads(
-            json.dumps(
-                asdict(
-                    AirflowRunFacet(
-                        dag=DagInfo(dag),
-                        dagRun=DagRunInfo(dag_run),
-                        taskInstance=TaskInstanceInfo(task_instance),
-                        task=TaskInfo(task),
-                        taskUuid=task_uuid,
+        "airflow": attrs.asdict(
+            AirflowRunFacet(
+                dag=DagInfo(dag),
+                dagRun=DagRunInfo(dag_run),
+                taskInstance=TaskInstanceInfo(task_instance),
+                task=TaskInfo(task),
+                taskUuid=task_uuid,
+            )
+        )
+    }
+
+
+def get_unknown_source_attribute_run_facet(task: BaseOperator, name: str | None = None):
+    if not name:
+        name = get_operator_class(task).__name__
+    return {
+        "unknownSourceAttribute": attrs.asdict(
+            UnknownOperatorAttributeRunFacet(
+                unknownItems=[
+                    UnknownOperatorInstance(
+                        name=name,
+                        properties=TaskInfo(task),
                     )
-                ),
-                default=str,
+                ]
             )
         )
     }
@@ -412,14 +374,6 @@ def print_warning(log):
     return decorator
 
 
-@cache
-def is_source_enabled() -> bool:
-    source_var = conf.get(
-        "openlineage", "disable_source_code", fallback=os.getenv("OPENLINEAGE_AIRFLOW_DISABLE_SOURCE_CODE")
-    )
-    return isinstance(source_var, str) and source_var.lower() not in ("true", "1", "t")
-
-
 def get_filtered_unknown_operator_keys(operator: BaseOperator) -> dict:
     not_required_keys = {"dag", "task_group"}
     return {attr: value for attr, value in operator.__dict__.items() if attr not in not_required_keys}

{apache_airflow_providers_openlineage-1.6.0 → apache_airflow_providers_openlineage-1.7.0}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-openlineage"
-version = "1.6.0"
+version = "1.7.0"
 description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -51,6 +51,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: System :: Monitoring",
 ]
 requires-python = "~=3.8"
@@ -63,8 +64,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.6.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.7.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

apache_airflow_providers_openlineage-1.6.0/airflow/providers/openlineage/plugins/macros.py

@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import os
-import typing
-
-from airflow.configuration import conf
-from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
-
-if typing.TYPE_CHECKING:
-    from airflow.models import TaskInstance
-
-_JOB_NAMESPACE = conf.get("openlineage", "namespace", fallback=os.getenv("OPENLINEAGE_NAMESPACE", "default"))
-
-
-def lineage_run_id(task_instance: TaskInstance):
-    """
-    Macro function which returns the generated run id for a given task.
-
-    This can be used to forward the run id from a task to a child run so the job hierarchy is preserved.
-
-    .. seealso::
-        For more information on how to use this operator, take a look at the guide:
-        :ref:`howto/macros:openlineage`
-    """
-    return OpenLineageAdapter.build_task_instance_run_id(
-        dag_id=task_instance.dag_id,
-        task_id=task_instance.task.task_id,
-        execution_date=task_instance.execution_date,
-        try_number=task_instance.try_number,
-    )
-
-
-def lineage_parent_id(run_id: str, task_instance: TaskInstance):
-    """
-    Macro function which returns the generated job and run id for a given task.
-
-    This can be used to forward the ids from a task to a child run so the job
-    hierarchy is preserved. Child run can create ParentRunFacet from those ids.
-
-    .. seealso::
-        For more information on how to use this macro, take a look at the guide:
-        :ref:`howto/macros:openlineage`
-    """
-    job_name = OpenLineageAdapter.build_task_instance_run_id(
-        dag_id=task_instance.dag_id,
-        task_id=task_instance.task.task_id,
-        execution_date=task_instance.execution_date,
-        try_number=task_instance.try_number,
-    )
-    return f"{_JOB_NAMESPACE}/{job_name}/{run_id}"