apache-airflow-providers-openlineage 2.0.0rc1__py3-none-any.whl → 2.1.0__py3-none-any.whl

airflow/providers/openlineage/LICENSE

@@ -199,55 +199,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-
-============================================================================
-   APACHE AIRFLOW SUBCOMPONENTS:
-
-   The Apache Airflow project contains subcomponents with separate copyright
-   notices and license terms. Your use of the source code for the these
-   subcomponents is subject to the terms and conditions of the following
-   licenses.
-
-
-========================================================================
-Third party Apache 2.0 licenses
-========================================================================
-
-The following components are provided under the Apache 2.0 License.
-See project link for details. The text of each license is also included
-at 3rd-party-licenses/LICENSE-[project].txt.
-
-    (ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
-    (ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
-    (ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
-    (ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
-
-========================================================================
-MIT licenses
-========================================================================
-
-The following components are provided under the MIT License. See project link for details.
-The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
-
-    (MIT License) jquery v3.5.1 (https://jquery.org/license/)
-    (MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
-    (MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
-    (MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
-    (MIT License) dataTables v1.10.25 (https://datatables.net)
-    (MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
-    (MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
-    (MIT License) MomentJS v2.24.0 (http://momentjs.com/)
-    (MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
-
-========================================================================
-BSD 3-Clause licenses
-========================================================================
-The following components are provided under the BSD 3-Clause license. See project links for details.
-The text of each license is also included at 3rd-party-licenses/LICENSE-[project].txt.
-
-    (BSD 3 License) d3 v5.16.0 (https://d3js.org)
-    (BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
-    (BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
-
-========================================================================
-See 3rd-party-licenses/LICENSES-ui.txt for packages used in `/airflow/www`

airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "2.0.0"
+__version__ = "2.1.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/openlineage/conf.py

@@ -83,6 +83,12 @@ def spark_inject_parent_job_info() -> bool:
     return conf.getboolean(_CONFIG_SECTION, "spark_inject_parent_job_info", fallback="False")
 
 
+@cache
+def spark_inject_transport_info() -> bool:
+    """[openlineage] spark_inject_transport_info."""
+    return conf.getboolean(_CONFIG_SECTION, "spark_inject_transport_info", fallback="False")
+
+
 @cache
 def custom_extractors() -> set[str]:
     """[openlineage] extractors."""

airflow/providers/openlineage/extractors/base.py

@@ -22,16 +22,16 @@ from abc import ABC, abstractmethod
 from typing import Generic, TypeVar, Union
 
 from attrs import Factory, define
+
 from openlineage.client.event_v2 import Dataset as OLDataset
 
 with warnings.catch_warnings():
     warnings.simplefilter("ignore", DeprecationWarning)
     from openlineage.client.facet import BaseFacet as BaseFacet_V1
-from openlineage.client.facet_v2 import JobFacet, RunFacet
-
 from airflow.providers.openlineage.utils.utils import AIRFLOW_V_2_10_PLUS
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import TaskInstanceState
+from openlineage.client.facet_v2 import JobFacet, RunFacet
 
 # this is not to break static checks compatibility with v1 OpenLineage facet classes
 DatasetSubclass = TypeVar("DatasetSubclass", bound=OLDataset)

airflow/providers/openlineage/extractors/bash.py

@@ -17,11 +17,10 @@
 
 from __future__ import annotations
 
-from openlineage.client.facet_v2 import source_code_job
-
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
 from airflow.providers.openlineage.utils.utils import get_unknown_source_attribute_run_facet
+from openlineage.client.facet_v2 import source_code_job
 
 """
 :meta private:

airflow/providers/openlineage/extractors/manager.py

@@ -34,10 +34,9 @@ from airflow.providers.openlineage.utils.utils import (
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
-    from openlineage.client.event_v2 import Dataset
-
-    from airflow.lineage.entities import Table
     from airflow.models import Operator
+    from airflow.providers.common.compat.lineage.entities import Table
+    from openlineage.client.event_v2 import Dataset
 
 
 def _iter_extractor_types() -> Iterator[type[BaseExtractor]]:
@@ -291,10 +290,9 @@ class ExtractorManager(LoggingMixin):
 
     @staticmethod
     def convert_to_ol_dataset(obj) -> Dataset | None:
+        from airflow.providers.common.compat.lineage.entities import File, Table
         from openlineage.client.event_v2 import Dataset
 
-        from airflow.lineage.entities import File, Table
-
         if isinstance(obj, Dataset):
             return obj
         elif isinstance(obj, Table):

airflow/providers/openlineage/extractors/python.py

@@ -20,11 +20,10 @@ from __future__ import annotations
 import inspect
 from typing import Callable
 
-from openlineage.client.facet_v2 import source_code_job
-
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
 from airflow.providers.openlineage.utils.utils import get_unknown_source_attribute_run_facet
+from openlineage.client.facet_v2 import source_code_job
 
 """
 :meta private:

airflow/providers/openlineage/get_provider_info.py

@@ -15,8 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
 #
 # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
 # `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
@@ -28,8 +27,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1734535974,
+        "source-date-epoch": 1739964022,
         "versions": [
+            "2.1.0",
             "2.0.0",
             "1.14.0",
             "1.13.0",
@@ -56,19 +56,11 @@ def get_provider_info():
             "1.0.1",
             "1.0.0",
         ],
-        "dependencies": [
-            "apache-airflow>=2.9.0",
-            "apache-airflow-providers-common-sql>=1.20.0",
-            "apache-airflow-providers-common-compat>=1.3.0",
-            "attrs>=22.2",
-            "openlineage-integration-common>=1.24.2",
-            "openlineage-python>=1.24.2",
-        ],
         "integrations": [
             {
                 "integration-name": "OpenLineage",
                 "external-doc-url": "https://openlineage.io",
-                "logo": "/integration-logos/openlineage/openlineage.svg",
+                "logo": "/docs/integration-logos/openlineage.svg",
                 "tags": ["protocol"],
             }
         ],
@@ -178,9 +170,25 @@ def get_provider_info():
                         "type": "boolean",
                         "default": "False",
                         "example": None,
-                        "version_added": "1.15.0",
+                        "version_added": "2.0.0",
+                    },
+                    "spark_inject_transport_info": {
+                        "description": "Automatically inject OpenLineage's transport information into Spark application properties\nfor supported Operators.\n",
+                        "type": "boolean",
+                        "default": "False",
+                        "example": None,
+                        "version_added": "2.1.0",
                     },
                 },
             }
         },
+        "dependencies": [
+            "apache-airflow>=2.9.0",
+            "apache-airflow-providers-common-sql>=1.20.0",
+            "apache-airflow-providers-common-compat>=1.4.0",
+            "attrs>=22.2",
+            "openlineage-integration-common>=1.24.2",
+            "openlineage-python>=1.24.2",
+            "uuid6>=2024.7.10",
+        ],
     }

airflow/providers/openlineage/plugins/adapter.py

@@ -21,6 +21,16 @@ from contextlib import ExitStack
 from typing import TYPE_CHECKING
 
 import yaml
+
+from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
+from airflow.providers.openlineage.utils.utils import (
+    OpenLineageRedactor,
+    get_airflow_debug_facet,
+    get_airflow_state_run_facet,
+    get_processing_engine_facet,
+)
+from airflow.stats import Stats
+from airflow.utils.log.logging_mixin import LoggingMixin
 from openlineage.client import OpenLineageClient, set_producer
 from openlineage.client.event_v2 import Job, Run, RunEvent, RunState
 from openlineage.client.facet_v2 import (
@@ -36,22 +46,17 @@ from openlineage.client.facet_v2 import (
 )
 from openlineage.client.uuid import generate_static_uuid
 
-from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VERSION, conf
-from airflow.providers.openlineage.utils.utils import (
-    OpenLineageRedactor,
-    get_airflow_debug_facet,
-    get_airflow_state_run_facet,
-    get_processing_engine_facet,
-)
-from airflow.stats import Stats
-from airflow.utils.log.logging_mixin import LoggingMixin
-
 if TYPE_CHECKING:
     from datetime import datetime
 
     from airflow.providers.openlineage.extractors import OperatorLineage
-    from airflow.utils.log.secrets_masker import SecretsMasker
+    from airflow.sdk.execution_time.secrets_masker import SecretsMasker, _secrets_masker
     from airflow.utils.state import DagRunState
+else:
+    try:
+        from airflow.sdk.execution_time.secrets_masker import SecretsMasker, _secrets_masker
+    except ImportError:
+        from airflow.utils.log.secrets_masker import SecretsMasker, _secrets_masker
 
 _PRODUCER = f"https://github.com/apache/airflow/tree/providers-openlineage/{OPENLINEAGE_PROVIDER_VERSION}"
 
@@ -71,8 +76,6 @@ class OpenLineageAdapter(LoggingMixin):
         super().__init__()
         self._client = client
         if not secrets_masker:
-            from airflow.utils.log.secrets_masker import _secrets_masker
-
             secrets_masker = _secrets_masker()
         self._redacter = OpenLineageRedactor.from_masker(secrets_masker)
 
@@ -251,6 +254,7 @@ class OpenLineageAdapter(LoggingMixin):
         run_facets = run_facets or {}
         if task:
             run_facets = {**task.run_facets, **run_facets}
+        run_facets = {**run_facets, **get_processing_engine_facet()}  # type: ignore
         event = RunEvent(
             eventType=RunState.COMPLETE,
             eventTime=end_time,
@@ -296,6 +300,7 @@ class OpenLineageAdapter(LoggingMixin):
         run_facets = run_facets or {}
         if task:
             run_facets = {**task.run_facets, **run_facets}
+        run_facets = {**run_facets, **get_processing_engine_facet()}  # type: ignore
 
         if error:
             stack_trace = None
@@ -377,6 +382,7 @@ class OpenLineageAdapter(LoggingMixin):
         clear_number: int,
         dag_run_state: DagRunState,
         task_ids: list[str],
+        run_facets: dict[str, RunFacet],
     ):
         try:
             event = RunEvent(
@@ -390,6 +396,8 @@ class OpenLineageAdapter(LoggingMixin):
                     facets={
                         **get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
                         **get_airflow_debug_facet(),
+                        **get_processing_engine_facet(),
+                        **run_facets,
                     },
                 ),
                 inputs=[],
@@ -413,6 +421,7 @@ class OpenLineageAdapter(LoggingMixin):
         dag_run_state: DagRunState,
         task_ids: list[str],
         msg: str,
+        run_facets: dict[str, RunFacet],
     ):
         try:
             event = RunEvent(
@@ -431,6 +440,8 @@ class OpenLineageAdapter(LoggingMixin):
                         ),
                         **get_airflow_state_run_facet(dag_id, run_id, task_ids, dag_run_state),
                         **get_airflow_debug_facet(),
+                        **get_processing_engine_facet(),
+                        **run_facets,
                     },
                 ),
                 inputs=[],

airflow/providers/openlineage/plugins/facets.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 from attrs import define
+
 from openlineage.client.facet_v2 import JobFacet, RunFacet
 from openlineage.client.utils import RedactMixin