apache-airflow-providers-openlineage 1.10.0rc1__tar.gz → 1.11.0rc1__tar.gz

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.10.0rc1
+Version: 1.11.0rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,14 +22,14 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow-providers-common-sql>=1.6.0rc0
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow>=2.8.0rc0
 Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.16.0
 Requires-Dist: openlineage-python>=1.16.0
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.10.0.rc1``
+Release: ``1.11.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/>`_.
 
 Installation
 ------------
@@ -110,7 +110,7 @@ Requirements
 =======================================  ==================
 PIP package                              Version required
 =======================================  ==================
-``apache-airflow``                       ``>=2.7.0``
+``apache-airflow``                       ``>=2.8.0``
 ``apache-airflow-providers-common-sql``  ``>=1.6.0``
 ``attrs``                                ``>=22.2``
 ``openlineage-integration-common``       ``>=1.16.0``
@@ -137,4 +137,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.10.0.rc1``
+Release: ``1.11.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/>`_.
 
 Installation
 ------------
@@ -72,7 +72,7 @@ Requirements
 =======================================  ==================
 PIP package                              Version required
 =======================================  ==================
-``apache-airflow``                       ``>=2.7.0``
+``apache-airflow``                       ``>=2.8.0``
 ``apache-airflow-providers-common-sql``  ``>=1.6.0``
 ``attrs``                                ``>=22.2``
 ``openlineage-integration-common``       ``>=1.16.0``
@@ -99,4 +99,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.10.0"
+__version__ = "1.11.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.7.0"
+    "2.8.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.7.0+"
+        f"The package `apache-airflow-providers-openlineage:{__version__}` needs Apache Airflow 2.8.0+"
     )

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/conf.py

@@ -145,3 +145,9 @@ def execution_timeout() -> int:
 def include_full_task_info() -> bool:
     """[openlineage] include_full_task_info."""
     return conf.getboolean(_CONFIG_SECTION, "include_full_task_info", fallback="False")
+
+
+@cache
+def debug_mode() -> bool:
+    """[openlineage] debug_mode."""
+    return conf.getboolean(_CONFIG_SECTION, "debug_mode", fallback="False")

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/extractors/base.py

@@ -113,7 +113,7 @@ class DefaultExtractor(BaseExtractor):
                 "Operator %s does not have the get_openlineage_facets_on_start method.",
                 self.operator.task_type,
             )
-            return None
+            return OperatorLineage()
 
     def extract_on_complete(self, task_instance) -> OperatorLineage | None:
         failed_states = [TaskInstanceState.FAILED, TaskInstanceState.UP_FOR_RETRY]

apache_airflow_providers_openlineage-1.11.0rc1/airflow/providers/openlineage/facets/AirflowDebugRunFacet.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$defs": {
+      "AirflowDebugRunFacet": {
+        "allOf": [
+          {
+            "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+          },
+          {
+            "type": "object",
+            "properties": {
+              "packages": {
+                "description": "The names and versions of all installed Python packages.",
+                "type": "object",
+                "additionalProperties": true
+              }
+            },
+            "required": ["packages"]
+          }
+        ],
+        "type": "object"
+      }
+    },
+    "type": "object",
+    "properties": {
+      "debug": {
+        "$ref": "#/$defs/AirflowDebugRunFacet"
+      }
+    }
+  }

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1722664661,
+        "source-date-epoch": 1723970474,
         "versions": [
+            "1.11.0",
             "1.10.0",
             "1.9.1",
             "1.9.0",
@@ -50,7 +51,7 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.7.0",
+            "apache-airflow>=2.8.0",
             "apache-airflow-providers-common-sql>=1.6.0",
             "attrs>=22.2",
             "openlineage-integration-common>=1.16.0",
@@ -158,6 +159,13 @@ def get_provider_info():
                         "type": "boolean",
                         "version_added": "1.10.0",
                     },
+                    "debug_mode": {
+                        "description": "If true, OpenLineage events will include information useful for debugging - potentially\ncontaining large fields e.g. all installed packages and their versions.\n",
+                        "default": "False",
+                        "example": None,
+                        "type": "boolean",
+                        "version_added": "1.11.0",
+                    },
                 },
             }
         },

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/plugins/adapter.py

@@ -41,6 +41,7 @@ from airflow.providers.openlineage import __version__ as OPENLINEAGE_PROVIDER_VE
 from airflow.providers.openlineage.utils.utils import (
     OpenLineageRedactor,
     get_airflow_dag_run_facet,
+    get_airflow_debug_facet,
     get_airflow_state_run_facet,
 )
 from airflow.stats import Stats
@@ -90,7 +91,7 @@ class OpenLineageAdapter(LoggingMixin):
                     "OpenLineage configuration not found directly in Airflow. "
                     "Looking for legacy environment configuration. "
                 )
-                self._client = OpenLineageClient.from_environment()
+                self._client = OpenLineageClient()
         return self._client
 
     def get_openlineage_config(self) -> dict | None:
@@ -361,7 +362,7 @@ class OpenLineageAdapter(LoggingMixin):
                     job_name=dag_run.dag_id,
                     nominal_start_time=nominal_start_time,
                     nominal_end_time=nominal_end_time,
-                    run_facets=get_airflow_dag_run_facet(dag_run),
+                    run_facets={**get_airflow_dag_run_facet(dag_run), **get_airflow_debug_facet()},
                 ),
                 inputs=[],
                 outputs=[],
@@ -385,7 +386,7 @@ class OpenLineageAdapter(LoggingMixin):
                         dag_id=dag_run.dag_id,
                         execution_date=dag_run.execution_date,
                     ),
-                    facets={**get_airflow_state_run_facet(dag_run)},
+                    facets={**get_airflow_state_run_facet(dag_run), **get_airflow_debug_facet()},
                 ),
                 inputs=[],
                 outputs=[],
@@ -414,6 +415,7 @@ class OpenLineageAdapter(LoggingMixin):
                             message=msg, programmingLanguage="python"
                         ),
                         **get_airflow_state_run_facet(dag_run),
+                        **get_airflow_debug_facet(),
                     },
                 ),
                 inputs=[],

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/plugins/facets.py

@@ -17,17 +17,10 @@
 from __future__ import annotations
 
 from attrs import define
-from deprecated import deprecated
 from openlineage.client.facet_v2 import JobFacet, RunFacet
 from openlineage.client.utils import RedactMixin
 
-from airflow.exceptions import AirflowProviderDeprecationWarning
 
-
-@deprecated(
-    reason="To be removed in the next release. Make sure to use information from AirflowRunFacet instead.",
-    category=AirflowProviderDeprecationWarning,
-)
 @define
 class AirflowMappedTaskRunFacet(RunFacet):
     """Run facet containing information about mapped tasks."""
@@ -108,6 +101,13 @@ class AirflowDagRunFacet(RunFacet):
     dagRun: dict
 
 
+@define
+class AirflowDebugRunFacet(RunFacet):
+    """Airflow Debug run facet."""
+
+    packages: dict
+
+
 @define
 class UnknownOperatorInstance(RedactMixin):
     """
@@ -123,10 +123,6 @@ class UnknownOperatorInstance(RedactMixin):
     _skip_redact = ["name", "type"]
 
 
-@deprecated(
-    reason="To be removed in the next release. Make sure to use information from AirflowRunFacet instead.",
-    category=AirflowProviderDeprecationWarning,
-)
 @define
 class UnknownOperatorAttributeRunFacet(RunFacet):
     """RunFacet that describes unknown operators in an Airflow DAG."""

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/plugins/listener.py

@@ -32,6 +32,7 @@ from airflow.providers.openlineage.extractors import ExtractorManager
 from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter, RunState
 from airflow.providers.openlineage.utils.utils import (
     IS_AIRFLOW_2_10_OR_HIGHER,
+    get_airflow_debug_facet,
     get_airflow_job_facet,
     get_airflow_mapped_task_facet,
     get_airflow_run_facet,
@@ -122,6 +123,9 @@ class OpenLineageListener:
             )
             return
 
+        # Needs to be calculated outside of inner method so that it gets cached for usage in fork processes
+        debug_facet = get_airflow_debug_facet()
+
         @print_warning(self.log)
         def on_running():
             # that's a workaround to detect task running from deferred state
@@ -166,6 +170,7 @@ class OpenLineageListener:
                     **get_user_provided_run_facets(task_instance, TaskInstanceState.RUNNING),
                     **get_airflow_mapped_task_facet(task_instance),
                     **get_airflow_run_facet(dagrun, dag, task_instance, task, task_uuid),
+                    **debug_facet,
                 },
             )
             Stats.gauge(
@@ -237,6 +242,7 @@ class OpenLineageListener:
                 run_facets={
                     **get_user_provided_run_facets(task_instance, TaskInstanceState.SUCCESS),
                     **get_airflow_run_facet(dagrun, dag, task_instance, task, task_uuid),
+                    **get_airflow_debug_facet(),
                 },
             )
             Stats.gauge(
@@ -336,6 +342,7 @@ class OpenLineageListener:
                 run_facets={
                     **get_user_provided_run_facets(task_instance, TaskInstanceState.FAILED),
                     **get_airflow_run_facet(dagrun, dag, task_instance, task, task_uuid),
+                    **get_airflow_debug_facet(),
                 },
             )
             Stats.gauge(

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/airflow/providers/openlineage/utils/utils.py

@@ -20,10 +20,9 @@ from __future__ import annotations
 import datetime
 import json
 import logging
-import re
-from contextlib import redirect_stdout, suppress
+from contextlib import suppress
 from functools import wraps
-from io import StringIO
+from importlib import metadata
 from typing import TYPE_CHECKING, Any, Callable, Iterable
 
 import attrs
@@ -34,10 +33,11 @@ from packaging.version import Version
 from airflow import __version__ as AIRFLOW_VERSION
 from airflow.datasets import Dataset
 from airflow.exceptions import AirflowProviderDeprecationWarning  # TODO: move this maybe to Airflow's logic?
-from airflow.models import DAG, BaseOperator, MappedOperator
+from airflow.models import DAG, BaseOperator, MappedOperator, Operator
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.plugins.facets import (
     AirflowDagRunFacet,
+    AirflowDebugRunFacet,
     AirflowJobFacet,
     AirflowMappedTaskRunFacet,
     AirflowRunFacet,
@@ -85,6 +85,10 @@ def get_job_name(task: TaskInstance) -> str:
 def get_airflow_mapped_task_facet(task_instance: TaskInstance) -> dict[str, Any]:
     # check for -1 comes from SmartSensor compatibility with dynamic task mapping
     # this comes from Airflow code
+    log.debug(
+        "AirflowMappedTaskRunFacet is deprecated and will be removed. "
+        "Use information from AirflowRunFacet instead."
+    )
     if hasattr(task_instance, "map_index") and getattr(task_instance, "map_index") != -1:
         return {"airflow_mappedTask": AirflowMappedTaskRunFacet.from_task_instance(task_instance)}
     return {}
@@ -240,7 +244,7 @@ class InfoJsonEncodable(dict):
 class DagInfo(InfoJsonEncodable):
     """Defines encoding DAG object to JSON."""
 
-    includes = ["dag_id", "description", "owner", "schedule_interval", "start_date", "tags"]
+    includes = ["dag_id", "description", "fileloc", "owner", "schedule_interval", "start_date", "tags"]
     casts = {"timetable": lambda dag: dag.timetable.serialize() if getattr(dag, "timetable", None) else None}
     renames = {"_dag_id": "dag_id"}
 
@@ -374,6 +378,28 @@ def get_airflow_dag_run_facet(dag_run: DagRun) -> dict[str, RunFacet]:
     }
 
 
+@conf.cache
+def _get_all_packages_installed() -> dict[str, str]:
+    """
+    Retrieve a dictionary of all installed packages and their versions.
+
+    This operation involves scanning the system's installed packages, which can be a heavy operation.
+    It is recommended to cache the result to avoid repeated, expensive lookups.
+    """
+    return {dist.metadata["Name"]: dist.version for dist in metadata.distributions()}
+
+
+def get_airflow_debug_facet() -> dict[str, AirflowDebugRunFacet]:
+    if not conf.debug_mode():
+        return {}
+    log.warning("OpenLineage debug_mode is enabled. Be aware that this may log and emit extensive details.")
+    return {
+        "debug": AirflowDebugRunFacet(
+            packages=_get_all_packages_installed(),
+        )
+    }
+
+
 def get_airflow_run_facet(
     dag_run: DagRun,
     dag: DAG,
@@ -413,16 +439,6 @@ def get_airflow_state_run_facet(dag_run: DagRun) -> dict[str, AirflowStateRunFac
     }
 
 
-def _safe_get_dag_tree_view(dag: DAG) -> list[str]:
-    # get_tree_view() has been added in Airflow 2.8.2
-    if hasattr(dag, "get_tree_view"):
-        return dag.get_tree_view().splitlines()
-
-    with redirect_stdout(StringIO()) as stdout:
-        dag.tree_view()
-        return stdout.getvalue().splitlines()
-
-
 def _get_parsed_dag_tree(dag: DAG) -> dict:
     """
     Get DAG's tasks hierarchy representation.
@@ -448,37 +464,15 @@ def _get_parsed_dag_tree(dag: DAG) -> dict:
         "task_6": {}
     }
     """
-    lines = _safe_get_dag_tree_view(dag)
-    task_dict: dict[str, dict] = {}
-    parent_map: dict[int, tuple[str, dict]] = {}
-
-    for line in lines:
-        stripped_line = line.strip()
-        if not stripped_line:
-            continue
-
-        # Determine the level by counting the leading spaces, assuming 4 spaces per level
-        # as defined in airflow.models.dag.DAG._generate_tree_view()
-        level = (len(line) - len(stripped_line)) // 4
-        # airflow.models.baseoperator.BaseOperator.__repr__ or
-        # airflow.models.mappedoperator.MappedOperator.__repr__ is used in DAG tree
-        # <Task({op_class}): {task_id}> or <Mapped({op_class}): {task_id}>
-        match = re.match(r"^<(?:Task|Mapped)\(.+\): (.+)>$", stripped_line)
-        if not match:
-            return {}
-        current_task_id = match[1]
-
-        if level == 0:  # It's a root task
-            task_dict[current_task_id] = {}
-            parent_map[level] = (current_task_id, task_dict[current_task_id])
-        else:
-            # Find the immediate parent task
-            parent_task, parent_dict = parent_map[(level - 1)]
-            # Create new dict for the current task
-            parent_dict[current_task_id] = {}
-            # Update this task in the parent map
-            parent_map[level] = (current_task_id, parent_dict[current_task_id])
 
+    def get_downstream(task: Operator, current_dict: dict):
+        current_dict[task.task_id] = {}
+        for tmp_task in sorted(task.downstream_list, key=lambda x: x.task_id):
+            get_downstream(tmp_task, current_dict[task.task_id])
+
+    task_dict: dict = {}
+    for t in sorted(dag.roots, key=lambda x: x.task_id):
+        get_downstream(t, task_dict)
     return task_dict
 
 
@@ -536,6 +530,10 @@ def _emits_ol_events(task: BaseOperator | MappedOperator) -> bool:
 def get_unknown_source_attribute_run_facet(task: BaseOperator, name: str | None = None):
     if not name:
         name = get_operator_class(task).__name__
+    log.debug(
+        "UnknownOperatorAttributeRunFacet is deprecated and will be removed. "
+        "Use information from AirflowRunFacet instead."
+    )
     return {
         "unknownSourceAttribute": attrs.asdict(
             UnknownOperatorAttributeRunFacet(

{apache_airflow_providers_openlineage-1.10.0rc1 → apache_airflow_providers_openlineage-1.11.0rc1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-openlineage"
-version = "1.10.0.rc1"
+version = "1.11.0.rc1"
 description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -57,15 +57,15 @@ classifiers = [
 requires-python = "~=3.8"
 dependencies = [
     "apache-airflow-providers-common-sql>=1.6.0rc0",
-    "apache-airflow>=2.7.0rc0",
+    "apache-airflow>=2.8.0rc0",
     "attrs>=22.2",
     "openlineage-integration-common>=1.16.0",
     "openlineage-python>=1.16.0",
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.11.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"