apache-airflow-providers-openlineage 1.9.1rc1__tar.gz → 1.10.0rc1__tar.gz

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-openlineage
-Version: 1.9.1rc1
+Version: 1.10.0rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -28,8 +28,8 @@ Requires-Dist: openlineage-integration-common>=1.16.0
 Requires-Dist: openlineage-python>=1.16.0
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Provides-Extra: common.sql
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.9.1.rc1``
+Release: ``1.10.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/>`_.
 
 Installation
 ------------
@@ -137,4 +137,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``1.9.1.rc1``
+Release: ``1.10.0.rc1``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/>`_.
 
 Installation
 ------------
@@ -99,4 +99,4 @@ Dependent package
 ============================================================================================================  ==============
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.9.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/1.10.0/changelog.html>`_.

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "1.9.1"
+__version__ = "1.10.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/conf.py

@@ -17,15 +17,9 @@
 """
 This module provides functions for safely retrieving and handling OpenLineage configurations.
 
-To prevent errors caused by invalid user-provided configuration values, we use ``conf.get()``
-to fetch values as strings and perform safe conversions using custom functions.
-
-Any invalid configuration values should be treated as incorrect and replaced with default values.
-For example, if the default for boolean ``custom_ol_var`` is False, any non-true value provided:
-``"asdf"``, ``12345``, ``{"key": 1}`` or empty string, will result in False being used.
-
-By using default values for invalid configuration values, we ensure that the configurations are handled
-safely, preventing potential runtime errors due to conversion issues.
+For the legacy boolean env variables `OPENLINEAGE_AIRFLOW_DISABLE_SOURCE_CODE` and `OPENLINEAGE_DISABLED`,
+any string not equal to "true", "1", or "t" should be treated as False, to maintain backward compatibility.
+Support for legacy variables will be removed in Airflow 3.
 """
 
 from __future__ import annotations
@@ -51,13 +45,6 @@ def _is_true(arg: Any) -> bool:
     return str(arg).lower().strip() in ("true", "1", "t")
 
 
-def _safe_int_convert(arg: Any, default: int) -> int:
-    try:
-        return int(arg)
-    except (ValueError, TypeError):
-        return default
-
-
 @cache
 def config_path(check_legacy_env_var: bool = True) -> str:
     """[openlineage] config_path."""
@@ -70,11 +57,11 @@ def config_path(check_legacy_env_var: bool = True) -> str:
 @cache
 def is_source_enabled() -> bool:
     """[openlineage] disable_source_code."""
-    option = conf.get(_CONFIG_SECTION, "disable_source_code", fallback="")
-    if not option:
-        option = os.getenv("OPENLINEAGE_AIRFLOW_DISABLE_SOURCE_CODE", "")
-    # when disable_source_code is True, is_source_enabled() should be False
-    return not _is_true(option)
+    option = conf.getboolean(_CONFIG_SECTION, "disable_source_code", fallback="False")
+    if option is False:  # Check legacy variable
+        option = _is_true(os.getenv("OPENLINEAGE_AIRFLOW_DISABLE_SOURCE_CODE", ""))
+    # when disable_source_code is True, is_source_enabled() should be False; hence the "not"
+    return not option
 
 
 @cache
@@ -87,8 +74,7 @@ def disabled_operators() -> set[str]:
 @cache
 def selective_enable() -> bool:
     """[openlineage] selective_enable."""
-    option = conf.get(_CONFIG_SECTION, "selective_enable", fallback="")
-    return _is_true(option)
+    return conf.getboolean(_CONFIG_SECTION, "selective_enable", fallback="False")
 
 
 @cache
@@ -100,6 +86,17 @@ def custom_extractors() -> set[str]:
     return set(extractor.strip() for extractor in option.split(";") if extractor.strip())
 
 
+@cache
+def custom_run_facets() -> set[str]:
+    """[openlineage] custom_run_facets."""
+    option = conf.get(_CONFIG_SECTION, "custom_run_facets", fallback="")
+    return set(
+        custom_facet_function.strip()
+        for custom_facet_function in option.split(";")
+        if custom_facet_function.strip()
+    )
+
+
 @cache
 def namespace() -> str:
     """[openlineage] namespace."""
@@ -121,13 +118,12 @@ def transport() -> dict[str, Any]:
 @cache
 def is_disabled() -> bool:
     """[openlineage] disabled + check if any configuration is present."""
-    option = conf.get(_CONFIG_SECTION, "disabled", fallback="")
-    if _is_true(option):
+    if conf.getboolean(_CONFIG_SECTION, "disabled", fallback="False"):
         return True
 
-    option = os.getenv("OPENLINEAGE_DISABLED", "")
-    if _is_true(option):
+    if _is_true(os.getenv("OPENLINEAGE_DISABLED", "")):  # Check legacy variable
         return True
+
     # Check if both 'transport' and 'config_path' are not present and also
     # if legacy 'OPENLINEAGE_URL' environment variables is not set
     return transport() == {} and config_path(True) == "" and os.getenv("OPENLINEAGE_URL", "") == ""
@@ -136,12 +132,16 @@ def is_disabled() -> bool:
 @cache
 def dag_state_change_process_pool_size() -> int:
     """[openlineage] dag_state_change_process_pool_size."""
-    option = conf.get(_CONFIG_SECTION, "dag_state_change_process_pool_size", fallback="")
-    return _safe_int_convert(str(option).strip(), default=1)
+    return conf.getint(_CONFIG_SECTION, "dag_state_change_process_pool_size", fallback="1")
 
 
 @cache
 def execution_timeout() -> int:
     """[openlineage] execution_timeout."""
-    option = conf.get(_CONFIG_SECTION, "execution_timeout", fallback="")
-    return _safe_int_convert(str(option).strip(), default=10)
+    return conf.getint(_CONFIG_SECTION, "execution_timeout", fallback="10")
+
+
+@cache
+def include_full_task_info() -> bool:
+    """[openlineage] include_full_task_info."""
+    return conf.getboolean(_CONFIG_SECTION, "include_full_task_info", fallback="False")

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/extractors/base.py

@@ -17,27 +17,35 @@
 
 from __future__ import annotations
 
+import warnings
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING
+from typing import Generic, TypeVar, Union
 
 from attrs import Factory, define
+from openlineage.client.event_v2 import Dataset as OLDataset
 
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from openlineage.client.facet import BaseFacet as BaseFacet_V1
+from openlineage.client.facet_v2 import JobFacet, RunFacet
+
+from airflow.providers.openlineage.utils.utils import IS_AIRFLOW_2_10_OR_HIGHER
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import TaskInstanceState
 
-if TYPE_CHECKING:
-    from openlineage.client.facet import BaseFacet
-    from openlineage.client.run import Dataset
+# this is not to break static checks compatibility with v1 OpenLineage facet classes
+DatasetSubclass = TypeVar("DatasetSubclass", bound=OLDataset)
+BaseFacetSubclass = TypeVar("BaseFacetSubclass", bound=Union[BaseFacet_V1, RunFacet, JobFacet])
 
 
 @define
-class OperatorLineage:
+class OperatorLineage(Generic[DatasetSubclass, BaseFacetSubclass]):
     """Structure returned from lineage extraction."""
 
-    inputs: list[Dataset] = Factory(list)
-    outputs: list[Dataset] = Factory(list)
-    run_facets: dict[str, BaseFacet] = Factory(dict)
-    job_facets: dict[str, BaseFacet] = Factory(dict)
+    inputs: list[DatasetSubclass] = Factory(list)
+    outputs: list[DatasetSubclass] = Factory(list)
+    run_facets: dict[str, BaseFacetSubclass] = Factory(dict)
+    job_facets: dict[str, BaseFacetSubclass] = Factory(dict)
 
 
 class BaseExtractor(ABC, LoggingMixin):
@@ -108,7 +116,14 @@ class DefaultExtractor(BaseExtractor):
             return None
 
     def extract_on_complete(self, task_instance) -> OperatorLineage | None:
-        if task_instance.state == TaskInstanceState.FAILED:
+        failed_states = [TaskInstanceState.FAILED, TaskInstanceState.UP_FOR_RETRY]
+        if not IS_AIRFLOW_2_10_OR_HIGHER:  # todo: remove when min airflow version >= 2.10.0
+            # Before fix (#41053) implemented in Airflow 2.10 TaskInstance's state was still RUNNING when
+            # being passed to listener's on_failure method. Since `extract_on_complete()` is only called
+            # after task completion, RUNNING state means that we are dealing with FAILED task in < 2.10
+            failed_states = [TaskInstanceState.RUNNING]
+
+        if task_instance.state in failed_states:
             on_failed = getattr(self.operator, "get_openlineage_facets_on_failure", None)
             if on_failed and callable(on_failed):
                 self.log.debug(

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/extractors/bash.py

@@ -17,7 +17,7 @@
 
 from __future__ import annotations
 
-from openlineage.client.facet import SourceCodeJobFacet
+from openlineage.client.facet_v2 import source_code_job
 
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
@@ -47,10 +47,10 @@ class BashExtractor(BaseExtractor):
         job_facets: dict = {}
         if conf.is_source_enabled():
             job_facets = {
-                "sourceCode": SourceCodeJobFacet(
+                "sourceCode": source_code_job.SourceCodeJobFacet(
                     language="bash",
                     # We're on worker and should have access to DAG files
-                    source=self.operator.bash_command,
+                    sourceCode=self.operator.bash_command,
                 )
             }
         else:

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/extractors/manager.py

@@ -30,7 +30,7 @@ from airflow.providers.openlineage.utils.utils import (
 from airflow.utils.log.logging_mixin import LoggingMixin
 
 if TYPE_CHECKING:
-    from openlineage.client.run import Dataset
+    from openlineage.client.event_v2 import Dataset
 
     from airflow.lineage.entities import Table
     from airflow.models import Operator
@@ -172,7 +172,7 @@ class ExtractorManager(LoggingMixin):
     def convert_to_ol_dataset_from_object_storage_uri(uri: str) -> Dataset | None:
         from urllib.parse import urlparse
 
-        from openlineage.client.run import Dataset
+        from openlineage.client.event_v2 import Dataset
 
         if "/" not in uri:
             return None
@@ -196,20 +196,19 @@ class ExtractorManager(LoggingMixin):
 
     @staticmethod
     def convert_to_ol_dataset_from_table(table: Table) -> Dataset:
-        from openlineage.client.facet import (
-            BaseFacet,
-            OwnershipDatasetFacet,
-            OwnershipDatasetFacetOwners,
-            SchemaDatasetFacet,
-            SchemaField,
+        from openlineage.client.event_v2 import Dataset
+        from openlineage.client.facet_v2 import (
+            DatasetFacet,
+            documentation_dataset,
+            ownership_dataset,
+            schema_dataset,
         )
-        from openlineage.client.run import Dataset
 
-        facets: dict[str, BaseFacet] = {}
+        facets: dict[str, DatasetFacet] = {}
         if table.columns:
-            facets["schema"] = SchemaDatasetFacet(
+            facets["schema"] = schema_dataset.SchemaDatasetFacet(
                 fields=[
-                    SchemaField(
+                    schema_dataset.SchemaDatasetFacetFields(
                         name=column.name,
                         type=column.data_type,
                         description=column.description,
@@ -218,9 +217,9 @@ class ExtractorManager(LoggingMixin):
                 ]
             )
         if table.owners:
-            facets["ownership"] = OwnershipDatasetFacet(
+            facets["ownership"] = ownership_dataset.OwnershipDatasetFacet(
                 owners=[
-                    OwnershipDatasetFacetOwners(
+                    ownership_dataset.Owner(
                         # f.e. "user:John Doe <jdoe@company.com>" or just "user:<jdoe@company.com>"
                         name=f"user:"
                         f"{user.first_name + ' ' if user.first_name else ''}"
@@ -231,6 +230,10 @@ class ExtractorManager(LoggingMixin):
                     for user in table.owners
                 ]
             )
+        if table.description:
+            facets["documentation"] = documentation_dataset.DocumentationDatasetFacet(
+                description=table.description
+            )
         return Dataset(
             namespace=f"{table.cluster}",
             name=f"{table.database}.{table.name}",
@@ -239,7 +242,7 @@ class ExtractorManager(LoggingMixin):
 
     @staticmethod
     def convert_to_ol_dataset(obj) -> Dataset | None:
-        from openlineage.client.run import Dataset
+        from openlineage.client.event_v2 import Dataset
 
         from airflow.lineage.entities import File, Table
 

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/extractors/python.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import inspect
 from typing import Callable
 
-from openlineage.client.facet import SourceCodeJobFacet
+from openlineage.client.facet_v2 import source_code_job
 
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.extractors.base import BaseExtractor, OperatorLineage
@@ -51,10 +51,10 @@ class PythonExtractor(BaseExtractor):
         job_facet: dict = {}
         if conf.is_source_enabled() and source_code:
             job_facet = {
-                "sourceCode": SourceCodeJobFacet(
+                "sourceCode": source_code_job.SourceCodeJobFacet(
                     language="python",
                     # We're on worker and should have access to DAG files
-                    source=source_code,
+                    sourceCode=source_code,
                 )
             }
         else:

apache_airflow_providers_openlineage-1.10.0rc1/airflow/providers/openlineage/facets/AirflowDagRunFacet.json

@@ -0,0 +1,105 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$defs": {
+    "AirflowDagRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "dag": {
+              "$ref": "#/$defs/DAG"
+            },
+            "dagRun": {
+              "$ref": "#/$defs/DagRun"
+            }
+          },
+          "required": [
+            "dag",
+            "dagRun"
+          ]
+        }
+      ]
+    },
+    "DAG": {
+      "type": "object",
+      "properties": {
+        "dag_id": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "owner": {
+          "type": "string"
+        },
+        "schedule_interval": {
+          "type": "string"
+        },
+        "start_date": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "tags": {
+          "type": "string"
+        },
+        "timetable": {
+          "description": "Describes timetable (successor of schedule_interval)",
+          "type": "object",
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "dag_id",
+        "start_date"
+      ]
+    },
+    "DagRun": {
+      "type": "object",
+      "properties": {
+        "conf": {
+          "type": "object",
+          "additionalProperties": true
+        },
+        "dag_id": {
+          "type": "string"
+        },
+        "data_interval_start": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "data_interval_end": {
+          "type": "string",
+          "format": "date-time"
+        },
+        "external_trigger": {
+          "type": "boolean"
+        },
+        "run_id": {
+          "type": "string"
+        },
+        "run_type": {
+          "type": "string"
+        },
+        "start_date": {
+          "type": "string",
+          "format": "date-time"
+        }
+      },
+      "additionalProperties": true,
+      "required": [
+        "dag_id",
+        "run_id"
+      ]
+    }
+  },
+  "type": "object",
+  "properties": {
+    "airflowDagRun": {
+      "$ref": "#/$defs/AirflowDagRunFacet"
+    }
+  }
+}

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/facets/AirflowRunFacet.json

@@ -205,6 +205,10 @@
         "queued_dttm": {
           "type": "string",
           "format": "date-time"
+        },
+        "log_url": {
+          "type": "string",
+          "format": "uri"
         }
       },
       "additionalProperties": true,

{apache_airflow_providers_openlineage-1.9.1rc1 → apache_airflow_providers_openlineage-1.10.0rc1}/airflow/providers/openlineage/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "OpenLineage Airflow",
         "description": "`OpenLineage <https://openlineage.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1720423723,
+        "source-date-epoch": 1722664661,
         "versions": [
+            "1.10.0",
             "1.9.1",
             "1.9.0",
             "1.8.0",
@@ -108,6 +109,13 @@ def get_provider_info():
                         "default": None,
                         "version_added": None,
                     },
+                    "custom_run_facets": {
+                        "description": "Register custom run facet functions by passing a string of semicolon separated full import paths.\n",
+                        "type": "string",
+                        "example": "full.path.to.custom_facet_function;full.path.to.another_custom_facet_function",
+                        "default": "",
+                        "version_added": "1.10.0",
+                    },
                     "config_path": {
                         "description": "Specify the path to the YAML configuration file.\nThis ensures backwards compatibility with passing config through the `openlineage.yml` file.\n",
                         "version_added": None,
@@ -143,6 +151,13 @@ def get_provider_info():
                         "type": "integer",
                         "version_added": "1.9.0",
                     },
+                    "include_full_task_info": {
+                        "description": "If true, OpenLineage event will include full task info - potentially containing large fields.\n",
+                        "default": "False",
+                        "example": None,
+                        "type": "boolean",
+                        "version_added": "1.10.0",
+                    },
                 },
             }
         },