apache-airflow-providers-openlineage 2.1.2b1__tar.gz → 2.1.2rc1__tar.gz

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-openlineage
-Version: 2.1.2b1
+Version: 2.1.2rc1
 Summary: Provider package apache-airflow-providers-openlineage for Apache Airflow
 Keywords: airflow-provider,openlineage,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -20,15 +20,15 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.9.0
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.4.0
+Requires-Dist: apache-airflow>=2.9.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.4.0rc0
 Requires-Dist: attrs>=22.2
 Requires-Dist: openlineage-integration-common>=1.24.2
 Requires-Dist: openlineage-python>=1.24.2
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -59,7 +59,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``2.1.2b1``
+Release: ``2.1.2``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -72,7 +72,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/>`_.
 
 Installation
 ------------
@@ -101,7 +101,7 @@ Cross provider package dependencies
 -----------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider packages in order to use them.
+You need to install the specified providers in order to use them.
 
 You can install such cross-provider dependencies when installing from PyPI. For example:
 
@@ -118,5 +118,5 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/changelog.html>`_.
 

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-openlineage``
 
-Release: ``2.1.2b1``
+Release: ``2.1.2``
 
 
 `OpenLineage <https://openlineage.io/>`__
@@ -36,7 +36,7 @@ This is a provider package for ``openlineage`` provider. All classes for this pr
 are in ``airflow.providers.openlineage`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/>`_.
 
 Installation
 ------------
@@ -65,7 +65,7 @@ Cross provider package dependencies
 -----------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider packages in order to use them.
+You need to install the specified providers in order to use them.
 
 You can install such cross-provider dependencies when installing from PyPI. For example:
 
@@ -82,4 +82,4 @@ Dependent package
 ==================================================================================================================  =================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/changelog.html>`_.

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/pyproject.toml

@@ -20,12 +20,12 @@
 # IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
 # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
 [build-system]
-requires = ["flit_core==3.11.0"]
+requires = ["flit_core==3.12.0"]
 build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-openlineage"
-version = "2.1.2b1"
+version = "2.1.2.rc1"
 description = "Provider package apache-airflow-providers-openlineage for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -57,9 +57,9 @@ requires-python = "~=3.9"
 # Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
-    "apache-airflow>=2.9.0",
-    "apache-airflow-providers-common-sql>=1.20.0",
-    "apache-airflow-providers-common-compat>=1.4.0",
+    "apache-airflow>=2.9.0rc0",
+    "apache-airflow-providers-common-sql>=1.20.0rc0",
+    "apache-airflow-providers-common-compat>=1.4.0rc0",
     "attrs>=22.2",
     "openlineage-integration-common>=1.24.2",
     "openlineage-python>=1.24.2",
@@ -87,8 +87,8 @@ apache-airflow-providers-fab = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2b1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-openlineage/2.1.2/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/src/airflow/providers/openlineage/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "2.1.2b1"
+__version__ = "2.1.2"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/src/airflow/providers/openlineage/get_provider_info.py

@@ -29,7 +29,7 @@ def get_provider_info():
         "state": "ready",
         "source-date-epoch": 1742478177,
         "versions": [
-            "2.1.2b1",
+            "2.1.2",
             "2.1.1",
             "2.1.0",
             "2.0.0",

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/src/airflow/providers/openlineage/plugins/listener.py

@@ -69,15 +69,16 @@ def _get_try_number_success(val):
 
 def _executor_initializer():
     """
-    Initialize worker processes for the executor used for DagRun listener.
+    Initialize processes for the executor used with DAGRun listener's methods (on scheduler).
 
     This function must be picklable, so it cannot be defined as an inner method or local function.
 
     Reconfigures the ORM engine to prevent issues that arise when multiple processes interact with
     the Airflow database.
     """
-    if not AIRFLOW_V_3_0_PLUS:
-        settings.configure_orm()
+    # This initializer is used only on the scheduler
+    # We can configure_orm regardless of the Airflow version, as DB access is always allowed from scheduler.
+    settings.configure_orm()
 
 
 class OpenLineageListener:

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/src/airflow/providers/openlineage/plugins/macros.py

@@ -21,6 +21,7 @@ from typing import TYPE_CHECKING
 from airflow.providers.openlineage import conf
 from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
 from airflow.providers.openlineage.utils.utils import get_job_name
+from airflow.providers.openlineage.version_compat import AIRFLOW_V_3_0_PLUS
 
 if TYPE_CHECKING:
     from airflow.models import TaskInstance
@@ -58,15 +59,25 @@ def lineage_run_id(task_instance: TaskInstance):
         For more information take a look at the guide:
         :ref:`howto/macros:openlineage`
     """
-    if hasattr(task_instance, "logical_date"):
-        logical_date = task_instance.logical_date
+    if AIRFLOW_V_3_0_PLUS:
+        context = task_instance.get_template_context()
+        if hasattr(task_instance, "dag_run"):
+            dag_run = task_instance.dag_run
+        elif hasattr(context, "dag_run"):
+            dag_run = context["dag_run"]
+        if hasattr(dag_run, "logical_date") and dag_run.logical_date:
+            date = dag_run.logical_date
+        else:
+            date = dag_run.run_after
+    elif hasattr(task_instance, "logical_date"):
+        date = task_instance.logical_date
     else:
-        logical_date = task_instance.execution_date
+        date = task_instance.execution_date
     return OpenLineageAdapter.build_task_instance_run_id(
         dag_id=task_instance.dag_id,
         task_id=task_instance.task_id,
         try_number=task_instance.try_number,
-        logical_date=logical_date,
+        logical_date=date,
         map_index=task_instance.map_index,
     )
 

{apache_airflow_providers_openlineage-2.1.2b1 → apache_airflow_providers_openlineage-2.1.2rc1}/src/airflow/providers/openlineage/utils/spark.py

@@ -53,35 +53,73 @@ def _get_parent_job_information_as_spark_properties(context: Context) -> dict:
 
 def _get_transport_information_as_spark_properties() -> dict:
     """Retrieve transport information as Spark properties."""
-    transport = get_openlineage_listener().adapter.get_or_create_openlineage_client().transport
-    if transport.kind != "http":
-        log.info(
-            "OpenLineage transport type `%s` does not support automatic "
-            "injection of OpenLineage transport information into Spark properties.",
-            transport.kind,
-        )
-        return {}
-
-    properties = {
-        "spark.openlineage.transport.type": transport.kind,
-        "spark.openlineage.transport.url": transport.url,
-        "spark.openlineage.transport.endpoint": transport.endpoint,
-        "spark.openlineage.transport.timeoutInMillis": str(
-            int(transport.timeout * 1000)  # convert to milliseconds, as required by Spark integration
-        ),
-    }
-    if transport.compression:
-        properties["spark.openlineage.transport.compression"] = str(transport.compression)
 
-    if hasattr(transport.config.auth, "api_key") and transport.config.auth.get_bearer():
-        properties["spark.openlineage.transport.auth.type"] = "api_key"
-        properties["spark.openlineage.transport.auth.apiKey"] = transport.config.auth.get_bearer()
+    def _get_transport_information(tp) -> dict:
+        properties = {
+            "type": tp.kind,
+            "url": tp.url,
+            "endpoint": tp.endpoint,
+            "timeoutInMillis": str(
+                int(tp.timeout) * 1000  # convert to milliseconds, as required by Spark integration
+            ),
+        }
+        if hasattr(tp, "compression") and tp.compression:
+            properties["compression"] = str(tp.compression)
+
+        if hasattr(tp.config.auth, "api_key") and tp.config.auth.get_bearer():
+            properties["auth.type"] = "api_key"
+            properties["auth.apiKey"] = tp.config.auth.get_bearer()
+
+        if hasattr(tp.config, "custom_headers") and tp.config.custom_headers:
+            for key, value in tp.config.custom_headers.items():
+                properties[f"headers.{key}"] = value
+        return properties
+
+    def _format_transport(props: dict, transport: dict, name: str | None):
+        for key, value in transport.items():
+            if name:
+                props[f"spark.openlineage.transport.transports.{name}.{key}"] = value
+            else:
+                props[f"spark.openlineage.transport.{key}"] = value
+        return props
 
-    if hasattr(transport.config, "custom_headers") and transport.config.custom_headers:
-        for key, value in transport.config.custom_headers.items():
-            properties[f"spark.openlineage.transport.headers.{key}"] = value
+    transport = get_openlineage_listener().adapter.get_or_create_openlineage_client().transport
 
-    return properties
+    if transport.kind == "composite":
+        http_transports = {}
+        for nested_transport in transport.transports:
+            if nested_transport.kind == "http":
+                http_transports[nested_transport.name] = _get_transport_information(nested_transport)
+            else:
+                name = nested_transport.name if hasattr(nested_transport, "name") else "no-name"
+                log.info(
+                    "OpenLineage transport type `%s` with name `%s` is not supported in composite transport.",
+                    nested_transport.kind,
+                    name,
+                )
+        if len(http_transports) == 0:
+            log.warning(
+                "OpenLineage transport type `composite` does not contain http transport. Skipping "
+                "injection of OpenLineage transport information into Spark properties.",
+            )
+            return {}
+        props = {
+            "spark.openlineage.transport.type": "composite",
+            "spark.openlineage.transport.continueOnFailure": str(transport.config.continue_on_failure),
+        }
+        for name, http_transport in http_transports.items():
+            props = _format_transport(props, http_transport, name)
+        return props
+
+    elif transport.kind == "http":
+        return _format_transport({}, _get_transport_information(transport), None)
+
+    log.info(
+        "OpenLineage transport type `%s` does not support automatic "
+        "injection of OpenLineage transport information into Spark properties.",
+        transport.kind,
+    )
+    return {}
 
 
 def _is_parent_job_information_present_in_spark_properties(properties: dict) -> bool: