openlineage-python 1.36.0__tar.gz → 1.37.0__tar.gz

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.36.0
+Version: 1.37.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/openlineage/client/client.py

@@ -67,11 +67,9 @@ class OpenLineageConfig:
         if "filters" in params:
             config.filters = [FilterConfig(**filter_config) for filter_config in params["filters"]]
         if "tags" in params:
-            job_tags = params["tags"].get("job", {})
-            run_tags = params["tags"].get("run", {})
             config.tags = TagsConfig(
-                job=[TagsJobFacetFields(key, value, "USER") for (key, value) in job_tags.items()],
-                run=[TagsRunFacetFields(key, value, "USER") for (key, value) in run_tags.items()],
+                job=params["tags"].get("job", {}),
+                run=params["tags"].get("run", {}),
             )
         return config
 
@@ -452,14 +450,16 @@ class OpenLineageClient:
         """
         run_event_types = (RunEvent, event_v2.RunEvent)
         run_and_job_event_types = (RunEvent, event_v2.RunEvent, JobEvent, event_v2.JobEvent)
-        tags_job = self.config.tags.job
+        # tags_job = self.config.tags.job
+        tags_job = [TagsJobFacetFields(key, value, "USER") for (key, value) in self.config.tags.job.items()]
         if isinstance(event, run_and_job_event_types) and tags_job:
             # Ensure facets exists
             event.job.facets = {} if not event.job.facets else event.job.facets
             tags_facet = event.job.facets.get("tags", TagsJobFacet())
             event.job.facets["tags"] = self._update_tag_facet(tags_facet, tags_job)  # type: ignore [arg-type, assignment]
 
-        tags_run = self.config.tags.run
+        # tags_run = self.config.tags.run
+        tags_run = [TagsRunFacetFields(key, value, "USER") for (key, value) in self.config.tags.run.items()]
         if isinstance(event, run_event_types) and tags_run:
             # Ensure facets exists
             event.run.facets = {} if not event.run.facets else event.run.facets

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.36.0"
+__version__ = "1.37.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

openlineage_python-1.37.0/openlineage/client/tags.py

@@ -0,0 +1,11 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import attr
+
+
+@attr.define
+class TagsConfig:
+    job: dict[str, str] = attr.field(factory=dict)
+    run: dict[str, str] = attr.field(factory=dict)

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/openlineage/client/transport/__init__.py

@@ -6,6 +6,7 @@ from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, A
 from openlineage.client.transport.async_http import AsyncHttpConfig, AsyncHttpTransport
 from openlineage.client.transport.composite import CompositeTransport
 from openlineage.client.transport.console import ConsoleTransport
+from openlineage.client.transport.datadog import DatadogConfig, DatadogTransport
 from openlineage.client.transport.factory import DefaultTransportFactory
 from openlineage.client.transport.file import FileTransport
 from openlineage.client.transport.http import HttpConfig, HttpTransport
@@ -29,6 +30,7 @@ _factory.register_transport(NoopTransport.kind, NoopTransport)
 _factory.register_transport(FileTransport.kind, FileTransport)
 _factory.register_transport(TransformTransport.kind, TransformTransport)
 _factory.register_transport(AmazonDataZoneTransport.kind, AmazonDataZoneTransport)
+_factory.register_transport(DatadogTransport.kind, DatadogTransport)
 
 
 def get_default_factory() -> DefaultTransportFactory:
@@ -50,6 +52,8 @@ __all__ = [
     "CompositeTransport",
     "Config",
     "ConsoleTransport",
+    "DatadogConfig",
+    "DatadogTransport",
     "FileTransport",
     "HttpConfig",
     "HttpTransport",

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/openlineage/client/transport/async_http.py

@@ -606,7 +606,9 @@ class AsyncHttpTransport(Transport):
         }
         if self.config.compression == HttpCompression.GZIP:
             headers["Content-Encoding"] = "gzip"
-            return gzip.compress(event.encode("utf-8")), headers
+            # levels higher than 3 are twice as slow:
+            # https://github.com/python/cpython/issues/91349#issuecomment-2737161048
+            return gzip.compress(event.encode("utf-8"), compresslevel=3), headers
 
         return event, headers
 

openlineage_python-1.37.0/openlineage/client/transport/datadog.py

@@ -0,0 +1,164 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
+
+import attr
+from openlineage.client.transport.async_http import AsyncHttpConfig, AsyncHttpTransport
+from openlineage.client.transport.http import HttpConfig, HttpTransport
+from openlineage.client.transport.transport import Config, Transport
+from openlineage.client.utils import get_only_specified_fields
+
+if TYPE_CHECKING:
+    from openlineage.client.client import Event
+
+
+# A mapping from https://docs.datadoghq.com/getting_started/site/ to
+# Datadog endpoint that accepts OL events.
+SITE_MAPPING = {
+    "datadoghq.com": "https://data-obs-intake.datadoghq.com",
+    "us3.datadoghq.com": "https://data-obs-intake.us3.datadoghq.com",
+    "us5.datadoghq.com": "https://data-obs-intake.us5.datadoghq.com",
+    "datadoghq.eu": "https://data-obs-intake.datadoghq.eu",
+    "ddog-gov.com": "https://data-obs-intake.ddog-gov.com",
+    "ap1.datadoghq.com": "https://data-obs-intake.ap1.datadoghq.com",
+    "ap2.datadoghq.com": "https://data-obs-intake.ap2.datadoghq.com",
+    "datad0g.com": "https://data-obs-intake.datad0g.com",
+}
+
+
+def _is_valid_url(url: str) -> bool:
+    try:
+        parsed = urlparse(url)
+        return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+@attr.define
+class DatadogConfig(Config):
+    apiKey: str = attr.field()  # noqa: N815
+    site: str = attr.field(default="datadoghq.com")
+    timeout: float = attr.field(default=5.0)
+    retry: dict[str, Any] = attr.field(
+        default={
+            "total": 5,
+            "read": 5,
+            "connect": 5,
+            "backoff_factor": 0.3,
+            "status_forcelist": [500, 502, 503, 504],
+            "allowed_methods": ["HEAD", "POST"],
+        }
+    )
+    max_queue_size: int = attr.field(default=10000)
+    max_concurrent_requests: int = attr.field(default=100)
+    async_transport_rules: dict[str, dict[str, bool]] = attr.field(default={"dbt": {"*": True}})
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> DatadogConfig:
+        api_key = params.get("apiKey") or params.get("apikey") or os.getenv("DD_API_KEY")
+        if not api_key:
+            msg = (
+                "apiKey is required for DatadogTransport. "
+                "Provide it in config or set DD_API_KEY environment variable."
+            )
+            raise ValueError(msg)
+
+        site = params.get("site") or os.getenv("DD_SITE", "datadoghq.com")
+
+        if site not in SITE_MAPPING and not _is_valid_url(site):  # type: ignore
+            msg = f"Invalid site '{site}'. Must be one of: {list(SITE_MAPPING.keys())} or a valid URL"
+            raise ValueError(msg)
+
+        config_params = params.copy()
+        config_params["apiKey"] = api_key
+        config_params["site"] = site
+
+        specified_dict = get_only_specified_fields(cls, config_params)
+        return cls(**specified_dict)
+
+
+class DatadogTransport(Transport):
+    kind = "datadog"
+    config_class = DatadogConfig
+
+    def __init__(self, config: DatadogConfig) -> None:
+        self.config = config
+
+        if _is_valid_url(config.site):
+            intake_url = config.site
+        else:
+            intake_url = SITE_MAPPING[config.site]
+
+        shared_config = {
+            "url": intake_url,
+            "timeout": config.timeout,
+            "retry": config.retry,
+            "compression": "gzip",
+            "auth": {"type": "api_key", "apiKey": config.apiKey},
+        }
+
+        http_config = HttpConfig.from_dict(shared_config)
+        self.http = HttpTransport(http_config)
+
+        async_config = shared_config.copy()
+        async_config.update(
+            {
+                "max_queue_size": config.max_queue_size,
+                "max_concurrent_requests": config.max_concurrent_requests,
+            }
+        )
+        async_http_config = AsyncHttpConfig.from_dict(async_config)
+        self.async_http = AsyncHttpTransport(async_http_config)
+
+    def emit(self, event: Event) -> Any:
+        # Check if event has JobTypeJobFacet with integration = "dbt"
+        should_use_async = self._should_use_async_transport(event)
+
+        if should_use_async:
+            return self.async_http.emit(event)
+        else:
+            return self.http.emit(event)
+
+    def close(self, timeout: float = -1) -> bool:
+        http_result = self.http.close(timeout)
+        async_result = self.async_http.close(timeout)
+        return http_result and async_result
+
+    def _should_use_async_transport(self, event: Event) -> bool:
+        """
+        Determine if event should use async transport based on async_transport_rules.
+        Returns True if the event matches the configured rules for async transport.
+        """
+
+        # Check for double wildcard first - if present, use async for all events
+        wildcard_rules = self.config.async_transport_rules.get("*", {})
+        if wildcard_rules.get("*"):
+            return True
+
+        if not hasattr(event, "job") or not hasattr(event.job, "facets"):
+            return False
+
+        job_facets = event.job.facets
+        if not job_facets:
+            return False
+
+        job_type_facet = job_facets.get("jobType")
+        if not job_type_facet:
+            return False
+
+        integration = getattr(job_type_facet, "integration", "").lower()
+        job_type = getattr(job_type_facet, "jobType", "").lower()
+
+        if not integration:
+            return False
+
+        for rule_integration, job_type_rules in self.config.async_transport_rules.items():
+            if rule_integration == "*" or rule_integration.lower() == integration:
+                for rule_job_type, use_async in job_type_rules.items():
+                    if rule_job_type == "*" or rule_job_type.lower() == job_type:
+                        return use_async
+        return False

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/openlineage/client/transport/http.py

@@ -226,6 +226,8 @@ class HttpTransport(Transport):
         }
         if self.compression == HttpCompression.GZIP:
             headers["Content-Encoding"] = "gzip"
-            return gzip.compress(event_str.encode("utf-8")), headers
+            # levels higher than 3 are twice as slow:
+            # https://github.com/python/cpython/issues/91349#issuecomment-2737161048
+            return gzip.compress(event_str.encode("utf-8"), compresslevel=3), headers
 
         return event_str, headers

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/pyproject.toml

@@ -6,7 +6,7 @@ requires = [
 
 [project]
 name = "openlineage-python"
-version = "1.36.0"
+version = "1.37.0"
 description = "OpenLineage Python Client"
 readme = "README.md"
 keywords = [

{openlineage_python-1.36.0 → openlineage_python-1.37.0}/tests/test_client.py

@@ -8,6 +8,7 @@ import re
 from typing import TYPE_CHECKING
 from unittest.mock import MagicMock, patch
 
+import attr
 import pytest
 from openlineage.client import event_v2
 from openlineage.client.client import OpenLineageClient, OpenLineageClientOptions, OpenLineageConfig
@@ -417,6 +418,16 @@ def test_ol_config_from_dict():
         OpenLineageConfig.from_dict({"facets": "invalid_data"})
 
 
+def test_ol_config_from_the_same_dict():
+    config_dict = {
+        "transport": {"url": "http://localhost:5050"},
+        "facets": {"environment_variables": ["VAR1", "VAR2"]},
+        "filters": [{"type": "exact", "match": "job_name"}],
+    }
+    config = OpenLineageConfig.from_dict(config_dict)
+    assert OpenLineageConfig.from_dict(attr.asdict(config)) == config
+
+
 @patch("yaml.safe_load", return_value=None)
 def test_config_file_content_empty_file(mock_yaml) -> None:  # noqa: ARG001
     assert OpenLineageClient().config == OpenLineageConfig()