openlineage-python 1.32.1__tar.gz → 1.34.0__tar.gz

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.32.1
+Version: 1.34.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage
@@ -18,6 +18,8 @@ Requires-Dist: packaging>=21.0
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: pyyaml>=5.4
 Requires-Dist: requests>=2.32.3
+Provides-Extra: datazone
+Requires-Dist: boto3>=1.34.134; extra == 'datazone'
 Provides-Extra: docs
 Requires-Dist: beautifulsoup4>=4.12.0; extra == 'docs'
 Requires-Dist: pydata-sphinx-theme>=0.13; extra == 'docs'

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/client.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+import contextlib
 import json
 import logging
 import os
@@ -10,19 +11,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
 
 import attr
 import yaml
-from openlineage.client.filter import Filter, FilterConfig, create_filter
-from openlineage.client.serde import Serde
-from openlineage.client.tags import TagsConfig
-from openlineage.client.utils import deep_merge_dicts
-
-if TYPE_CHECKING:
-    from requests import Session
-    from requests.adapters import HTTPAdapter
-
-import contextlib
-
 from openlineage.client import event_v2
 from openlineage.client.facets import FacetsConfig
+from openlineage.client.filter import Filter, FilterConfig, create_filter
 from openlineage.client.generated.environment_variables_run import (
     EnvironmentVariable,
     EnvironmentVariablesRunFacet,
@@ -30,6 +21,8 @@ from openlineage.client.generated.environment_variables_run import (
 from openlineage.client.generated.tags_job import TagsJobFacet, TagsJobFacetFields
 from openlineage.client.generated.tags_run import TagsRunFacet, TagsRunFacetFields
 from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
+from openlineage.client.serde import Serde
+from openlineage.client.tags import TagsConfig
 from openlineage.client.transport import (
     Transport,
     TransportFactory,
@@ -37,26 +30,32 @@ from openlineage.client.transport import (
 )
 from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
 from openlineage.client.transport.noop import NoopConfig, NoopTransport
+from openlineage.client.utils import deep_merge_dicts
+
+if TYPE_CHECKING:
+    from requests import Session
+    from requests.adapters import HTTPAdapter
+
 
 Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
 Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
 Event = Union[Event_v1, Event_v2]
 
 
-@attr.s
+@attr.define
 class OpenLineageClientOptions:
-    timeout: float = attr.ib(default=5.0)
-    verify: bool = attr.ib(default=True)
-    api_key: str = attr.ib(default=None)
-    adapter: HTTPAdapter = attr.ib(default=None)
+    timeout: float = 5.0
+    verify: bool = True
+    api_key: str | None = None
+    adapter: HTTPAdapter | None = None
 
 
-@attr.s
+@attr.define
 class OpenLineageConfig:
-    transport: dict[str, Any] | None = attr.ib(factory=dict)
-    facets: FacetsConfig = attr.ib(factory=FacetsConfig)
-    filters: list[FilterConfig] = attr.ib(factory=list)
-    tags: TagsConfig = attr.ib(factory=TagsConfig)
+    transport: dict[str, Any] | None = attr.field(factory=dict)
+    facets: FacetsConfig = attr.field(factory=FacetsConfig)
+    filters: list[FilterConfig] = attr.field(factory=list)
+    tags: TagsConfig = attr.field(factory=TagsConfig)
 
     @classmethod
     def from_dict(cls, params: dict[str, Any]) -> OpenLineageConfig:

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.32.1"
+__version__ = "1.34.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/facet_v2.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from openlineage.client.generated import (
+    catalog_dataset,
     column_lineage_dataset,
     data_quality_assertions_dataset,
     data_quality_metrics_input_dataset,
@@ -53,6 +54,7 @@ __all__ = [
     "OutputDatasetFacet",
     "RunFacet",
     "set_producer",
+    "catalog_dataset",
     "column_lineage_dataset",
     "data_quality_assertions_dataset",
     "data_quality_metrics_input_dataset",

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/facets.py

@@ -5,6 +5,6 @@ from __future__ import annotations
 import attr
 
 
-@attr.s
+@attr.define
 class FacetsConfig:
-    environment_variables: list[str] = attr.ib(factory=list)
+    environment_variables: list[str] = attr.field(factory=list)

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/filter.py

@@ -14,11 +14,11 @@ log = logging.getLogger(__name__)
 RunEventType = typing.Union[RunEvent, RunEvent_v2]
 
 
-@attr.s
+@attr.define
 class FilterConfig:
-    type: str | None = attr.ib(default=None)
-    match: str | None = attr.ib(default=None)
-    regex: str | None = attr.ib(default=None)
+    type: str | None = None
+    match: str | None = None
+    regex: str | None = None
 
 
 class Filter:

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/generated/base.py

@@ -92,7 +92,7 @@ class BaseFacet(RedactMixin):
 
         new_class = attr.make_class(
             self.__class__.__name__,
-            {k: attr.field() for k in kwargs if k not in current_attrs},
+            {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
             bases=(self.__class__,),
         )
         new_class.__module__ = self.__class__.__module__

openlineage_python-1.34.0/openlineage/client/generated/catalog_dataset.py

@@ -0,0 +1,32 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import attr
+from openlineage.client.generated.base import DatasetFacet
+
+
+@attr.define
+class CatalogDatasetFacet(DatasetFacet):
+    framework: str
+    """The storage framework for which the catalog is configured"""
+
+    type: str
+    """Type of the catalog."""
+
+    name: str
+    """Name of the catalog, as configured in the source system."""
+
+    metadataUri: str | None = attr.field(default=None)  # noqa: N815
+    """URI or connection string to the catalog, if applicable."""
+
+    warehouseUri: str | None = attr.field(default=None)  # noqa: N815
+    """URI or connection string to the physical location of the data that the catalog describes."""
+
+    source: str | None = attr.field(default=None)
+    """Source system where the catalog is configured."""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/facets/1-0-0/CatalogDatasetFacet.json#/$defs/CatalogDatasetFacet"

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/generated/column_lineage_dataset.py

@@ -44,7 +44,7 @@ class Fields(RedactMixin):
 
         new_class = attr.make_class(
             self.__class__.__name__,
-            {k: attr.field() for k in kwargs if k not in current_attrs},
+            {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
             bases=(self.__class__,),
         )
         new_class.__module__ = self.__class__.__module__
@@ -81,7 +81,7 @@ class InputField(RedactMixin):
 
         new_class = attr.make_class(
             self.__class__.__name__,
-            {k: attr.field() for k in kwargs if k not in current_attrs},
+            {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
             bases=(self.__class__,),
         )
         new_class.__module__ = self.__class__.__module__
@@ -122,7 +122,7 @@ class Transformation(RedactMixin):
 
         new_class = attr.make_class(
             self.__class__.__name__,
-            {k: attr.field() for k in kwargs if k not in current_attrs},
+            {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
             bases=(self.__class__,),
         )
         new_class.__module__ = self.__class__.__module__

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/generated/documentation_dataset.py

@@ -12,6 +12,9 @@ class DocumentationDatasetFacet(DatasetFacet):
     description: str
     """The description of the dataset."""
 
+    contentType: str | None = attr.field(default=None)  # noqa: N815
+    """MIME type of the description field content."""
+
     @staticmethod
     def _get_schema() -> str:
-        return "https://openlineage.io/spec/facets/1-0-1/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet"
+        return "https://openlineage.io/spec/facets/1-1-0/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet"

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/generated/documentation_job.py

@@ -12,8 +12,11 @@ class DocumentationJobFacet(JobFacet):
     description: str
     """The description of the job."""
 
+    contentType: str | None = attr.field(default=None)  # noqa: N815
+    """MIME type of the description field content."""
+
     @staticmethod
     def _get_schema() -> str:
         return (
-            "https://openlineage.io/spec/facets/1-0-1/DocumentationJobFacet.json#/$defs/DocumentationJobFacet"
+            "https://openlineage.io/spec/facets/1-1-0/DocumentationJobFacet.json#/$defs/DocumentationJobFacet"
         )

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/generator/templates/dataclass.jinja2

@@ -157,7 +157,7 @@ class {{ class_name }}:
 
         new_class = attr.make_class(
                 self.__class__.__name__,
-                {k: attr.field() for k in kwargs if k not in current_attrs},
+                {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
                 bases=(self.__class__,),
             )
         new_class.__module__ = self.__class__.__module__

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/tags.py

@@ -11,7 +11,7 @@ if TYPE_CHECKING:
     from openlineage.client.generated.tags_run import TagsRunFacetFields
 
 
-@attr.s
+@attr.define
 class TagsConfig:
-    job: list[TagsJobFacetFields] = attr.ib(factory=list)
-    run: list[TagsRunFacetFields] = attr.ib(factory=list)
+    job: list[TagsJobFacetFields] = attr.field(factory=list)
+    run: list[TagsRunFacetFields] = attr.field(factory=list)

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/transport/__init__.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, AmazonDataZoneTransport
 from openlineage.client.transport.composite import CompositeTransport
 from openlineage.client.transport.console import ConsoleTransport
 from openlineage.client.transport.factory import DefaultTransportFactory
@@ -10,6 +11,10 @@ from openlineage.client.transport.http import HttpConfig, HttpTransport
 from openlineage.client.transport.kafka import KafkaConfig, KafkaTransport
 from openlineage.client.transport.msk_iam import MSKIAMConfig, MSKIAMTransport
 from openlineage.client.transport.noop import NoopTransport
+from openlineage.client.transport.transform.transform import (
+    TransformConfig,
+    TransformTransport,
+)
 from openlineage.client.transport.transport import Config, Transport, TransportFactory
 
 _factory = DefaultTransportFactory()
@@ -20,6 +25,8 @@ _factory.register_transport(MSKIAMTransport.kind, MSKIAMTransport)
 _factory.register_transport(ConsoleTransport.kind, ConsoleTransport)
 _factory.register_transport(NoopTransport.kind, NoopTransport)
 _factory.register_transport(FileTransport.kind, FileTransport)
+_factory.register_transport(TransformTransport.kind, TransformTransport)
+_factory.register_transport(AmazonDataZoneTransport.kind, AmazonDataZoneTransport)
 
 
 def get_default_factory() -> DefaultTransportFactory:
@@ -34,17 +41,23 @@ def register_transport(clazz: type[Transport]) -> type[Transport]:
 
 
 __all__ = [
+    "register_transport",
+    "get_default_factory",
     "Config",
+    "Transport",
     "TransportFactory",
+    "AmazonDataZoneConfig",
+    "AmazonDataZoneTransport",
+    "CompositeTransport",
+    "ConsoleTransport",
+    "FileTransport",
     "HttpConfig",
     "HttpTransport",
     "KafkaConfig",
     "KafkaTransport",
-    "MSKIAMTransport",
     "MSKIAMConfig",
-    "ConsoleTransport",
+    "MSKIAMTransport",
     "NoopTransport",
-    "Transport",
-    "register_transport",
-    "get_default_factory",
+    "TransformConfig",
+    "TransformTransport",
 ]

openlineage_python-1.34.0/openlineage/client/transport/amazon_datazone.py

@@ -0,0 +1,77 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from openlineage.client import event_v2
+from openlineage.client.run import RunEvent
+from openlineage.client.serde import Serde
+from openlineage.client.transport.transport import Config, Transport
+
+if TYPE_CHECKING:
+    from openlineage.client.client import Event
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class AmazonDataZoneConfig(Config):
+    domain_id: str
+    endpoint_override: str | None = None
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> AmazonDataZoneConfig:
+        if "domainId" not in params:
+            msg = "`domainId` key not passed to AmazonDataZoneConfig"
+            raise RuntimeError(msg)
+        return cls(domain_id=params["domainId"], endpoint_override=params.get("endpointOverride"))
+
+
+class AmazonDataZoneTransport(Transport):
+    kind = "amazon_datazone_api"
+    config_class = AmazonDataZoneConfig
+
+    def __init__(self, config: AmazonDataZoneConfig) -> None:
+        self.config = config
+        self.datazone = None
+
+        self._setup_datazone(self.config.endpoint_override)
+        log.debug(
+            "Constructing OpenLineage transport that will send events to Amazon DataZone domain `%s`.",
+            config.domain_id,
+        )
+
+    def emit(self, event: Event) -> None:
+        if not isinstance(event, (RunEvent, event_v2.RunEvent)):
+            # DataZone only supports RunEvent
+            log.warning("DataZone only supports RunEvent")
+            return
+        try:
+            response = self.datazone.post_lineage_event(  # type: ignore[attr-defined]
+                domainIdentifier=self.config.domain_id, event=Serde.to_json(event).encode("utf-8")
+            )
+            log.info(
+                "Successfully posted a LineageEvent: %s in Domain: %s", response["id"], response["domainId"]
+            )
+        except Exception as error:  # noqa: BLE001
+            msg = f"Failed to send lineage event to DataZone Domain {self.config.domain_id}: {event}"
+            raise RuntimeError(msg) from error
+
+    def _setup_datazone(self, endpoint_url: str | None = None) -> None:
+        try:
+            import boto3  # type: ignore[import-untyped]
+
+            self.datazone = boto3.client("datazone", endpoint_url=endpoint_url)
+        except (ImportError, ModuleNotFoundError):
+            log.exception(
+                "OpenLineage client could not import the boto3 module. "
+                "This could be due to boto3 not being installed or being corrupted. "
+                "DataZoneTransport requires boto3 to function properly. "
+                "Please ensure boto3 is installed correctly. "
+                "You can install or reinstall it via `pip install boto3`"
+            )
+            raise

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/transport/composite.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 
-@attr.s
+@attr.define
 class CompositeConfig(Config):
     """
     CompositeConfig is a configuration class for CompositeTransport.
@@ -36,8 +36,8 @@ class CompositeConfig(Config):
             transport will halt the emission process for subsequent transports.
     """
 
-    transports: list[dict[str, Any]] | dict[str, dict[str, Any]] = attr.ib()
-    continue_on_failure: bool = attr.ib(default=True)
+    transports: list[dict[str, Any]] | dict[str, dict[str, Any]]
+    continue_on_failure: bool = True
 
     @classmethod
     def from_dict(cls, params: dict[str, Any]) -> CompositeConfig:
@@ -86,6 +86,7 @@ class CompositeTransport(Transport):
             except Exception as e:  # noqa: BLE001
                 if self.config.continue_on_failure:
                     log.warning("Transport %s failed to emit event with error: %s", transport, e)
+                    log.debug("OpenLineage emission failure details:", exc_info=True)
                 else:
                     msg = f"Transport {transport} failed to emit event"
                     raise RuntimeError(msg) from e

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/transport/http.py

@@ -76,23 +76,23 @@ def get_session() -> Session:
     return Session()
 
 
-@attr.s
+@attr.define
 class HttpConfig(Config):
-    url: str = attr.ib()
-    endpoint: str = attr.ib(default="api/v1/lineage")
-    timeout: float = attr.ib(default=5.0)
+    url: str
+    endpoint: str = "api/v1/lineage"
+    timeout: float = 5.0
     # check TLS certificates
-    verify: bool = attr.ib(default=True)
-    auth: TokenProvider = attr.ib(factory=lambda: TokenProvider({}))
-    compression: HttpCompression | None = attr.ib(default=None)
+    verify: bool = True
+    auth: TokenProvider = attr.field(factory=lambda: TokenProvider({}))
+    compression: HttpCompression | None = None
     # not set by TransportFactory
-    session: Session | None = attr.ib(default=None)
+    session: Session | None = None
     # not set by TransportFactory
-    adapter: HTTPAdapter | None = attr.ib(default=None)
+    adapter: HTTPAdapter | None = None
     # custom headers support
-    custom_headers: dict[str, str] = attr.ib(factory=dict)
+    custom_headers: dict[str, str] = attr.field(factory=dict)
     # retry settings
-    retry: dict[str, Any] = attr.ib(
+    retry: dict[str, Any] = attr.field(
         default={
             "total": 5,
             "read": 5,

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/transport/kafka.py

@@ -24,21 +24,21 @@ log = logging.getLogger(__name__)
 _T = TypeVar("_T", bound="KafkaConfig")
 
 
-@attr.s
+@attr.define
 class KafkaConfig(Config):
     # Kafka producer config
     # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration
-    config: dict[str, str] = attr.ib()
+    config: dict[str, str]
 
     # Topic on which we should send messages
-    topic: str = attr.ib()
+    topic: str
 
     # Explicit key for Kafka producer
-    messageKey: str | None = attr.ib(default=None)  # noqa: N815
+    messageKey: str | None = None  # noqa: N815
 
     # Set to true if Kafka should flush after each event. The process that emits can be killed in
     # some cases - for example in Airflow integration, so flushing is desirable there.
-    flush: bool = attr.ib(default=True)
+    flush: bool = True
 
     @classmethod
     def from_dict(cls: type[_T], params: dict[str, Any]) -> _T:

{openlineage_python-1.32.1 → openlineage_python-1.34.0}/openlineage/client/transport/msk_iam.py

@@ -32,15 +32,15 @@ def _detect_running_region() -> None | str:
     return None
 
 
-@attr.s
+@attr.define
 class MSKIAMConfig(KafkaConfig):
     # MSK producer config
     # https://github.com/aws/aws-msk-iam-sasl-signer-python
 
-    region: str = attr.ib(default=None)
-    aws_profile: None | str = attr.ib(default=None)
-    role_arn: None | str = attr.ib(default=None)
-    aws_debug_creds: bool = attr.ib(default=False)
+    region: str | None = None
+    aws_profile: None | str = None
+    role_arn: None | str = None
+    aws_debug_creds: bool = False
 
 
 def _oauth_cb(config: MSKIAMConfig, *_: Any) -> tuple[str, float]:

openlineage_python-1.34.0/openlineage/client/transport/transform/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from openlineage.client.transport.transform.transform import (
+    EventTransformer,
+    TransformConfig,
+    TransformTransport,
+)
+from openlineage.client.transport.transform.transformers.job_namespace_replace_transformer import (
+    JobNamespaceReplaceTransformer,
+)
+
+__all__ = [
+    "EventTransformer",
+    "TransformConfig",
+    "TransformTransport",
+    "JobNamespaceReplaceTransformer",
+]