openlineage-python 1.32.0__tar.gz → 1.33.0__tar.gz

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.32.0
+Version: 1.33.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage
@@ -18,6 +18,8 @@ Requires-Dist: packaging>=21.0
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: pyyaml>=5.4
 Requires-Dist: requests>=2.32.3
+Provides-Extra: datazone
+Requires-Dist: boto3>=1.34.134; extra == 'datazone'
 Provides-Extra: docs
 Requires-Dist: beautifulsoup4>=4.12.0; extra == 'docs'
 Requires-Dist: pydata-sphinx-theme>=0.13; extra == 'docs'

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.32.0"
+__version__ = "1.33.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/openlineage/client/facet_v2.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from openlineage.client.generated import (
+    catalog_dataset,
     column_lineage_dataset,
     data_quality_assertions_dataset,
     data_quality_metrics_input_dataset,
@@ -53,6 +54,7 @@ __all__ = [
     "OutputDatasetFacet",
     "RunFacet",
     "set_producer",
+    "catalog_dataset",
     "column_lineage_dataset",
     "data_quality_assertions_dataset",
     "data_quality_metrics_input_dataset",

openlineage_python-1.33.0/openlineage/client/generated/catalog_dataset.py

@@ -0,0 +1,32 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import attr
+from openlineage.client.generated.base import DatasetFacet
+
+
+@attr.define
+class CatalogDatasetFacet(DatasetFacet):
+    framework: str
+    """The storage framework for which the catalog is configured"""
+
+    type: str
+    """Type of the catalog."""
+
+    name: str
+    """Name of the catalog, as configured in the source system."""
+
+    metadataUri: str | None = attr.field(default=None)  # noqa: N815
+    """URI or connection string to the catalog, if applicable."""
+
+    warehouseUri: str | None = attr.field(default=None)  # noqa: N815
+    """URI or connection string to the physical location of the data that the catalog describes."""
+
+    source: str | None = attr.field(default=None)
+    """Source system where the catalog is configured."""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/facets/1-0-0/CatalogDatasetFacet.json#/$defs/CatalogDatasetFacet"

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/openlineage/client/transport/__init__.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, AmazonDataZoneTransport
 from openlineage.client.transport.composite import CompositeTransport
 from openlineage.client.transport.console import ConsoleTransport
 from openlineage.client.transport.factory import DefaultTransportFactory
@@ -10,6 +11,10 @@ from openlineage.client.transport.http import HttpConfig, HttpTransport
 from openlineage.client.transport.kafka import KafkaConfig, KafkaTransport
 from openlineage.client.transport.msk_iam import MSKIAMConfig, MSKIAMTransport
 from openlineage.client.transport.noop import NoopTransport
+from openlineage.client.transport.transform.transform import (
+    TransformConfig,
+    TransformTransport,
+)
 from openlineage.client.transport.transport import Config, Transport, TransportFactory
 
 _factory = DefaultTransportFactory()
@@ -20,6 +25,8 @@ _factory.register_transport(MSKIAMTransport.kind, MSKIAMTransport)
 _factory.register_transport(ConsoleTransport.kind, ConsoleTransport)
 _factory.register_transport(NoopTransport.kind, NoopTransport)
 _factory.register_transport(FileTransport.kind, FileTransport)
+_factory.register_transport(TransformTransport.kind, TransformTransport)
+_factory.register_transport(AmazonDataZoneTransport.kind, AmazonDataZoneTransport)
 
 
 def get_default_factory() -> DefaultTransportFactory:
@@ -34,17 +41,23 @@ def register_transport(clazz: type[Transport]) -> type[Transport]:
 
 
 __all__ = [
+    "register_transport",
+    "get_default_factory",
     "Config",
+    "Transport",
     "TransportFactory",
+    "AmazonDataZoneConfig",
+    "AmazonDataZoneTransport",
+    "CompositeTransport",
+    "ConsoleTransport",
+    "FileTransport",
     "HttpConfig",
     "HttpTransport",
     "KafkaConfig",
     "KafkaTransport",
-    "MSKIAMTransport",
     "MSKIAMConfig",
-    "ConsoleTransport",
+    "MSKIAMTransport",
     "NoopTransport",
-    "Transport",
-    "register_transport",
-    "get_default_factory",
+    "TransformConfig",
+    "TransformTransport",
 ]

openlineage_python-1.33.0/openlineage/client/transport/amazon_datazone.py

@@ -0,0 +1,77 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from openlineage.client import event_v2
+from openlineage.client.run import RunEvent
+from openlineage.client.serde import Serde
+from openlineage.client.transport.transport import Config, Transport
+
+if TYPE_CHECKING:
+    from openlineage.client.client import Event
+
+log = logging.getLogger(__name__)
+
+
+@dataclass
+class AmazonDataZoneConfig(Config):
+    domain_id: str
+    endpoint_override: str | None = None
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> AmazonDataZoneConfig:
+        if "domainId" not in params:
+            msg = "`domainId` key not passed to AmazonDataZoneConfig"
+            raise RuntimeError(msg)
+        return cls(domain_id=params["domainId"], endpoint_override=params.get("endpointOverride"))
+
+
+class AmazonDataZoneTransport(Transport):
+    kind = "amazon_datazone_api"
+    config_class = AmazonDataZoneConfig
+
+    def __init__(self, config: AmazonDataZoneConfig) -> None:
+        self.config = config
+        self.datazone = None
+
+        self._setup_datazone(self.config.endpoint_override)
+        log.debug(
+            "Constructing OpenLineage transport that will send events to Amazon DataZone domain `%s`.",
+            config.domain_id,
+        )
+
+    def emit(self, event: Event) -> None:
+        if not isinstance(event, (RunEvent, event_v2.RunEvent)):
+            # DataZone only supports RunEvent
+            log.warning("DataZone only supports RunEvent")
+            return
+        try:
+            response = self.datazone.post_lineage_event(  # type: ignore[attr-defined]
+                domainIdentifier=self.config.domain_id, event=Serde.to_json(event).encode("utf-8")
+            )
+            log.info(
+                "Successfully posted a LineageEvent: %s in Domain: %s", response["id"], response["domainId"]
+            )
+        except Exception as error:  # noqa: BLE001
+            msg = f"Failed to send lineage event to DataZone Domain {self.config.domain_id}: {event}"
+            raise RuntimeError(msg) from error
+
+    def _setup_datazone(self, endpoint_url: str | None = None) -> None:
+        try:
+            import boto3  # type: ignore[import-untyped]
+
+            self.datazone = boto3.client("datazone", endpoint_url=endpoint_url)
+        except (ImportError, ModuleNotFoundError):
+            log.exception(
+                "OpenLineage client could not import the boto3 module. "
+                "This could be due to boto3 not being installed or being corrupted. "
+                "DataZoneTransport requires boto3 to function properly. "
+                "Please ensure boto3 is installed correctly. "
+                "You can install or reinstall it via `pip install boto3`"
+            )
+            raise

openlineage_python-1.33.0/openlineage/client/transport/transform/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from openlineage.client.transport.transform.transform import (
+    EventTransformer,
+    TransformConfig,
+    TransformTransport,
+)
+from openlineage.client.transport.transform.transformers.job_namespace_replace_transformer import (
+    JobNamespaceReplaceTransformer,
+)
+
+__all__ = [
+    "EventTransformer",
+    "TransformConfig",
+    "TransformTransport",
+    "JobNamespaceReplaceTransformer",
+]

openlineage_python-1.33.0/openlineage/client/transport/transform/transform.py

@@ -0,0 +1,141 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import copy
+import inspect
+import logging
+from typing import TYPE_CHECKING, Any
+
+import attr
+from openlineage.client.serde import Serde
+from openlineage.client.transport.transport import Config, Transport
+from openlineage.client.utils import get_only_specified_fields, import_from_string
+
+if TYPE_CHECKING:
+    from openlineage.client.client import Event
+
+log = logging.getLogger(__name__)
+
+
+class EventTransformer:
+    """Base class for transformers of OpenLineage events."""
+
+    def __init__(self, properties: dict[str, Any]) -> None:
+        """Initialize the transformer with a dictionary of properties.
+
+        These properties can be used by subclasses to configure the transformation behavior.
+        """
+        self.properties = properties
+
+    def transform(self, event: Event) -> Event | None:
+        """Transform an OpenLineage event.
+
+        Subclasses must override this method to apply specific transformations
+        to the given event.
+
+        Args:
+            event: The event to be transformed.
+
+        Returns:
+            The transformed event, or None to not emit the event.
+        """
+        raise NotImplementedError
+
+    def __str__(self) -> str:
+        return f"<{self.__class__.__name__}({self.properties})>"
+
+
+@attr.s
+class TransformConfig(Config):
+    """Configuration class for transform transport.
+
+    Holds configuration details including inner transport settings, the name of the
+    transformer class to use, and optional transformer-specific properties.
+    """
+
+    transport: dict[str, Any] = attr.ib()
+    transformer_class: str = attr.ib()
+    transformer_properties: dict[str, Any] = attr.ib(default={})
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> TransformConfig:
+        """Create a TransformConfig instance from a dictionary of parameters.
+
+        Validates that required fields (`transport` and `transformer_class`) are present
+        and filters out unspecified fields based on the class definition.
+
+        Args:
+            params: Dictionary containing configuration parameters.
+
+        Returns:
+            An instance of TransformConfig populated with the provided data.
+
+        Raises:
+            RuntimeError: If required keys (`transport` or `transformer_class`) are missing.
+        """
+        if "transport" not in params:
+            msg = "'transport' not passed to TransformConfig"
+            raise RuntimeError(msg)
+        if "transformer_class" not in params:
+            msg = "'transformer_class' not passed to TransformConfig"
+            raise RuntimeError(msg)
+        return cls(**get_only_specified_fields(cls, params))
+
+
+class TransformTransport(Transport):
+    """Transport that transforms events before emitting them.
+
+    This transport wraps another transport (defined in the configuration) and intercepts
+    events for transformation using a user-defined EventTransformer class. Events can be
+    modified or discarded prior to being sent.
+    """
+
+    kind = "transform"
+    config_class = TransformConfig
+
+    def __init__(self, config: TransformConfig) -> None:
+        """Initialize the TransformTransport with the given configuration.
+
+        Dynamically loads the transformer class specified in the config, validates it,
+        and initializes both the transformer and the inner transport.
+
+        Args:
+            config: Configuration object containing transport and transformer settings.
+
+        Raises:
+            TypeError: If the transformer is not a valid subclass of EventTransformer.
+        """
+        # Import within method to avoid circular import error
+        from openlineage.client.transport import get_default_factory
+
+        transformer_class = import_from_string(config.transformer_class)
+        if not inspect.isclass(transformer_class) or not issubclass(transformer_class, EventTransformer):
+            msg = f"Transformer `{transformer_class}` has to be class, and subclass of EventTransformer"
+            raise TypeError(msg)
+        self.transformer = transformer_class(properties=config.transformer_properties)
+        self.transport = get_default_factory().create(config.transport)
+
+    def emit(self, event: Event) -> Any:
+        """Transform and emit an OpenLineage event.
+
+        Applies the configured EventTransformer to the event. If the result is not `None`,
+        the transformed event is passed to the wrapped transport's `emit` method.
+
+        Args:
+            event: The OpenLineage event to be transformed and emitted.
+
+        Returns:
+            The result of the inner transport's emit call, or None if the event is dropped.
+        """
+        log.debug("Event before transformation: %s", Serde.to_json(event))
+        event = copy.deepcopy(event)  # Copy to make sure the transformer does not modify original event
+
+        log.debug("Transforming OpenLineage event with %s", self.transformer)
+        transformed = self.transformer.transform(event)
+        if transformed is None:
+            log.info("Transformed OpenLineage event is None. Skipping emission.")
+            return None
+
+        log.debug("Event after transformation: %s", Serde.to_json(transformed))
+        return self.transport.emit(transformed)

openlineage_python-1.33.0/openlineage/client/transport/transform/transformers/job_namespace_replace_transformer.py

@@ -0,0 +1,74 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from openlineage.client import event_v2
+from openlineage.client.transport.transform.transform import EventTransformer
+
+if TYPE_CHECKING:
+    from openlineage.client.client import Event
+    from openlineage.client.facet_v2 import parent_run
+
+log = logging.getLogger(__name__)
+
+
+class JobNamespaceReplaceTransformer(EventTransformer):
+    """EventTransformer that replaces the Job namespace in OpenLineage events."""
+
+    def __init__(self, properties: dict[str, Any]) -> None:
+        if not properties.get("new_job_namespace"):
+            msg = f"`new_job_namespace` not passed to {self.__class__.__name__}"
+            raise RuntimeError(msg)
+        self.new_job_namespace = properties["new_job_namespace"]
+        self.include_parent_facet = str(properties.get("include_parent_facet", "true")).lower() == "true"
+        super().__init__(properties=properties)
+
+    def transform(self, event: Event) -> Event | None:
+        if not isinstance(event, (event_v2.RunEvent, event_v2.JobEvent)):  # These events have Job
+            log.warning(
+                "Unsupported type: expected `RunEvent` or `JobEvent`, got `%s`. Skipping transformation.",
+                type(event),
+            )
+            return event
+
+        log.debug(
+            "Replacing OpenLineage job namespace: `%s` with: `%s` for single event.",
+            event.job.namespace,
+            self.new_job_namespace,
+        )
+        event.job.namespace = self.new_job_namespace
+
+        if not self.include_parent_facet:
+            log.debug("Set `include_parent_facet` to True to include replacement in ParentRunFacet.")
+            return event
+
+        if not isinstance(event, event_v2.RunEvent):
+            log.debug("Only `RunEvent` can have ParentRunFacet, skipping replacement in ParentRunFacet.")
+            return event
+
+        if not event.run.facets or not event.run.facets.get("parent"):
+            log.debug("Parent run facet not found, skipping replacement in ParentRunFacet.")
+            return event
+
+        # Mypy sees RunFacet, but we assume ParentRunFacet, so we ignore the check.
+        parent_facet: parent_run.ParentRunFacet = event.run.facets["parent"]  # type: ignore[assignment]
+        log.debug(
+            "Replacing parent OpenLineage job namespace: `%s` with: `%s` for single event.",
+            parent_facet.job.namespace,
+            self.new_job_namespace,
+        )
+        parent_facet.job.namespace = self.new_job_namespace
+
+        if parent_facet.root:
+            log.debug(
+                "Replacing root OpenLineage job namespace: `%s` with: `%s` for single event.",
+                parent_facet.root.job.namespace,
+                self.new_job_namespace,
+            )
+            parent_facet.root.job.namespace = self.new_job_namespace
+
+        return event

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/pyproject.toml

@@ -6,7 +6,7 @@ requires = [
 
 [project]
 name = "openlineage-python"
-version = "1.32.0"
+version = "1.33.0"
 description = "OpenLineage Python Client"
 readme = "README.md"
 keywords = [
@@ -38,6 +38,9 @@ optional-dependencies.msk-iam = [
   "aws-msk-iam-sasl-signer-python>=1.0.1",
   "confluent-kafka>=2.1.1",
 ]
+optional-dependencies.datazone = [
+  "boto3>=1.34.134"
+]
 optional-dependencies.test = [
   "covdefaults>=2.3",
   "pytest>=7.3.1",

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/redact_fields.yml

@@ -239,3 +239,7 @@
       redact_fields: []
     - class_name: TagsRunFacet
       redact_fields: []
+- module: catalog_dataset
+  classes:
+    - class_name: CatalogDatasetFacet
+      redact_fields: []

openlineage_python-1.33.0/tests/generator/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0

openlineage_python-1.33.0/tests/test_amazon_datazone.py

@@ -0,0 +1,123 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from openlineage.client import OpenLineageClient
+from openlineage.client.run import (
+    Dataset,
+    DatasetEvent,
+    Job,
+    JobEvent,
+    Run,
+    RunEvent,
+    RunState,
+)
+from openlineage.client.serde import Serde
+from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, AmazonDataZoneTransport
+
+if TYPE_CHECKING:
+    from pytest_mock import MockerFixture
+
+
+@pytest.fixture()
+def domain_id() -> str:
+    return "dzd_a1b2c3d4e5f6g7"
+
+
+@pytest.fixture()
+def run_event() -> RunEvent:
+    return RunEvent(
+        eventType=RunState.START,
+        eventTime="2024-08-20T11:08:01.123456",
+        run=Run(runId="f8b61441-a9fb-4f65-a156-be466eb29832"),
+        job=Job(namespace="test-namespace", name="test-job"),
+        producer="prod",
+        schemaURL="schema",
+    )
+
+
+@pytest.fixture()
+def dataset_event() -> DatasetEvent:
+    return DatasetEvent(
+        eventTime="2024-08-20T11:08:01.123456",
+        dataset=Dataset(namespace="test-namespace", name="test-dataset"),
+        producer="prod",
+        schemaURL="schema",
+    )
+
+
+@pytest.fixture()
+def job_event() -> JobEvent:
+    return JobEvent(
+        eventTime="2024-08-20T11:08:01.123456",
+        job=Job(namespace="test-namespace", name="test-job"),
+        producer="prod",
+        schemaURL="schema",
+    )
+
+
+def test_datazone_config_raises_when_missing_domain_id() -> None:
+    with pytest.raises(TypeError):
+        AmazonDataZoneConfig()
+
+
+def test_client_with_amazon_datazone_transport_emits_run_event(
+    run_event: RunEvent, domain_id: str, mocker: MockerFixture
+) -> None:
+    mocker.patch("boto3.client")
+    config = AmazonDataZoneConfig(domain_id=domain_id)
+    transport = AmazonDataZoneTransport(config)
+    client = OpenLineageClient(transport=transport)
+
+    client.emit(run_event)
+
+    transport.datazone.post_lineage_event.assert_called_once_with(
+        domainIdentifier=domain_id,
+        event=Serde.to_json(run_event).encode("utf-8"),
+    )
+
+
+def test_client_with_amazon_datazone_transport_skips_job_event(
+    job_event: JobEvent, mocker: MockerFixture
+) -> None:
+    mocker.patch("boto3.client")
+    config = AmazonDataZoneConfig(domain_id=domain_id)
+    transport = AmazonDataZoneTransport(config)
+    client = OpenLineageClient(transport=transport)
+
+    client.emit(job_event)
+
+    transport.datazone.post_lineage_event.assert_not_called()
+
+
+def test_client_with_amazon_datazone_transport_skips_dataset_event(
+    dataset_event: DatasetEvent, mocker: MockerFixture
+) -> None:
+    mocker.patch("boto3.client")
+    config = AmazonDataZoneConfig(domain_id=domain_id)
+    transport = AmazonDataZoneTransport(config)
+    client = OpenLineageClient(transport=transport)
+
+    client.emit(dataset_event)
+
+    transport.datazone.post_lineage_event.assert_not_called()
+
+
+def test_setup_datazone_raises_when_boto3_missing(mocker: MockerFixture) -> None:
+    mocker.patch("boto3.client", side_effect=ModuleNotFoundError("No module named 'boto3'"))
+    config = AmazonDataZoneConfig(domain_id=domain_id)
+
+    with pytest.raises(ModuleNotFoundError):
+        AmazonDataZoneTransport(config)
+
+
+def test_setup_datazone_raises_when_boto3_corrupted(mocker: MockerFixture) -> None:
+    mocker.patch("boto3.client", side_effect=ImportError("Cannot import 'boto3'"))
+    config = AmazonDataZoneConfig(domain_id=domain_id)
+
+    with pytest.raises(ImportError):
+        AmazonDataZoneTransport(config)

{openlineage_python-1.32.0 → openlineage_python-1.33.0}/tests/test_uuid.py

@@ -12,7 +12,11 @@ def test_generate_new_uuid_returns_uuidv7():
     assert generate_new_uuid().version == 7  # noqa: PLR2004
 
 
-def test_generate_static_uuid_returns_different_result_on_each_call():
+def test_generate_new_uuid_returns_different_result_on_each_call():
+    result1 = generate_new_uuid()
+    result2 = generate_new_uuid()
+    assert result1 != result2
+
     instant = datetime.now()
     result1 = generate_new_uuid(instant)
     result2 = generate_new_uuid(instant)

openlineage_python-1.33.0/tests/transform/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0