openlineage-python 1.18.0__tar.gz → 1.20.3__tar.gz

{openlineage_python-1.18.0 → openlineage_python-1.20.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: openlineage-python
-Version: 1.18.0
+Version: 1.20.3
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage

{openlineage_python-1.18.0 → openlineage_python-1.20.3}/docs/source/openlineage.client.rst

@@ -273,6 +273,14 @@ openlineage.client.generated.symlinks\_dataset module
    :undoc-members:
    :show-inheritance:
 
+openlineage.client.transport.composite module
+----------------------------------------
+
+.. automodule:: openlineage.client.transport.composite
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 openlineage.client.transport.console module
 -------------------------------------------
 

{openlineage_python-1.18.0 → openlineage_python-1.20.3}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.18.0"
+__version__ = "1.20.3"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

{openlineage_python-1.18.0 → openlineage_python-1.20.3}/openlineage/client/transport/__init__.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+from openlineage.client.transport.composite import CompositeTransport
 from openlineage.client.transport.console import ConsoleTransport
 from openlineage.client.transport.factory import DefaultTransportFactory
 from openlineage.client.transport.file import FileTransport
@@ -12,6 +13,7 @@ from openlineage.client.transport.noop import NoopTransport
 from openlineage.client.transport.transport import Config, Transport, TransportFactory
 
 _factory = DefaultTransportFactory()
+_factory.register_transport(CompositeTransport.kind, CompositeTransport)
 _factory.register_transport(HttpTransport.kind, HttpTransport)
 _factory.register_transport(KafkaTransport.kind, KafkaTransport)
 _factory.register_transport(MSKIAMTransport.kind, MSKIAMTransport)

openlineage_python-1.20.3/openlineage/client/transport/composite.py

@@ -0,0 +1,81 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+from functools import cached_property
+from typing import TYPE_CHECKING, Any
+
+import attr
+from openlineage.client.transport.transport import Config, Transport
+from openlineage.client.utils import get_only_specified_fields
+
+if TYPE_CHECKING:
+    from openlineage.client.client import Event
+
+log = logging.getLogger(__name__)
+
+
+@attr.s
+class CompositeConfig(Config):
+    """
+    CompositeConfig is a configuration class for CompositeTransport.
+
+    Attributes:
+        transports:
+            A list of dictionaries, where each dictionary represents the configuration
+            for a child transport. Each dictionary should contain the necessary parameters
+            to initialize a specific transport instance.
+
+        continue_on_failure:
+            If set to True, the CompositeTransport will attempt to emit the event using
+            all configured transports, regardless of whether any previous transport
+            in the list failed to emit the event. If set to False, an error in one
+            transport will halt the emission process for subsequent transports.
+    """
+
+    transports: list[dict[str, Any]] = attr.ib()
+    continue_on_failure: bool = attr.ib(default=True)
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> CompositeConfig:
+        """Create a CompositeConfig object from a dictionary."""
+        if "transports" not in params:
+            msg = "composite `transports` not passed to CompositeConfig"
+            raise RuntimeError(msg)
+        return cls(**get_only_specified_fields(cls, params))
+
+
+class CompositeTransport(Transport):
+    """CompositeTransport is a transport class that emits events using multiple transports."""
+
+    kind = "composite"
+    config_class = CompositeConfig
+
+    def __init__(self, config: CompositeConfig) -> None:
+        """Initialize a CompositeTransport object."""
+        self.config = config
+
+    @cached_property
+    def transports(self) -> list[Transport]:
+        """Create and return a list of transports based on the config."""
+        from openlineage.client.transport import get_default_factory
+
+        transports = []
+        for transport_config in self.config.transports:
+            transports.append(get_default_factory().create(transport_config))
+        return transports
+
+    def emit(self, event: Event) -> None:
+        """Emit an event using all transports in the config."""
+        for transport in self.transports:
+            try:
+                log.debug("Emitting event using %s", transport.kind)
+                transport.emit(event)
+            except Exception as e:  # noqa: BLE001
+                if self.config.continue_on_failure:
+                    log.warning("Transport %s failed to emit event with error: %s", transport.kind, e)
+                else:
+                    msg = f"Transport {transport.kind} failed to emit event"
+                    raise RuntimeError(msg) from e

{openlineage_python-1.18.0 → openlineage_python-1.20.3}/pyproject.toml

@@ -6,7 +6,7 @@ requires = [
 
 [project]
 name = "openlineage-python"
-version = "1.18.0"
+version = "1.20.3"
 description = "OpenLineage Python Client"
 readme = "README.md"
 keywords = [

openlineage_python-1.20.3/tests/test_composite.py

@@ -0,0 +1,130 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from unittest import mock
+from unittest.mock import MagicMock
+
+import pytest
+from openlineage.client.transport.composite import CompositeConfig, CompositeTransport
+from openlineage.client.transport.transport import Transport
+
+
+def test_composite_loads_full_config() -> None:
+    config = CompositeConfig.from_dict(
+        {
+            "type": "composite",
+            "transports": {
+                "kafka": {
+                    "type": "kafka",
+                    "config": {"bootstrap.servers": "localhost:9092"},
+                    "topic": "random-topic",
+                    "messageKey": "key",
+                    "flush": False,
+                },
+                "console": {"type": "console"},
+            },
+            "continue_on_failure": False,
+        },
+    )
+
+    assert config.transports["kafka"]["config"]["bootstrap.servers"] == "localhost:9092"
+    assert config.transports["kafka"]["topic"] == "random-topic"
+    assert config.transports["kafka"]["messageKey"] == "key"
+    assert config.transports["kafka"]["flush"] is False
+    assert config.transports["console"] == {"type": "console"}
+    assert config.continue_on_failure is False
+
+
+def test_composite_loads_partial_config_with_defaults() -> None:
+    config = CompositeConfig.from_dict(
+        {
+            "type": "composite",
+            "transports": [
+                {
+                    "type": "kafka",
+                    "config": {"bootstrap.servers": "localhost:9092"},
+                    "topic": "random-topic",
+                    "messageKey": "key",
+                    "flush": False,
+                },
+                {"type": "console"},
+            ],
+        },
+    )
+    assert config.transports[0]["config"]["bootstrap.servers"] == "localhost:9092"
+    assert config.transports[0]["topic"] == "random-topic"
+    assert config.transports[0]["messageKey"] == "key"
+    assert config.transports[0]["flush"] is False
+    assert config.transports[1] == {"type": "console"}
+    assert config.continue_on_failure is True
+
+
+def test_composite_transport_create_transports():
+    config = CompositeConfig(transports=[{"type": "tests.transport.FakeTransport"}], continue_on_failure=True)
+    transport = CompositeTransport(config)
+    assert len(transport.transports) == 1
+    assert transport.transports[0].kind == "fake"
+
+
+@mock.patch("openlineage.client.transport.get_default_factory")
+def test_transports(mock_factory):
+    mock_transport1 = MagicMock(spec=Transport)
+    mock_transport2 = MagicMock(spec=Transport)
+    mock_factory.return_value.create.side_effect = [mock_transport1, mock_transport2]
+
+    config = CompositeConfig(transports=[{}, {}])
+    transport = CompositeTransport(config)
+
+    assert transport.transports == [mock_transport1, mock_transport2]
+
+
+@mock.patch("openlineage.client.transport.get_default_factory")
+def test_emit_success(mock_factory):
+    mock_transport1 = MagicMock(spec=Transport)
+    mock_transport2 = MagicMock(spec=Transport)
+    mock_factory.return_value.create.side_effect = [mock_transport1, mock_transport2]
+
+    config = CompositeConfig(transports=[{}, {}])
+    transport = CompositeTransport(config)
+    event = MagicMock()
+
+    transport.emit(event)
+
+    mock_transport1.emit.assert_called_once_with(event)
+    mock_transport2.emit.assert_called_once_with(event)
+
+
+@mock.patch("openlineage.client.transport.get_default_factory")
+def test_emit_failure_no_continue_on_failure(mock_factory):
+    mock_transport1 = MagicMock(spec=Transport)
+    mock_transport2 = MagicMock(spec=Transport)
+    mock_transport2.emit.side_effect = Exception("Error")
+    mock_factory.return_value.create.side_effect = [mock_transport1, mock_transport2]
+
+    config = CompositeConfig(transports=[{}, {}], continue_on_failure=False)
+    transport = CompositeTransport(config)
+    event = MagicMock()
+
+    with pytest.raises(RuntimeError):
+        transport.emit(event)
+
+    mock_transport1.emit.assert_called_once_with(event)
+    mock_transport2.emit.assert_called_once_with(event)
+
+
+@mock.patch("openlineage.client.transport.get_default_factory")
+def test_emit_failure_continue_on_failure(mock_factory):
+    mock_transport1 = MagicMock(spec=Transport)
+    mock_transport2 = MagicMock(spec=Transport)
+    mock_factory.return_value.create.side_effect = [mock_transport1, mock_transport2]
+
+    config = CompositeConfig(transports=[{}, {}], continue_on_failure=False)
+    transport = CompositeTransport(config)
+    event = MagicMock()
+
+    # Emit should not raise an exception because continue_on_failure is True.
+    transport.emit(event)
+
+    mock_transport1.emit.assert_called_once_with(event)
+    mock_transport2.emit.assert_called_once_with(event)