openlineage-python 1.28.0__tar.gz → 1.30.0__tar.gz

{openlineage_python-1.28.0 → openlineage_python-1.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.28.0
+Version: 1.30.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage

{openlineage_python-1.28.0 → openlineage_python-1.30.0}/openlineage/client/client.py

@@ -12,6 +12,7 @@ import attr
 import yaml
 from openlineage.client.filter import Filter, FilterConfig, create_filter
 from openlineage.client.serde import Serde
+from openlineage.client.tags import TagsConfig
 from openlineage.client.utils import deep_merge_dicts
 
 if TYPE_CHECKING:
@@ -26,6 +27,8 @@ from openlineage.client.generated.environment_variables_run import (
     EnvironmentVariable,
     EnvironmentVariablesRunFacet,
 )
+from openlineage.client.generated.tags_job import TagsJobFacet, TagsJobFacetFields
+from openlineage.client.generated.tags_run import TagsRunFacet, TagsRunFacetFields
 from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
 from openlineage.client.transport import (
     Transport,
@@ -53,6 +56,7 @@ class OpenLineageConfig:
     transport: dict[str, Any] | None = attr.ib(factory=dict)
     facets: FacetsConfig = attr.ib(factory=FacetsConfig)
     filters: list[FilterConfig] = attr.ib(factory=list)
+    tags: TagsConfig = attr.ib(factory=TagsConfig)
 
     @classmethod
     def from_dict(cls, params: dict[str, Any]) -> OpenLineageConfig:
@@ -63,6 +67,13 @@ class OpenLineageConfig:
             config.facets = FacetsConfig(**params["facets"])
         if "filters" in params:
             config.filters = [FilterConfig(**filter_config) for filter_config in params["filters"]]
+        if "tags" in params:
+            job_tags = params["tags"].get("job", {})
+            run_tags = params["tags"].get("run", {})
+            config.tags = TagsConfig(
+                job=[TagsJobFacetFields(key, value, "USER") for (key, value) in job_tags.items()],
+                run=[TagsRunFacetFields(key, value, "USER") for (key, value) in run_tags.items()],
+            )
         return config
 
 
@@ -163,6 +174,7 @@ class OpenLineageClient:
             return
 
         event = self.add_environment_facets(event)
+        event = self.update_event_tags_facets(event)
 
         if log.isEnabledFor(logging.DEBUG):
             val = Serde.to_json(event).encode("utf-8")
@@ -362,7 +374,6 @@ class OpenLineageClient:
             # Parse value (try to parse as JSON, otherwise lowercase the value)
             with contextlib.suppress(json.JSONDecodeError):
                 env_value = json.loads(env_value)  # noqa: PLW2901
-
             cls._insert_into_config(config, keys, env_value)
 
         return config
@@ -404,3 +415,50 @@ class OpenLineageClient:
                 missing_vars,
             )
         return filtered_vars
+
+    def update_event_tags_facets(self, event: Event) -> Event:
+        """
+        Creates or updates job and run tag facets based on user-supplied environment variables
+        """
+        tags_job = self.config.tags.job
+        if (isinstance(event, (JobEvent, RunEvent))) and tags_job:
+            tags_facet = event.job.facets.get("tags", TagsJobFacet())
+            event.job.facets["tags"] = self._update_tag_facet(tags_facet, tags_job)  # type: ignore [arg-type]
+
+        tags_run = self.config.tags.run
+        if (isinstance(event, RunEvent)) and tags_run:
+            tags_facet = event.run.facets.get("tags", TagsRunFacet())
+            event.run.facets["tags"] = self._update_tag_facet(tags_facet, tags_run)  # type: ignore [arg-type]
+
+        return event
+
+    def _update_tag_facet(
+        self,
+        tags_facet: TagsJobFacet | TagsRunFacet,
+        user_tags: list[TagsJobFacetFields | TagsRunFacetFields],
+    ) -> TagsJobFacet | TagsRunFacet:
+        """
+        Handles updating tags in an existing tag facet
+        """
+
+        # Get tags from the facet that will not be updated (Do not have the same key as a user tag)
+        user_tag_keys = [tag.key for tag in user_tags]
+        keep_tags = []
+        if tags_facet.tags is not None:
+            keep_tags = [tag for tag in tags_facet.tags if tag.key.lower() not in user_tag_keys]
+
+        # Update tag key for any user tags that should override a facet tag.
+        # This preserves case of the facet tag key while updating value and source.
+        facet_tag_keys = {}
+        if tags_facet.tags is not None:
+            facet_tag_keys = {tag.key.lower(): tag.key for tag in tags_facet.tags}
+
+        for user_tag in user_tags:
+            if user_tag.key in facet_tag_keys:
+                facet_tag_key = facet_tag_keys[user_tag.key]
+                log.info("Overriding integration-supplied tag `%s` with user-supplied tag", facet_tag_key)
+                user_tag.key = facet_tag_key
+
+        all_tags = keep_tags + user_tags
+        tags_facet.tags = all_tags  # type: ignore [assignment]
+        return tags_facet

{openlineage_python-1.28.0 → openlineage_python-1.30.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.28.0"
+__version__ = "1.30.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

openlineage_python-1.30.0/openlineage/client/tags.py

@@ -0,0 +1,17 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import attr
+
+if TYPE_CHECKING:
+    from openlineage.client.generated.tags_job import TagsJobFacetFields
+    from openlineage.client.generated.tags_run import TagsRunFacetFields
+
+
+@attr.s
+class TagsConfig:
+    job: list[TagsJobFacetFields] = attr.ib(factory=list)
+    run: list[TagsRunFacetFields] = attr.ib(factory=list)

{openlineage_python-1.28.0 → openlineage_python-1.30.0}/pyproject.toml

@@ -6,7 +6,7 @@ requires = [
 
 [project]
 name = "openlineage-python"
-version = "1.28.0"
+version = "1.30.0"
 description = "OpenLineage Python Client"
 readme = "README.md"
 keywords = [

openlineage_python-1.30.0/tests/conftest.py

@@ -0,0 +1,54 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Union
+
+import pytest
+from openlineage.client import event_v2, set_producer
+from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
+from openlineage.client.transport.transport import Config, Transport
+
+
+@pytest.fixture(scope="session")
+def root() -> Path:
+    return Path(__file__).parent
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _setup_producer() -> None:
+    set_producer("https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python")
+
+
+# For testing events emitted by the client
+
+Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
+Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
+Event = Union[Event_v1, Event_v2]
+
+
+class NoOutputConfig(Config):
+    ...
+
+
+class NoOutputTransport(Transport):
+    """Minimal transport that stores the event for assertions"""
+
+    kind = "no_output"
+    config_class = NoOutputConfig
+
+    def __init__(self, config: NoOutputConfig) -> None:  # noqa: ARG002
+        self._event = None
+
+    def emit(self, event: Event) -> None:
+        self._event = event
+
+    @property
+    def event(self):
+        return self._event
+
+
+@pytest.fixture(scope="session")
+def transport():
+    return NoOutputTransport(config=NoOutputConfig())

{openlineage_python-1.28.0 → openlineage_python-1.30.0}/tests/test_client.py

@@ -15,6 +15,8 @@ from openlineage.client.generated.environment_variables_run import (
     EnvironmentVariable,
     EnvironmentVariablesRunFacet,
 )
+from openlineage.client.generated.tags_job import TagsJobFacet, TagsJobFacetFields
+from openlineage.client.generated.tags_run import TagsRunFacet, TagsRunFacetFields
 from openlineage.client.run import (
     SCHEMA_URL,
     Dataset,
@@ -82,7 +84,6 @@ def test_client_passes_to_create_with_valid_url(url: str, res: str) -> None:
 def test_client_sends_proper_json_with_minimal_run_event() -> None:
     session = MagicMock()
     client = OpenLineageClient(url="http://example.com", session=session)
-
     client.emit(
         RunEvent(
             RunState.START,
@@ -839,6 +840,31 @@ class TestOpenLineageConfigLoader:
                 },
                 {"transport": {"type": "console"}},
             ),
+            (
+                {
+                    "OPENLINEAGE__TAGS__JOB__ENVIRONMENT": "PRODUCTION",
+                    "OPENLINEAGE__TAGS__JOB__pipeline": "finance",
+                    "OPENLINEAGE__TAGS__RUN__environment": "PRODUCTION",
+                    "OPENLINEAGE__TAGS__RUN__pipeline": "finance",
+                },
+                {
+                    "tags": {
+                        "job": {"environment": "PRODUCTION", "pipeline": "finance"},
+                        "run": {"environment": "PRODUCTION", "pipeline": "finance"},
+                    }
+                },
+            ),
+            (
+                {
+                    "OPENLINEAGE__TAGS": '{"job": {"a": "b", "c": "d"}, "run": {"a": "b", "c": "d"}}',
+                },
+                {
+                    "tags": {
+                        "job": {"a": "b", "c": "d"},
+                        "run": {"a": "b", "c": "d"},
+                    }
+                },
+            ),
         ],
     )
     @patch.dict(os.environ, {})
@@ -846,3 +872,178 @@ class TestOpenLineageConfigLoader:
         with patch.dict(os.environ, env_vars):
             config = OpenLineageClient._load_config_from_env_variables()  # noqa: SLF001
             assert config == expected_config
+
+
+def test_update_job_tag_facet_creates_new_facet(transport):
+    tag_environment_variables = {
+        "OPENLINEAGE__TAGS__JOB__ENVIRONMENT": "PRODUCTION",
+        "OPENLINEAGE__TAGS__JOB__pipeline": "SALES",
+    }
+
+    run_event = RunEvent(
+        RunState.START,
+        "2021-11-03T10:53:52.427343",
+        Run("69f4acab-b87d-4fc0-b27b-8ea950370ff3"),
+        Job("openlineage", "job"),
+        "producer",
+    )
+
+    tags = [
+        TagsJobFacetFields("environment", "PRODUCTION", "USER"),
+        TagsJobFacetFields("pipeline", "SALES", "USER"),
+    ]
+
+    with patch.dict(os.environ, tag_environment_variables):
+        client = OpenLineageClient(transport=transport)
+        client.emit(run_event)
+        assert transport.event.job.facets.get("tags")
+        event_tags = sorted(transport.event.job.facets["tags"].tags, key=lambda x: x.key)
+        expected_tags = sorted(tags, key=lambda x: x.key)
+        assert event_tags == expected_tags
+
+
+def test_update_job_tag_facet_updates_existing_facet(transport):
+    tag_environment_variables = {
+        "OPENLINEAGE__TAGS__JOB__ENVIRONMENT": "PRODUCTION",
+        "OPENLINEAGE__TAGS__JOB__pipeline": "SALES",
+    }
+
+    run_event = RunEvent(
+        RunState.START,
+        "2021-11-03T10:53:52.427343",
+        Run("69f4acab-b87d-4fc0-b27b-8ea950370ff3"),
+        Job(
+            namespace="openlineage",
+            name="job",
+            facets={
+                "tags": TagsRunFacet(
+                    tags=[
+                        TagsJobFacetFields("environment", "STAGING", "USER"),
+                        TagsJobFacetFields("foo", "bar", "USER"),
+                    ]
+                )
+            },
+        ),
+        "producer",
+    )
+
+    # One existing tag (not updated), one existing tag (updated), one new tag from the user
+    tags = [
+        TagsJobFacetFields("foo", "bar", "USER"),
+        TagsJobFacetFields("environment", "PRODUCTION", "USER"),
+        TagsJobFacetFields("pipeline", "SALES", "USER"),
+    ]
+
+    with patch.dict(os.environ, tag_environment_variables):
+        client = OpenLineageClient(transport=transport)
+        client.emit(run_event)
+        assert transport.event.job.facets.get("tags")
+        event_tags = sorted(transport.event.job.facets["tags"].tags, key=lambda x: x.key)
+        expected_tags = sorted(tags, key=lambda x: x.key)
+        assert event_tags == expected_tags
+
+
+def test_update_run_tag_facet_creates_new_facet(transport):
+    tag_environment_variables = {
+        "OPENLINEAGE__TAGS__RUN__ENVIRONMENT": "PRODUCTION",
+        "OPENLINEAGE__TAGS__RUN__pipeline": "SALES",
+    }
+
+    run_event = RunEvent(
+        RunState.START,
+        "2021-11-03T10:53:52.427343",
+        Run("69f4acab-b87d-4fc0-b27b-8ea950370ff3"),
+        Job("openlineage", "job"),
+        "producer",
+    )
+
+    tags = [
+        TagsRunFacetFields("environment", "PRODUCTION", "USER"),
+        TagsRunFacetFields("pipeline", "SALES", "USER"),
+    ]
+
+    with patch.dict(os.environ, tag_environment_variables):
+        client = OpenLineageClient(transport=transport)
+        client.emit(run_event)
+        assert transport.event.run.facets.get("tags")
+        event_tags = sorted(transport.event.run.facets["tags"].tags, key=lambda x: x.key)
+        expected_tags = sorted(tags, key=lambda x: x.key)
+        assert event_tags == expected_tags
+
+
+def test_update_run_tag_facet_updates_existing_facet(transport):
+    tag_environment_variables = {
+        "OPENLINEAGE__TAGS__RUN__ENVIRONMENT": "PRODUCTION",
+        "OPENLINEAGE__TAGS__RUN__pipeline": "SALES",
+    }
+
+    run_event = RunEvent(
+        RunState.START,
+        "2021-11-03T10:53:52.427343",
+        Run(
+            runId="69f4acab-b87d-4fc0-b27b-8ea950370ff3",
+            facets={
+                "tags": TagsJobFacet(
+                    tags=[
+                        TagsRunFacetFields("ENVIRONMENT", "STAGING", "USER"),
+                        TagsRunFacetFields("foo", "bar", "USER"),
+                    ]
+                )
+            },
+        ),
+        Job("openlineage", "job"),
+        "producer",
+    )
+
+    # One existing tag (not updated), one existing tag (updated), one new tag from the user
+    tags = [
+        TagsRunFacetFields("foo", "bar", "USER"),
+        TagsRunFacetFields("ENVIRONMENT", "PRODUCTION", "USER"),
+        TagsRunFacetFields("pipeline", "SALES", "USER"),
+    ]
+
+    with patch.dict(os.environ, tag_environment_variables):
+        client = OpenLineageClient(transport=transport)
+        client.emit(run_event)
+        assert transport.event.run.facets.get("tags")
+        event_tags = sorted(transport.event.run.facets["tags"].tags, key=lambda x: x.key)
+        expected_tags = sorted(tags, key=lambda x: x.key)
+        assert event_tags == expected_tags
+
+
+def test_update_tag_facet_keeps_key_case(transport):
+    tag_environment_variables = {
+        "OPENLINEAGE__TAGS__RUN__ENVIRONMENT": "PRODUCTION",
+        "OPENLINEAGE__TAGS__RUN__pipeline": "SALES",
+    }
+
+    run_event = RunEvent(
+        RunState.START,
+        "2021-11-03T10:53:52.427343",
+        Run(
+            runId="69f4acab-b87d-4fc0-b27b-8ea950370ff3",
+            facets={
+                "tags": TagsJobFacet(
+                    tags=[
+                        TagsRunFacetFields("environment", "STAGING", "USER"),
+                        TagsRunFacetFields("PIPELINE", "FINANCE", "USER"),
+                    ]
+                )
+            },
+        ),
+        Job("openlineage", "job"),
+        "producer",
+    )
+
+    tags = [
+        TagsRunFacetFields("environment", "PRODUCTION", "USER"),
+        TagsRunFacetFields("PIPELINE", "SALES", "USER"),
+    ]
+
+    with patch.dict(os.environ, tag_environment_variables):
+        client = OpenLineageClient(transport=transport)
+        client.emit(run_event)
+        assert transport.event.run.facets.get("tags")
+        event_tags = sorted(transport.event.run.facets["tags"].tags, key=lambda x: x.key)
+        expected_tags = sorted(tags, key=lambda x: x.key)
+        assert event_tags == expected_tags

openlineage_python-1.28.0/tests/conftest.py

@@ -1,18 +0,0 @@
-# Copyright 2018-2025 contributors to the OpenLineage project
-# SPDX-License-Identifier: Apache-2.0
-from __future__ import annotations
-
-from pathlib import Path
-
-import pytest
-from openlineage.client import set_producer
-
-
-@pytest.fixture(scope="session")
-def root() -> Path:
-    return Path(__file__).parent
-
-
-@pytest.fixture(scope="session", autouse=True)
-def _setup_producer() -> None:
-    set_producer("https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python")