openlineage-python 1.33.0__tar.gz → 1.34.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/PKG-INFO +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/client.py +20 -21
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/constants.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/facets.py +2 -2
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/filter.py +4 -4
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/base.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/column_lineage_dataset.py +3 -3
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/documentation_dataset.py +4 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/documentation_job.py +4 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/templates/dataclass.jinja2 +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/tags.py +3 -3
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/composite.py +4 -3
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/http.py +11 -11
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/kafka.py +5 -5
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/msk_iam.py +5 -5
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/transform/transform.py +12 -9
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/transport.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/pyproject.toml +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_events.py +9 -9
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_facet.py +18 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_facet_v2.py +48 -3
- openlineage_python-1.34.0/tests/test_uuid.py +102 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/transform/test_transform.py +88 -1
- openlineage_python-1.33.0/tests/test_uuid.py +0 -54
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/.gitignore +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/README.md +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/docs/Makefile +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/docs/conf.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/docs/convert.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/docs/index.rst +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/docs/make.bat +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/docs/source/openlineage.client.rst +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/event_v2.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/facet.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/facet_v2.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/catalog_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/data_quality_assertions_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/dataset_type_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/dataset_version_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/datasource_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/environment_variables_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/error_message_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/external_query_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/extraction_error_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/input_statistics_input_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/job_type_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/lifecycle_state_change_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/nominal_time_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/output_statistics_output_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/ownership_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/ownership_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/parent_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/processing_engine_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/schema_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/source_code_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/source_code_location_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/sql_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/storage_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/symlinks_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/tags_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/tags_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/tags_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/base.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/cli.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/generate.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/header.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/templates/Enum.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/templates/facet_v2.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/templates/root.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/templates/validators.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/py.typed +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/serde.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/amazon_datazone.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/console.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/factory.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/file.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/noop.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/transform/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/transform/transformers/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/transform/transformers/job_namespace_replace_transformer.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/utils.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/uuid.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/redact_fields.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/config/config.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/config/exact_filter.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/config/http.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/config/openlineage.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/config/regex_filter.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/conftest.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/example_full_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/generator/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/generator/test_base.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/generator/test_cli.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/nominal_time_without_end.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/serde_example_dataset_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/serde_example_job_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/serde_example_run_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_amazon_datazone.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_client.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_composite.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_factory.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_file.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_http.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_kafka.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_msk_iam.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/test_utils.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/transform/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/transform/transformers/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/transform/transformers/test_job_namespace_replace_transformer.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/transport.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.34.0}/tox.ini +0 -0
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import contextlib
|
|
5
6
|
import json
|
|
6
7
|
import logging
|
|
7
8
|
import os
|
|
@@ -10,19 +11,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
|
|
|
10
11
|
|
|
11
12
|
import attr
|
|
12
13
|
import yaml
|
|
13
|
-
from openlineage.client.filter import Filter, FilterConfig, create_filter
|
|
14
|
-
from openlineage.client.serde import Serde
|
|
15
|
-
from openlineage.client.tags import TagsConfig
|
|
16
|
-
from openlineage.client.utils import deep_merge_dicts
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from requests import Session
|
|
20
|
-
from requests.adapters import HTTPAdapter
|
|
21
|
-
|
|
22
|
-
import contextlib
|
|
23
|
-
|
|
24
14
|
from openlineage.client import event_v2
|
|
25
15
|
from openlineage.client.facets import FacetsConfig
|
|
16
|
+
from openlineage.client.filter import Filter, FilterConfig, create_filter
|
|
26
17
|
from openlineage.client.generated.environment_variables_run import (
|
|
27
18
|
EnvironmentVariable,
|
|
28
19
|
EnvironmentVariablesRunFacet,
|
|
@@ -30,6 +21,8 @@ from openlineage.client.generated.environment_variables_run import (
|
|
|
30
21
|
from openlineage.client.generated.tags_job import TagsJobFacet, TagsJobFacetFields
|
|
31
22
|
from openlineage.client.generated.tags_run import TagsRunFacet, TagsRunFacetFields
|
|
32
23
|
from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
|
|
24
|
+
from openlineage.client.serde import Serde
|
|
25
|
+
from openlineage.client.tags import TagsConfig
|
|
33
26
|
from openlineage.client.transport import (
|
|
34
27
|
Transport,
|
|
35
28
|
TransportFactory,
|
|
@@ -37,26 +30,32 @@ from openlineage.client.transport import (
|
|
|
37
30
|
)
|
|
38
31
|
from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
|
|
39
32
|
from openlineage.client.transport.noop import NoopConfig, NoopTransport
|
|
33
|
+
from openlineage.client.utils import deep_merge_dicts
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from requests import Session
|
|
37
|
+
from requests.adapters import HTTPAdapter
|
|
38
|
+
|
|
40
39
|
|
|
41
40
|
Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
|
|
42
41
|
Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
|
|
43
42
|
Event = Union[Event_v1, Event_v2]
|
|
44
43
|
|
|
45
44
|
|
|
46
|
-
@attr.
|
|
45
|
+
@attr.define
|
|
47
46
|
class OpenLineageClientOptions:
|
|
48
|
-
timeout: float =
|
|
49
|
-
verify: bool =
|
|
50
|
-
api_key: str =
|
|
51
|
-
adapter: HTTPAdapter =
|
|
47
|
+
timeout: float = 5.0
|
|
48
|
+
verify: bool = True
|
|
49
|
+
api_key: str | None = None
|
|
50
|
+
adapter: HTTPAdapter | None = None
|
|
52
51
|
|
|
53
52
|
|
|
54
|
-
@attr.
|
|
53
|
+
@attr.define
|
|
55
54
|
class OpenLineageConfig:
|
|
56
|
-
transport: dict[str, Any] | None = attr.
|
|
57
|
-
facets: FacetsConfig = attr.
|
|
58
|
-
filters: list[FilterConfig] = attr.
|
|
59
|
-
tags: TagsConfig = attr.
|
|
55
|
+
transport: dict[str, Any] | None = attr.field(factory=dict)
|
|
56
|
+
facets: FacetsConfig = attr.field(factory=FacetsConfig)
|
|
57
|
+
filters: list[FilterConfig] = attr.field(factory=list)
|
|
58
|
+
tags: TagsConfig = attr.field(factory=TagsConfig)
|
|
60
59
|
|
|
61
60
|
@classmethod
|
|
62
61
|
def from_dict(cls, params: dict[str, Any]) -> OpenLineageConfig:
|
|
@@ -14,11 +14,11 @@ log = logging.getLogger(__name__)
|
|
|
14
14
|
RunEventType = typing.Union[RunEvent, RunEvent_v2]
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
@attr.
|
|
17
|
+
@attr.define
|
|
18
18
|
class FilterConfig:
|
|
19
|
-
type: str | None =
|
|
20
|
-
match: str | None =
|
|
21
|
-
regex: str | None =
|
|
19
|
+
type: str | None = None
|
|
20
|
+
match: str | None = None
|
|
21
|
+
regex: str | None = None
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class Filter:
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/base.py
RENAMED
|
@@ -92,7 +92,7 @@ class BaseFacet(RedactMixin):
|
|
|
92
92
|
|
|
93
93
|
new_class = attr.make_class(
|
|
94
94
|
self.__class__.__name__,
|
|
95
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
95
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
96
96
|
bases=(self.__class__,),
|
|
97
97
|
)
|
|
98
98
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -44,7 +44,7 @@ class Fields(RedactMixin):
|
|
|
44
44
|
|
|
45
45
|
new_class = attr.make_class(
|
|
46
46
|
self.__class__.__name__,
|
|
47
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
47
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
48
48
|
bases=(self.__class__,),
|
|
49
49
|
)
|
|
50
50
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -81,7 +81,7 @@ class InputField(RedactMixin):
|
|
|
81
81
|
|
|
82
82
|
new_class = attr.make_class(
|
|
83
83
|
self.__class__.__name__,
|
|
84
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
84
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
85
85
|
bases=(self.__class__,),
|
|
86
86
|
)
|
|
87
87
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -122,7 +122,7 @@ class Transformation(RedactMixin):
|
|
|
122
122
|
|
|
123
123
|
new_class = attr.make_class(
|
|
124
124
|
self.__class__.__name__,
|
|
125
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
125
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
126
126
|
bases=(self.__class__,),
|
|
127
127
|
)
|
|
128
128
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -12,6 +12,9 @@ class DocumentationDatasetFacet(DatasetFacet):
|
|
|
12
12
|
description: str
|
|
13
13
|
"""The description of the dataset."""
|
|
14
14
|
|
|
15
|
+
contentType: str | None = attr.field(default=None) # noqa: N815
|
|
16
|
+
"""MIME type of the description field content."""
|
|
17
|
+
|
|
15
18
|
@staticmethod
|
|
16
19
|
def _get_schema() -> str:
|
|
17
|
-
return "https://openlineage.io/spec/facets/1-0
|
|
20
|
+
return "https://openlineage.io/spec/facets/1-1-0/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet"
|
|
@@ -12,8 +12,11 @@ class DocumentationJobFacet(JobFacet):
|
|
|
12
12
|
description: str
|
|
13
13
|
"""The description of the job."""
|
|
14
14
|
|
|
15
|
+
contentType: str | None = attr.field(default=None) # noqa: N815
|
|
16
|
+
"""MIME type of the description field content."""
|
|
17
|
+
|
|
15
18
|
@staticmethod
|
|
16
19
|
def _get_schema() -> str:
|
|
17
20
|
return (
|
|
18
|
-
"https://openlineage.io/spec/facets/1-0
|
|
21
|
+
"https://openlineage.io/spec/facets/1-1-0/DocumentationJobFacet.json#/$defs/DocumentationJobFacet"
|
|
19
22
|
)
|
|
@@ -157,7 +157,7 @@ class {{ class_name }}:
|
|
|
157
157
|
|
|
158
158
|
new_class = attr.make_class(
|
|
159
159
|
self.__class__.__name__,
|
|
160
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
160
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
161
161
|
bases=(self.__class__,),
|
|
162
162
|
)
|
|
163
163
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
|
|
|
11
11
|
from openlineage.client.generated.tags_run import TagsRunFacetFields
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
@attr.
|
|
14
|
+
@attr.define
|
|
15
15
|
class TagsConfig:
|
|
16
|
-
job: list[TagsJobFacetFields] = attr.
|
|
17
|
-
run: list[TagsRunFacetFields] = attr.
|
|
16
|
+
job: list[TagsJobFacetFields] = attr.field(factory=list)
|
|
17
|
+
run: list[TagsRunFacetFields] = attr.field(factory=list)
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/composite.py
RENAMED
|
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
|
|
|
17
17
|
log = logging.getLogger(__name__)
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
@attr.
|
|
20
|
+
@attr.define
|
|
21
21
|
class CompositeConfig(Config):
|
|
22
22
|
"""
|
|
23
23
|
CompositeConfig is a configuration class for CompositeTransport.
|
|
@@ -36,8 +36,8 @@ class CompositeConfig(Config):
|
|
|
36
36
|
transport will halt the emission process for subsequent transports.
|
|
37
37
|
"""
|
|
38
38
|
|
|
39
|
-
transports: list[dict[str, Any]] | dict[str, dict[str, Any]]
|
|
40
|
-
continue_on_failure: bool =
|
|
39
|
+
transports: list[dict[str, Any]] | dict[str, dict[str, Any]]
|
|
40
|
+
continue_on_failure: bool = True
|
|
41
41
|
|
|
42
42
|
@classmethod
|
|
43
43
|
def from_dict(cls, params: dict[str, Any]) -> CompositeConfig:
|
|
@@ -86,6 +86,7 @@ class CompositeTransport(Transport):
|
|
|
86
86
|
except Exception as e: # noqa: BLE001
|
|
87
87
|
if self.config.continue_on_failure:
|
|
88
88
|
log.warning("Transport %s failed to emit event with error: %s", transport, e)
|
|
89
|
+
log.debug("OpenLineage emission failure details:", exc_info=True)
|
|
89
90
|
else:
|
|
90
91
|
msg = f"Transport {transport} failed to emit event"
|
|
91
92
|
raise RuntimeError(msg) from e
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/http.py
RENAMED
|
@@ -76,23 +76,23 @@ def get_session() -> Session:
|
|
|
76
76
|
return Session()
|
|
77
77
|
|
|
78
78
|
|
|
79
|
-
@attr.
|
|
79
|
+
@attr.define
|
|
80
80
|
class HttpConfig(Config):
|
|
81
|
-
url: str
|
|
82
|
-
endpoint: str =
|
|
83
|
-
timeout: float =
|
|
81
|
+
url: str
|
|
82
|
+
endpoint: str = "api/v1/lineage"
|
|
83
|
+
timeout: float = 5.0
|
|
84
84
|
# check TLS certificates
|
|
85
|
-
verify: bool =
|
|
86
|
-
auth: TokenProvider = attr.
|
|
87
|
-
compression: HttpCompression | None =
|
|
85
|
+
verify: bool = True
|
|
86
|
+
auth: TokenProvider = attr.field(factory=lambda: TokenProvider({}))
|
|
87
|
+
compression: HttpCompression | None = None
|
|
88
88
|
# not set by TransportFactory
|
|
89
|
-
session: Session | None =
|
|
89
|
+
session: Session | None = None
|
|
90
90
|
# not set by TransportFactory
|
|
91
|
-
adapter: HTTPAdapter | None =
|
|
91
|
+
adapter: HTTPAdapter | None = None
|
|
92
92
|
# custom headers support
|
|
93
|
-
custom_headers: dict[str, str] = attr.
|
|
93
|
+
custom_headers: dict[str, str] = attr.field(factory=dict)
|
|
94
94
|
# retry settings
|
|
95
|
-
retry: dict[str, Any] = attr.
|
|
95
|
+
retry: dict[str, Any] = attr.field(
|
|
96
96
|
default={
|
|
97
97
|
"total": 5,
|
|
98
98
|
"read": 5,
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/kafka.py
RENAMED
|
@@ -24,21 +24,21 @@ log = logging.getLogger(__name__)
|
|
|
24
24
|
_T = TypeVar("_T", bound="KafkaConfig")
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
@attr.
|
|
27
|
+
@attr.define
|
|
28
28
|
class KafkaConfig(Config):
|
|
29
29
|
# Kafka producer config
|
|
30
30
|
# https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration
|
|
31
|
-
config: dict[str, str]
|
|
31
|
+
config: dict[str, str]
|
|
32
32
|
|
|
33
33
|
# Topic on which we should send messages
|
|
34
|
-
topic: str
|
|
34
|
+
topic: str
|
|
35
35
|
|
|
36
36
|
# Explicit key for Kafka producer
|
|
37
|
-
messageKey: str | None =
|
|
37
|
+
messageKey: str | None = None # noqa: N815
|
|
38
38
|
|
|
39
39
|
# Set to true if Kafka should flush after each event. The process that emits can be killed in
|
|
40
40
|
# some cases - for example in Airflow integration, so flushing is desirable there.
|
|
41
|
-
flush: bool =
|
|
41
|
+
flush: bool = True
|
|
42
42
|
|
|
43
43
|
@classmethod
|
|
44
44
|
def from_dict(cls: type[_T], params: dict[str, Any]) -> _T:
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/msk_iam.py
RENAMED
|
@@ -32,15 +32,15 @@ def _detect_running_region() -> None | str:
|
|
|
32
32
|
return None
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
@attr.
|
|
35
|
+
@attr.define
|
|
36
36
|
class MSKIAMConfig(KafkaConfig):
|
|
37
37
|
# MSK producer config
|
|
38
38
|
# https://github.com/aws/aws-msk-iam-sasl-signer-python
|
|
39
39
|
|
|
40
|
-
region: str =
|
|
41
|
-
aws_profile: None | str =
|
|
42
|
-
role_arn: None | str =
|
|
43
|
-
aws_debug_creds: bool =
|
|
40
|
+
region: str | None = None
|
|
41
|
+
aws_profile: None | str = None
|
|
42
|
+
role_arn: None | str = None
|
|
43
|
+
aws_debug_creds: bool = False
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def _oauth_cb(config: MSKIAMConfig, *_: Any) -> tuple[str, float]:
|
|
@@ -46,7 +46,7 @@ class EventTransformer:
|
|
|
46
46
|
return f"<{self.__class__.__name__}({self.properties})>"
|
|
47
47
|
|
|
48
48
|
|
|
49
|
-
@attr.
|
|
49
|
+
@attr.define
|
|
50
50
|
class TransformConfig(Config):
|
|
51
51
|
"""Configuration class for transform transport.
|
|
52
52
|
|
|
@@ -54,9 +54,9 @@ class TransformConfig(Config):
|
|
|
54
54
|
transformer class to use, and optional transformer-specific properties.
|
|
55
55
|
"""
|
|
56
56
|
|
|
57
|
-
transport: dict[str, Any]
|
|
58
|
-
transformer_class: str
|
|
59
|
-
transformer_properties: dict[str, Any] =
|
|
57
|
+
transport: dict[str, Any]
|
|
58
|
+
transformer_class: str
|
|
59
|
+
transformer_properties: dict[str, Any] | None = None
|
|
60
60
|
|
|
61
61
|
@classmethod
|
|
62
62
|
def from_dict(cls, params: dict[str, Any]) -> TransformConfig:
|
|
@@ -113,7 +113,7 @@ class TransformTransport(Transport):
|
|
|
113
113
|
if not inspect.isclass(transformer_class) or not issubclass(transformer_class, EventTransformer):
|
|
114
114
|
msg = f"Transformer `{transformer_class}` has to be class, and subclass of EventTransformer"
|
|
115
115
|
raise TypeError(msg)
|
|
116
|
-
self.transformer = transformer_class(properties=config.transformer_properties)
|
|
116
|
+
self.transformer = transformer_class(properties=config.transformer_properties or {})
|
|
117
117
|
self.transport = get_default_factory().create(config.transport)
|
|
118
118
|
|
|
119
119
|
def emit(self, event: Event) -> Any:
|
|
@@ -128,14 +128,17 @@ class TransformTransport(Transport):
|
|
|
128
128
|
Returns:
|
|
129
129
|
The result of the inner transport's emit call, or None if the event is dropped.
|
|
130
130
|
"""
|
|
131
|
-
log.debug("Event before
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
log.debug("Event before copy: %s", Serde.to_json(event))
|
|
132
|
+
try:
|
|
133
|
+
event = copy.deepcopy(event) # Copy to make sure the transformer does not modify original event
|
|
134
|
+
log.debug("Event after copy, before transformation: %s", Serde.to_json(event))
|
|
135
|
+
except AttributeError as e:
|
|
136
|
+
msg = "Error serializing copy of event."
|
|
137
|
+
raise ValueError(msg) from e
|
|
134
138
|
log.debug("Transforming OpenLineage event with %s", self.transformer)
|
|
135
139
|
transformed = self.transformer.transform(event)
|
|
136
140
|
if transformed is None:
|
|
137
141
|
log.info("Transformed OpenLineage event is None. Skipping emission.")
|
|
138
142
|
return None
|
|
139
|
-
|
|
140
143
|
log.debug("Event after transformation: %s", Serde.to_json(transformed))
|
|
141
144
|
return self.transport.emit(transformed)
|
|
@@ -103,25 +103,25 @@ def test_schema_field_default() -> None:
|
|
|
103
103
|
)
|
|
104
104
|
|
|
105
105
|
|
|
106
|
-
@attr.
|
|
106
|
+
@attr.define
|
|
107
107
|
class NestedObject:
|
|
108
|
-
value: int | None =
|
|
108
|
+
value: int | None = None
|
|
109
109
|
|
|
110
110
|
|
|
111
|
-
@attr.
|
|
111
|
+
@attr.define
|
|
112
112
|
class NestingObject:
|
|
113
|
-
nested: list[NestedObject]
|
|
114
|
-
optional: int | None =
|
|
113
|
+
nested: list[NestedObject]
|
|
114
|
+
optional: int | None = None
|
|
115
115
|
|
|
116
116
|
|
|
117
|
-
@attr.
|
|
117
|
+
@attr.define
|
|
118
118
|
class ListOfStrings:
|
|
119
|
-
values: list[str]
|
|
119
|
+
values: list[str]
|
|
120
120
|
|
|
121
121
|
|
|
122
|
-
@attr.
|
|
122
|
+
@attr.define
|
|
123
123
|
class NestedListOfStrings:
|
|
124
|
-
nested: list[ListOfStrings]
|
|
124
|
+
nested: list[ListOfStrings]
|
|
125
125
|
|
|
126
126
|
|
|
127
127
|
def test_serde_list_of_strings() -> None:
|
|
@@ -10,6 +10,7 @@ from unittest.mock import MagicMock
|
|
|
10
10
|
import pytest
|
|
11
11
|
from openlineage.client.client import OpenLineageClient
|
|
12
12
|
from openlineage.client.facet import (
|
|
13
|
+
BaseFacet,
|
|
13
14
|
ColumnLineageDatasetFacet,
|
|
14
15
|
ColumnLineageDatasetFacetFieldsAdditional,
|
|
15
16
|
ColumnLineageDatasetFacetFieldsAdditionalInputFields,
|
|
@@ -27,6 +28,7 @@ from openlineage.client.facet import (
|
|
|
27
28
|
SymlinksDatasetFacetIdentifiers,
|
|
28
29
|
)
|
|
29
30
|
from openlineage.client.run import SCHEMA_URL, Dataset, Job, Run, RunEvent, RunState
|
|
31
|
+
from openlineage.client.serde import Serde
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
@pytest.fixture()
|
|
@@ -465,3 +467,19 @@ def test_job_type_job_facet(event: dict[str, Any]) -> None:
|
|
|
465
467
|
expected_event["job"]["facets"]["jobType"] = job_type_facet
|
|
466
468
|
|
|
467
469
|
assert expected_event == event_sent
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def test_facet_copy_serialization_base_facet():
|
|
473
|
+
facet = BaseFacet()
|
|
474
|
+
facet_copy = copy.deepcopy(facet)
|
|
475
|
+
assert Serde.to_json(facet) == Serde.to_json(facet_copy)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def test_facet_copy_serialization_parent_run_facet():
|
|
479
|
+
facet = OwnershipJobFacet(
|
|
480
|
+
owners=[
|
|
481
|
+
OwnershipJobFacetOwners("some-owner", "some-owner-type"),
|
|
482
|
+
],
|
|
483
|
+
)
|
|
484
|
+
facet_copy = copy.deepcopy(facet)
|
|
485
|
+
assert Serde.to_json(facet) == Serde.to_json(facet_copy)
|
|
@@ -2,10 +2,12 @@
|
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import copy
|
|
5
6
|
import json
|
|
6
7
|
import os
|
|
7
8
|
from unittest import mock
|
|
8
9
|
|
|
10
|
+
import pytest
|
|
9
11
|
from attr import asdict, define
|
|
10
12
|
from openlineage.client.client import OpenLineageClient
|
|
11
13
|
from openlineage.client.event_v2 import (
|
|
@@ -29,6 +31,7 @@ from openlineage.client.facet_v2 import (
|
|
|
29
31
|
schema_dataset,
|
|
30
32
|
set_producer,
|
|
31
33
|
)
|
|
34
|
+
from openlineage.client.serde import Serde
|
|
32
35
|
|
|
33
36
|
|
|
34
37
|
def test_set_producer():
|
|
@@ -102,8 +105,6 @@ def test_custom_facet() -> None:
|
|
|
102
105
|
assert expected_event == event_sent
|
|
103
106
|
|
|
104
107
|
|
|
105
|
-
#
|
|
106
|
-
# def test_full_core_event_serializes_properly(facet_mocker, event_mocker) -> None:
|
|
107
108
|
def test_full_core_event_serializes_properly() -> None:
|
|
108
109
|
session = mock.MagicMock()
|
|
109
110
|
client = OpenLineageClient(url="http://example.com", session=session)
|
|
@@ -233,9 +234,10 @@ def test_with_additional_properties_works_with_attr_asdict():
|
|
|
233
234
|
|
|
234
235
|
assert asdict(changed_facet) == {
|
|
235
236
|
"_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
|
|
236
|
-
"_schemaURL": "https://openlineage.io/spec/facets/1-0
|
|
237
|
+
"_schemaURL": "https://openlineage.io/spec/facets/1-1-0/DocumentationJobFacet.json#/$defs/DocumentationJobFacet",
|
|
237
238
|
"_deleted": None,
|
|
238
239
|
"description": "desc",
|
|
240
|
+
"contentType": None,
|
|
239
241
|
"new_prop": "new_value",
|
|
240
242
|
}
|
|
241
243
|
|
|
@@ -246,3 +248,46 @@ def test_with_additional_properties_isinstance_works():
|
|
|
246
248
|
|
|
247
249
|
assert isinstance(changed_facet, documentation_job.DocumentationJobFacet)
|
|
248
250
|
assert isinstance(changed_facet, BaseFacet)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_facet_copy_serialization_base_facet():
|
|
254
|
+
facet = BaseFacet(producer="producer")
|
|
255
|
+
facet_copy = copy.deepcopy(facet)
|
|
256
|
+
assert Serde.to_json(facet) == Serde.to_json(facet_copy)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def test_facet_copy_serialization_parent_run_facet():
|
|
260
|
+
facet = parent_run.ParentRunFacet(
|
|
261
|
+
run=parent_run.Run(runId="3bb703d1-09c1-4a42-8da5-35a0b3216072"),
|
|
262
|
+
job=parent_run.Job(namespace="default", name="parent_job_name"),
|
|
263
|
+
root=parent_run.Root(
|
|
264
|
+
run=parent_run.RootRun("3bb703d1-09c1-4a42-8da5-35a0b3216071"),
|
|
265
|
+
job=parent_run.RootJob(namespace="root_job_namespace", name="root_job_name"),
|
|
266
|
+
),
|
|
267
|
+
)
|
|
268
|
+
facet_copy = copy.deepcopy(facet)
|
|
269
|
+
assert Serde.to_json(facet) == Serde.to_json(facet_copy)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def test_custom_facet_copy_serialization_success():
|
|
273
|
+
@define
|
|
274
|
+
class SomeFacet(BaseFacet):
|
|
275
|
+
version: str
|
|
276
|
+
|
|
277
|
+
facet = SomeFacet(version="1")
|
|
278
|
+
facet_copy = copy.deepcopy(facet)
|
|
279
|
+
assert Serde.to_json(facet) == Serde.to_json(facet_copy)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def test_custom_facet_copy_serialization_fails_when_mixing_attr_classes():
|
|
283
|
+
"""This will fail as BaseFacet class uses attr.define and SomeFacet attr.s"""
|
|
284
|
+
import attr
|
|
285
|
+
|
|
286
|
+
@attr.s
|
|
287
|
+
class SomeFacet(BaseFacet):
|
|
288
|
+
version: str = attr.ib()
|
|
289
|
+
|
|
290
|
+
facet = SomeFacet(version="1")
|
|
291
|
+
facet_copy = copy.deepcopy(facet)
|
|
292
|
+
with pytest.raises(AttributeError, match="'SomeFacet' object has no attribute 'version'"):
|
|
293
|
+
Serde.to_json(facet_copy)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timedelta, timezone
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from openlineage.client.uuid import generate_new_uuid, generate_static_uuid
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_generate_new_uuid_returns_uuidv7():
|
|
12
|
+
assert generate_new_uuid().version == 7 # noqa: PLR2004
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_generate_new_uuid_returns_different_result_on_each_call():
|
|
16
|
+
result1 = generate_new_uuid()
|
|
17
|
+
result2 = generate_new_uuid()
|
|
18
|
+
assert result1 != result2
|
|
19
|
+
|
|
20
|
+
instant = datetime.now()
|
|
21
|
+
result1 = generate_new_uuid(instant)
|
|
22
|
+
result2 = generate_new_uuid(instant)
|
|
23
|
+
assert result1 != result2
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_generate_new_uuid_result_is_increasing_with_instant_increment():
|
|
27
|
+
result1 = generate_new_uuid(datetime.now())
|
|
28
|
+
result2 = generate_new_uuid(datetime.now() + timedelta(seconds=0.001))
|
|
29
|
+
assert result1 != result2
|
|
30
|
+
assert result1 < result2
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_generate_new_uuid_returns_prefix_based_on_instant_milliseconds():
|
|
34
|
+
instant_milliseconds = datetime(2025, 5, 20, 10, 52, 33, 881000, tzinfo=timezone.utc)
|
|
35
|
+
instant_microseconds = datetime(2025, 5, 20, 10, 52, 33, 881863, tzinfo=timezone.utc)
|
|
36
|
+
|
|
37
|
+
uuid1 = generate_new_uuid(instant_milliseconds)
|
|
38
|
+
uuid2 = generate_new_uuid(instant_microseconds)
|
|
39
|
+
|
|
40
|
+
assert str(uuid1).startswith("0196ed52-e0d9-7")
|
|
41
|
+
assert str(uuid2).startswith("0196ed52-e0d9-7")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_generate_static_uuid_returns_uuidv7():
|
|
45
|
+
assert generate_static_uuid(datetime.now(), b"some").version == 7 # noqa: PLR2004
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_generate_static_uuid_returns_same_result_for_same_input():
|
|
49
|
+
instant = datetime.now()
|
|
50
|
+
data = b"some"
|
|
51
|
+
assert generate_static_uuid(instant, data) == generate_static_uuid(instant, data)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@pytest.mark.parametrize(
|
|
55
|
+
"data",
|
|
56
|
+
[
|
|
57
|
+
b"some",
|
|
58
|
+
b"other",
|
|
59
|
+
],
|
|
60
|
+
)
|
|
61
|
+
def test_generate_static_uuid_result_prefix_depends_on_instant_milliseconds(data):
|
|
62
|
+
instant_milliseconds = datetime(2025, 5, 20, 10, 52, 33, 881000, tzinfo=timezone.utc)
|
|
63
|
+
instant_microseconds = datetime(2025, 5, 20, 10, 52, 33, 881863, tzinfo=timezone.utc)
|
|
64
|
+
|
|
65
|
+
uuid1 = generate_static_uuid(instant_milliseconds, data)
|
|
66
|
+
uuid2 = generate_static_uuid(instant_microseconds, data)
|
|
67
|
+
|
|
68
|
+
assert str(uuid1).startswith("0196ed52-e0d9-7")
|
|
69
|
+
assert str(uuid2).startswith("0196ed52-e0d9-7")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@pytest.mark.parametrize(
|
|
73
|
+
("data1", "data2"),
|
|
74
|
+
[
|
|
75
|
+
(b"some", b"other"),
|
|
76
|
+
(b"some", b"some"),
|
|
77
|
+
],
|
|
78
|
+
)
|
|
79
|
+
def test_generate_static_uuid_result_is_increasing_with_instant_increment(data1, data2):
|
|
80
|
+
instant = datetime.now()
|
|
81
|
+
result1 = generate_static_uuid(instant, data1)
|
|
82
|
+
result2 = generate_static_uuid(instant + timedelta(seconds=0.001), data2)
|
|
83
|
+
assert result1 != result2
|
|
84
|
+
assert result1 < result2
|
|
85
|
+
|
|
86
|
+
# both timestamp and random parts are different
|
|
87
|
+
assert result1.bytes != result2.bytes
|
|
88
|
+
assert result1.bytes_le != result2.bytes_le
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_generate_static_uuid_result_is_different_for_different_data():
|
|
92
|
+
timestamp = datetime.now()
|
|
93
|
+
result1 = generate_static_uuid(timestamp, b"some")
|
|
94
|
+
result2 = generate_static_uuid(timestamp, b"other")
|
|
95
|
+
assert result1 != result2
|
|
96
|
+
|
|
97
|
+
# timestamp part is the same
|
|
98
|
+
assert int.from_bytes(result1.bytes, "big") & 0x0000 == int.from_bytes(result2.bytes, "big") & 0x0000
|
|
99
|
+
|
|
100
|
+
# random part is different
|
|
101
|
+
assert int.from_bytes(result1.bytes, "big") & 0xFFFF != int.from_bytes(result2.bytes, "big") & 0xFFFF
|
|
102
|
+
assert result1.bytes_le != result2.bytes_le
|
|
@@ -4,12 +4,15 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import datetime
|
|
6
6
|
import os
|
|
7
|
+
import warnings
|
|
7
8
|
from typing import TYPE_CHECKING
|
|
8
9
|
from unittest.mock import MagicMock, patch
|
|
9
10
|
|
|
11
|
+
import attr
|
|
10
12
|
import pytest
|
|
11
13
|
from openlineage.client import OpenLineageClient
|
|
12
14
|
from openlineage.client.event_v2 import BaseEvent, Job, Run, RunEvent, RunState
|
|
15
|
+
from openlineage.client.facet_v2 import external_query_run
|
|
13
16
|
from openlineage.client.serde import Serde
|
|
14
17
|
from openlineage.client.transport.transform import EventTransformer, TransformConfig, TransformTransport
|
|
15
18
|
from openlineage.client.uuid import generate_new_uuid
|
|
@@ -91,7 +94,7 @@ def test_transform_config_from_dict_minimal_config() -> None:
|
|
|
91
94
|
)
|
|
92
95
|
assert config.transport == {"type": "console"}
|
|
93
96
|
assert config.transformer_class == "some.class.to.be.Imported"
|
|
94
|
-
assert config.transformer_properties
|
|
97
|
+
assert config.transformer_properties is None
|
|
95
98
|
|
|
96
99
|
|
|
97
100
|
def test_transform_config_from_dict_missing_config() -> None:
|
|
@@ -205,6 +208,90 @@ def test_client_with_transform_transport_emits_modified_event(mocker: MockerFixt
|
|
|
205
208
|
)
|
|
206
209
|
|
|
207
210
|
|
|
211
|
+
def test_client_with_transform_transport_emits_modified_event_with_older_facets(
|
|
212
|
+
mocker: MockerFixture,
|
|
213
|
+
) -> None:
|
|
214
|
+
with warnings.catch_warnings():
|
|
215
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
216
|
+
from openlineage.client.facet import BaseFacet as OldBaseFacet
|
|
217
|
+
|
|
218
|
+
@attr.s
|
|
219
|
+
class SomeOldRunFacet(OldBaseFacet):
|
|
220
|
+
version: str = attr.ib()
|
|
221
|
+
|
|
222
|
+
session = mocker.patch("requests.Session")
|
|
223
|
+
config = TransformConfig.from_dict(
|
|
224
|
+
{
|
|
225
|
+
"transport": {
|
|
226
|
+
"type": "http",
|
|
227
|
+
"url": "http://backend:5000",
|
|
228
|
+
"session": session,
|
|
229
|
+
},
|
|
230
|
+
"transformer_class": "tests.transform.test_transform.SampleEventTransformer",
|
|
231
|
+
}
|
|
232
|
+
)
|
|
233
|
+
transport = TransformTransport(config)
|
|
234
|
+
client = OpenLineageClient(transport=transport)
|
|
235
|
+
now = datetime.datetime.now().isoformat()
|
|
236
|
+
run_id = str(generate_new_uuid())
|
|
237
|
+
event = RunEvent(
|
|
238
|
+
eventType=RunState.START,
|
|
239
|
+
eventTime=now,
|
|
240
|
+
run=Run(
|
|
241
|
+
runId=run_id,
|
|
242
|
+
facets={
|
|
243
|
+
"externalQuery": external_query_run.ExternalQueryRunFacet(
|
|
244
|
+
externalQueryId="queryid", source="source"
|
|
245
|
+
),
|
|
246
|
+
"old_facet": SomeOldRunFacet(version="2"),
|
|
247
|
+
},
|
|
248
|
+
),
|
|
249
|
+
job=Job(namespace="http", name="test"),
|
|
250
|
+
producer="prod",
|
|
251
|
+
)
|
|
252
|
+
modified_event = RunEvent(
|
|
253
|
+
eventType=RunState.START,
|
|
254
|
+
eventTime=now,
|
|
255
|
+
run=Run(
|
|
256
|
+
runId=run_id,
|
|
257
|
+
facets={
|
|
258
|
+
"externalQuery": external_query_run.ExternalQueryRunFacet(
|
|
259
|
+
externalQueryId="queryid", source="source"
|
|
260
|
+
),
|
|
261
|
+
"old_facet": SomeOldRunFacet(version="2"),
|
|
262
|
+
},
|
|
263
|
+
),
|
|
264
|
+
job=Job(namespace="new_value", name="test"),
|
|
265
|
+
producer="prod",
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
client.emit(event)
|
|
269
|
+
transport.transport.session.post.assert_called_once_with(
|
|
270
|
+
url="http://backend:5000/api/v1/lineage",
|
|
271
|
+
data=Serde.to_json(modified_event),
|
|
272
|
+
headers={"Content-Type": "application/json"},
|
|
273
|
+
timeout=5.0,
|
|
274
|
+
verify=True,
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
# Assert the original event is unchanged
|
|
278
|
+
assert event == RunEvent(
|
|
279
|
+
eventType=RunState.START,
|
|
280
|
+
eventTime=now,
|
|
281
|
+
run=Run(
|
|
282
|
+
runId=run_id,
|
|
283
|
+
facets={
|
|
284
|
+
"externalQuery": external_query_run.ExternalQueryRunFacet(
|
|
285
|
+
externalQueryId="queryid", source="source"
|
|
286
|
+
),
|
|
287
|
+
"old_facet": SomeOldRunFacet(version="2"),
|
|
288
|
+
},
|
|
289
|
+
),
|
|
290
|
+
job=Job(namespace="http", name="test"),
|
|
291
|
+
producer="prod",
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
|
|
208
295
|
def test_client_with_transform_transport_skips_emission_when_transformed_event_is_none(
|
|
209
296
|
mocker: MockerFixture,
|
|
210
297
|
) -> None:
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from datetime import datetime, timedelta
|
|
6
|
-
|
|
7
|
-
import pytest
|
|
8
|
-
from openlineage.client.uuid import generate_new_uuid, generate_static_uuid
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def test_generate_new_uuid_returns_uuidv7():
|
|
12
|
-
assert generate_new_uuid().version == 7 # noqa: PLR2004
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def test_generate_new_uuid_returns_different_result_on_each_call():
|
|
16
|
-
result1 = generate_new_uuid()
|
|
17
|
-
result2 = generate_new_uuid()
|
|
18
|
-
assert result1 != result2
|
|
19
|
-
|
|
20
|
-
instant = datetime.now()
|
|
21
|
-
result1 = generate_new_uuid(instant)
|
|
22
|
-
result2 = generate_new_uuid(instant)
|
|
23
|
-
assert result1 != result2
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def test_generate_new_uuid_result_is_increasing_with_instant_increment():
|
|
27
|
-
result1 = generate_new_uuid(datetime.now())
|
|
28
|
-
result2 = generate_new_uuid(datetime.now() + timedelta(seconds=0.001))
|
|
29
|
-
assert result1 != result2
|
|
30
|
-
assert result1 < result2
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def test_generate_static_uuid_returns_uuidv7():
|
|
34
|
-
assert generate_static_uuid(datetime.now(), b"some").version == 7 # noqa: PLR2004
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def test_generate_static_uuid_returns_same_result_for_same_input():
|
|
38
|
-
instant = datetime.now()
|
|
39
|
-
data = b"some"
|
|
40
|
-
assert generate_static_uuid(instant, data) == generate_static_uuid(instant, data)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
@pytest.mark.parametrize(
|
|
44
|
-
("data1", "data2"),
|
|
45
|
-
[
|
|
46
|
-
(b"some", b"other"),
|
|
47
|
-
(b"some", b"some"),
|
|
48
|
-
],
|
|
49
|
-
)
|
|
50
|
-
def test_generate_static_uuid_result_is_increasing_with_instant_increment(data1, data2):
|
|
51
|
-
result1 = generate_static_uuid(datetime.now(), data1)
|
|
52
|
-
result2 = generate_static_uuid(datetime.now() + timedelta(seconds=0.001), data2)
|
|
53
|
-
assert result1 != result2
|
|
54
|
-
assert result1 < result2
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/job_type_job.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/parent_run.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/sql_job.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/tags_dataset.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/tags_job.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generated/tags_run.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/base.py
RENAMED
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/generate.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/generator/header.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/console.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/factory.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/file.py
RENAMED
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/openlineage/client/transport/noop.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/serde_example_dataset_event.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.33.0 → openlineage_python-1.34.0}/tests/transform/transformers/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|