openlineage-python 1.26.0__tar.gz → 1.27.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/PKG-INFO +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/docs/conf.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/docs/convert.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/__init__.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/client.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/constants.py +2 -2
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/event_v2.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/facet.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/facet_v2.py +9 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/facets.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/filter.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/base.py +22 -2
- openlineage_python-1.27.0/openlineage/client/generated/column_lineage_dataset.py +137 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/data_quality_assertions_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py +1 -1
- openlineage_python-1.27.0/openlineage/client/generated/dataset_type_dataset.py +24 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/dataset_version_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/datasource_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/documentation_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/documentation_job.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/environment_variables_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/error_message_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/external_query_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/extraction_error_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/input_statistics_input_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/job_type_job.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/lifecycle_state_change_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/nominal_time_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/output_statistics_output_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/ownership_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/ownership_job.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/parent_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/processing_engine_run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/schema_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/source_code_job.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/source_code_location_job.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/sql_job.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/storage_dataset.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/symlinks_dataset.py +1 -1
- openlineage_python-1.27.0/openlineage/client/generated/tags_dataset.py +37 -0
- openlineage_python-1.27.0/openlineage/client/generated/tags_job.py +34 -0
- openlineage_python-1.27.0/openlineage/client/generated/tags_run.py +34 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/run.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/serde.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/__init__.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/composite.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/console.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/factory.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/file.py +2 -2
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/http.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/kafka.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/msk_iam.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/noop.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/transport.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/utils.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/uuid.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/pyproject.toml +3 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/redact_fields.yml +24 -0
- openlineage_python-1.27.0/tests/__init__.py +2 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/conftest.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_client.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_composite.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_events.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_facet.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_facet_v2.py +46 -2
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_factory.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_file.py +28 -2
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_http.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_kafka.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_msk_iam.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_utils.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/test_uuid.py +1 -1
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/transport.py +1 -1
- openlineage_python-1.26.0/openlineage/client/generated/column_lineage_dataset.py +0 -77
- openlineage_python-1.26.0/tests/__init__.py +0 -2
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/.gitignore +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/README.md +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/docs/Makefile +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/docs/index.rst +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/docs/make.bat +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/docs/source/openlineage.client.rst +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/py.typed +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/config/config.yml +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/config/exact_filter.yml +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/config/http.yml +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/config/openlineage.yml +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/config/regex_filter.yml +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/example_full_event.json +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/nominal_time_without_end.json +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/serde_example_dataset_event.json +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/serde_example_job_event.json +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/serde_example_run_event.json +0 -0
- {openlineage_python-1.26.0 → openlineage_python-1.27.0}/tox.ini +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
# Copyright 2018-
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
__version__ = "1.
|
|
5
|
+
__version__ = "1.27.0"
|
|
6
6
|
|
|
7
7
|
DEFAULT_TIMEOUT_MS = 5000
|
|
8
8
|
DEFAULT_NAMESPACE_NAME = "default"
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
# Copyright 2018-
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
4
|
from openlineage.client.generated import (
|
|
5
5
|
column_lineage_dataset,
|
|
6
6
|
data_quality_assertions_dataset,
|
|
7
7
|
data_quality_metrics_input_dataset,
|
|
8
|
+
dataset_type_dataset,
|
|
8
9
|
dataset_version_dataset,
|
|
9
10
|
datasource_dataset,
|
|
10
11
|
documentation_dataset,
|
|
@@ -28,6 +29,9 @@ from openlineage.client.generated import (
|
|
|
28
29
|
sql_job,
|
|
29
30
|
storage_dataset,
|
|
30
31
|
symlinks_dataset,
|
|
32
|
+
tags_dataset,
|
|
33
|
+
tags_job,
|
|
34
|
+
tags_run,
|
|
31
35
|
)
|
|
32
36
|
from openlineage.client.generated.base import (
|
|
33
37
|
PRODUCER,
|
|
@@ -52,6 +56,7 @@ __all__ = [
|
|
|
52
56
|
"column_lineage_dataset",
|
|
53
57
|
"data_quality_assertions_dataset",
|
|
54
58
|
"data_quality_metrics_input_dataset",
|
|
59
|
+
"dataset_type_dataset",
|
|
55
60
|
"dataset_version_dataset",
|
|
56
61
|
"datasource_dataset",
|
|
57
62
|
"documentation_dataset",
|
|
@@ -75,4 +80,7 @@ __all__ = [
|
|
|
75
80
|
"sql_job",
|
|
76
81
|
"storage_dataset",
|
|
77
82
|
"symlinks_dataset",
|
|
83
|
+
"tags_dataset",
|
|
84
|
+
"tags_job",
|
|
85
|
+
"tags_run",
|
|
78
86
|
]
|
{openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/generated/base.py
RENAMED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
# Copyright 2018-
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
from enum import Enum
|
|
7
|
-
from typing import ClassVar
|
|
7
|
+
from typing import Any, ClassVar, cast
|
|
8
8
|
|
|
9
9
|
import attr
|
|
10
10
|
from openlineage.client.constants import DEFAULT_PRODUCER
|
|
@@ -86,6 +86,26 @@ class BaseFacet(RedactMixin):
|
|
|
86
86
|
def skip_redact(self) -> list[str]:
|
|
87
87
|
return self._base_skip_redact + self._additional_skip_redact
|
|
88
88
|
|
|
89
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> BaseFacet:
|
|
90
|
+
"""Add additional properties to updated class instance."""
|
|
91
|
+
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
92
|
+
|
|
93
|
+
new_class = attr.make_class(
|
|
94
|
+
self.__class__.__name__,
|
|
95
|
+
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
96
|
+
bases=(self.__class__,),
|
|
97
|
+
)
|
|
98
|
+
new_class.__module__ = self.__class__.__module__
|
|
99
|
+
attrs = attr.fields(self.__class__)
|
|
100
|
+
for a in attrs:
|
|
101
|
+
if not a.init:
|
|
102
|
+
continue
|
|
103
|
+
attr_name = a.name # To deal with private attributes.
|
|
104
|
+
init_name = a.alias
|
|
105
|
+
if init_name not in kwargs:
|
|
106
|
+
kwargs[init_name] = getattr(self, attr_name)
|
|
107
|
+
return cast(BaseFacet, new_class(**kwargs))
|
|
108
|
+
|
|
89
109
|
@staticmethod
|
|
90
110
|
def _get_schema() -> str:
|
|
91
111
|
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet"
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import Any, ClassVar, cast
|
|
7
|
+
|
|
8
|
+
import attr
|
|
9
|
+
from openlineage.client.generated.base import DatasetFacet
|
|
10
|
+
from openlineage.client.utils import RedactMixin
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@attr.define
|
|
14
|
+
class ColumnLineageDatasetFacet(DatasetFacet):
|
|
15
|
+
fields: dict[str, Fields]
|
|
16
|
+
"""Column level lineage that maps output fields into input fields used to evaluate them."""
|
|
17
|
+
|
|
18
|
+
dataset: list[InputField] | None = attr.field(factory=list)
|
|
19
|
+
"""
|
|
20
|
+
Column level lineage that affects the whole dataset. This includes filtering, sorting, grouping
|
|
21
|
+
(aggregates), joining, window functions, etc.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@staticmethod
|
|
25
|
+
def _get_schema() -> str:
|
|
26
|
+
return "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@attr.define
|
|
30
|
+
class Fields(RedactMixin):
|
|
31
|
+
inputFields: list[InputField] # noqa: N815
|
|
32
|
+
transformationDescription: str | None = attr.field(default=None) # noqa: N815
|
|
33
|
+
"""a string representation of the transformation applied"""
|
|
34
|
+
|
|
35
|
+
transformationType: str | None = attr.field(default=None) # noqa: N815
|
|
36
|
+
"""
|
|
37
|
+
IDENTITY|MASKED reflects a clearly defined behavior. IDENTITY: exact same as input; MASKED: no
|
|
38
|
+
original data available (like a hash of PII for example)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> Fields:
|
|
42
|
+
"""Add additional properties to updated class instance."""
|
|
43
|
+
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
44
|
+
|
|
45
|
+
new_class = attr.make_class(
|
|
46
|
+
self.__class__.__name__,
|
|
47
|
+
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
48
|
+
bases=(self.__class__,),
|
|
49
|
+
)
|
|
50
|
+
new_class.__module__ = self.__class__.__module__
|
|
51
|
+
attrs = attr.fields(self.__class__)
|
|
52
|
+
for a in attrs:
|
|
53
|
+
if not a.init:
|
|
54
|
+
continue
|
|
55
|
+
attr_name = a.name # To deal with private attributes.
|
|
56
|
+
init_name = a.alias
|
|
57
|
+
if init_name not in kwargs:
|
|
58
|
+
kwargs[init_name] = getattr(self, attr_name)
|
|
59
|
+
return cast(Fields, new_class(**kwargs))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@attr.define
|
|
63
|
+
class InputField(RedactMixin):
|
|
64
|
+
"""Represents a single dependency on some field (column)."""
|
|
65
|
+
|
|
66
|
+
namespace: str
|
|
67
|
+
"""The input dataset namespace"""
|
|
68
|
+
|
|
69
|
+
name: str
|
|
70
|
+
"""The input dataset name"""
|
|
71
|
+
|
|
72
|
+
field: str
|
|
73
|
+
"""The input field"""
|
|
74
|
+
|
|
75
|
+
transformations: list[Transformation] | None = attr.field(factory=list)
|
|
76
|
+
_skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
|
|
77
|
+
|
|
78
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> InputField:
|
|
79
|
+
"""Add additional properties to updated class instance."""
|
|
80
|
+
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
81
|
+
|
|
82
|
+
new_class = attr.make_class(
|
|
83
|
+
self.__class__.__name__,
|
|
84
|
+
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
85
|
+
bases=(self.__class__,),
|
|
86
|
+
)
|
|
87
|
+
new_class.__module__ = self.__class__.__module__
|
|
88
|
+
attrs = attr.fields(self.__class__)
|
|
89
|
+
for a in attrs:
|
|
90
|
+
if not a.init:
|
|
91
|
+
continue
|
|
92
|
+
attr_name = a.name # To deal with private attributes.
|
|
93
|
+
init_name = a.alias
|
|
94
|
+
if init_name not in kwargs:
|
|
95
|
+
kwargs[init_name] = getattr(self, attr_name)
|
|
96
|
+
return cast(InputField, new_class(**kwargs))
|
|
97
|
+
|
|
98
|
+
@staticmethod
|
|
99
|
+
def _get_schema() -> str:
|
|
100
|
+
return "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/InputField"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@attr.define
|
|
104
|
+
class Transformation(RedactMixin):
|
|
105
|
+
type: str
|
|
106
|
+
"""The type of the transformation. Allowed values are: DIRECT, INDIRECT"""
|
|
107
|
+
|
|
108
|
+
subtype: str | None = attr.field(default=None)
|
|
109
|
+
"""The subtype of the transformation"""
|
|
110
|
+
|
|
111
|
+
description: str | None = attr.field(default=None)
|
|
112
|
+
"""a string representation of the transformation applied"""
|
|
113
|
+
|
|
114
|
+
masking: bool | None = attr.field(default=None)
|
|
115
|
+
"""is transformation masking the data or not"""
|
|
116
|
+
|
|
117
|
+
_skip_redact: ClassVar[list[str]] = ["type", "subtype", "masking"]
|
|
118
|
+
|
|
119
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> Transformation:
|
|
120
|
+
"""Add additional properties to updated class instance."""
|
|
121
|
+
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
122
|
+
|
|
123
|
+
new_class = attr.make_class(
|
|
124
|
+
self.__class__.__name__,
|
|
125
|
+
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
126
|
+
bases=(self.__class__,),
|
|
127
|
+
)
|
|
128
|
+
new_class.__module__ = self.__class__.__module__
|
|
129
|
+
attrs = attr.fields(self.__class__)
|
|
130
|
+
for a in attrs:
|
|
131
|
+
if not a.init:
|
|
132
|
+
continue
|
|
133
|
+
attr_name = a.name # To deal with private attributes.
|
|
134
|
+
init_name = a.alias
|
|
135
|
+
if init_name not in kwargs:
|
|
136
|
+
kwargs[init_name] = getattr(self, attr_name)
|
|
137
|
+
return cast(Transformation, new_class(**kwargs))
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import ClassVar
|
|
7
|
+
|
|
8
|
+
import attr
|
|
9
|
+
from openlineage.client.generated.base import DatasetFacet
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@attr.define
|
|
13
|
+
class DatasetTypeDatasetFacet(DatasetFacet):
|
|
14
|
+
datasetType: str # noqa: N815
|
|
15
|
+
"""Dataset type, for example: FILE|STREAM|MODEL|TABLE|VIEW."""
|
|
16
|
+
|
|
17
|
+
subType: str | None = attr.field(default=None) # noqa: N815
|
|
18
|
+
"""Optional sub-type within the dataset type (e.g., BEST VALIDATION under MODEL CHECKPOINT)."""
|
|
19
|
+
|
|
20
|
+
_additional_skip_redact: ClassVar[list[str]] = ["datasetType", "subType"]
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def _get_schema() -> str:
|
|
24
|
+
return "https://openlineage.io/spec/facets/1-0-0/DatasetTypeDatasetFacet.json#/$defs/DatasetTypeDatasetFacet"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import attr
|
|
7
|
+
from openlineage.client.generated.base import DatasetFacet
|
|
8
|
+
from openlineage.client.utils import RedactMixin
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@attr.define
|
|
12
|
+
class TagsDatasetFacet(DatasetFacet):
|
|
13
|
+
tags: list[TagsDatasetFacetFields] | None = attr.field(factory=list)
|
|
14
|
+
"""The tags applied to the dataset facet"""
|
|
15
|
+
|
|
16
|
+
@staticmethod
|
|
17
|
+
def _get_schema() -> str:
|
|
18
|
+
return "https://openlineage.io/spec/facets/1-0-0/TagsDatasetFacet.json#/$defs/TagsDatasetFacet"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@attr.define
|
|
22
|
+
class TagsDatasetFacetFields(RedactMixin):
|
|
23
|
+
key: str
|
|
24
|
+
"""Key that identifies the tag"""
|
|
25
|
+
|
|
26
|
+
value: str
|
|
27
|
+
"""The value of the field"""
|
|
28
|
+
|
|
29
|
+
source: str | None = attr.field(default=None)
|
|
30
|
+
"""The source of the tag. INTEGRATION|USER|DBT CORE|SPARK|etc."""
|
|
31
|
+
|
|
32
|
+
field: str | None = attr.field(default=None)
|
|
33
|
+
"""Identifies the field in a dataset if a tag applies to one"""
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def _get_schema() -> str:
|
|
37
|
+
return "https://openlineage.io/spec/facets/1-0-0/TagsDatasetFacet.json#/$defs/TagsDatasetFacetFields"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import attr
|
|
7
|
+
from openlineage.client.generated.base import JobFacet
|
|
8
|
+
from openlineage.client.utils import RedactMixin
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@attr.define
|
|
12
|
+
class TagsJobFacet(JobFacet):
|
|
13
|
+
tags: list[TagsJobFacetFields] | None = attr.field(factory=list)
|
|
14
|
+
"""The tags applied to the job facet"""
|
|
15
|
+
|
|
16
|
+
@staticmethod
|
|
17
|
+
def _get_schema() -> str:
|
|
18
|
+
return "https://openlineage.io/spec/facets/1-0-0/TagsJobFacet.json#/$defs/TagsJobFacet"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@attr.define
|
|
22
|
+
class TagsJobFacetFields(RedactMixin):
|
|
23
|
+
key: str
|
|
24
|
+
"""Key that identifies the tag"""
|
|
25
|
+
|
|
26
|
+
value: str
|
|
27
|
+
"""The value of the field"""
|
|
28
|
+
|
|
29
|
+
source: str | None = attr.field(default=None)
|
|
30
|
+
"""The source of the tag. INTEGRATION|USER|DBT CORE|SPARK|etc."""
|
|
31
|
+
|
|
32
|
+
@staticmethod
|
|
33
|
+
def _get_schema() -> str:
|
|
34
|
+
return "https://openlineage.io/spec/facets/1-0-0/TagsJobFacet.json#/$defs/TagsJobFacetFields"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import attr
|
|
7
|
+
from openlineage.client.generated.base import RunFacet
|
|
8
|
+
from openlineage.client.utils import RedactMixin
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@attr.define
|
|
12
|
+
class TagsRunFacet(RunFacet):
|
|
13
|
+
tags: list[TagsRunFacetFields] | None = attr.field(factory=list)
|
|
14
|
+
"""The tags applied to the run facet"""
|
|
15
|
+
|
|
16
|
+
@staticmethod
|
|
17
|
+
def _get_schema() -> str:
|
|
18
|
+
return "https://openlineage.io/spec/facets/1-0-0/TagsRunFacet.json#/$defs/TagsRunFacet"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@attr.define
|
|
22
|
+
class TagsRunFacetFields(RedactMixin):
|
|
23
|
+
key: str
|
|
24
|
+
"""Key that identifies the tag"""
|
|
25
|
+
|
|
26
|
+
value: str
|
|
27
|
+
"""The value of the field"""
|
|
28
|
+
|
|
29
|
+
source: str | None = attr.field(default=None)
|
|
30
|
+
"""The source of the tag. INTEGRATION|USER|DBT CORE|SPARK|etc."""
|
|
31
|
+
|
|
32
|
+
@staticmethod
|
|
33
|
+
def _get_schema() -> str:
|
|
34
|
+
return "https://openlineage.io/spec/facets/1-0-0/TagsRunFacet.json#/$defs/TagsRunFacetFields"
|
{openlineage_python-1.26.0 → openlineage_python-1.27.0}/openlineage/client/transport/file.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2018-
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
@@ -49,7 +49,7 @@ class FileTransport(Transport):
|
|
|
49
49
|
log_file_path = self.config.log_file_path
|
|
50
50
|
else:
|
|
51
51
|
time_str = datetime.now().strftime("%Y%m%d-%H%M%S.%f")
|
|
52
|
-
log_file_path = f"{self.config.log_file_path}-{time_str}"
|
|
52
|
+
log_file_path = f"{self.config.log_file_path}-{time_str}.json"
|
|
53
53
|
|
|
54
54
|
log.debug("Openlineage event will be emitted to file: `%s`", log_file_path)
|
|
55
55
|
try:
|
|
@@ -6,7 +6,7 @@ requires = [
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "openlineage-python"
|
|
9
|
-
version = "1.
|
|
9
|
+
version = "1.27.0"
|
|
10
10
|
description = "OpenLineage Python Client"
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
keywords = [
|
|
@@ -66,9 +66,11 @@ run.plugins = ["covdefaults"]
|
|
|
66
66
|
|
|
67
67
|
[tool.mypy]
|
|
68
68
|
disable_error_code = "no-redef"
|
|
69
|
+
warn_redundant_casts = true
|
|
69
70
|
show_error_codes = true
|
|
70
71
|
overrides = [
|
|
71
72
|
{ ignore_missing_imports = true, module = [
|
|
72
73
|
"confluent_kafka.*",
|
|
73
74
|
] } ]
|
|
74
75
|
strict = true
|
|
76
|
+
pretty = true
|
|
@@ -74,6 +74,12 @@
|
|
|
74
74
|
redact_fields: []
|
|
75
75
|
- class_name: DataQualityMetricsInputDatasetFacet
|
|
76
76
|
redact_fields: []
|
|
77
|
+
- module: dataset_type_dataset
|
|
78
|
+
classes:
|
|
79
|
+
- class_name: DatasetTypeDatasetFacet
|
|
80
|
+
redact_fields:
|
|
81
|
+
- datasetType
|
|
82
|
+
- subType
|
|
77
83
|
- module: dataset_version_dataset
|
|
78
84
|
classes:
|
|
79
85
|
- class_name: DatasetVersionDatasetFacet
|
|
@@ -204,3 +210,21 @@
|
|
|
204
210
|
redact_fields: []
|
|
205
211
|
- class_name: SymlinksDatasetFacet
|
|
206
212
|
redact_fields: []
|
|
213
|
+
- module: tags_dataset
|
|
214
|
+
classes:
|
|
215
|
+
- class_name: TagsDatasetFacetFields
|
|
216
|
+
redact_fields: []
|
|
217
|
+
- class_name: TagsDatasetFacet
|
|
218
|
+
redact_fields: []
|
|
219
|
+
- module: tags_job
|
|
220
|
+
classes:
|
|
221
|
+
- class_name: TagsJobFacetFields
|
|
222
|
+
redact_fields: []
|
|
223
|
+
- class_name: TagsJobFacet
|
|
224
|
+
redact_fields: []
|
|
225
|
+
- module: tags_run
|
|
226
|
+
classes:
|
|
227
|
+
- class_name: TagsRunFacetFields
|
|
228
|
+
redact_fields: []
|
|
229
|
+
- class_name: TagsRunFacet
|
|
230
|
+
redact_fields: []
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2018-
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -6,7 +6,7 @@ import json
|
|
|
6
6
|
import os
|
|
7
7
|
from unittest import mock
|
|
8
8
|
|
|
9
|
-
from attr import define
|
|
9
|
+
from attr import asdict, define
|
|
10
10
|
from openlineage.client.client import OpenLineageClient
|
|
11
11
|
from openlineage.client.event_v2 import (
|
|
12
12
|
BaseEvent,
|
|
@@ -202,3 +202,47 @@ def test_full_core_event_serializes_properly() -> None:
|
|
|
202
202
|
expected_event = json.load(f)
|
|
203
203
|
|
|
204
204
|
assert expected_event == event_sent
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_with_additional_properties_adds_new_properties():
|
|
208
|
+
base_facet = BaseFacet()
|
|
209
|
+
changed_facet = base_facet.with_additional_properties(new_prop="new_value")
|
|
210
|
+
|
|
211
|
+
assert hasattr(changed_facet, "new_prop")
|
|
212
|
+
assert changed_facet.new_prop == "new_value"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def test_with_additional_properties_updates_existing_properties():
|
|
216
|
+
documentation_facet = documentation_job.DocumentationJobFacet(description="desc")
|
|
217
|
+
changed_facet = documentation_facet.with_additional_properties(description="new_value")
|
|
218
|
+
|
|
219
|
+
assert changed_facet.description == "new_value"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def test_with_additional_properties_does_not_overwrite_class_level_attributes():
|
|
223
|
+
base_facet = BaseFacet()
|
|
224
|
+
original_attrs = base_facet.__class__.__attrs_attrs__
|
|
225
|
+
base_facet.with_additional_properties(new_prop="new_value")
|
|
226
|
+
|
|
227
|
+
assert base_facet.__class__.__attrs_attrs__ == original_attrs
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def test_with_additional_properties_works_with_attr_asdict():
|
|
231
|
+
documentation_facet = documentation_job.DocumentationJobFacet(description="desc")
|
|
232
|
+
changed_facet = documentation_facet.with_additional_properties(new_prop="new_value")
|
|
233
|
+
|
|
234
|
+
assert asdict(changed_facet) == {
|
|
235
|
+
"_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
|
|
236
|
+
"_schemaURL": "https://openlineage.io/spec/facets/1-0-1/DocumentationJobFacet.json#/$defs/DocumentationJobFacet",
|
|
237
|
+
"_deleted": None,
|
|
238
|
+
"description": "desc",
|
|
239
|
+
"new_prop": "new_value",
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def test_with_additional_properties_isinstance_works():
|
|
244
|
+
documentation_facet = documentation_job.DocumentationJobFacet(description="desc")
|
|
245
|
+
changed_facet = documentation_facet.with_additional_properties(new_prop="new_value")
|
|
246
|
+
|
|
247
|
+
assert isinstance(changed_facet, documentation_job.DocumentationJobFacet)
|
|
248
|
+
assert isinstance(changed_facet, BaseFacet)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2018-
|
|
1
|
+
# Copyright 2018-2025 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
import datetime
|
|
4
4
|
import json
|
|
@@ -119,7 +119,7 @@ def test_file_transport_raises_error_with_non_writeable_file(mock_dt, append) ->
|
|
|
119
119
|
config = FileConfig(log_file_path=log_base_file_path, append=append)
|
|
120
120
|
transport = FileTransport(config)
|
|
121
121
|
|
|
122
|
-
log_file_path = log_base_file_path if append else f"{log_base_file_path}-timestr"
|
|
122
|
+
log_file_path = log_base_file_path if append else f"{log_base_file_path}-timestr.json"
|
|
123
123
|
|
|
124
124
|
# Create the file and make it read-only
|
|
125
125
|
with open(log_file_path, "w") as f:
|
|
@@ -132,6 +132,32 @@ def test_file_transport_raises_error_with_non_writeable_file(mock_dt, append) ->
|
|
|
132
132
|
os.remove(log_file_path)
|
|
133
133
|
|
|
134
134
|
|
|
135
|
+
@mock.patch("openlineage.client.transport.file.datetime")
|
|
136
|
+
def test_file_transport_file_path_has_valid_extension_when_not_in_append_mode(mock_dt) -> None:
|
|
137
|
+
event = RunEvent(
|
|
138
|
+
eventType=RunState.START,
|
|
139
|
+
eventTime=datetime.datetime.now().isoformat(),
|
|
140
|
+
run=Run(runId=str(generate_new_uuid())),
|
|
141
|
+
job=Job(namespace="file", name="test"),
|
|
142
|
+
producer="prod",
|
|
143
|
+
)
|
|
144
|
+
mock_dt.now().strftime.return_value = "timestr"
|
|
145
|
+
|
|
146
|
+
with tempfile.TemporaryDirectory() as log_dir:
|
|
147
|
+
log_base_file_path = join(log_dir, "logtest")
|
|
148
|
+
|
|
149
|
+
config = FileConfig(log_file_path=log_base_file_path, append=False)
|
|
150
|
+
transport = FileTransport(config)
|
|
151
|
+
|
|
152
|
+
log_file_path = f"{log_base_file_path}-timestr.json"
|
|
153
|
+
|
|
154
|
+
assert not os.path.exists(log_file_path)
|
|
155
|
+
transport.emit(event)
|
|
156
|
+
assert os.path.exists(log_file_path)
|
|
157
|
+
|
|
158
|
+
os.remove(log_file_path)
|
|
159
|
+
|
|
160
|
+
|
|
135
161
|
@pytest.mark.parametrize(
|
|
136
162
|
"append",
|
|
137
163
|
[
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
-
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
from typing import ClassVar
|
|
7
|
-
|
|
8
|
-
import attr
|
|
9
|
-
from openlineage.client.generated.base import DatasetFacet
|
|
10
|
-
from openlineage.client.utils import RedactMixin
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@attr.define
|
|
14
|
-
class ColumnLineageDatasetFacet(DatasetFacet):
|
|
15
|
-
fields: dict[str, Fields]
|
|
16
|
-
"""Column level lineage that maps output fields into input fields used to evaluate them."""
|
|
17
|
-
|
|
18
|
-
dataset: list[InputField] | None = attr.field(factory=list)
|
|
19
|
-
"""
|
|
20
|
-
Column level lineage that affects the whole dataset. This includes filtering, sorting, grouping
|
|
21
|
-
(aggregates), joining, window functions, etc.
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
@staticmethod
|
|
25
|
-
def _get_schema() -> str:
|
|
26
|
-
return "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet"
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@attr.define
|
|
30
|
-
class Fields(RedactMixin):
|
|
31
|
-
inputFields: list[InputField] # noqa: N815
|
|
32
|
-
transformationDescription: str | None = attr.field(default=None) # noqa: N815
|
|
33
|
-
"""a string representation of the transformation applied"""
|
|
34
|
-
|
|
35
|
-
transformationType: str | None = attr.field(default=None) # noqa: N815
|
|
36
|
-
"""
|
|
37
|
-
IDENTITY|MASKED reflects a clearly defined behavior. IDENTITY: exact same as input; MASKED: no
|
|
38
|
-
original data available (like a hash of PII for example)
|
|
39
|
-
"""
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
@attr.define
|
|
43
|
-
class InputField(RedactMixin):
|
|
44
|
-
"""Represents a single dependency on some field (column)."""
|
|
45
|
-
|
|
46
|
-
namespace: str
|
|
47
|
-
"""The input dataset namespace"""
|
|
48
|
-
|
|
49
|
-
name: str
|
|
50
|
-
"""The input dataset name"""
|
|
51
|
-
|
|
52
|
-
field: str
|
|
53
|
-
"""The input field"""
|
|
54
|
-
|
|
55
|
-
transformations: list[Transformation] | None = attr.field(factory=list)
|
|
56
|
-
_skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
|
|
57
|
-
|
|
58
|
-
@staticmethod
|
|
59
|
-
def _get_schema() -> str:
|
|
60
|
-
return "https://openlineage.io/spec/facets/1-2-0/ColumnLineageDatasetFacet.json#/$defs/InputField"
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
@attr.define
|
|
64
|
-
class Transformation(RedactMixin):
|
|
65
|
-
type: str
|
|
66
|
-
"""The type of the transformation. Allowed values are: DIRECT, INDIRECT"""
|
|
67
|
-
|
|
68
|
-
subtype: str | None = attr.field(default=None)
|
|
69
|
-
"""The subtype of the transformation"""
|
|
70
|
-
|
|
71
|
-
description: str | None = attr.field(default=None)
|
|
72
|
-
"""a string representation of the transformation applied"""
|
|
73
|
-
|
|
74
|
-
masking: bool | None = attr.field(default=None)
|
|
75
|
-
"""is transformation masking the data or not"""
|
|
76
|
-
|
|
77
|
-
_skip_redact: ClassVar[list[str]] = ["type", "subtype", "masking"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{openlineage_python-1.26.0 → openlineage_python-1.27.0}/tests/serde_example_dataset_event.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|