openlineage-python 1.12.0__tar.gz → 1.13.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/PKG-INFO +1 -1
- openlineage_python-1.13.1/openlineage/client/__init__.py +15 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/client.py +14 -7
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/constants.py +1 -1
- openlineage_python-1.13.1/openlineage/client/event_v2.py +35 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/facet.py +14 -9
- openlineage_python-1.13.1/openlineage/client/facet_v2.py +74 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/filter.py +7 -5
- openlineage_python-1.13.1/openlineage/client/generated/base.py +321 -0
- openlineage_python-1.13.1/openlineage/client/generated/column_lineage_dataset.py +47 -0
- openlineage_python-1.13.1/openlineage/client/generated/data_quality_assertions_dataset.py +35 -0
- openlineage_python-1.13.1/openlineage/client/generated/data_quality_metrics_input_dataset.py +47 -0
- openlineage_python-1.13.1/openlineage/client/generated/dataset_version_dataset.py +17 -0
- openlineage_python-1.13.1/openlineage/client/generated/datasource_dataset.py +29 -0
- openlineage_python-1.13.1/openlineage/client/generated/documentation_dataset.py +17 -0
- openlineage_python-1.13.1/openlineage/client/generated/documentation_job.py +19 -0
- openlineage_python-1.13.1/openlineage/client/generated/error_message_run.py +29 -0
- openlineage_python-1.13.1/openlineage/client/generated/external_query_run.py +22 -0
- openlineage_python-1.13.1/openlineage/client/generated/extraction_error_run.py +44 -0
- openlineage_python-1.13.1/openlineage/client/generated/job_type_job.py +23 -0
- openlineage_python-1.13.1/openlineage/client/generated/lifecycle_state_change_dataset.py +42 -0
- openlineage_python-1.13.1/openlineage/client/generated/nominal_time_run.py +48 -0
- openlineage_python-1.13.1/openlineage/client/generated/output_statistics_output_dataset.py +27 -0
- openlineage_python-1.13.1/openlineage/client/generated/ownership_dataset.py +31 -0
- openlineage_python-1.13.1/openlineage/client/generated/ownership_job.py +29 -0
- openlineage_python-1.13.1/openlineage/client/generated/parent_run.py +58 -0
- openlineage_python-1.13.1/openlineage/client/generated/processing_engine_run.py +23 -0
- openlineage_python-1.13.1/openlineage/client/generated/schema_dataset.py +39 -0
- openlineage_python-1.13.1/openlineage/client/generated/source_code_job.py +24 -0
- openlineage_python-1.13.1/openlineage/client/generated/source_code_location_job.py +48 -0
- openlineage_python-1.13.1/openlineage/client/generated/sql_job.py +16 -0
- openlineage_python-1.13.1/openlineage/client/generated/storage_dataset.py +20 -0
- openlineage_python-1.13.1/openlineage/client/generated/symlinks_dataset.py +31 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/run.py +5 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/console.py +3 -3
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/file.py +2 -2
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/http.py +31 -12
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/kafka.py +62 -4
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/noop.py +3 -3
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/transport.py +3 -3
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/utils.py +2 -2
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/pyproject.toml +1 -1
- openlineage_python-1.13.1/redact_fields.yml +188 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/config/http.yml +1 -0
- openlineage_python-1.13.1/tests/example_full_event.json +104 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_client.py +22 -9
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_facet.py +8 -8
- openlineage_python-1.13.1/tests/test_facet_v2.py +199 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_http.py +48 -5
- openlineage_python-1.13.1/tests/test_kafka.py +322 -0
- openlineage_python-1.13.1/tests/test_utils.py +38 -0
- openlineage_python-1.12.0/openlineage/client/__init__.py +0 -12
- openlineage_python-1.12.0/tests/test_kafka.py +0 -139
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/.gitignore +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/README.md +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/py.typed +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/serde.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/__init__.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/factory.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/openlineage/client/transport/msk_iam.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/__init__.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/config/config.yml +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/config/exact_filter.yml +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/config/openlineage.yml +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/config/regex_filter.yml +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/conftest.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/nominal_time_without_end.json +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/serde_example_dataset_event.json +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/serde_example_job_event.json +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/serde_example_run_event.json +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_events.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_factory.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_file.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/test_msk_iam.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tests/transport.py +0 -0
- {openlineage_python-1.12.0 → openlineage_python-1.13.1}/tox.ini +0 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from openlineage.client.client import OpenLineageClient, OpenLineageClientOptions
|
|
6
|
+
from openlineage.client.facet import set_producer as set_producer_v1
|
|
7
|
+
from openlineage.client.facet_v2 import set_producer as set_producer_v2
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def set_producer(producer: str) -> None:
|
|
11
|
+
set_producer_v1(producer)
|
|
12
|
+
set_producer_v2(producer)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
__all__ = ["OpenLineageClient", "OpenLineageClientOptions", "set_producer"]
|
|
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
6
|
import os
|
|
7
|
-
from typing import TYPE_CHECKING, Any, TypeVar
|
|
7
|
+
from typing import TYPE_CHECKING, Any, TypeVar, Union
|
|
8
8
|
|
|
9
9
|
import attr
|
|
10
10
|
from openlineage.client.filter import Filter, create_filter
|
|
@@ -15,10 +15,15 @@ if TYPE_CHECKING:
|
|
|
15
15
|
from requests import Session
|
|
16
16
|
from requests.adapters import HTTPAdapter
|
|
17
17
|
|
|
18
|
+
from openlineage.client import event_v2
|
|
18
19
|
from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
|
|
19
20
|
from openlineage.client.transport import Transport, TransportFactory, get_default_factory
|
|
20
21
|
from openlineage.client.transport.http import HttpConfig, HttpTransport
|
|
21
22
|
|
|
23
|
+
Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
|
|
24
|
+
Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
|
|
25
|
+
Event = Union[Event_v1, Event_v2]
|
|
26
|
+
|
|
22
27
|
|
|
23
28
|
@attr.s
|
|
24
29
|
class OpenLineageClientOptions:
|
|
@@ -62,7 +67,7 @@ class OpenLineageClient:
|
|
|
62
67
|
elif transport:
|
|
63
68
|
self.transport = transport
|
|
64
69
|
else:
|
|
65
|
-
transport_config =
|
|
70
|
+
transport_config = self.config.get("transport", None)
|
|
66
71
|
self.transport = factory.create(transport_config)
|
|
67
72
|
|
|
68
73
|
self._filters: list[Filter] = []
|
|
@@ -86,10 +91,12 @@ class OpenLineageClient:
|
|
|
86
91
|
),
|
|
87
92
|
)
|
|
88
93
|
|
|
89
|
-
def emit(self, event:
|
|
90
|
-
|
|
94
|
+
def emit(self, event: Event) -> None:
|
|
95
|
+
from typing import get_args
|
|
96
|
+
|
|
97
|
+
if type(event) not in get_args(Event):
|
|
91
98
|
msg = "`emit` only accepts RunEvent, DatasetEvent, JobEvent classes"
|
|
92
|
-
raise ValueError(msg)
|
|
99
|
+
raise ValueError(msg)
|
|
93
100
|
if self._filters and self.filter_event(event) is None:
|
|
94
101
|
return
|
|
95
102
|
if not self.transport:
|
|
@@ -112,8 +119,8 @@ class OpenLineageClient:
|
|
|
112
119
|
|
|
113
120
|
def filter_event(
|
|
114
121
|
self,
|
|
115
|
-
event:
|
|
116
|
-
) ->
|
|
122
|
+
event: Event,
|
|
123
|
+
) -> Event | None:
|
|
117
124
|
"""Filters jobs according to config-defined events"""
|
|
118
125
|
for _filter in self._filters:
|
|
119
126
|
if isinstance(event, RunEvent) and _filter.filter_event(event) is None:
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
from openlineage.client.generated.base import (
|
|
6
|
+
PRODUCER,
|
|
7
|
+
BaseEvent,
|
|
8
|
+
Dataset,
|
|
9
|
+
DatasetEvent,
|
|
10
|
+
InputDataset,
|
|
11
|
+
Job,
|
|
12
|
+
JobEvent,
|
|
13
|
+
OutputDataset,
|
|
14
|
+
Run,
|
|
15
|
+
RunEvent,
|
|
16
|
+
set_producer,
|
|
17
|
+
)
|
|
18
|
+
from openlineage.client.generated.base import (
|
|
19
|
+
EventType as RunState,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"BaseEvent",
|
|
24
|
+
"RunEvent",
|
|
25
|
+
"JobEvent",
|
|
26
|
+
"DatasetEvent",
|
|
27
|
+
"RunState",
|
|
28
|
+
"Dataset",
|
|
29
|
+
"InputDataset",
|
|
30
|
+
"OutputDataset",
|
|
31
|
+
"Run",
|
|
32
|
+
"Job",
|
|
33
|
+
"PRODUCER",
|
|
34
|
+
"set_producer",
|
|
35
|
+
]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import warnings
|
|
3
4
|
from enum import Enum
|
|
4
5
|
from typing import Any, ClassVar, Dict, List, Optional
|
|
5
6
|
|
|
@@ -7,6 +8,10 @@ import attr
|
|
|
7
8
|
from openlineage.client.constants import DEFAULT_PRODUCER
|
|
8
9
|
from openlineage.client.utils import RedactMixin
|
|
9
10
|
|
|
11
|
+
warnings.warn(
|
|
12
|
+
"This module is deprecated. Please use `openlineage.client.facet_v2`.", DeprecationWarning, stacklevel=2
|
|
13
|
+
)
|
|
14
|
+
|
|
10
15
|
SCHEMA_URI = "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json"
|
|
11
16
|
|
|
12
17
|
PRODUCER = DEFAULT_PRODUCER
|
|
@@ -85,7 +90,7 @@ class DocumentationJobFacet(BaseFacet):
|
|
|
85
90
|
|
|
86
91
|
@attr.s
|
|
87
92
|
class SourceCodeLocationJobFacet(BaseFacet):
|
|
88
|
-
type: str = attr.ib()
|
|
93
|
+
type: str = attr.ib()
|
|
89
94
|
url: str = attr.ib()
|
|
90
95
|
|
|
91
96
|
_additional_skip_redact: ClassVar[List[str]] = ["type", "url"]
|
|
@@ -116,7 +121,7 @@ class DocumentationDatasetFacet(BaseFacet):
|
|
|
116
121
|
@attr.s
|
|
117
122
|
class SchemaField(RedactMixin):
|
|
118
123
|
name: str = attr.ib()
|
|
119
|
-
type: str = attr.ib()
|
|
124
|
+
type: str = attr.ib()
|
|
120
125
|
description: Optional[str] = attr.ib(default=None)
|
|
121
126
|
|
|
122
127
|
_do_not_redact: ClassVar[List[str]] = ["name", "type"]
|
|
@@ -160,17 +165,17 @@ class OutputStatisticsOutputDatasetFacet(BaseFacet):
|
|
|
160
165
|
class ColumnMetric:
|
|
161
166
|
nullCount: Optional[int] = attr.ib(default=None) # noqa: N815
|
|
162
167
|
distinctCount: Optional[int] = attr.ib(default=None) # noqa: N815
|
|
163
|
-
sum: Optional[int] = attr.ib(default=None)
|
|
168
|
+
sum: Optional[int] = attr.ib(default=None)
|
|
164
169
|
count: Optional[int] = attr.ib(default=None)
|
|
165
|
-
min: Optional[float] = attr.ib(default=None)
|
|
166
|
-
max: Optional[float] = attr.ib(default=None)
|
|
170
|
+
min: Optional[float] = attr.ib(default=None)
|
|
171
|
+
max: Optional[float] = attr.ib(default=None)
|
|
167
172
|
quantiles: Optional[Dict[str, float]] = attr.ib(default=None)
|
|
168
173
|
|
|
169
174
|
|
|
170
175
|
@attr.s
|
|
171
176
|
class DataQualityMetricsInputDatasetFacet(BaseFacet):
|
|
172
177
|
rowCount: Optional[int] = attr.ib(default=None) # noqa: N815
|
|
173
|
-
bytes: Optional[int] = attr.ib(default=None)
|
|
178
|
+
bytes: Optional[int] = attr.ib(default=None)
|
|
174
179
|
fileCount: Optional[int] = attr.ib(default=None) # noqa: N815
|
|
175
180
|
columnMetrics: Dict[str, ColumnMetric] = attr.ib(factory=dict) # noqa: N815
|
|
176
181
|
|
|
@@ -241,7 +246,7 @@ class ErrorMessageRunFacet(BaseFacet):
|
|
|
241
246
|
class SymlinksDatasetFacetIdentifiers:
|
|
242
247
|
namespace: str = attr.ib()
|
|
243
248
|
name: str = attr.ib()
|
|
244
|
-
type: str = attr.ib()
|
|
249
|
+
type: str = attr.ib()
|
|
245
250
|
|
|
246
251
|
|
|
247
252
|
@attr.s
|
|
@@ -272,7 +277,7 @@ class StorageDatasetFacet(BaseFacet):
|
|
|
272
277
|
@attr.s
|
|
273
278
|
class OwnershipJobFacetOwners:
|
|
274
279
|
name: str = attr.ib()
|
|
275
|
-
type: Optional[str] = attr.ib(default=None)
|
|
280
|
+
type: Optional[str] = attr.ib(default=None)
|
|
276
281
|
|
|
277
282
|
|
|
278
283
|
@attr.s
|
|
@@ -344,7 +349,7 @@ class LifecycleStateChangeDatasetFacet(BaseFacet):
|
|
|
344
349
|
@attr.s
|
|
345
350
|
class OwnershipDatasetFacetOwners:
|
|
346
351
|
name: str = attr.ib()
|
|
347
|
-
type: str = attr.ib()
|
|
352
|
+
type: str = attr.ib()
|
|
348
353
|
|
|
349
354
|
|
|
350
355
|
@attr.s
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from openlineage.client.generated import (
|
|
5
|
+
column_lineage_dataset,
|
|
6
|
+
data_quality_assertions_dataset,
|
|
7
|
+
data_quality_metrics_input_dataset,
|
|
8
|
+
dataset_version_dataset,
|
|
9
|
+
datasource_dataset,
|
|
10
|
+
documentation_dataset,
|
|
11
|
+
documentation_job,
|
|
12
|
+
error_message_run,
|
|
13
|
+
external_query_run,
|
|
14
|
+
extraction_error_run,
|
|
15
|
+
job_type_job,
|
|
16
|
+
lifecycle_state_change_dataset,
|
|
17
|
+
nominal_time_run,
|
|
18
|
+
output_statistics_output_dataset,
|
|
19
|
+
ownership_dataset,
|
|
20
|
+
ownership_job,
|
|
21
|
+
parent_run,
|
|
22
|
+
processing_engine_run,
|
|
23
|
+
schema_dataset,
|
|
24
|
+
source_code_job,
|
|
25
|
+
source_code_location_job,
|
|
26
|
+
sql_job,
|
|
27
|
+
storage_dataset,
|
|
28
|
+
symlinks_dataset,
|
|
29
|
+
)
|
|
30
|
+
from openlineage.client.generated.base import (
|
|
31
|
+
PRODUCER,
|
|
32
|
+
BaseFacet,
|
|
33
|
+
DatasetFacet,
|
|
34
|
+
InputDatasetFacet,
|
|
35
|
+
JobFacet,
|
|
36
|
+
OutputDatasetFacet,
|
|
37
|
+
RunFacet,
|
|
38
|
+
set_producer,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
__all__ = [
|
|
42
|
+
"PRODUCER",
|
|
43
|
+
"BaseFacet",
|
|
44
|
+
"DatasetFacet",
|
|
45
|
+
"InputDatasetFacet",
|
|
46
|
+
"JobFacet",
|
|
47
|
+
"OutputDatasetFacet",
|
|
48
|
+
"RunFacet",
|
|
49
|
+
"set_producer",
|
|
50
|
+
"column_lineage_dataset",
|
|
51
|
+
"data_quality_assertions_dataset",
|
|
52
|
+
"data_quality_metrics_input_dataset",
|
|
53
|
+
"dataset_version_dataset",
|
|
54
|
+
"datasource_dataset",
|
|
55
|
+
"documentation_dataset",
|
|
56
|
+
"documentation_job",
|
|
57
|
+
"error_message_run",
|
|
58
|
+
"external_query_run",
|
|
59
|
+
"extraction_error_run",
|
|
60
|
+
"job_type_job",
|
|
61
|
+
"lifecycle_state_change_dataset",
|
|
62
|
+
"nominal_time_run",
|
|
63
|
+
"output_statistics_output_dataset",
|
|
64
|
+
"ownership_dataset",
|
|
65
|
+
"ownership_job",
|
|
66
|
+
"parent_run",
|
|
67
|
+
"processing_engine_run",
|
|
68
|
+
"schema_dataset",
|
|
69
|
+
"source_code_job",
|
|
70
|
+
"source_code_location_job",
|
|
71
|
+
"sql_job",
|
|
72
|
+
"storage_dataset",
|
|
73
|
+
"symlinks_dataset",
|
|
74
|
+
]
|
|
@@ -5,12 +5,14 @@ from __future__ import annotations
|
|
|
5
5
|
import re
|
|
6
6
|
import typing
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
from openlineage.client.event_v2 import RunEvent as RunEvent_v2
|
|
9
|
+
from openlineage.client.run import RunEvent
|
|
10
|
+
|
|
11
|
+
RunEventType = typing.Union[RunEvent, RunEvent_v2]
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
class Filter:
|
|
13
|
-
def filter_event(self, event:
|
|
15
|
+
def filter_event(self, event: RunEventType) -> RunEventType | None:
|
|
14
16
|
...
|
|
15
17
|
|
|
16
18
|
|
|
@@ -18,7 +20,7 @@ class ExactMatchFilter(Filter):
|
|
|
18
20
|
def __init__(self, match: str) -> None:
|
|
19
21
|
self.match = match
|
|
20
22
|
|
|
21
|
-
def filter_event(self, event:
|
|
23
|
+
def filter_event(self, event: RunEventType) -> RunEventType | None:
|
|
22
24
|
if self.match == event.job.name:
|
|
23
25
|
return None
|
|
24
26
|
return event
|
|
@@ -28,7 +30,7 @@ class RegexFilter(Filter):
|
|
|
28
30
|
def __init__(self, regex: str) -> None:
|
|
29
31
|
self.pattern = re.compile(regex)
|
|
30
32
|
|
|
31
|
-
def filter_event(self, event:
|
|
33
|
+
def filter_event(self, event: RunEventType) -> RunEventType | None:
|
|
32
34
|
if self.pattern.match(event.job.name):
|
|
33
35
|
return None
|
|
34
36
|
return event
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import ClassVar
|
|
8
|
+
|
|
9
|
+
from attr import define, field
|
|
10
|
+
from openlineage.client.constants import DEFAULT_PRODUCER
|
|
11
|
+
from openlineage.client.utils import RedactMixin
|
|
12
|
+
|
|
13
|
+
PRODUCER = DEFAULT_PRODUCER
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def set_producer(producer: str) -> None:
|
|
17
|
+
global PRODUCER # noqa: PLW0603
|
|
18
|
+
PRODUCER = producer
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@define(kw_only=True)
|
|
22
|
+
class BaseEvent(RedactMixin):
|
|
23
|
+
eventTime: str = field() # noqa: N815
|
|
24
|
+
"""the time the event occurred at"""
|
|
25
|
+
|
|
26
|
+
producer: str = field(default="", kw_only=True)
|
|
27
|
+
schemaURL: str = field( # noqa: N815
|
|
28
|
+
default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent", init=False
|
|
29
|
+
)
|
|
30
|
+
_base_skip_redact: ClassVar[list[str]] = ["producer", "schemaURL"]
|
|
31
|
+
_additional_skip_redact: ClassVar[list[str]] = []
|
|
32
|
+
|
|
33
|
+
def __attrs_post_init__(self) -> None:
|
|
34
|
+
if not self.producer:
|
|
35
|
+
self.producer = PRODUCER
|
|
36
|
+
self.schemaURL = self._get_schema()
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def skip_redact(self) -> list[str]:
|
|
40
|
+
return self._base_skip_redact + self._additional_skip_redact
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def _get_schema() -> str:
|
|
44
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent"
|
|
45
|
+
|
|
46
|
+
@eventTime.validator
|
|
47
|
+
def eventtime_check(self, attribute: str, value: str) -> None: # noqa: ARG002
|
|
48
|
+
from dateutil import parser
|
|
49
|
+
|
|
50
|
+
parser.isoparse(value)
|
|
51
|
+
if "t" not in value.lower():
|
|
52
|
+
# make sure date-time contains time
|
|
53
|
+
msg = f"Parsed date-time has to contain time: {value}"
|
|
54
|
+
raise ValueError(msg)
|
|
55
|
+
|
|
56
|
+
@producer.validator
|
|
57
|
+
def producer_check(self, attribute: str, value: str) -> None: # noqa: ARG002
|
|
58
|
+
from urllib.parse import urlparse
|
|
59
|
+
|
|
60
|
+
result = urlparse(value)
|
|
61
|
+
if value and not all([result.scheme, result.netloc]):
|
|
62
|
+
msg = "producer is not a valid URI"
|
|
63
|
+
raise ValueError(msg)
|
|
64
|
+
|
|
65
|
+
@schemaURL.validator
|
|
66
|
+
def schemaurl_check(self, attribute: str, value: str) -> None: # noqa: ARG002
|
|
67
|
+
from urllib.parse import urlparse
|
|
68
|
+
|
|
69
|
+
result = urlparse(value)
|
|
70
|
+
if value and not all([result.scheme, result.netloc]):
|
|
71
|
+
msg = "schemaURL is not a valid URI"
|
|
72
|
+
raise ValueError(msg)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@define
|
|
76
|
+
class BaseFacet(RedactMixin):
|
|
77
|
+
"""all fields of the base facet are prefixed with _ to avoid name conflicts in facets"""
|
|
78
|
+
|
|
79
|
+
_producer: str = field(default="", kw_only=True)
|
|
80
|
+
_schemaURL: str = field( # noqa: N815
|
|
81
|
+
default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet", init=False
|
|
82
|
+
)
|
|
83
|
+
_base_skip_redact: ClassVar[list[str]] = ["_producer", "_schemaURL"]
|
|
84
|
+
_additional_skip_redact: ClassVar[list[str]] = []
|
|
85
|
+
|
|
86
|
+
def __attrs_post_init__(self) -> None:
|
|
87
|
+
if not self._producer:
|
|
88
|
+
self._producer = PRODUCER
|
|
89
|
+
self._schemaURL = self._get_schema()
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def skip_redact(self) -> list[str]:
|
|
93
|
+
return self._base_skip_redact + self._additional_skip_redact
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def _get_schema() -> str:
|
|
97
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet"
|
|
98
|
+
|
|
99
|
+
@_producer.validator
|
|
100
|
+
def _producer_check(self, attribute: str, value: str) -> None: # noqa: ARG002
|
|
101
|
+
from urllib.parse import urlparse
|
|
102
|
+
|
|
103
|
+
result = urlparse(value)
|
|
104
|
+
if value and not all([result.scheme, result.netloc]):
|
|
105
|
+
msg = "_producer is not a valid URI"
|
|
106
|
+
raise ValueError(msg)
|
|
107
|
+
|
|
108
|
+
@_schemaURL.validator
|
|
109
|
+
def _schemaurl_check(self, attribute: str, value: str) -> None: # noqa: ARG002
|
|
110
|
+
from urllib.parse import urlparse
|
|
111
|
+
|
|
112
|
+
result = urlparse(value)
|
|
113
|
+
if value and not all([result.scheme, result.netloc]):
|
|
114
|
+
msg = "_schemaURL is not a valid URI"
|
|
115
|
+
raise ValueError(msg)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@define
|
|
119
|
+
class Dataset(RedactMixin):
|
|
120
|
+
namespace: str
|
|
121
|
+
"""The namespace containing that dataset"""
|
|
122
|
+
|
|
123
|
+
name: str
|
|
124
|
+
"""The unique name for that dataset within that namespace"""
|
|
125
|
+
|
|
126
|
+
facets: dict[str, DatasetFacet] | None = field(factory=dict, kw_only=True) # type: ignore[assignment]
|
|
127
|
+
"""The facets for this dataset"""
|
|
128
|
+
|
|
129
|
+
_skip_redact: ClassVar[list[str]] = ["namespace", "name"]
|
|
130
|
+
|
|
131
|
+
@staticmethod
|
|
132
|
+
def _get_schema() -> str:
|
|
133
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/Dataset"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@define(kw_only=True)
|
|
137
|
+
class DatasetEvent(BaseEvent):
|
|
138
|
+
dataset: StaticDataset
|
|
139
|
+
|
|
140
|
+
@staticmethod
|
|
141
|
+
def _get_schema() -> str:
|
|
142
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetEvent"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@define
|
|
146
|
+
class DatasetFacet(BaseFacet):
|
|
147
|
+
"""A Dataset Facet"""
|
|
148
|
+
|
|
149
|
+
_deleted: bool | None = field(default=None, kw_only=True)
|
|
150
|
+
"""set to true to delete a facet"""
|
|
151
|
+
|
|
152
|
+
@staticmethod
|
|
153
|
+
def _get_schema() -> str:
|
|
154
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class EventType(Enum):
|
|
158
|
+
"""
|
|
159
|
+
the current transition of the run state. It is required to issue 1 START event and 1 of [ COMPLETE,
|
|
160
|
+
ABORT, FAIL ] event per run. Additional events with OTHER eventType can be added to the same run.
|
|
161
|
+
For example to send additional metadata after the run is complete
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
START = "START"
|
|
165
|
+
RUNNING = "RUNNING"
|
|
166
|
+
COMPLETE = "COMPLETE"
|
|
167
|
+
ABORT = "ABORT"
|
|
168
|
+
FAIL = "FAIL"
|
|
169
|
+
OTHER = "OTHER"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@define
|
|
173
|
+
class InputDataset(Dataset):
|
|
174
|
+
"""An input dataset"""
|
|
175
|
+
|
|
176
|
+
inputFacets: dict[str, InputDatasetFacet] | None = field(factory=dict) # type: ignore[assignment]# noqa: N815
|
|
177
|
+
"""The input facets for this dataset."""
|
|
178
|
+
|
|
179
|
+
_additional_skip_redact: ClassVar[list[str]] = ["namespace", "name"]
|
|
180
|
+
|
|
181
|
+
@staticmethod
|
|
182
|
+
def _get_schema() -> str:
|
|
183
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/InputDataset"
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@define
|
|
187
|
+
class InputDatasetFacet(BaseFacet):
|
|
188
|
+
"""An Input Dataset Facet"""
|
|
189
|
+
|
|
190
|
+
@staticmethod
|
|
191
|
+
def _get_schema() -> str:
|
|
192
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/InputDatasetFacet"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@define
|
|
196
|
+
class Job(RedactMixin):
|
|
197
|
+
namespace: str
|
|
198
|
+
"""The namespace containing that job"""
|
|
199
|
+
|
|
200
|
+
name: str
|
|
201
|
+
"""The unique name for that job within that namespace"""
|
|
202
|
+
|
|
203
|
+
facets: dict[str, JobFacet] | None = field(factory=dict) # type: ignore[assignment]
|
|
204
|
+
"""The job facets."""
|
|
205
|
+
|
|
206
|
+
_skip_redact: ClassVar[list[str]] = ["namespace", "name"]
|
|
207
|
+
|
|
208
|
+
@staticmethod
|
|
209
|
+
def _get_schema() -> str:
|
|
210
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/Job"
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@define(kw_only=True)
|
|
214
|
+
class JobEvent(BaseEvent):
|
|
215
|
+
job: Job
|
|
216
|
+
inputs: list[InputDataset] | None = field(factory=list) # type: ignore[assignment]
|
|
217
|
+
"""The set of **input** datasets."""
|
|
218
|
+
|
|
219
|
+
outputs: list[OutputDataset] | None = field(factory=list) # type: ignore[assignment]
|
|
220
|
+
"""The set of **output** datasets."""
|
|
221
|
+
|
|
222
|
+
@staticmethod
|
|
223
|
+
def _get_schema() -> str:
|
|
224
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobEvent"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@define
|
|
228
|
+
class JobFacet(BaseFacet):
|
|
229
|
+
"""A Job Facet"""
|
|
230
|
+
|
|
231
|
+
_deleted: bool | None = field(default=None, kw_only=True)
|
|
232
|
+
"""set to true to delete a facet"""
|
|
233
|
+
|
|
234
|
+
@staticmethod
|
|
235
|
+
def _get_schema() -> str:
|
|
236
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@define
|
|
240
|
+
class OutputDataset(Dataset):
|
|
241
|
+
"""An output dataset"""
|
|
242
|
+
|
|
243
|
+
outputFacets: dict[str, OutputDatasetFacet] | None = field(factory=dict) # type: ignore[assignment]# noqa: N815
|
|
244
|
+
"""The output facets for this dataset"""
|
|
245
|
+
|
|
246
|
+
_additional_skip_redact: ClassVar[list[str]] = ["namespace", "name"]
|
|
247
|
+
|
|
248
|
+
@staticmethod
|
|
249
|
+
def _get_schema() -> str:
|
|
250
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/OutputDataset"
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
@define
|
|
254
|
+
class OutputDatasetFacet(BaseFacet):
|
|
255
|
+
"""An Output Dataset Facet"""
|
|
256
|
+
|
|
257
|
+
@staticmethod
|
|
258
|
+
def _get_schema() -> str:
|
|
259
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/OutputDatasetFacet"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
@define
|
|
263
|
+
class Run(RedactMixin):
|
|
264
|
+
runId: str = field() # noqa: N815
|
|
265
|
+
"""The globally unique ID of the run associated with the job."""
|
|
266
|
+
|
|
267
|
+
facets: dict[str, RunFacet] | None = field(factory=dict) # type: ignore[assignment]
|
|
268
|
+
"""The run facets."""
|
|
269
|
+
|
|
270
|
+
_skip_redact: ClassVar[list[str]] = ["runId"]
|
|
271
|
+
|
|
272
|
+
@staticmethod
|
|
273
|
+
def _get_schema() -> str:
|
|
274
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/Run"
|
|
275
|
+
|
|
276
|
+
@runId.validator
|
|
277
|
+
def runid_check(self, attribute: str, value: str) -> None: # noqa: ARG002
|
|
278
|
+
from uuid import UUID
|
|
279
|
+
|
|
280
|
+
UUID(value)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@define(kw_only=True)
|
|
284
|
+
class RunEvent(BaseEvent):
|
|
285
|
+
run: Run
|
|
286
|
+
job: Job
|
|
287
|
+
eventType: EventType | None = field(default=None) # noqa: N815
|
|
288
|
+
"""
|
|
289
|
+
the current transition of the run state. It is required to issue 1 START event and 1 of [ COMPLETE,
|
|
290
|
+
ABORT, FAIL ] event per run. Additional events with OTHER eventType can be added to the same run.
|
|
291
|
+
For example to send additional metadata after the run is complete
|
|
292
|
+
"""
|
|
293
|
+
inputs: list[InputDataset] | None = field(factory=list) # type: ignore[assignment]
|
|
294
|
+
"""The set of **input** datasets."""
|
|
295
|
+
|
|
296
|
+
outputs: list[OutputDataset] | None = field(factory=list) # type: ignore[assignment]
|
|
297
|
+
"""The set of **output** datasets."""
|
|
298
|
+
|
|
299
|
+
_additional_skip_redact: ClassVar[list[str]] = ["eventType", "eventTime"]
|
|
300
|
+
|
|
301
|
+
@staticmethod
|
|
302
|
+
def _get_schema() -> str:
|
|
303
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent"
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@define
|
|
307
|
+
class RunFacet(BaseFacet):
|
|
308
|
+
"""A Run Facet"""
|
|
309
|
+
|
|
310
|
+
@staticmethod
|
|
311
|
+
def _get_schema() -> str:
|
|
312
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@define
|
|
316
|
+
class StaticDataset(Dataset):
|
|
317
|
+
"""A Dataset sent within static metadata events"""
|
|
318
|
+
|
|
319
|
+
@staticmethod
|
|
320
|
+
def _get_schema() -> str:
|
|
321
|
+
return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/StaticDataset"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import ClassVar
|
|
7
|
+
|
|
8
|
+
from attr import define, field
|
|
9
|
+
from openlineage.client.generated.base import DatasetFacet
|
|
10
|
+
from openlineage.client.utils import RedactMixin
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@define
|
|
14
|
+
class ColumnLineageDatasetFacet(DatasetFacet):
|
|
15
|
+
fields: dict[str, Fields]
|
|
16
|
+
"""Column level lineage that maps output fields into input fields used to evaluate them."""
|
|
17
|
+
|
|
18
|
+
@staticmethod
|
|
19
|
+
def _get_schema() -> str:
|
|
20
|
+
return "https://openlineage.io/spec/facets/1-0-2/ColumnLineageDatasetFacet.json#/$defs/ColumnLineageDatasetFacet"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@define
|
|
24
|
+
class Fields(RedactMixin):
|
|
25
|
+
inputFields: list[InputField] # noqa: N815
|
|
26
|
+
transformationDescription: str | None = field(default=None) # noqa: N815
|
|
27
|
+
"""a string representation of the transformation applied"""
|
|
28
|
+
|
|
29
|
+
transformationType: str | None = field(default=None) # noqa: N815
|
|
30
|
+
"""
|
|
31
|
+
IDENTITY|MASKED reflects a clearly defined behavior. IDENTITY: exact same as input; MASKED: no
|
|
32
|
+
original data available (like a hash of PII for example)
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@define
|
|
37
|
+
class InputField(RedactMixin):
|
|
38
|
+
namespace: str
|
|
39
|
+
"""The input dataset namespace"""
|
|
40
|
+
|
|
41
|
+
name: str
|
|
42
|
+
"""The input dataset name"""
|
|
43
|
+
|
|
44
|
+
field: str
|
|
45
|
+
"""The input field"""
|
|
46
|
+
|
|
47
|
+
_skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
|