openlineage-python 1.14.0__tar.gz → 1.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/PKG-INFO +1 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/release-api-reference.sh +1 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/source/openlineage.client.rst +8 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/__init__.py +5 -0
- openlineage_python-1.15.0/openlineage/client/client.py +247 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/constants.py +1 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/__init__.py +1 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/console.py +3 -2
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/factory.py +13 -35
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/file.py +9 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/http.py +6 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/kafka.py +6 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/msk_iam.py +1 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/noop.py +1 -1
- openlineage_python-1.15.0/openlineage/client/utils.py +44 -0
- openlineage_python-1.15.0/openlineage/client/uuid.py +89 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/pyproject.toml +1 -1
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_client.py +161 -3
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_factory.py +6 -17
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_file.py +3 -3
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_http.py +3 -3
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_msk_iam.py +2 -2
- openlineage_python-1.15.0/tests/test_utils.py +30 -0
- openlineage_python-1.15.0/tests/test_uuid.py +50 -0
- openlineage_python-1.14.0/openlineage/client/client.py +0 -134
- openlineage_python-1.14.0/openlineage/client/utils.py +0 -91
- openlineage_python-1.14.0/tests/test_utils.py +0 -38
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/.gitignore +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/README.md +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/Makefile +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/conf.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/convert.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/index.rst +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/make.bat +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/event_v2.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/facet.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/facet_v2.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/filter.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/base.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/column_lineage_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/data_quality_assertions_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/dataset_version_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/datasource_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/documentation_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/documentation_job.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/error_message_run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/external_query_run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/extraction_error_run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/job_type_job.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/lifecycle_state_change_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/nominal_time_run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/output_statistics_output_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/ownership_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/ownership_job.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/parent_run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/processing_engine_run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/schema_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/source_code_job.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/source_code_location_job.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/sql_job.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/storage_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/symlinks_dataset.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/py.typed +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/run.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/serde.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/transport.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/redact_fields.yml +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/__init__.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/config.yml +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/exact_filter.yml +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/http.yml +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/openlineage.yml +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/regex_filter.yml +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/conftest.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/example_full_event.json +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/nominal_time_without_end.json +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/serde_example_dataset_event.json +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/serde_example_job_event.json +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/serde_example_run_event.json +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_events.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_facet.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_facet_v2.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_kafka.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/transport.py +0 -0
- {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tox.ini +0 -0
|
@@ -65,6 +65,14 @@ openlineage.client.utils module
|
|
|
65
65
|
:undoc-members:
|
|
66
66
|
:show-inheritance:
|
|
67
67
|
|
|
68
|
+
openlineage.client.uuid module
|
|
69
|
+
-------------------------------
|
|
70
|
+
|
|
71
|
+
.. automodule:: openlineage.client.uuid
|
|
72
|
+
:members:
|
|
73
|
+
:undoc-members:
|
|
74
|
+
:show-inheritance:
|
|
75
|
+
|
|
68
76
|
openlineage.client.generated.base module
|
|
69
77
|
----------------------------------------
|
|
70
78
|
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
5
7
|
from openlineage.client.client import OpenLineageClient, OpenLineageClientOptions
|
|
8
|
+
|
|
9
|
+
with warnings.catch_warnings():
|
|
10
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
6
11
|
from openlineage.client.facet import set_producer as set_producer_v1
|
|
7
12
|
from openlineage.client.facet_v2 import set_producer as set_producer_v2
|
|
8
13
|
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import warnings
|
|
8
|
+
from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
|
|
9
|
+
|
|
10
|
+
import attr
|
|
11
|
+
import yaml
|
|
12
|
+
from openlineage.client.filter import Filter, create_filter
|
|
13
|
+
from openlineage.client.serde import Serde
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from requests import Session
|
|
17
|
+
from requests.adapters import HTTPAdapter
|
|
18
|
+
|
|
19
|
+
from openlineage.client import event_v2
|
|
20
|
+
from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
|
|
21
|
+
from openlineage.client.transport import Transport, TransportFactory, get_default_factory
|
|
22
|
+
from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
|
|
23
|
+
from openlineage.client.transport.noop import NoopConfig, NoopTransport
|
|
24
|
+
|
|
25
|
+
Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
|
|
26
|
+
Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
|
|
27
|
+
Event = Union[Event_v1, Event_v2]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@attr.s
|
|
31
|
+
class OpenLineageClientOptions:
|
|
32
|
+
timeout: float = attr.ib(default=5.0)
|
|
33
|
+
verify: bool = attr.ib(default=True)
|
|
34
|
+
api_key: str = attr.ib(default=None)
|
|
35
|
+
adapter: HTTPAdapter = attr.ib(default=None)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
log = logging.getLogger(__name__)
|
|
39
|
+
_T = TypeVar("_T", bound="OpenLineageClient")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class OpenLineageClient:
|
|
43
|
+
def __init__( # noqa: PLR0913
|
|
44
|
+
self,
|
|
45
|
+
url: str | None = None,
|
|
46
|
+
options: OpenLineageClientOptions | None = None,
|
|
47
|
+
session: Session | None = None,
|
|
48
|
+
transport: Transport | None = None,
|
|
49
|
+
factory: TransportFactory | None = None,
|
|
50
|
+
) -> None:
|
|
51
|
+
# Set parent's logging level if environment variable is present
|
|
52
|
+
custom_logging_level = os.getenv("OPENLINEAGE_CLIENT_LOGGING", None)
|
|
53
|
+
if custom_logging_level:
|
|
54
|
+
logging.getLogger(__name__.rpartition(".")[0]).setLevel(custom_logging_level)
|
|
55
|
+
|
|
56
|
+
if url:
|
|
57
|
+
warnings.warn(
|
|
58
|
+
message="Initializing OpenLineageClient with url, options and session is deprecated.",
|
|
59
|
+
category=DeprecationWarning,
|
|
60
|
+
stacklevel=2,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Make config ellipsis - as a guard value to not try to
|
|
64
|
+
# reload yaml each time config is referred to.
|
|
65
|
+
self._config: dict[str, dict[str, str]] | None = None
|
|
66
|
+
|
|
67
|
+
self.transport = self._resolve_transport(
|
|
68
|
+
url=url, options=options, session=session, transport=transport, factory=factory
|
|
69
|
+
)
|
|
70
|
+
log.info("OpenLineageClient will use `%s` transport", self.transport.kind)
|
|
71
|
+
|
|
72
|
+
self._filters: list[Filter] = []
|
|
73
|
+
for conf in self.config.get("filters", []):
|
|
74
|
+
_filter = create_filter(conf)
|
|
75
|
+
if _filter:
|
|
76
|
+
self._filters.append(_filter)
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def from_environment(cls: type[_T]) -> _T:
|
|
80
|
+
warnings.warn(
|
|
81
|
+
message="`OpenLineageClient.from_environment()` is deprecated. Use `OpenLineageClient()`.",
|
|
82
|
+
category=DeprecationWarning,
|
|
83
|
+
stacklevel=2,
|
|
84
|
+
)
|
|
85
|
+
return cls()
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def from_dict(cls: type[_T], config: dict[str, str]) -> _T:
|
|
89
|
+
return cls(transport=get_default_factory().create(config=config))
|
|
90
|
+
|
|
91
|
+
def filter_event(
|
|
92
|
+
self,
|
|
93
|
+
event: Event,
|
|
94
|
+
) -> Event | None:
|
|
95
|
+
"""Filters jobs according to config-defined events"""
|
|
96
|
+
for _filter in self._filters:
|
|
97
|
+
if isinstance(event, RunEvent) and _filter.filter_event(event) is None:
|
|
98
|
+
return None
|
|
99
|
+
return event
|
|
100
|
+
|
|
101
|
+
def emit(self, event: Event) -> None:
|
|
102
|
+
from typing import get_args
|
|
103
|
+
|
|
104
|
+
if type(event) not in get_args(Event):
|
|
105
|
+
msg = "`emit` only accepts RunEvent, DatasetEvent, JobEvent classes"
|
|
106
|
+
raise ValueError(msg)
|
|
107
|
+
if not self.transport:
|
|
108
|
+
log.error("Tried to emit OpenLineage event, but transport is not configured.")
|
|
109
|
+
return
|
|
110
|
+
if self.transport.kind == NoopTransport.kind:
|
|
111
|
+
log.debug("OpenLineage is disabled. No events will be emitted.")
|
|
112
|
+
return
|
|
113
|
+
if self._filters and self.filter_event(event) is None:
|
|
114
|
+
log.debug("OpenLineage event has been filtered out and will not be emitted.")
|
|
115
|
+
return
|
|
116
|
+
|
|
117
|
+
if log.isEnabledFor(logging.DEBUG):
|
|
118
|
+
val = Serde.to_json(event).encode("utf-8")
|
|
119
|
+
log.debug("OpenLineageClient will emit event %s", val)
|
|
120
|
+
self.transport.emit(event)
|
|
121
|
+
log.debug("OpenLineage event successfully emitted.")
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def config(self) -> dict[str, Any]:
|
|
125
|
+
"""Content of OpenLineage YAML config file."""
|
|
126
|
+
if self._config is None:
|
|
127
|
+
config_path = self._find_yaml_config_path()
|
|
128
|
+
if config_path:
|
|
129
|
+
self._config = self._get_config_file_content(config_path)
|
|
130
|
+
else:
|
|
131
|
+
self._config = {}
|
|
132
|
+
return self._config
|
|
133
|
+
|
|
134
|
+
def _resolve_transport(self, **kwargs: Any) -> Transport:
|
|
135
|
+
"""
|
|
136
|
+
Resolves the transport mechanism based on the provided arguments or environment settings.
|
|
137
|
+
|
|
138
|
+
This method determines the appropriate transport by executing a sequence of checks:
|
|
139
|
+
1. Verifies if OpenLineage is disabled through environment variable.
|
|
140
|
+
2. Looks for a transport object provided in the arguments.
|
|
141
|
+
3. Attempts to configure the transport from a YAML config file.
|
|
142
|
+
4. Tries to initialize HTTP transport with an url argument (deprecated).
|
|
143
|
+
5. Tries to set up HTTP transport using environment variables.
|
|
144
|
+
6. If no configuration is found, defaults to a console transport and logs a warning message.
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
The transport object that will be used to send lineage events.
|
|
148
|
+
"""
|
|
149
|
+
# 1. Check if OpenLineage is disabled
|
|
150
|
+
if os.getenv("OPENLINEAGE_DISABLED", "").lower().strip() == "true":
|
|
151
|
+
log.info("OpenLineage is disabled. No events will be emitted.")
|
|
152
|
+
return NoopTransport(NoopConfig())
|
|
153
|
+
|
|
154
|
+
# 2. Check if transport is provided explicitly
|
|
155
|
+
if kwargs.get("transport"):
|
|
156
|
+
return cast(Transport, kwargs["transport"])
|
|
157
|
+
|
|
158
|
+
# 3. Check if transport configuration is provided in YAML config file
|
|
159
|
+
if self.config.get("transport"):
|
|
160
|
+
factory = kwargs.get("factory") or get_default_factory()
|
|
161
|
+
return factory.create(self.config["transport"])
|
|
162
|
+
|
|
163
|
+
# 4. Check legacy HTTP transport initialization with url and options
|
|
164
|
+
if kwargs.get("url"):
|
|
165
|
+
return self._http_transport_from_url(
|
|
166
|
+
url=kwargs["url"], options=kwargs.get("options"), session=kwargs.get("session")
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
# 5. Check HTTP transport initialization with env variables
|
|
170
|
+
if os.environ.get("OPENLINEAGE_URL"):
|
|
171
|
+
return self._http_transport_from_env_variables()
|
|
172
|
+
|
|
173
|
+
# 6. If all else fails, print events to console
|
|
174
|
+
from openlineage.client.transport.console import ConsoleConfig, ConsoleTransport
|
|
175
|
+
|
|
176
|
+
log.warning("Couldn't find any OpenLineage transport configuration; will print events to console.")
|
|
177
|
+
return ConsoleTransport(ConsoleConfig())
|
|
178
|
+
|
|
179
|
+
@staticmethod
|
|
180
|
+
def _get_config_file_content(config_path: str) -> dict[str, Any]:
|
|
181
|
+
try:
|
|
182
|
+
with open(config_path) as f:
|
|
183
|
+
config: dict[str, Any] | None = yaml.safe_load(f)
|
|
184
|
+
if not config:
|
|
185
|
+
log.error("Empty OpenLineage config file: `%s`", config_path)
|
|
186
|
+
return {}
|
|
187
|
+
log.debug("Using content of OpenLineage config file: `%s`", config_path)
|
|
188
|
+
return config
|
|
189
|
+
except Exception:
|
|
190
|
+
log.exception("Couldn't read content of the OpenLineage config file `%s`: ", config_path)
|
|
191
|
+
return {}
|
|
192
|
+
|
|
193
|
+
@staticmethod
|
|
194
|
+
def _find_yaml_config_path() -> str | None:
|
|
195
|
+
paths_to_check = (
|
|
196
|
+
# Check OPENLINEAGE_CONFIG env variable
|
|
197
|
+
(os.getenv("OPENLINEAGE_CONFIG", None), True),
|
|
198
|
+
# Check current working directory for `openlineage.yml` file
|
|
199
|
+
(os.path.join(os.getcwd(), "openlineage.yml"), False),
|
|
200
|
+
# Check $HOME/.openlineage dir for `openlineage.yml` file
|
|
201
|
+
(os.path.join(os.path.expanduser("~/.openlineage"), "openlineage.yml"), False),
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
for path, verbose in paths_to_check:
|
|
205
|
+
try:
|
|
206
|
+
if path and os.path.isfile(path) and os.access(path, os.R_OK):
|
|
207
|
+
return path
|
|
208
|
+
if path and verbose:
|
|
209
|
+
log.debug("OpenLineage config file is missing or not readable: `%s`.", path)
|
|
210
|
+
except Exception: # noqa: BLE001
|
|
211
|
+
# We can get different errors depending on system
|
|
212
|
+
if verbose:
|
|
213
|
+
log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
|
|
214
|
+
return None
|
|
215
|
+
|
|
216
|
+
@staticmethod
|
|
217
|
+
def _http_transport_from_env_variables() -> HttpTransport:
|
|
218
|
+
config = HttpConfig(
|
|
219
|
+
url=os.environ["OPENLINEAGE_URL"],
|
|
220
|
+
auth=create_token_provider(
|
|
221
|
+
{
|
|
222
|
+
"type": "api_key",
|
|
223
|
+
"apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
|
|
224
|
+
},
|
|
225
|
+
),
|
|
226
|
+
)
|
|
227
|
+
endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
|
|
228
|
+
if endpoint is not None:
|
|
229
|
+
config.endpoint = endpoint
|
|
230
|
+
|
|
231
|
+
return HttpTransport(config)
|
|
232
|
+
|
|
233
|
+
@staticmethod
|
|
234
|
+
def _http_transport_from_url(
|
|
235
|
+
url: str,
|
|
236
|
+
options: OpenLineageClientOptions | None,
|
|
237
|
+
session: Session | None,
|
|
238
|
+
) -> HttpTransport:
|
|
239
|
+
if not options:
|
|
240
|
+
options = OpenLineageClientOptions()
|
|
241
|
+
return HttpTransport(
|
|
242
|
+
HttpConfig.from_options(
|
|
243
|
+
url=url,
|
|
244
|
+
options=options,
|
|
245
|
+
session=session,
|
|
246
|
+
),
|
|
247
|
+
)
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/__init__.py
RENAMED
|
@@ -7,7 +7,7 @@ from openlineage.client.transport.factory import DefaultTransportFactory
|
|
|
7
7
|
from openlineage.client.transport.file import FileTransport
|
|
8
8
|
from openlineage.client.transport.http import HttpConfig, HttpTransport
|
|
9
9
|
from openlineage.client.transport.kafka import KafkaConfig, KafkaTransport
|
|
10
|
-
from openlineage.client.transport.msk_iam import MSKIAMTransport
|
|
10
|
+
from openlineage.client.transport.msk_iam import MSKIAMConfig, MSKIAMTransport
|
|
11
11
|
from openlineage.client.transport.noop import NoopTransport
|
|
12
12
|
from openlineage.client.transport.transport import Config, Transport, TransportFactory
|
|
13
13
|
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/console.py
RENAMED
|
@@ -22,8 +22,9 @@ class ConsoleTransport(Transport):
|
|
|
22
22
|
|
|
23
23
|
def __init__(self, config: ConsoleConfig) -> None: # noqa: ARG002
|
|
24
24
|
self.log = logging.getLogger(__name__)
|
|
25
|
-
self.log.debug("Constructing
|
|
25
|
+
self.log.debug("Constructing OpenLineage transport that will send events to console or logs")
|
|
26
26
|
|
|
27
27
|
def emit(self, event: Event) -> None:
|
|
28
|
-
#
|
|
28
|
+
# Note: When the logging level is set to DEBUG, the content of events is logged twice:
|
|
29
|
+
# here on the INFO level and in client.py on the DEBUG level for when different transport is used.
|
|
29
30
|
self.log.info(Serde.to_json(event))
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/factory.py
RENAMED
|
@@ -21,20 +21,26 @@ class DefaultTransportFactory(TransportFactory):
|
|
|
21
21
|
self.transports[of_type] = clazz
|
|
22
22
|
|
|
23
23
|
def create(self, config: dict[str, str] | None = None) -> Transport:
|
|
24
|
-
|
|
24
|
+
"""
|
|
25
|
+
Initializes and returns a transport mechanism based on the provided configuration.
|
|
26
|
+
|
|
27
|
+
If 'OPENLINEAGE_DISABLED' is set to 'true', a NoopTransport instance is returned,
|
|
28
|
+
effectively disabling transport.
|
|
29
|
+
If a configuration dictionary is provided, transport specified by the config is initialized.
|
|
30
|
+
If no configuration is provided, the function defaults to a console-based transport, logging
|
|
31
|
+
a warning and printing events to the console.
|
|
32
|
+
"""
|
|
33
|
+
if os.getenv("OPENLINEAGE_DISABLED", "").lower().strip() == "true":
|
|
34
|
+
log.info("OpenLineage is disabled. No events will be emitted.")
|
|
25
35
|
return NoopTransport(NoopConfig())
|
|
26
36
|
|
|
27
37
|
if config:
|
|
28
38
|
return self._create_transport(config)
|
|
29
39
|
|
|
30
|
-
#
|
|
31
|
-
http = self._try_http_from_env_config()
|
|
32
|
-
if http:
|
|
33
|
-
return http
|
|
34
|
-
# If there is no HTTP transport, log events to console
|
|
40
|
+
# If no config is passed, log events to console
|
|
35
41
|
from openlineage.client.transport.console import ConsoleConfig, ConsoleTransport
|
|
36
42
|
|
|
37
|
-
log.warning("Couldn't initialize transport; will print events to console.")
|
|
43
|
+
log.warning("Couldn't initialize OpenLineage transport; will print events to console.")
|
|
38
44
|
return ConsoleTransport(ConsoleConfig())
|
|
39
45
|
|
|
40
46
|
def _create_transport(self, config: dict[str, str]) -> Transport:
|
|
@@ -63,31 +69,3 @@ class DefaultTransportFactory(TransportFactory):
|
|
|
63
69
|
raise TypeError(msg)
|
|
64
70
|
|
|
65
71
|
return transport_class(config_class.from_dict(config)) # type: ignore[call-arg]
|
|
66
|
-
|
|
67
|
-
@staticmethod
|
|
68
|
-
def _try_http_from_env_config() -> Transport | None:
|
|
69
|
-
from openlineage.client.transport.http import (
|
|
70
|
-
HttpConfig,
|
|
71
|
-
HttpTransport,
|
|
72
|
-
create_token_provider,
|
|
73
|
-
)
|
|
74
|
-
|
|
75
|
-
# backwards compatibility: create Transport from
|
|
76
|
-
# OPENLINEAGE_URL, OPENLINEAGE_ENDPOINT, and OPENLINEAGE_API_KEY
|
|
77
|
-
if "OPENLINEAGE_URL" not in os.environ:
|
|
78
|
-
log.error("Did not find openlineage.yml and OPENLINEAGE_URL is not set")
|
|
79
|
-
return None
|
|
80
|
-
config = HttpConfig(
|
|
81
|
-
url=os.environ["OPENLINEAGE_URL"],
|
|
82
|
-
auth=create_token_provider(
|
|
83
|
-
{
|
|
84
|
-
"type": "api_key",
|
|
85
|
-
"apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
|
|
86
|
-
},
|
|
87
|
-
),
|
|
88
|
-
)
|
|
89
|
-
endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
|
|
90
|
-
if endpoint is not None:
|
|
91
|
-
config.endpoint = endpoint
|
|
92
|
-
|
|
93
|
-
return HttpTransport(config)
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/file.py
RENAMED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import io
|
|
7
|
+
import logging
|
|
7
8
|
from dataclasses import dataclass
|
|
8
9
|
from datetime import datetime
|
|
9
10
|
from typing import TYPE_CHECKING, Any
|
|
@@ -14,6 +15,8 @@ if TYPE_CHECKING:
|
|
|
14
15
|
from openlineage.client.client import Event
|
|
15
16
|
from openlineage.client.serde import Serde
|
|
16
17
|
|
|
18
|
+
log = logging.getLogger(__name__)
|
|
19
|
+
|
|
17
20
|
|
|
18
21
|
@dataclass
|
|
19
22
|
class FileConfig(Config):
|
|
@@ -35,6 +38,11 @@ class FileTransport(Transport):
|
|
|
35
38
|
|
|
36
39
|
def __init__(self, config: FileConfig) -> None:
|
|
37
40
|
self.config = config
|
|
41
|
+
log.debug(
|
|
42
|
+
"Constructing OpenLineage transport that will send events "
|
|
43
|
+
"to file(s) using the following config: %s",
|
|
44
|
+
self.config,
|
|
45
|
+
)
|
|
38
46
|
|
|
39
47
|
def emit(self, event: Event) -> None:
|
|
40
48
|
if self.config.append:
|
|
@@ -43,6 +51,7 @@ class FileTransport(Transport):
|
|
|
43
51
|
time_str = datetime.now().strftime("%Y%m%d-%H%M%S.%f")
|
|
44
52
|
log_file_path = f"{self.config.log_file_path}-{time_str}"
|
|
45
53
|
|
|
54
|
+
log.debug("Openlineage event will be emitted to file: `%s`", log_file_path)
|
|
46
55
|
try:
|
|
47
56
|
with open(log_file_path, "a" if self.config.append else "w") as log_file_handle:
|
|
48
57
|
log_file_handle.write(Serde.to_json(event) + "\n")
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/http.py
RENAMED
|
@@ -123,7 +123,12 @@ class HttpTransport(Transport):
|
|
|
123
123
|
url = config.url.strip()
|
|
124
124
|
self.config = config
|
|
125
125
|
|
|
126
|
-
log.debug(
|
|
126
|
+
log.debug(
|
|
127
|
+
"Constructing OpenLineage transport that will send events "
|
|
128
|
+
"to HTTP endpoint `%s` using the following config: %s",
|
|
129
|
+
urljoin(url, config.endpoint),
|
|
130
|
+
config,
|
|
131
|
+
)
|
|
127
132
|
try:
|
|
128
133
|
from urllib3.util import parse_url
|
|
129
134
|
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/kafka.py
RENAMED
|
@@ -70,7 +70,12 @@ class KafkaTransport(Transport):
|
|
|
70
70
|
self.producer = None
|
|
71
71
|
if not self._is_airflow_sqlalchemy:
|
|
72
72
|
self._setup_producer(self.kafka_config.config)
|
|
73
|
-
log.debug(
|
|
73
|
+
log.debug(
|
|
74
|
+
"Constructing OpenLineage transport that will send events "
|
|
75
|
+
"to kafka topic `%s` using the following config: %s",
|
|
76
|
+
self.topic,
|
|
77
|
+
self.kafka_config,
|
|
78
|
+
)
|
|
74
79
|
|
|
75
80
|
def _get_message_key(self, event: Event) -> str:
|
|
76
81
|
if isinstance(event, (DatasetEvent, event_v2.DatasetEvent)):
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/msk_iam.py
RENAMED
|
@@ -78,7 +78,7 @@ class MSKIAMTransport(KafkaTransport):
|
|
|
78
78
|
|
|
79
79
|
def _setup_producer(self, config: dict) -> None: # type: ignore[type-arg]
|
|
80
80
|
try:
|
|
81
|
-
log.debug("
|
|
81
|
+
log.debug("Use MSK producer with the following config: %s", self.msk_config)
|
|
82
82
|
|
|
83
83
|
# https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
|
|
84
84
|
if self.msk_config.region is None:
|
{openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/noop.py
RENAMED
|
@@ -22,7 +22,7 @@ class NoopTransport(Transport):
|
|
|
22
22
|
config_class = NoopConfig
|
|
23
23
|
|
|
24
24
|
def __init__(self, config: NoopConfig) -> None: # noqa: ARG002
|
|
25
|
-
log.
|
|
25
|
+
log.debug("Constructing OpenLineage transport that will NOT send any events.")
|
|
26
26
|
|
|
27
27
|
def emit(self, event: Event) -> None: # noqa: ARG002
|
|
28
28
|
return None
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Any, ClassVar, Type, cast
|
|
8
|
+
|
|
9
|
+
import attr
|
|
10
|
+
|
|
11
|
+
log = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def import_from_string(path: str) -> type[Any]:
|
|
15
|
+
try:
|
|
16
|
+
module_path, target = path.rsplit(".", 1)
|
|
17
|
+
module = importlib.import_module(module_path)
|
|
18
|
+
return cast(Type[Any], getattr(module, target))
|
|
19
|
+
except Exception as e: # noqa: BLE001
|
|
20
|
+
log.warning(e)
|
|
21
|
+
msg = f"Failed to import {path}"
|
|
22
|
+
raise ImportError(msg) from e
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def try_import_from_string(path: str) -> type[Any] | None:
|
|
26
|
+
try:
|
|
27
|
+
return import_from_string(path)
|
|
28
|
+
except ImportError:
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Filter dictionary to get only those key: value pairs that have
|
|
33
|
+
# key specified in passed attr class
|
|
34
|
+
def get_only_specified_fields(clazz: type[Any], params: dict[str, Any]) -> dict[str, Any]:
|
|
35
|
+
field_keys = [item.name for item in attr.fields(clazz)]
|
|
36
|
+
return {key: value for key, value in params.items() if key in field_keys}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class RedactMixin:
|
|
40
|
+
_skip_redact: ClassVar[list[str]] = []
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def skip_redact(self) -> list[str]:
|
|
44
|
+
return self._skip_redact
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# Copyright 2018-2024 contributors to the OpenLineage project
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import secrets
|
|
6
|
+
import time
|
|
7
|
+
from datetime import timezone
|
|
8
|
+
from hashlib import sha1
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
from uuid import UUID
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def generate_new_uuid(instant: datetime | None = None) -> UUID:
|
|
17
|
+
"""Generate new UUID for an instant of time. Each function call returns a new UUID value.
|
|
18
|
+
|
|
19
|
+
UUID version is an implementation detail, and **should not** be relied on.
|
|
20
|
+
For now it is [UUIDv7](https://datatracker.ietf.org/doc/rfc9562/), so for increasing instant values,
|
|
21
|
+
returned UUID is always greater than previous one.
|
|
22
|
+
|
|
23
|
+
Using uuid6 lib implementation (MIT License), with few changes:
|
|
24
|
+
* https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L128-L147
|
|
25
|
+
* https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L46-L51
|
|
26
|
+
|
|
27
|
+
Added in v1.15.0
|
|
28
|
+
|
|
29
|
+
:param instant: instant of time used to generate UUID. If not provided, current time is used.
|
|
30
|
+
:return: UUID
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
timestamp_ms = int(instant.timestamp() * 1000) if instant else time.time_ns() // 10**6
|
|
34
|
+
node = secrets.randbits(76)
|
|
35
|
+
return _build_uuidv7(timestamp_ms, node)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def generate_static_uuid(
|
|
39
|
+
instant: datetime,
|
|
40
|
+
data: bytes,
|
|
41
|
+
) -> UUID:
|
|
42
|
+
"""Generate UUID for instant of time and input data.
|
|
43
|
+
Calling function with same arguments always produces the same result.
|
|
44
|
+
|
|
45
|
+
UUID version is an implementation detail, and **should not* be relied on.
|
|
46
|
+
For now it is [UUIDv7](https://datatracker.ietf.org/doc/rfc9562/), so for increasing instant values,
|
|
47
|
+
returned UUID is always greater than previous one. The only difference from RFC 9562 is that
|
|
48
|
+
least significant bytes are not random, but instead a SHA-1 hash of input data.
|
|
49
|
+
|
|
50
|
+
Using uuid6 lib implementation (MIT License), with few changes:
|
|
51
|
+
* https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L128-L147
|
|
52
|
+
* https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L46-L51
|
|
53
|
+
|
|
54
|
+
Added in v1.15.0
|
|
55
|
+
|
|
56
|
+
:param instant: instant of time used to generate UUID. If not provided, current time is used.
|
|
57
|
+
:param data: input data to generate random part from.
|
|
58
|
+
:return: UUID
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
instant_utc = instant.astimezone(timezone.utc)
|
|
62
|
+
timestamp_ms = int(instant_utc.timestamp() * 1000)
|
|
63
|
+
|
|
64
|
+
# generate the rest of bytes using some hash.
|
|
65
|
+
# can be used to generate consistent UUIDs for some input, e.g. external runId.
|
|
66
|
+
# if data is some static value, e.g. job name, mix it with timestamp to make it more random
|
|
67
|
+
digest = sha1(instant_utc.isoformat().encode("utf-8") + data).digest() # noqa: S324
|
|
68
|
+
# sha1 returns 160bit hash, we need only first 76 bits
|
|
69
|
+
node = int(digest.hex(), 16) >> 84
|
|
70
|
+
|
|
71
|
+
return _build_uuidv7(timestamp_ms, node)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _build_uuidv7(timestamp: int, node: int) -> UUID:
|
|
75
|
+
# merge timestamp and node into 128-bit UUID
|
|
76
|
+
# timestamp is first 48 bits, node is last 80 bits
|
|
77
|
+
uuid_int = (timestamp & 0xFFFFFFFFFFFF) << 80
|
|
78
|
+
uuid_int |= node & 0xFFFFFFFFFFFFFFFFFFFF
|
|
79
|
+
|
|
80
|
+
# Set the version number (4 bit).
|
|
81
|
+
version = 7
|
|
82
|
+
uuid_int &= ~(0xF000 << 64)
|
|
83
|
+
uuid_int |= version << 76
|
|
84
|
+
|
|
85
|
+
# Set the variant (2 bit) to RFC 4122.
|
|
86
|
+
uuid_int &= ~(0xC000 << 48)
|
|
87
|
+
uuid_int |= 0x8000 << 48
|
|
88
|
+
|
|
89
|
+
return UUID(int=uuid_int)
|