openlineage-python 1.14.0__tar.gz → 1.15.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/PKG-INFO +1 -1
  2. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/release-api-reference.sh +1 -1
  3. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/source/openlineage.client.rst +8 -0
  4. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/__init__.py +5 -0
  5. openlineage_python-1.15.0/openlineage/client/client.py +247 -0
  6. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/constants.py +1 -1
  7. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/__init__.py +1 -1
  8. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/console.py +3 -2
  9. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/factory.py +13 -35
  10. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/file.py +9 -0
  11. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/http.py +6 -1
  12. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/kafka.py +6 -1
  13. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/msk_iam.py +1 -1
  14. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/noop.py +1 -1
  15. openlineage_python-1.15.0/openlineage/client/utils.py +44 -0
  16. openlineage_python-1.15.0/openlineage/client/uuid.py +89 -0
  17. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/pyproject.toml +1 -1
  18. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_client.py +161 -3
  19. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_factory.py +6 -17
  20. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_file.py +3 -3
  21. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_http.py +3 -3
  22. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_msk_iam.py +2 -2
  23. openlineage_python-1.15.0/tests/test_utils.py +30 -0
  24. openlineage_python-1.15.0/tests/test_uuid.py +50 -0
  25. openlineage_python-1.14.0/openlineage/client/client.py +0 -134
  26. openlineage_python-1.14.0/openlineage/client/utils.py +0 -91
  27. openlineage_python-1.14.0/tests/test_utils.py +0 -38
  28. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/.gitignore +0 -0
  29. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/README.md +0 -0
  30. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/Makefile +0 -0
  31. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/conf.py +0 -0
  32. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/convert.py +0 -0
  33. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/index.rst +0 -0
  34. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/docs/make.bat +0 -0
  35. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/event_v2.py +0 -0
  36. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/facet.py +0 -0
  37. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/facet_v2.py +0 -0
  38. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/filter.py +0 -0
  39. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/base.py +0 -0
  40. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/column_lineage_dataset.py +0 -0
  41. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/data_quality_assertions_dataset.py +0 -0
  42. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py +0 -0
  43. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/dataset_version_dataset.py +0 -0
  44. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/datasource_dataset.py +0 -0
  45. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/documentation_dataset.py +0 -0
  46. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/documentation_job.py +0 -0
  47. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/error_message_run.py +0 -0
  48. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/external_query_run.py +0 -0
  49. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/extraction_error_run.py +0 -0
  50. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/job_type_job.py +0 -0
  51. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/lifecycle_state_change_dataset.py +0 -0
  52. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/nominal_time_run.py +0 -0
  53. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/output_statistics_output_dataset.py +0 -0
  54. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/ownership_dataset.py +0 -0
  55. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/ownership_job.py +0 -0
  56. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/parent_run.py +0 -0
  57. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/processing_engine_run.py +0 -0
  58. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/schema_dataset.py +0 -0
  59. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/source_code_job.py +0 -0
  60. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/source_code_location_job.py +0 -0
  61. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/sql_job.py +0 -0
  62. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/storage_dataset.py +0 -0
  63. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/generated/symlinks_dataset.py +0 -0
  64. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/py.typed +0 -0
  65. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/run.py +0 -0
  66. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/serde.py +0 -0
  67. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/openlineage/client/transport/transport.py +0 -0
  68. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/redact_fields.yml +0 -0
  69. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/__init__.py +0 -0
  70. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/config.yml +0 -0
  71. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/exact_filter.yml +0 -0
  72. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/http.yml +0 -0
  73. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/openlineage.yml +0 -0
  74. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/config/regex_filter.yml +0 -0
  75. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/conftest.py +0 -0
  76. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/example_full_event.json +0 -0
  77. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/nominal_time_without_end.json +0 -0
  78. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/serde_example_dataset_event.json +0 -0
  79. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/serde_example_job_event.json +0 -0
  80. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/serde_example_run_event.json +0 -0
  81. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_events.py +0 -0
  82. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_facet.py +0 -0
  83. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_facet_v2.py +0 -0
  84. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/test_kafka.py +0 -0
  85. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tests/transport.py +0 -0
  86. {openlineage_python-1.14.0 → openlineage_python-1.15.0}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: openlineage-python
3
- Version: 1.14.0
3
+ Version: 1.15.0
4
4
  Summary: OpenLineage Python Client
5
5
  Author-email: OpenLineage <info@openlineage.io>
6
6
  Keywords: openlineage
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env bash
2
2
  #
3
- # Copyright 2018-2023 contributors to the OpenLineage project
3
+ # Copyright 2018-2024 contributors to the OpenLineage project
4
4
  # SPDX-License-Identifier: Apache-2.0
5
5
  #
6
6
 
@@ -65,6 +65,14 @@ openlineage.client.utils module
65
65
  :undoc-members:
66
66
  :show-inheritance:
67
67
 
68
+ openlineage.client.uuid module
69
+ -------------------------------
70
+
71
+ .. automodule:: openlineage.client.uuid
72
+ :members:
73
+ :undoc-members:
74
+ :show-inheritance:
75
+
68
76
  openlineage.client.generated.base module
69
77
  ----------------------------------------
70
78
 
@@ -2,7 +2,12 @@
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
5
+ import warnings
6
+
5
7
  from openlineage.client.client import OpenLineageClient, OpenLineageClientOptions
8
+
9
+ with warnings.catch_warnings():
10
+ warnings.simplefilter("ignore", DeprecationWarning)
6
11
  from openlineage.client.facet import set_producer as set_producer_v1
7
12
  from openlineage.client.facet_v2 import set_producer as set_producer_v2
8
13
 
@@ -0,0 +1,247 @@
1
+ # Copyright 2018-2024 contributors to the OpenLineage project
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import os
7
+ import warnings
8
+ from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
9
+
10
+ import attr
11
+ import yaml
12
+ from openlineage.client.filter import Filter, create_filter
13
+ from openlineage.client.serde import Serde
14
+
15
+ if TYPE_CHECKING:
16
+ from requests import Session
17
+ from requests.adapters import HTTPAdapter
18
+
19
+ from openlineage.client import event_v2
20
+ from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
21
+ from openlineage.client.transport import Transport, TransportFactory, get_default_factory
22
+ from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
23
+ from openlineage.client.transport.noop import NoopConfig, NoopTransport
24
+
25
+ Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
26
+ Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
27
+ Event = Union[Event_v1, Event_v2]
28
+
29
+
30
+ @attr.s
31
+ class OpenLineageClientOptions:
32
+ timeout: float = attr.ib(default=5.0)
33
+ verify: bool = attr.ib(default=True)
34
+ api_key: str = attr.ib(default=None)
35
+ adapter: HTTPAdapter = attr.ib(default=None)
36
+
37
+
38
+ log = logging.getLogger(__name__)
39
+ _T = TypeVar("_T", bound="OpenLineageClient")
40
+
41
+
42
+ class OpenLineageClient:
43
+ def __init__( # noqa: PLR0913
44
+ self,
45
+ url: str | None = None,
46
+ options: OpenLineageClientOptions | None = None,
47
+ session: Session | None = None,
48
+ transport: Transport | None = None,
49
+ factory: TransportFactory | None = None,
50
+ ) -> None:
51
+ # Set parent's logging level if environment variable is present
52
+ custom_logging_level = os.getenv("OPENLINEAGE_CLIENT_LOGGING", None)
53
+ if custom_logging_level:
54
+ logging.getLogger(__name__.rpartition(".")[0]).setLevel(custom_logging_level)
55
+
56
+ if url:
57
+ warnings.warn(
58
+ message="Initializing OpenLineageClient with url, options and session is deprecated.",
59
+ category=DeprecationWarning,
60
+ stacklevel=2,
61
+ )
62
+
63
+ # Make config ellipsis - as a guard value to not try to
64
+ # reload yaml each time config is referred to.
65
+ self._config: dict[str, dict[str, str]] | None = None
66
+
67
+ self.transport = self._resolve_transport(
68
+ url=url, options=options, session=session, transport=transport, factory=factory
69
+ )
70
+ log.info("OpenLineageClient will use `%s` transport", self.transport.kind)
71
+
72
+ self._filters: list[Filter] = []
73
+ for conf in self.config.get("filters", []):
74
+ _filter = create_filter(conf)
75
+ if _filter:
76
+ self._filters.append(_filter)
77
+
78
+ @classmethod
79
+ def from_environment(cls: type[_T]) -> _T:
80
+ warnings.warn(
81
+ message="`OpenLineageClient.from_environment()` is deprecated. Use `OpenLineageClient()`.",
82
+ category=DeprecationWarning,
83
+ stacklevel=2,
84
+ )
85
+ return cls()
86
+
87
+ @classmethod
88
+ def from_dict(cls: type[_T], config: dict[str, str]) -> _T:
89
+ return cls(transport=get_default_factory().create(config=config))
90
+
91
+ def filter_event(
92
+ self,
93
+ event: Event,
94
+ ) -> Event | None:
95
+ """Filters jobs according to config-defined events"""
96
+ for _filter in self._filters:
97
+ if isinstance(event, RunEvent) and _filter.filter_event(event) is None:
98
+ return None
99
+ return event
100
+
101
+ def emit(self, event: Event) -> None:
102
+ from typing import get_args
103
+
104
+ if type(event) not in get_args(Event):
105
+ msg = "`emit` only accepts RunEvent, DatasetEvent, JobEvent classes"
106
+ raise ValueError(msg)
107
+ if not self.transport:
108
+ log.error("Tried to emit OpenLineage event, but transport is not configured.")
109
+ return
110
+ if self.transport.kind == NoopTransport.kind:
111
+ log.debug("OpenLineage is disabled. No events will be emitted.")
112
+ return
113
+ if self._filters and self.filter_event(event) is None:
114
+ log.debug("OpenLineage event has been filtered out and will not be emitted.")
115
+ return
116
+
117
+ if log.isEnabledFor(logging.DEBUG):
118
+ val = Serde.to_json(event).encode("utf-8")
119
+ log.debug("OpenLineageClient will emit event %s", val)
120
+ self.transport.emit(event)
121
+ log.debug("OpenLineage event successfully emitted.")
122
+
123
+ @property
124
+ def config(self) -> dict[str, Any]:
125
+ """Content of OpenLineage YAML config file."""
126
+ if self._config is None:
127
+ config_path = self._find_yaml_config_path()
128
+ if config_path:
129
+ self._config = self._get_config_file_content(config_path)
130
+ else:
131
+ self._config = {}
132
+ return self._config
133
+
134
+ def _resolve_transport(self, **kwargs: Any) -> Transport:
135
+ """
136
+ Resolves the transport mechanism based on the provided arguments or environment settings.
137
+
138
+ This method determines the appropriate transport by executing a sequence of checks:
139
+ 1. Verifies if OpenLineage is disabled through environment variable.
140
+ 2. Looks for a transport object provided in the arguments.
141
+ 3. Attempts to configure the transport from a YAML config file.
142
+ 4. Tries to initialize HTTP transport with an url argument (deprecated).
143
+ 5. Tries to set up HTTP transport using environment variables.
144
+ 6. If no configuration is found, defaults to a console transport and logs a warning message.
145
+
146
+ Returns:
147
+ The transport object that will be used to send lineage events.
148
+ """
149
+ # 1. Check if OpenLineage is disabled
150
+ if os.getenv("OPENLINEAGE_DISABLED", "").lower().strip() == "true":
151
+ log.info("OpenLineage is disabled. No events will be emitted.")
152
+ return NoopTransport(NoopConfig())
153
+
154
+ # 2. Check if transport is provided explicitly
155
+ if kwargs.get("transport"):
156
+ return cast(Transport, kwargs["transport"])
157
+
158
+ # 3. Check if transport configuration is provided in YAML config file
159
+ if self.config.get("transport"):
160
+ factory = kwargs.get("factory") or get_default_factory()
161
+ return factory.create(self.config["transport"])
162
+
163
+ # 4. Check legacy HTTP transport initialization with url and options
164
+ if kwargs.get("url"):
165
+ return self._http_transport_from_url(
166
+ url=kwargs["url"], options=kwargs.get("options"), session=kwargs.get("session")
167
+ )
168
+
169
+ # 5. Check HTTP transport initialization with env variables
170
+ if os.environ.get("OPENLINEAGE_URL"):
171
+ return self._http_transport_from_env_variables()
172
+
173
+ # 6. If all else fails, print events to console
174
+ from openlineage.client.transport.console import ConsoleConfig, ConsoleTransport
175
+
176
+ log.warning("Couldn't find any OpenLineage transport configuration; will print events to console.")
177
+ return ConsoleTransport(ConsoleConfig())
178
+
179
+ @staticmethod
180
+ def _get_config_file_content(config_path: str) -> dict[str, Any]:
181
+ try:
182
+ with open(config_path) as f:
183
+ config: dict[str, Any] | None = yaml.safe_load(f)
184
+ if not config:
185
+ log.error("Empty OpenLineage config file: `%s`", config_path)
186
+ return {}
187
+ log.debug("Using content of OpenLineage config file: `%s`", config_path)
188
+ return config
189
+ except Exception:
190
+ log.exception("Couldn't read content of the OpenLineage config file `%s`: ", config_path)
191
+ return {}
192
+
193
+ @staticmethod
194
+ def _find_yaml_config_path() -> str | None:
195
+ paths_to_check = (
196
+ # Check OPENLINEAGE_CONFIG env variable
197
+ (os.getenv("OPENLINEAGE_CONFIG", None), True),
198
+ # Check current working directory for `openlineage.yml` file
199
+ (os.path.join(os.getcwd(), "openlineage.yml"), False),
200
+ # Check $HOME/.openlineage dir for `openlineage.yml` file
201
+ (os.path.join(os.path.expanduser("~/.openlineage"), "openlineage.yml"), False),
202
+ )
203
+
204
+ for path, verbose in paths_to_check:
205
+ try:
206
+ if path and os.path.isfile(path) and os.access(path, os.R_OK):
207
+ return path
208
+ if path and verbose:
209
+ log.debug("OpenLineage config file is missing or not readable: `%s`.", path)
210
+ except Exception: # noqa: BLE001
211
+ # We can get different errors depending on system
212
+ if verbose:
213
+ log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
214
+ return None
215
+
216
+ @staticmethod
217
+ def _http_transport_from_env_variables() -> HttpTransport:
218
+ config = HttpConfig(
219
+ url=os.environ["OPENLINEAGE_URL"],
220
+ auth=create_token_provider(
221
+ {
222
+ "type": "api_key",
223
+ "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
224
+ },
225
+ ),
226
+ )
227
+ endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
228
+ if endpoint is not None:
229
+ config.endpoint = endpoint
230
+
231
+ return HttpTransport(config)
232
+
233
+ @staticmethod
234
+ def _http_transport_from_url(
235
+ url: str,
236
+ options: OpenLineageClientOptions | None,
237
+ session: Session | None,
238
+ ) -> HttpTransport:
239
+ if not options:
240
+ options = OpenLineageClientOptions()
241
+ return HttpTransport(
242
+ HttpConfig.from_options(
243
+ url=url,
244
+ options=options,
245
+ session=session,
246
+ ),
247
+ )
@@ -2,7 +2,7 @@
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
5
- __version__ = "1.14.0"
5
+ __version__ = "1.15.0"
6
6
 
7
7
  DEFAULT_TIMEOUT_MS = 5000
8
8
  DEFAULT_NAMESPACE_NAME = "default"
@@ -7,7 +7,7 @@ from openlineage.client.transport.factory import DefaultTransportFactory
7
7
  from openlineage.client.transport.file import FileTransport
8
8
  from openlineage.client.transport.http import HttpConfig, HttpTransport
9
9
  from openlineage.client.transport.kafka import KafkaConfig, KafkaTransport
10
- from openlineage.client.transport.msk_iam import MSKIAMTransport
10
+ from openlineage.client.transport.msk_iam import MSKIAMConfig, MSKIAMTransport
11
11
  from openlineage.client.transport.noop import NoopTransport
12
12
  from openlineage.client.transport.transport import Config, Transport, TransportFactory
13
13
 
@@ -22,8 +22,9 @@ class ConsoleTransport(Transport):
22
22
 
23
23
  def __init__(self, config: ConsoleConfig) -> None: # noqa: ARG002
24
24
  self.log = logging.getLogger(__name__)
25
- self.log.debug("Constructing openlineage client to send events to console or logs")
25
+ self.log.debug("Constructing OpenLineage transport that will send events to console or logs")
26
26
 
27
27
  def emit(self, event: Event) -> None:
28
- # If logging is set to DEBUG, this will also log event in client.py
28
+ # Note: When the logging level is set to DEBUG, the content of events is logged twice:
29
+ # here on the INFO level and in client.py on the DEBUG level for when different transport is used.
29
30
  self.log.info(Serde.to_json(event))
@@ -21,20 +21,26 @@ class DefaultTransportFactory(TransportFactory):
21
21
  self.transports[of_type] = clazz
22
22
 
23
23
  def create(self, config: dict[str, str] | None = None) -> Transport:
24
- if os.getenv("OPENLINEAGE_DISABLED", "").lower() == "true":
24
+ """
25
+ Initializes and returns a transport mechanism based on the provided configuration.
26
+
27
+ If 'OPENLINEAGE_DISABLED' is set to 'true', a NoopTransport instance is returned,
28
+ effectively disabling transport.
29
+ If a configuration dictionary is provided, transport specified by the config is initialized.
30
+ If no configuration is provided, the function defaults to a console-based transport, logging
31
+ a warning and printing events to the console.
32
+ """
33
+ if os.getenv("OPENLINEAGE_DISABLED", "").lower().strip() == "true":
34
+ log.info("OpenLineage is disabled. No events will be emitted.")
25
35
  return NoopTransport(NoopConfig())
26
36
 
27
37
  if config:
28
38
  return self._create_transport(config)
29
39
 
30
- # Fallback to setting HTTP transport from env variables
31
- http = self._try_http_from_env_config()
32
- if http:
33
- return http
34
- # If there is no HTTP transport, log events to console
40
+ # If no config is passed, log events to console
35
41
  from openlineage.client.transport.console import ConsoleConfig, ConsoleTransport
36
42
 
37
- log.warning("Couldn't initialize transport; will print events to console.")
43
+ log.warning("Couldn't initialize OpenLineage transport; will print events to console.")
38
44
  return ConsoleTransport(ConsoleConfig())
39
45
 
40
46
  def _create_transport(self, config: dict[str, str]) -> Transport:
@@ -63,31 +69,3 @@ class DefaultTransportFactory(TransportFactory):
63
69
  raise TypeError(msg)
64
70
 
65
71
  return transport_class(config_class.from_dict(config)) # type: ignore[call-arg]
66
-
67
- @staticmethod
68
- def _try_http_from_env_config() -> Transport | None:
69
- from openlineage.client.transport.http import (
70
- HttpConfig,
71
- HttpTransport,
72
- create_token_provider,
73
- )
74
-
75
- # backwards compatibility: create Transport from
76
- # OPENLINEAGE_URL, OPENLINEAGE_ENDPOINT, and OPENLINEAGE_API_KEY
77
- if "OPENLINEAGE_URL" not in os.environ:
78
- log.error("Did not find openlineage.yml and OPENLINEAGE_URL is not set")
79
- return None
80
- config = HttpConfig(
81
- url=os.environ["OPENLINEAGE_URL"],
82
- auth=create_token_provider(
83
- {
84
- "type": "api_key",
85
- "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
86
- },
87
- ),
88
- )
89
- endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
90
- if endpoint is not None:
91
- config.endpoint = endpoint
92
-
93
- return HttpTransport(config)
@@ -4,6 +4,7 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  import io
7
+ import logging
7
8
  from dataclasses import dataclass
8
9
  from datetime import datetime
9
10
  from typing import TYPE_CHECKING, Any
@@ -14,6 +15,8 @@ if TYPE_CHECKING:
14
15
  from openlineage.client.client import Event
15
16
  from openlineage.client.serde import Serde
16
17
 
18
+ log = logging.getLogger(__name__)
19
+
17
20
 
18
21
  @dataclass
19
22
  class FileConfig(Config):
@@ -35,6 +38,11 @@ class FileTransport(Transport):
35
38
 
36
39
  def __init__(self, config: FileConfig) -> None:
37
40
  self.config = config
41
+ log.debug(
42
+ "Constructing OpenLineage transport that will send events "
43
+ "to file(s) using the following config: %s",
44
+ self.config,
45
+ )
38
46
 
39
47
  def emit(self, event: Event) -> None:
40
48
  if self.config.append:
@@ -43,6 +51,7 @@ class FileTransport(Transport):
43
51
  time_str = datetime.now().strftime("%Y%m%d-%H%M%S.%f")
44
52
  log_file_path = f"{self.config.log_file_path}-{time_str}"
45
53
 
54
+ log.debug("Openlineage event will be emitted to file: `%s`", log_file_path)
46
55
  try:
47
56
  with open(log_file_path, "a" if self.config.append else "w") as log_file_handle:
48
57
  log_file_handle.write(Serde.to_json(event) + "\n")
@@ -123,7 +123,12 @@ class HttpTransport(Transport):
123
123
  url = config.url.strip()
124
124
  self.config = config
125
125
 
126
- log.debug("Constructing openlineage client to send events to %s - config %s", url, config)
126
+ log.debug(
127
+ "Constructing OpenLineage transport that will send events "
128
+ "to HTTP endpoint `%s` using the following config: %s",
129
+ urljoin(url, config.endpoint),
130
+ config,
131
+ )
127
132
  try:
128
133
  from urllib3.util import parse_url
129
134
 
@@ -70,7 +70,12 @@ class KafkaTransport(Transport):
70
70
  self.producer = None
71
71
  if not self._is_airflow_sqlalchemy:
72
72
  self._setup_producer(self.kafka_config.config)
73
- log.debug("Constructing openlineage client to send events to topic %s", config.topic)
73
+ log.debug(
74
+ "Constructing OpenLineage transport that will send events "
75
+ "to kafka topic `%s` using the following config: %s",
76
+ self.topic,
77
+ self.kafka_config,
78
+ )
74
79
 
75
80
  def _get_message_key(self, event: Event) -> str:
76
81
  if isinstance(event, (DatasetEvent, event_v2.DatasetEvent)):
@@ -78,7 +78,7 @@ class MSKIAMTransport(KafkaTransport):
78
78
 
79
79
  def _setup_producer(self, config: dict) -> None: # type: ignore[type-arg]
80
80
  try:
81
- log.debug("Setup the MSK transport with this configuration: %s", self.msk_config)
81
+ log.debug("Use MSK producer with the following config: %s", self.msk_config)
82
82
 
83
83
  # https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
84
84
  if self.msk_config.region is None:
@@ -22,7 +22,7 @@ class NoopTransport(Transport):
22
22
  config_class = NoopConfig
23
23
 
24
24
  def __init__(self, config: NoopConfig) -> None: # noqa: ARG002
25
- log.info("OpenLineage client is disabled. NoopTransport.")
25
+ log.debug("Constructing OpenLineage transport that will NOT send any events.")
26
26
 
27
27
  def emit(self, event: Event) -> None: # noqa: ARG002
28
28
  return None
@@ -0,0 +1,44 @@
1
+ # Copyright 2018-2024 contributors to the OpenLineage project
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ from __future__ import annotations
4
+
5
+ import importlib
6
+ import logging
7
+ from typing import Any, ClassVar, Type, cast
8
+
9
+ import attr
10
+
11
+ log = logging.getLogger(__name__)
12
+
13
+
14
+ def import_from_string(path: str) -> type[Any]:
15
+ try:
16
+ module_path, target = path.rsplit(".", 1)
17
+ module = importlib.import_module(module_path)
18
+ return cast(Type[Any], getattr(module, target))
19
+ except Exception as e: # noqa: BLE001
20
+ log.warning(e)
21
+ msg = f"Failed to import {path}"
22
+ raise ImportError(msg) from e
23
+
24
+
25
+ def try_import_from_string(path: str) -> type[Any] | None:
26
+ try:
27
+ return import_from_string(path)
28
+ except ImportError:
29
+ return None
30
+
31
+
32
+ # Filter dictionary to get only those key: value pairs that have
33
+ # key specified in passed attr class
34
+ def get_only_specified_fields(clazz: type[Any], params: dict[str, Any]) -> dict[str, Any]:
35
+ field_keys = [item.name for item in attr.fields(clazz)]
36
+ return {key: value for key, value in params.items() if key in field_keys}
37
+
38
+
39
+ class RedactMixin:
40
+ _skip_redact: ClassVar[list[str]] = []
41
+
42
+ @property
43
+ def skip_redact(self) -> list[str]:
44
+ return self._skip_redact
@@ -0,0 +1,89 @@
1
+ # Copyright 2018-2024 contributors to the OpenLineage project
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ from __future__ import annotations
4
+
5
+ import secrets
6
+ import time
7
+ from datetime import timezone
8
+ from hashlib import sha1
9
+ from typing import TYPE_CHECKING
10
+ from uuid import UUID
11
+
12
+ if TYPE_CHECKING:
13
+ from datetime import datetime
14
+
15
+
16
+ def generate_new_uuid(instant: datetime | None = None) -> UUID:
17
+ """Generate new UUID for an instant of time. Each function call returns a new UUID value.
18
+
19
+ UUID version is an implementation detail, and **should not** be relied on.
20
+ For now it is [UUIDv7](https://datatracker.ietf.org/doc/rfc9562/), so for increasing instant values,
21
+ returned UUID is always greater than previous one.
22
+
23
+ Using uuid6 lib implementation (MIT License), with few changes:
24
+ * https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L128-L147
25
+ * https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L46-L51
26
+
27
+ Added in v1.15.0
28
+
29
+ :param instant: instant of time used to generate UUID. If not provided, current time is used.
30
+ :return: UUID
31
+ """
32
+
33
+ timestamp_ms = int(instant.timestamp() * 1000) if instant else time.time_ns() // 10**6
34
+ node = secrets.randbits(76)
35
+ return _build_uuidv7(timestamp_ms, node)
36
+
37
+
38
+ def generate_static_uuid(
39
+ instant: datetime,
40
+ data: bytes,
41
+ ) -> UUID:
42
+ """Generate UUID for instant of time and input data.
43
+ Calling function with same arguments always produces the same result.
44
+
45
+ UUID version is an implementation detail, and **should not* be relied on.
46
+ For now it is [UUIDv7](https://datatracker.ietf.org/doc/rfc9562/), so for increasing instant values,
47
+ returned UUID is always greater than previous one. The only difference from RFC 9562 is that
48
+ least significant bytes are not random, but instead a SHA-1 hash of input data.
49
+
50
+ Using uuid6 lib implementation (MIT License), with few changes:
51
+ * https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L128-L147
52
+ * https://github.com/oittaa/uuid6-python/blob/4f879849178b8a7a564f7cb76c3f7a6e5228d9ed/src/uuid6/__init__.py#L46-L51
53
+
54
+ Added in v1.15.0
55
+
56
+ :param instant: instant of time used to generate UUID. If not provided, current time is used.
57
+ :param data: input data to generate random part from.
58
+ :return: UUID
59
+ """
60
+
61
+ instant_utc = instant.astimezone(timezone.utc)
62
+ timestamp_ms = int(instant_utc.timestamp() * 1000)
63
+
64
+ # generate the rest of bytes using some hash.
65
+ # can be used to generate consistent UUIDs for some input, e.g. external runId.
66
+ # if data is some static value, e.g. job name, mix it with timestamp to make it more random
67
+ digest = sha1(instant_utc.isoformat().encode("utf-8") + data).digest() # noqa: S324
68
+ # sha1 returns 160bit hash, we need only first 76 bits
69
+ node = int(digest.hex(), 16) >> 84
70
+
71
+ return _build_uuidv7(timestamp_ms, node)
72
+
73
+
74
+ def _build_uuidv7(timestamp: int, node: int) -> UUID:
75
+ # merge timestamp and node into 128-bit UUID
76
+ # timestamp is first 48 bits, node is last 80 bits
77
+ uuid_int = (timestamp & 0xFFFFFFFFFFFF) << 80
78
+ uuid_int |= node & 0xFFFFFFFFFFFFFFFFFFFF
79
+
80
+ # Set the version number (4 bit).
81
+ version = 7
82
+ uuid_int &= ~(0xF000 << 64)
83
+ uuid_int |= version << 76
84
+
85
+ # Set the variant (2 bit) to RFC 4122.
86
+ uuid_int &= ~(0xC000 << 48)
87
+ uuid_int |= 0x8000 << 48
88
+
89
+ return UUID(int=uuid_int)
@@ -6,7 +6,7 @@ requires = [
6
6
 
7
7
  [project]
8
8
  name = "openlineage-python"
9
- version = "1.14.0"
9
+ version = "1.15.0"
10
10
  description = "OpenLineage Python Client"
11
11
  readme = "README.md"
12
12
  keywords = [