openlineage-python 1.33.0__tar.gz → 1.35.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/PKG-INFO +6 -2
  2. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/__init__.py +1 -1
  3. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/client.py +66 -42
  4. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/constants.py +1 -1
  5. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/facets.py +2 -2
  6. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/filter.py +9 -5
  7. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/base.py +4 -4
  8. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/catalog_dataset.py +1 -1
  9. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/column_lineage_dataset.py +7 -7
  10. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py +4 -4
  11. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/documentation_dataset.py +4 -1
  12. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/documentation_job.py +4 -1
  13. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/ownership_dataset.py +1 -1
  14. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/ownership_job.py +1 -1
  15. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/parent_run.py +1 -1
  16. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/schema_dataset.py +1 -1
  17. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/source_code_location_job.py +1 -1
  18. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/sql_job.py +2 -1
  19. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/symlinks_dataset.py +1 -1
  20. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/cli.py +2 -2
  21. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/dataclass.jinja2 +1 -1
  22. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/run.py +5 -1
  23. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/serde.py +1 -1
  24. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/tags.py +3 -3
  25. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/__init__.py +9 -5
  26. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/amazon_datazone.py +7 -1
  27. openlineage_python-1.35.0/openlineage/client/transport/async_http.py +556 -0
  28. openlineage_python-1.35.0/openlineage/client/transport/composite.py +170 -0
  29. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/factory.py +8 -0
  30. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/http.py +46 -41
  31. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/kafka.py +32 -12
  32. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/msk_iam.py +5 -5
  33. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/transform.py +15 -9
  34. openlineage_python-1.35.0/openlineage/client/transport/transport.py +70 -0
  35. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/utils.py +2 -2
  36. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/pyproject.toml +42 -3
  37. openlineage_python-1.35.0/tests/config/test-transport.yaml +7 -0
  38. openlineage_python-1.35.0/tests/conftest.py +364 -0
  39. openlineage_python-1.35.0/tests/fixtures/events.py +134 -0
  40. openlineage_python-1.35.0/tests/fixtures/mock_responses.py +92 -0
  41. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/generator/test_base.py +1 -1
  42. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/generator/test_cli.py +1 -1
  43. openlineage_python-1.35.0/tests/integration/docker-compose.yml +21 -0
  44. openlineage_python-1.35.0/tests/integration/test_server/Dockerfile +25 -0
  45. openlineage_python-1.35.0/tests/integration/test_server/requirements.txt +3 -0
  46. openlineage_python-1.35.0/tests/integration/test_server/test_server.py +642 -0
  47. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_amazon_datazone.py +16 -4
  48. openlineage_python-1.35.0/tests/test_async_http.py +971 -0
  49. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_client.py +128 -34
  50. openlineage_python-1.35.0/tests/test_composite.py +330 -0
  51. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_events.py +9 -9
  52. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_facet.py +83 -34
  53. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_facet_v2.py +73 -20
  54. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_factory.py +93 -0
  55. openlineage_python-1.35.0/tests/test_http.py +510 -0
  56. openlineage_python-1.35.0/tests/test_http_integration.py +503 -0
  57. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_kafka.py +123 -40
  58. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_msk_iam.py +9 -2
  59. openlineage_python-1.35.0/tests/test_uuid.py +102 -0
  60. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/test_transform.py +106 -1
  61. openlineage_python-1.33.0/openlineage/client/transport/composite.py +0 -91
  62. openlineage_python-1.33.0/openlineage/client/transport/transport.py +0 -53
  63. openlineage_python-1.33.0/tests/conftest.py +0 -54
  64. openlineage_python-1.33.0/tests/test_composite.py +0 -140
  65. openlineage_python-1.33.0/tests/test_http.py +0 -350
  66. openlineage_python-1.33.0/tests/test_uuid.py +0 -54
  67. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/.gitignore +0 -0
  68. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/README.md +0 -0
  69. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/Makefile +0 -0
  70. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/conf.py +0 -0
  71. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/convert.py +0 -0
  72. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/index.rst +0 -0
  73. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/make.bat +0 -0
  74. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/source/openlineage.client.rst +0 -0
  75. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/event_v2.py +6 -6
  76. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/facet.py +0 -0
  77. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/facet_v2.py +0 -0
  78. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/data_quality_assertions_dataset.py +0 -0
  79. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/dataset_type_dataset.py +0 -0
  80. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/dataset_version_dataset.py +0 -0
  81. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/datasource_dataset.py +0 -0
  82. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/environment_variables_run.py +0 -0
  83. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/error_message_run.py +0 -0
  84. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/external_query_run.py +0 -0
  85. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/extraction_error_run.py +0 -0
  86. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/input_statistics_input_dataset.py +0 -0
  87. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/job_type_job.py +0 -0
  88. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/lifecycle_state_change_dataset.py +0 -0
  89. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/nominal_time_run.py +0 -0
  90. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/output_statistics_output_dataset.py +0 -0
  91. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/processing_engine_run.py +0 -0
  92. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/source_code_job.py +0 -0
  93. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/storage_dataset.py +0 -0
  94. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/tags_dataset.py +0 -0
  95. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/tags_job.py +0 -0
  96. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/tags_run.py +0 -0
  97. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/base.py +0 -0
  98. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/generate.py +0 -0
  99. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/header.py +0 -0
  100. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/Enum.jinja2 +0 -0
  101. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/facet_v2.jinja2 +0 -0
  102. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/root.jinja2 +0 -0
  103. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/validators.jinja2 +0 -0
  104. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/py.typed +0 -0
  105. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/console.py +0 -0
  106. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/file.py +0 -0
  107. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/noop.py +0 -0
  108. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/__init__.py +1 -1
  109. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/transformers/__init__.py +0 -0
  110. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/transformers/job_namespace_replace_transformer.py +0 -0
  111. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/uuid.py +0 -0
  112. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/redact_fields.yml +0 -0
  113. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/__init__.py +0 -0
  114. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/config.yml +0 -0
  115. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/exact_filter.yml +0 -0
  116. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/http.yml +0 -0
  117. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/openlineage.yml +0 -0
  118. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/regex_filter.yml +0 -0
  119. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/example_full_event.json +0 -0
  120. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/generator/__init__.py +0 -0
  121. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/nominal_time_without_end.json +0 -0
  122. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/serde_example_dataset_event.json +0 -0
  123. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/serde_example_job_event.json +0 -0
  124. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/serde_example_run_event.json +0 -0
  125. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_file.py +0 -0
  126. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_utils.py +0 -0
  127. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/__init__.py +0 -0
  128. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/transformers/__init__.py +0 -0
  129. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/transformers/test_job_namespace_replace_transformer.py +0 -0
  130. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transport.py +0 -0
  131. {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: openlineage-python
3
- Version: 1.33.0
3
+ Version: 1.35.0
4
4
  Summary: OpenLineage Python Client
5
5
  Author-email: OpenLineage <info@openlineage.io>
6
6
  Keywords: openlineage
@@ -14,10 +14,12 @@ Classifier: Programming Language :: Python :: 3.12
14
14
  Classifier: Programming Language :: Python :: 3.13
15
15
  Requires-Python: >=3.9
16
16
  Requires-Dist: attrs>=20.0
17
+ Requires-Dist: build>=1.2.2
18
+ Requires-Dist: httpx>=0.27.0
17
19
  Requires-Dist: packaging>=21.0
18
20
  Requires-Dist: python-dateutil>=2.8.2
19
21
  Requires-Dist: pyyaml>=5.4
20
- Requires-Dist: requests>=2.32.3
22
+ Requires-Dist: requests>=2.32.4
21
23
  Provides-Extra: datazone
22
24
  Requires-Dist: boto3>=1.34.134; extra == 'datazone'
23
25
  Provides-Extra: docs
@@ -36,10 +38,12 @@ Requires-Dist: aws-msk-iam-sasl-signer-python>=1.0.1; extra == 'msk-iam'
36
38
  Requires-Dist: confluent-kafka>=2.1.1; extra == 'msk-iam'
37
39
  Provides-Extra: test
38
40
  Requires-Dist: covdefaults>=2.3; extra == 'test'
41
+ Requires-Dist: fastapi>=0.100.0; extra == 'test'
39
42
  Requires-Dist: pytest-cov>=4.1; extra == 'test'
40
43
  Requires-Dist: pytest-mock>=3.10; extra == 'test'
41
44
  Requires-Dist: pytest>=7.3.1; extra == 'test'
42
45
  Requires-Dist: pyyaml>=5.4; extra == 'test'
46
+ Requires-Dist: uvicorn>=0.20.0; extra == 'test'
43
47
  Description-Content-Type: text/markdown
44
48
 
45
49
  # OpenLineage Python Client
@@ -8,7 +8,7 @@ from openlineage.client.client import OpenLineageClient, OpenLineageClientOption
8
8
 
9
9
  with warnings.catch_warnings():
10
10
  warnings.simplefilter("ignore", DeprecationWarning)
11
- from openlineage.client.facet import set_producer as set_producer_v1
11
+ from openlineage.client.facet import set_producer as set_producer_v1
12
12
  from openlineage.client.facet_v2 import set_producer as set_producer_v2
13
13
 
14
14
 
@@ -2,6 +2,7 @@
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
5
+ import contextlib
5
6
  import json
6
7
  import logging
7
8
  import os
@@ -10,19 +11,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
10
11
 
11
12
  import attr
12
13
  import yaml
13
- from openlineage.client.filter import Filter, FilterConfig, create_filter
14
- from openlineage.client.serde import Serde
15
- from openlineage.client.tags import TagsConfig
16
- from openlineage.client.utils import deep_merge_dicts
17
-
18
- if TYPE_CHECKING:
19
- from requests import Session
20
- from requests.adapters import HTTPAdapter
21
-
22
- import contextlib
23
-
24
14
  from openlineage.client import event_v2
25
15
  from openlineage.client.facets import FacetsConfig
16
+ from openlineage.client.filter import Filter, FilterConfig, create_filter
26
17
  from openlineage.client.generated.environment_variables_run import (
27
18
  EnvironmentVariable,
28
19
  EnvironmentVariablesRunFacet,
@@ -30,33 +21,41 @@ from openlineage.client.generated.environment_variables_run import (
30
21
  from openlineage.client.generated.tags_job import TagsJobFacet, TagsJobFacetFields
31
22
  from openlineage.client.generated.tags_run import TagsRunFacet, TagsRunFacetFields
32
23
  from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
24
+ from openlineage.client.serde import Serde
25
+ from openlineage.client.tags import TagsConfig
33
26
  from openlineage.client.transport import (
34
27
  Transport,
35
28
  TransportFactory,
36
29
  get_default_factory,
37
30
  )
38
- from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
31
+ from openlineage.client.transport.http import HttpConfig, HttpTransport
39
32
  from openlineage.client.transport.noop import NoopConfig, NoopTransport
33
+ from openlineage.client.utils import deep_merge_dicts
34
+
35
+ if TYPE_CHECKING:
36
+ from requests import Session
37
+ from requests.adapters import HTTPAdapter
38
+
40
39
 
41
40
  Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
42
41
  Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
43
42
  Event = Union[Event_v1, Event_v2]
44
43
 
45
44
 
46
- @attr.s
45
+ @attr.define
47
46
  class OpenLineageClientOptions:
48
- timeout: float = attr.ib(default=5.0)
49
- verify: bool = attr.ib(default=True)
50
- api_key: str = attr.ib(default=None)
51
- adapter: HTTPAdapter = attr.ib(default=None)
47
+ timeout: float = 5.0
48
+ verify: bool = True
49
+ api_key: str | None = None
50
+ adapter: HTTPAdapter | None = None
52
51
 
53
52
 
54
- @attr.s
53
+ @attr.define
55
54
  class OpenLineageConfig:
56
- transport: dict[str, Any] | None = attr.ib(factory=dict)
57
- facets: FacetsConfig = attr.ib(factory=FacetsConfig)
58
- filters: list[FilterConfig] = attr.ib(factory=list)
59
- tags: TagsConfig = attr.ib(factory=TagsConfig)
55
+ transport: dict[str, Any] | None = attr.field(factory=dict)
56
+ facets: FacetsConfig = attr.field(factory=FacetsConfig)
57
+ filters: list[FilterConfig] = attr.field(factory=list)
58
+ tags: TagsConfig = attr.field(factory=TagsConfig)
60
59
 
61
60
  @classmethod
62
61
  def from_dict(cls, params: dict[str, Any]) -> OpenLineageConfig:
@@ -184,6 +183,15 @@ class OpenLineageClient:
184
183
  self.transport.emit(event)
185
184
  log.debug("OpenLineage event successfully emitted.")
186
185
 
186
+ def close(self, timeout: float = -1.0) -> bool:
187
+ """
188
+ Closes down the transport until all events are processed or timeout is reached.
189
+ Params:
190
+ timeout: Timeout in seconds. `-1` means to block until last event is processed, 0 means no timeout.
191
+
192
+ """
193
+ return self.transport.close(timeout)
194
+
187
195
  @property
188
196
  def config(self) -> OpenLineageConfig:
189
197
  """
@@ -239,7 +247,7 @@ class OpenLineageClient:
239
247
 
240
248
  # 2. Check if transport is provided explicitly
241
249
  if kwargs.get("transport"):
242
- return cast(Transport, kwargs["transport"])
250
+ return cast("Transport", kwargs["transport"])
243
251
 
244
252
  # 3. Check if transport configuration is provided in YAML config file
245
253
  if self.config.transport and self.config.transport.get("type"):
@@ -293,28 +301,26 @@ class OpenLineageClient:
293
301
  return path
294
302
  if path and verbose:
295
303
  log.debug("OpenLineage config file is missing or not readable: `%s`.", path)
296
- except Exception: # noqa: BLE001
304
+ except Exception:
297
305
  # We can get different errors depending on system
298
306
  if verbose:
299
307
  log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
300
308
  return None
301
309
 
302
- @staticmethod
303
- def _http_transport_from_env_variables() -> HttpTransport:
304
- config = HttpConfig(
305
- url=os.environ["OPENLINEAGE_URL"],
306
- auth=create_token_provider(
307
- {
308
- "type": "api_key",
309
- "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
310
- },
311
- ),
312
- )
313
- endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
314
- if endpoint is not None:
315
- config.endpoint = endpoint
316
-
317
- return HttpTransport(config)
310
+ def _http_transport_from_env_variables(self) -> HttpTransport:
311
+ """
312
+ Create HTTP transport from legacy environment variables
313
+ """
314
+ # Start with basic config from legacy environment variables
315
+ config_dict = {
316
+ "url": os.environ["OPENLINEAGE_URL"],
317
+ "auth": {
318
+ "type": "api_key",
319
+ "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
320
+ },
321
+ "endpoint": os.environ.get("OPENLINEAGE_ENDPOINT", "api/v1/lineage"),
322
+ }
323
+ return HttpTransport(HttpConfig.from_dict(config_dict))
318
324
 
319
325
  @staticmethod
320
326
  def _http_transport_from_url(
@@ -344,9 +350,13 @@ class OpenLineageClient:
344
350
  default_transport_name,
345
351
  )
346
352
  return
353
+
354
+ api_key = os.environ.get("OPENLINEAGE_API_KEY")
355
+ endpoint = os.environ.get("OPENLINEAGE_ENDPOINT")
356
+
347
357
  os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__TYPE"] = "http"
348
358
  os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__URL"] = url
349
- if api_key := os.environ.get("OPENLINEAGE_API_KEY"):
359
+ if api_key:
350
360
  os.environ[
351
361
  f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__AUTH"
352
362
  ] = json.dumps(
@@ -355,11 +365,25 @@ class OpenLineageClient:
355
365
  "apiKey": api_key,
356
366
  }
357
367
  )
358
- if endpoint := os.environ.get("OPENLINEAGE_ENDPOINT"):
368
+ if endpoint:
359
369
  os.environ[
360
370
  f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__ENDPOINT"
361
371
  ] = endpoint
362
372
 
373
+ if os.environ.get("OPENLINEAGE__TRANSPORT__TYPE") == "async_http":
374
+ # Special case - for seamless switch to async transport
375
+ if not os.getenv("OPENLINEAGE__TRANSPORT__URL"):
376
+ os.environ["OPENLINEAGE__TRANSPORT__URL"] = url
377
+ if api_key and not os.getenv("OPENLINEAGE__TRANSPORT__AUTH"):
378
+ os.environ["OPENLINEAGE__TRANSPORT__AUTH"] = json.dumps(
379
+ {
380
+ "type": "api_key",
381
+ "apiKey": api_key,
382
+ }
383
+ )
384
+ if endpoint and not os.getenv("OPENLINEAGE__TRANSPORT__ENDPOINT"):
385
+ os.environ["OPENLINEAGE__TRANSPORT__ENDPOINT"] = endpoint
386
+
363
387
  @classmethod
364
388
  def _load_config_from_env_variables(cls) -> dict[str, Any] | None:
365
389
  config: dict[str, Any] = {}
@@ -2,7 +2,7 @@
2
2
  # SPDX-License-Identifier: Apache-2.0
3
3
  from __future__ import annotations
4
4
 
5
- __version__ = "1.33.0"
5
+ __version__ = "1.35.0"
6
6
 
7
7
  DEFAULT_TIMEOUT_MS = 5000
8
8
  DEFAULT_NAMESPACE_NAME = "default"
@@ -5,6 +5,6 @@ from __future__ import annotations
5
5
  import attr
6
6
 
7
7
 
8
- @attr.s
8
+ @attr.define
9
9
  class FacetsConfig:
10
- environment_variables: list[str] = attr.ib(factory=list)
10
+ environment_variables: list[str] = attr.field(factory=list)
@@ -5,20 +5,24 @@ from __future__ import annotations
5
5
  import logging
6
6
  import re
7
7
  import typing
8
+ import warnings
8
9
 
9
10
  import attr
10
11
  from openlineage.client.event_v2 import RunEvent as RunEvent_v2
11
- from openlineage.client.run import RunEvent
12
+
13
+ with warnings.catch_warnings():
14
+ warnings.simplefilter("ignore", DeprecationWarning)
15
+ from openlineage.client.run import RunEvent
12
16
 
13
17
  log = logging.getLogger(__name__)
14
18
  RunEventType = typing.Union[RunEvent, RunEvent_v2]
15
19
 
16
20
 
17
- @attr.s
21
+ @attr.define
18
22
  class FilterConfig:
19
- type: str | None = attr.ib(default=None)
20
- match: str | None = attr.ib(default=None)
21
- regex: str | None = attr.ib(default=None)
23
+ type: str | None = None
24
+ match: str | None = None
25
+ regex: str | None = None
22
26
 
23
27
 
24
28
  class Filter:
@@ -23,7 +23,7 @@ class BaseEvent(RedactMixin):
23
23
  eventTime: str = attr.field() # noqa: N815
24
24
  """the time the event occurred at"""
25
25
 
26
- producer: str = attr.field(default="", kw_only=True)
26
+ producer: str = attr.field(default="", kw_only=True) # noqa: N815
27
27
  schemaURL: str = attr.field( # noqa: N815
28
28
  default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent", init=False
29
29
  )
@@ -70,7 +70,7 @@ class BaseEvent(RedactMixin):
70
70
  class BaseFacet(RedactMixin):
71
71
  """all fields of the base facet are prefixed with _ to avoid name conflicts in facets"""
72
72
 
73
- _producer: str = attr.field(default="", kw_only=True)
73
+ _producer: str = attr.field(default="", kw_only=True) # noqa: N815
74
74
  _schemaURL: str = attr.field( # noqa: N815
75
75
  default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet", init=False
76
76
  )
@@ -86,13 +86,13 @@ class BaseFacet(RedactMixin):
86
86
  def skip_redact(self) -> list[str]:
87
87
  return self._base_skip_redact + self._additional_skip_redact
88
88
 
89
- def with_additional_properties(self, **kwargs: dict[str, Any]) -> BaseFacet:
89
+ def with_additional_properties(self, **kwargs: dict[str, Any]) -> "BaseFacet":
90
90
  """Add additional properties to updated class instance."""
91
91
  current_attrs = [a.name for a in attr.fields(self.__class__)]
92
92
 
93
93
  new_class = attr.make_class(
94
94
  self.__class__.__name__,
95
- {k: attr.field() for k in kwargs if k not in current_attrs},
95
+ {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
96
96
  bases=(self.__class__,),
97
97
  )
98
98
  new_class.__module__ = self.__class__.__module__
@@ -12,7 +12,7 @@ class CatalogDatasetFacet(DatasetFacet):
12
12
  framework: str
13
13
  """The storage framework for which the catalog is configured"""
14
14
 
15
- type: str
15
+ type: str # noqa: A003
16
16
  """Type of the catalog."""
17
17
 
18
18
  name: str
@@ -38,13 +38,13 @@ class Fields(RedactMixin):
38
38
  original data available (like a hash of PII for example)
39
39
  """
40
40
 
41
- def with_additional_properties(self, **kwargs: dict[str, Any]) -> Fields:
41
+ def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Fields":
42
42
  """Add additional properties to updated class instance."""
43
43
  current_attrs = [a.name for a in attr.fields(self.__class__)]
44
44
 
45
45
  new_class = attr.make_class(
46
46
  self.__class__.__name__,
47
- {k: attr.field() for k in kwargs if k not in current_attrs},
47
+ {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
48
48
  bases=(self.__class__,),
49
49
  )
50
50
  new_class.__module__ = self.__class__.__module__
@@ -75,13 +75,13 @@ class InputField(RedactMixin):
75
75
  transformations: list[Transformation] | None = attr.field(factory=list)
76
76
  _skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
77
77
 
78
- def with_additional_properties(self, **kwargs: dict[str, Any]) -> InputField:
78
+ def with_additional_properties(self, **kwargs: dict[str, Any]) -> "InputField":
79
79
  """Add additional properties to updated class instance."""
80
80
  current_attrs = [a.name for a in attr.fields(self.__class__)]
81
81
 
82
82
  new_class = attr.make_class(
83
83
  self.__class__.__name__,
84
- {k: attr.field() for k in kwargs if k not in current_attrs},
84
+ {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
85
85
  bases=(self.__class__,),
86
86
  )
87
87
  new_class.__module__ = self.__class__.__module__
@@ -102,7 +102,7 @@ class InputField(RedactMixin):
102
102
 
103
103
  @attr.define
104
104
  class Transformation(RedactMixin):
105
- type: str
105
+ type: str # noqa: A003
106
106
  """The type of the transformation. Allowed values are: DIRECT, INDIRECT"""
107
107
 
108
108
  subtype: str | None = attr.field(default=None)
@@ -116,13 +116,13 @@ class Transformation(RedactMixin):
116
116
 
117
117
  _skip_redact: ClassVar[list[str]] = ["type", "subtype", "masking"]
118
118
 
119
- def with_additional_properties(self, **kwargs: dict[str, Any]) -> Transformation:
119
+ def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Transformation":
120
120
  """Add additional properties to updated class instance."""
121
121
  current_attrs = [a.name for a in attr.fields(self.__class__)]
122
122
 
123
123
  new_class = attr.make_class(
124
124
  self.__class__.__name__,
125
- {k: attr.field() for k in kwargs if k not in current_attrs},
125
+ {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
126
126
  bases=(self.__class__,),
127
127
  )
128
128
  new_class.__module__ = self.__class__.__module__
@@ -16,14 +16,14 @@ class ColumnMetrics(RedactMixin):
16
16
  distinctCount: int | None = attr.field(default=None) # noqa: N815
17
17
  """The number of distinct values in this column for the rows evaluated"""
18
18
 
19
- sum: float | None = attr.field(default=None)
19
+ sum: float | None = attr.field(default=None) # noqa: A003
20
20
  """The total sum of values in this column for the rows evaluated"""
21
21
 
22
22
  count: float | None = attr.field(default=None)
23
23
  """The number of values in this column"""
24
24
 
25
- min: float | None = attr.field(default=None)
26
- max: float | None = attr.field(default=None)
25
+ min: float | None = attr.field(default=None) # noqa: A003
26
+ max: float | None = attr.field(default=None) # noqa: A003
27
27
  quantiles: dict[str, float] | None = attr.field(factory=dict)
28
28
  """The property key is the quantile. Examples: 0.1 0.25 0.5 0.75 1"""
29
29
 
@@ -36,7 +36,7 @@ class DataQualityMetricsInputDatasetFacet(InputDatasetFacet):
36
36
  rowCount: int | None = attr.field(default=None) # noqa: N815
37
37
  """The number of rows evaluated"""
38
38
 
39
- bytes: int | None = attr.field(default=None)
39
+ bytes: int | None = attr.field(default=None) # noqa: A003
40
40
  """The size in bytes"""
41
41
 
42
42
  fileCount: int | None = attr.field(default=None) # noqa: N815
@@ -12,6 +12,9 @@ class DocumentationDatasetFacet(DatasetFacet):
12
12
  description: str
13
13
  """The description of the dataset."""
14
14
 
15
+ contentType: str | None = attr.field(default=None) # noqa: N815
16
+ """MIME type of the description field content."""
17
+
15
18
  @staticmethod
16
19
  def _get_schema() -> str:
17
- return "https://openlineage.io/spec/facets/1-0-1/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet"
20
+ return "https://openlineage.io/spec/facets/1-1-0/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet"
@@ -12,8 +12,11 @@ class DocumentationJobFacet(JobFacet):
12
12
  description: str
13
13
  """The description of the job."""
14
14
 
15
+ contentType: str | None = attr.field(default=None) # noqa: N815
16
+ """MIME type of the description field content."""
17
+
15
18
  @staticmethod
16
19
  def _get_schema() -> str:
17
20
  return (
18
- "https://openlineage.io/spec/facets/1-0-1/DocumentationJobFacet.json#/$defs/DocumentationJobFacet"
21
+ "https://openlineage.io/spec/facets/1-1-0/DocumentationJobFacet.json#/$defs/DocumentationJobFacet"
19
22
  )
@@ -15,7 +15,7 @@ class Owner(RedactMixin):
15
15
  the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example
16
16
  application:foo, user:jdoe, team:data
17
17
  """
18
- type: str | None = attr.field(default=None)
18
+ type: str | None = attr.field(default=None) # noqa: A003
19
19
  """The type of ownership (optional)"""
20
20
 
21
21
 
@@ -15,7 +15,7 @@ class Owner(RedactMixin):
15
15
  the identifier of the owner of the Job. It is recommended to define this as a URN. For example
16
16
  application:foo, user:jdoe, team:data
17
17
  """
18
- type: str | None = attr.field(default=None)
18
+ type: str | None = attr.field(default=None) # noqa: A003
19
19
  """The type of ownership (optional)"""
20
20
 
21
21
 
@@ -36,7 +36,7 @@ class ParentRunFacet(RunFacet):
36
36
  return "https://openlineage.io/spec/facets/1-1-0/ParentRunFacet.json#/$defs/ParentRunFacet"
37
37
 
38
38
  @classmethod
39
- def create(cls, runId: str, namespace: str, name: str) -> ParentRunFacet: # noqa: N803
39
+ def create(cls, runId: str, namespace: str, name: str) -> "ParentRunFacet": # noqa: N803
40
40
  import warnings
41
41
 
42
42
  warnings.warn(
@@ -23,7 +23,7 @@ class SchemaDatasetFacetFields(RedactMixin):
23
23
  name: str
24
24
  """The name of the field."""
25
25
 
26
- type: str | None = attr.field(default=None)
26
+ type: str | None = attr.field(default=None) # noqa: A003
27
27
  """The type of the field."""
28
28
 
29
29
  description: str | None = attr.field(default=None)
@@ -11,7 +11,7 @@ from openlineage.client.generated.base import JobFacet
11
11
 
12
12
  @attr.define
13
13
  class SourceCodeLocationJobFacet(JobFacet):
14
- type: str
14
+ type: str # noqa: A003
15
15
  """the source control system"""
16
16
 
17
17
  url: str = attr.field()
@@ -10,7 +10,8 @@ from openlineage.client.generated.base import JobFacet
10
10
  @attr.define
11
11
  class SQLJobFacet(JobFacet):
12
12
  query: str
13
+ dialect: str | None = attr.field(default=None)
13
14
 
14
15
  @staticmethod
15
16
  def _get_schema() -> str:
16
- return "https://openlineage.io/spec/facets/1-0-1/SQLJobFacet.json#/$defs/SQLJobFacet"
17
+ return "https://openlineage.io/spec/facets/1-1-0/SQLJobFacet.json#/$defs/SQLJobFacet"
@@ -16,7 +16,7 @@ class Identifier(RedactMixin):
16
16
  name: str
17
17
  """The dataset name"""
18
18
 
19
- type: str
19
+ type: str # noqa: A003
20
20
  """Identifier type"""
21
21
 
22
22
 
@@ -6,7 +6,7 @@ import pathlib
6
6
  import tempfile
7
7
 
8
8
  import click
9
- import requests
9
+ import httpx
10
10
  from openlineage.client.generator.base import (
11
11
  camel_to_snake,
12
12
  format_and_save_output,
@@ -19,7 +19,7 @@ BASE_SPEC_URL = "https://openlineage.io/spec/2-0-2/OpenLineage.json"
19
19
 
20
20
  def get_base_spec() -> pathlib.Path:
21
21
  """Get the base spec from the OpenLineage repository."""
22
- response = requests.get(BASE_SPEC_URL, timeout=10)
22
+ response = httpx.get(BASE_SPEC_URL, timeout=10)
23
23
  response.raise_for_status() # Raise an HTTPError for bad responses
24
24
  tmpfile = pathlib.Path(tempfile.mkdtemp()) / "OpenLineage.json"
25
25
  tmpfile.write_text(response.text)
@@ -157,7 +157,7 @@ class {{ class_name }}:
157
157
 
158
158
  new_class = attr.make_class(
159
159
  self.__class__.__name__,
160
- {k: attr.field() for k in kwargs if k not in current_attrs},
160
+ {k: attr.field(default=None) for k in kwargs if k not in current_attrs},
161
161
  bases=(self.__class__,),
162
162
  )
163
163
  new_class.__module__ = self.__class__.__module__
@@ -7,9 +7,13 @@ from typing import Any, ClassVar, Optional
7
7
 
8
8
  import attr
9
9
  from dateutil import parser
10
- from openlineage.client.facet import NominalTimeRunFacet, ParentRunFacet
11
10
  from openlineage.client.utils import RedactMixin
12
11
 
12
+ with warnings.catch_warnings():
13
+ warnings.simplefilter("ignore", DeprecationWarning)
14
+ from openlineage.client.facet import NominalTimeRunFacet, ParentRunFacet
15
+
16
+
13
17
  warnings.warn(
14
18
  "This module is deprecated. Please use `openlineage.client.event_v2`.", DeprecationWarning, stacklevel=2
15
19
  )
@@ -47,7 +47,7 @@ class Serde:
47
47
  def to_dict(cls, obj: Any) -> dict[Any, Any]:
48
48
  if not isinstance(obj, dict):
49
49
  obj = attr.asdict(obj)
50
- return cast(dict[Any, Any], cls.remove_nulls_and_enums(obj))
50
+ return cast("dict[Any, Any]", cls.remove_nulls_and_enums(obj))
51
51
 
52
52
  @classmethod
53
53
  def to_json(cls, obj: Any) -> str:
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
11
11
  from openlineage.client.generated.tags_run import TagsRunFacetFields
12
12
 
13
13
 
14
- @attr.s
14
+ @attr.define
15
15
  class TagsConfig:
16
- job: list[TagsJobFacetFields] = attr.ib(factory=list)
17
- run: list[TagsRunFacetFields] = attr.ib(factory=list)
16
+ job: list[TagsJobFacetFields] = attr.field(factory=list)
17
+ run: list[TagsRunFacetFields] = attr.field(factory=list)
@@ -3,6 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, AmazonDataZoneTransport
6
+ from openlineage.client.transport.async_http import AsyncHttpConfig, AsyncHttpTransport
6
7
  from openlineage.client.transport.composite import CompositeTransport
7
8
  from openlineage.client.transport.console import ConsoleTransport
8
9
  from openlineage.client.transport.factory import DefaultTransportFactory
@@ -20,6 +21,7 @@ from openlineage.client.transport.transport import Config, Transport, TransportF
20
21
  _factory = DefaultTransportFactory()
21
22
  _factory.register_transport(CompositeTransport.kind, CompositeTransport)
22
23
  _factory.register_transport(HttpTransport.kind, HttpTransport)
24
+ _factory.register_transport(AsyncHttpTransport.kind, AsyncHttpTransport)
23
25
  _factory.register_transport(KafkaTransport.kind, KafkaTransport)
24
26
  _factory.register_transport(MSKIAMTransport.kind, MSKIAMTransport)
25
27
  _factory.register_transport(ConsoleTransport.kind, ConsoleTransport)
@@ -41,14 +43,12 @@ def register_transport(clazz: type[Transport]) -> type[Transport]:
41
43
 
42
44
 
43
45
  __all__ = [
44
- "register_transport",
45
- "get_default_factory",
46
- "Config",
47
- "Transport",
48
- "TransportFactory",
49
46
  "AmazonDataZoneConfig",
50
47
  "AmazonDataZoneTransport",
48
+ "AsyncHttpConfig",
49
+ "AsyncHttpTransport",
51
50
  "CompositeTransport",
51
+ "Config",
52
52
  "ConsoleTransport",
53
53
  "FileTransport",
54
54
  "HttpConfig",
@@ -60,4 +60,8 @@ __all__ = [
60
60
  "NoopTransport",
61
61
  "TransformConfig",
62
62
  "TransformTransport",
63
+ "Transport",
64
+ "TransportFactory",
65
+ "get_default_factory",
66
+ "register_transport",
63
67
  ]
@@ -57,10 +57,16 @@ class AmazonDataZoneTransport(Transport):
57
57
  log.info(
58
58
  "Successfully posted a LineageEvent: %s in Domain: %s", response["id"], response["domainId"]
59
59
  )
60
- except Exception as error: # noqa: BLE001
60
+ except Exception as error:
61
61
  msg = f"Failed to send lineage event to DataZone Domain {self.config.domain_id}: {event}"
62
62
  raise RuntimeError(msg) from error
63
63
 
64
+ def close(self, timeout: float = -1) -> bool:
65
+ if timeout >= 0:
66
+ log.warning("AmazonDataZoneTransport does not support timeout")
67
+ self.datazone.close() # type: ignore[attr-defined]
68
+ return True
69
+
64
70
  def _setup_datazone(self, endpoint_url: str | None = None) -> None:
65
71
  try:
66
72
  import boto3 # type: ignore[import-untyped]