openlineage-python 1.33.0__tar.gz → 1.35.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/PKG-INFO +6 -2
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/__init__.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/client.py +66 -42
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/constants.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/facets.py +2 -2
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/filter.py +9 -5
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/base.py +4 -4
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/catalog_dataset.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/column_lineage_dataset.py +7 -7
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py +4 -4
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/documentation_dataset.py +4 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/documentation_job.py +4 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/ownership_dataset.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/ownership_job.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/parent_run.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/schema_dataset.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/source_code_location_job.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/sql_job.py +2 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/symlinks_dataset.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/cli.py +2 -2
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/dataclass.jinja2 +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/run.py +5 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/serde.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/tags.py +3 -3
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/__init__.py +9 -5
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/amazon_datazone.py +7 -1
- openlineage_python-1.35.0/openlineage/client/transport/async_http.py +556 -0
- openlineage_python-1.35.0/openlineage/client/transport/composite.py +170 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/factory.py +8 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/http.py +46 -41
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/kafka.py +32 -12
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/msk_iam.py +5 -5
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/transform.py +15 -9
- openlineage_python-1.35.0/openlineage/client/transport/transport.py +70 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/utils.py +2 -2
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/pyproject.toml +42 -3
- openlineage_python-1.35.0/tests/config/test-transport.yaml +7 -0
- openlineage_python-1.35.0/tests/conftest.py +364 -0
- openlineage_python-1.35.0/tests/fixtures/events.py +134 -0
- openlineage_python-1.35.0/tests/fixtures/mock_responses.py +92 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/generator/test_base.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/generator/test_cli.py +1 -1
- openlineage_python-1.35.0/tests/integration/docker-compose.yml +21 -0
- openlineage_python-1.35.0/tests/integration/test_server/Dockerfile +25 -0
- openlineage_python-1.35.0/tests/integration/test_server/requirements.txt +3 -0
- openlineage_python-1.35.0/tests/integration/test_server/test_server.py +642 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_amazon_datazone.py +16 -4
- openlineage_python-1.35.0/tests/test_async_http.py +971 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_client.py +128 -34
- openlineage_python-1.35.0/tests/test_composite.py +330 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_events.py +9 -9
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_facet.py +83 -34
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_facet_v2.py +73 -20
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_factory.py +93 -0
- openlineage_python-1.35.0/tests/test_http.py +510 -0
- openlineage_python-1.35.0/tests/test_http_integration.py +503 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_kafka.py +123 -40
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_msk_iam.py +9 -2
- openlineage_python-1.35.0/tests/test_uuid.py +102 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/test_transform.py +106 -1
- openlineage_python-1.33.0/openlineage/client/transport/composite.py +0 -91
- openlineage_python-1.33.0/openlineage/client/transport/transport.py +0 -53
- openlineage_python-1.33.0/tests/conftest.py +0 -54
- openlineage_python-1.33.0/tests/test_composite.py +0 -140
- openlineage_python-1.33.0/tests/test_http.py +0 -350
- openlineage_python-1.33.0/tests/test_uuid.py +0 -54
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/.gitignore +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/README.md +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/Makefile +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/conf.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/convert.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/index.rst +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/make.bat +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/docs/source/openlineage.client.rst +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/event_v2.py +6 -6
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/facet.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/facet_v2.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/data_quality_assertions_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/dataset_type_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/dataset_version_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/datasource_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/environment_variables_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/error_message_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/external_query_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/extraction_error_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/input_statistics_input_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/job_type_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/lifecycle_state_change_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/nominal_time_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/output_statistics_output_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/processing_engine_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/source_code_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/storage_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/tags_dataset.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/tags_job.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/tags_run.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/base.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/generate.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/header.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/Enum.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/facet_v2.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/root.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generator/templates/validators.jinja2 +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/py.typed +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/console.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/file.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/noop.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/__init__.py +1 -1
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/transformers/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/transform/transformers/job_namespace_replace_transformer.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/uuid.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/redact_fields.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/config.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/exact_filter.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/http.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/openlineage.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/config/regex_filter.yml +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/example_full_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/generator/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/nominal_time_without_end.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/serde_example_dataset_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/serde_example_job_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/serde_example_run_event.json +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_file.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/test_utils.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/transformers/__init__.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transform/transformers/test_job_namespace_replace_transformer.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tests/transport.py +0 -0
- {openlineage_python-1.33.0 → openlineage_python-1.35.0}/tox.ini +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openlineage-python
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.35.0
|
|
4
4
|
Summary: OpenLineage Python Client
|
|
5
5
|
Author-email: OpenLineage <info@openlineage.io>
|
|
6
6
|
Keywords: openlineage
|
|
@@ -14,10 +14,12 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Requires-Python: >=3.9
|
|
16
16
|
Requires-Dist: attrs>=20.0
|
|
17
|
+
Requires-Dist: build>=1.2.2
|
|
18
|
+
Requires-Dist: httpx>=0.27.0
|
|
17
19
|
Requires-Dist: packaging>=21.0
|
|
18
20
|
Requires-Dist: python-dateutil>=2.8.2
|
|
19
21
|
Requires-Dist: pyyaml>=5.4
|
|
20
|
-
Requires-Dist: requests>=2.32.
|
|
22
|
+
Requires-Dist: requests>=2.32.4
|
|
21
23
|
Provides-Extra: datazone
|
|
22
24
|
Requires-Dist: boto3>=1.34.134; extra == 'datazone'
|
|
23
25
|
Provides-Extra: docs
|
|
@@ -36,10 +38,12 @@ Requires-Dist: aws-msk-iam-sasl-signer-python>=1.0.1; extra == 'msk-iam'
|
|
|
36
38
|
Requires-Dist: confluent-kafka>=2.1.1; extra == 'msk-iam'
|
|
37
39
|
Provides-Extra: test
|
|
38
40
|
Requires-Dist: covdefaults>=2.3; extra == 'test'
|
|
41
|
+
Requires-Dist: fastapi>=0.100.0; extra == 'test'
|
|
39
42
|
Requires-Dist: pytest-cov>=4.1; extra == 'test'
|
|
40
43
|
Requires-Dist: pytest-mock>=3.10; extra == 'test'
|
|
41
44
|
Requires-Dist: pytest>=7.3.1; extra == 'test'
|
|
42
45
|
Requires-Dist: pyyaml>=5.4; extra == 'test'
|
|
46
|
+
Requires-Dist: uvicorn>=0.20.0; extra == 'test'
|
|
43
47
|
Description-Content-Type: text/markdown
|
|
44
48
|
|
|
45
49
|
# OpenLineage Python Client
|
|
@@ -8,7 +8,7 @@ from openlineage.client.client import OpenLineageClient, OpenLineageClientOption
|
|
|
8
8
|
|
|
9
9
|
with warnings.catch_warnings():
|
|
10
10
|
warnings.simplefilter("ignore", DeprecationWarning)
|
|
11
|
-
from openlineage.client.facet import set_producer as set_producer_v1
|
|
11
|
+
from openlineage.client.facet import set_producer as set_producer_v1
|
|
12
12
|
from openlineage.client.facet_v2 import set_producer as set_producer_v2
|
|
13
13
|
|
|
14
14
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import contextlib
|
|
5
6
|
import json
|
|
6
7
|
import logging
|
|
7
8
|
import os
|
|
@@ -10,19 +11,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
|
|
|
10
11
|
|
|
11
12
|
import attr
|
|
12
13
|
import yaml
|
|
13
|
-
from openlineage.client.filter import Filter, FilterConfig, create_filter
|
|
14
|
-
from openlineage.client.serde import Serde
|
|
15
|
-
from openlineage.client.tags import TagsConfig
|
|
16
|
-
from openlineage.client.utils import deep_merge_dicts
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from requests import Session
|
|
20
|
-
from requests.adapters import HTTPAdapter
|
|
21
|
-
|
|
22
|
-
import contextlib
|
|
23
|
-
|
|
24
14
|
from openlineage.client import event_v2
|
|
25
15
|
from openlineage.client.facets import FacetsConfig
|
|
16
|
+
from openlineage.client.filter import Filter, FilterConfig, create_filter
|
|
26
17
|
from openlineage.client.generated.environment_variables_run import (
|
|
27
18
|
EnvironmentVariable,
|
|
28
19
|
EnvironmentVariablesRunFacet,
|
|
@@ -30,33 +21,41 @@ from openlineage.client.generated.environment_variables_run import (
|
|
|
30
21
|
from openlineage.client.generated.tags_job import TagsJobFacet, TagsJobFacetFields
|
|
31
22
|
from openlineage.client.generated.tags_run import TagsRunFacet, TagsRunFacetFields
|
|
32
23
|
from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
|
|
24
|
+
from openlineage.client.serde import Serde
|
|
25
|
+
from openlineage.client.tags import TagsConfig
|
|
33
26
|
from openlineage.client.transport import (
|
|
34
27
|
Transport,
|
|
35
28
|
TransportFactory,
|
|
36
29
|
get_default_factory,
|
|
37
30
|
)
|
|
38
|
-
from openlineage.client.transport.http import HttpConfig, HttpTransport
|
|
31
|
+
from openlineage.client.transport.http import HttpConfig, HttpTransport
|
|
39
32
|
from openlineage.client.transport.noop import NoopConfig, NoopTransport
|
|
33
|
+
from openlineage.client.utils import deep_merge_dicts
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from requests import Session
|
|
37
|
+
from requests.adapters import HTTPAdapter
|
|
38
|
+
|
|
40
39
|
|
|
41
40
|
Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
|
|
42
41
|
Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
|
|
43
42
|
Event = Union[Event_v1, Event_v2]
|
|
44
43
|
|
|
45
44
|
|
|
46
|
-
@attr.
|
|
45
|
+
@attr.define
|
|
47
46
|
class OpenLineageClientOptions:
|
|
48
|
-
timeout: float =
|
|
49
|
-
verify: bool =
|
|
50
|
-
api_key: str =
|
|
51
|
-
adapter: HTTPAdapter =
|
|
47
|
+
timeout: float = 5.0
|
|
48
|
+
verify: bool = True
|
|
49
|
+
api_key: str | None = None
|
|
50
|
+
adapter: HTTPAdapter | None = None
|
|
52
51
|
|
|
53
52
|
|
|
54
|
-
@attr.
|
|
53
|
+
@attr.define
|
|
55
54
|
class OpenLineageConfig:
|
|
56
|
-
transport: dict[str, Any] | None = attr.
|
|
57
|
-
facets: FacetsConfig = attr.
|
|
58
|
-
filters: list[FilterConfig] = attr.
|
|
59
|
-
tags: TagsConfig = attr.
|
|
55
|
+
transport: dict[str, Any] | None = attr.field(factory=dict)
|
|
56
|
+
facets: FacetsConfig = attr.field(factory=FacetsConfig)
|
|
57
|
+
filters: list[FilterConfig] = attr.field(factory=list)
|
|
58
|
+
tags: TagsConfig = attr.field(factory=TagsConfig)
|
|
60
59
|
|
|
61
60
|
@classmethod
|
|
62
61
|
def from_dict(cls, params: dict[str, Any]) -> OpenLineageConfig:
|
|
@@ -184,6 +183,15 @@ class OpenLineageClient:
|
|
|
184
183
|
self.transport.emit(event)
|
|
185
184
|
log.debug("OpenLineage event successfully emitted.")
|
|
186
185
|
|
|
186
|
+
def close(self, timeout: float = -1.0) -> bool:
|
|
187
|
+
"""
|
|
188
|
+
Closes down the transport until all events are processed or timeout is reached.
|
|
189
|
+
Params:
|
|
190
|
+
timeout: Timeout in seconds. `-1` means to block until last event is processed, 0 means no timeout.
|
|
191
|
+
|
|
192
|
+
"""
|
|
193
|
+
return self.transport.close(timeout)
|
|
194
|
+
|
|
187
195
|
@property
|
|
188
196
|
def config(self) -> OpenLineageConfig:
|
|
189
197
|
"""
|
|
@@ -239,7 +247,7 @@ class OpenLineageClient:
|
|
|
239
247
|
|
|
240
248
|
# 2. Check if transport is provided explicitly
|
|
241
249
|
if kwargs.get("transport"):
|
|
242
|
-
return cast(Transport, kwargs["transport"])
|
|
250
|
+
return cast("Transport", kwargs["transport"])
|
|
243
251
|
|
|
244
252
|
# 3. Check if transport configuration is provided in YAML config file
|
|
245
253
|
if self.config.transport and self.config.transport.get("type"):
|
|
@@ -293,28 +301,26 @@ class OpenLineageClient:
|
|
|
293
301
|
return path
|
|
294
302
|
if path and verbose:
|
|
295
303
|
log.debug("OpenLineage config file is missing or not readable: `%s`.", path)
|
|
296
|
-
except Exception:
|
|
304
|
+
except Exception:
|
|
297
305
|
# We can get different errors depending on system
|
|
298
306
|
if verbose:
|
|
299
307
|
log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
|
|
300
308
|
return None
|
|
301
309
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
return HttpTransport(config)
|
|
310
|
+
def _http_transport_from_env_variables(self) -> HttpTransport:
|
|
311
|
+
"""
|
|
312
|
+
Create HTTP transport from legacy environment variables
|
|
313
|
+
"""
|
|
314
|
+
# Start with basic config from legacy environment variables
|
|
315
|
+
config_dict = {
|
|
316
|
+
"url": os.environ["OPENLINEAGE_URL"],
|
|
317
|
+
"auth": {
|
|
318
|
+
"type": "api_key",
|
|
319
|
+
"apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
|
|
320
|
+
},
|
|
321
|
+
"endpoint": os.environ.get("OPENLINEAGE_ENDPOINT", "api/v1/lineage"),
|
|
322
|
+
}
|
|
323
|
+
return HttpTransport(HttpConfig.from_dict(config_dict))
|
|
318
324
|
|
|
319
325
|
@staticmethod
|
|
320
326
|
def _http_transport_from_url(
|
|
@@ -344,9 +350,13 @@ class OpenLineageClient:
|
|
|
344
350
|
default_transport_name,
|
|
345
351
|
)
|
|
346
352
|
return
|
|
353
|
+
|
|
354
|
+
api_key = os.environ.get("OPENLINEAGE_API_KEY")
|
|
355
|
+
endpoint = os.environ.get("OPENLINEAGE_ENDPOINT")
|
|
356
|
+
|
|
347
357
|
os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__TYPE"] = "http"
|
|
348
358
|
os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__URL"] = url
|
|
349
|
-
if api_key
|
|
359
|
+
if api_key:
|
|
350
360
|
os.environ[
|
|
351
361
|
f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__AUTH"
|
|
352
362
|
] = json.dumps(
|
|
@@ -355,11 +365,25 @@ class OpenLineageClient:
|
|
|
355
365
|
"apiKey": api_key,
|
|
356
366
|
}
|
|
357
367
|
)
|
|
358
|
-
if endpoint
|
|
368
|
+
if endpoint:
|
|
359
369
|
os.environ[
|
|
360
370
|
f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__ENDPOINT"
|
|
361
371
|
] = endpoint
|
|
362
372
|
|
|
373
|
+
if os.environ.get("OPENLINEAGE__TRANSPORT__TYPE") == "async_http":
|
|
374
|
+
# Special case - for seamless switch to async transport
|
|
375
|
+
if not os.getenv("OPENLINEAGE__TRANSPORT__URL"):
|
|
376
|
+
os.environ["OPENLINEAGE__TRANSPORT__URL"] = url
|
|
377
|
+
if api_key and not os.getenv("OPENLINEAGE__TRANSPORT__AUTH"):
|
|
378
|
+
os.environ["OPENLINEAGE__TRANSPORT__AUTH"] = json.dumps(
|
|
379
|
+
{
|
|
380
|
+
"type": "api_key",
|
|
381
|
+
"apiKey": api_key,
|
|
382
|
+
}
|
|
383
|
+
)
|
|
384
|
+
if endpoint and not os.getenv("OPENLINEAGE__TRANSPORT__ENDPOINT"):
|
|
385
|
+
os.environ["OPENLINEAGE__TRANSPORT__ENDPOINT"] = endpoint
|
|
386
|
+
|
|
363
387
|
@classmethod
|
|
364
388
|
def _load_config_from_env_variables(cls) -> dict[str, Any] | None:
|
|
365
389
|
config: dict[str, Any] = {}
|
|
@@ -5,20 +5,24 @@ from __future__ import annotations
|
|
|
5
5
|
import logging
|
|
6
6
|
import re
|
|
7
7
|
import typing
|
|
8
|
+
import warnings
|
|
8
9
|
|
|
9
10
|
import attr
|
|
10
11
|
from openlineage.client.event_v2 import RunEvent as RunEvent_v2
|
|
11
|
-
|
|
12
|
+
|
|
13
|
+
with warnings.catch_warnings():
|
|
14
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
15
|
+
from openlineage.client.run import RunEvent
|
|
12
16
|
|
|
13
17
|
log = logging.getLogger(__name__)
|
|
14
18
|
RunEventType = typing.Union[RunEvent, RunEvent_v2]
|
|
15
19
|
|
|
16
20
|
|
|
17
|
-
@attr.
|
|
21
|
+
@attr.define
|
|
18
22
|
class FilterConfig:
|
|
19
|
-
type: str | None =
|
|
20
|
-
match: str | None =
|
|
21
|
-
regex: str | None =
|
|
23
|
+
type: str | None = None
|
|
24
|
+
match: str | None = None
|
|
25
|
+
regex: str | None = None
|
|
22
26
|
|
|
23
27
|
|
|
24
28
|
class Filter:
|
{openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/base.py
RENAMED
|
@@ -23,7 +23,7 @@ class BaseEvent(RedactMixin):
|
|
|
23
23
|
eventTime: str = attr.field() # noqa: N815
|
|
24
24
|
"""the time the event occurred at"""
|
|
25
25
|
|
|
26
|
-
producer: str = attr.field(default="", kw_only=True)
|
|
26
|
+
producer: str = attr.field(default="", kw_only=True) # noqa: N815
|
|
27
27
|
schemaURL: str = attr.field( # noqa: N815
|
|
28
28
|
default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent", init=False
|
|
29
29
|
)
|
|
@@ -70,7 +70,7 @@ class BaseEvent(RedactMixin):
|
|
|
70
70
|
class BaseFacet(RedactMixin):
|
|
71
71
|
"""all fields of the base facet are prefixed with _ to avoid name conflicts in facets"""
|
|
72
72
|
|
|
73
|
-
_producer: str = attr.field(default="", kw_only=True)
|
|
73
|
+
_producer: str = attr.field(default="", kw_only=True) # noqa: N815
|
|
74
74
|
_schemaURL: str = attr.field( # noqa: N815
|
|
75
75
|
default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet", init=False
|
|
76
76
|
)
|
|
@@ -86,13 +86,13 @@ class BaseFacet(RedactMixin):
|
|
|
86
86
|
def skip_redact(self) -> list[str]:
|
|
87
87
|
return self._base_skip_redact + self._additional_skip_redact
|
|
88
88
|
|
|
89
|
-
def with_additional_properties(self, **kwargs: dict[str, Any]) -> BaseFacet:
|
|
89
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> "BaseFacet":
|
|
90
90
|
"""Add additional properties to updated class instance."""
|
|
91
91
|
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
92
92
|
|
|
93
93
|
new_class = attr.make_class(
|
|
94
94
|
self.__class__.__name__,
|
|
95
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
95
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
96
96
|
bases=(self.__class__,),
|
|
97
97
|
)
|
|
98
98
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -38,13 +38,13 @@ class Fields(RedactMixin):
|
|
|
38
38
|
original data available (like a hash of PII for example)
|
|
39
39
|
"""
|
|
40
40
|
|
|
41
|
-
def with_additional_properties(self, **kwargs: dict[str, Any]) -> Fields:
|
|
41
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Fields":
|
|
42
42
|
"""Add additional properties to updated class instance."""
|
|
43
43
|
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
44
44
|
|
|
45
45
|
new_class = attr.make_class(
|
|
46
46
|
self.__class__.__name__,
|
|
47
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
47
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
48
48
|
bases=(self.__class__,),
|
|
49
49
|
)
|
|
50
50
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -75,13 +75,13 @@ class InputField(RedactMixin):
|
|
|
75
75
|
transformations: list[Transformation] | None = attr.field(factory=list)
|
|
76
76
|
_skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
|
|
77
77
|
|
|
78
|
-
def with_additional_properties(self, **kwargs: dict[str, Any]) -> InputField:
|
|
78
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> "InputField":
|
|
79
79
|
"""Add additional properties to updated class instance."""
|
|
80
80
|
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
81
81
|
|
|
82
82
|
new_class = attr.make_class(
|
|
83
83
|
self.__class__.__name__,
|
|
84
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
84
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
85
85
|
bases=(self.__class__,),
|
|
86
86
|
)
|
|
87
87
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -102,7 +102,7 @@ class InputField(RedactMixin):
|
|
|
102
102
|
|
|
103
103
|
@attr.define
|
|
104
104
|
class Transformation(RedactMixin):
|
|
105
|
-
type: str
|
|
105
|
+
type: str # noqa: A003
|
|
106
106
|
"""The type of the transformation. Allowed values are: DIRECT, INDIRECT"""
|
|
107
107
|
|
|
108
108
|
subtype: str | None = attr.field(default=None)
|
|
@@ -116,13 +116,13 @@ class Transformation(RedactMixin):
|
|
|
116
116
|
|
|
117
117
|
_skip_redact: ClassVar[list[str]] = ["type", "subtype", "masking"]
|
|
118
118
|
|
|
119
|
-
def with_additional_properties(self, **kwargs: dict[str, Any]) -> Transformation:
|
|
119
|
+
def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Transformation":
|
|
120
120
|
"""Add additional properties to updated class instance."""
|
|
121
121
|
current_attrs = [a.name for a in attr.fields(self.__class__)]
|
|
122
122
|
|
|
123
123
|
new_class = attr.make_class(
|
|
124
124
|
self.__class__.__name__,
|
|
125
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
125
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
126
126
|
bases=(self.__class__,),
|
|
127
127
|
)
|
|
128
128
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -16,14 +16,14 @@ class ColumnMetrics(RedactMixin):
|
|
|
16
16
|
distinctCount: int | None = attr.field(default=None) # noqa: N815
|
|
17
17
|
"""The number of distinct values in this column for the rows evaluated"""
|
|
18
18
|
|
|
19
|
-
sum: float | None = attr.field(default=None)
|
|
19
|
+
sum: float | None = attr.field(default=None) # noqa: A003
|
|
20
20
|
"""The total sum of values in this column for the rows evaluated"""
|
|
21
21
|
|
|
22
22
|
count: float | None = attr.field(default=None)
|
|
23
23
|
"""The number of values in this column"""
|
|
24
24
|
|
|
25
|
-
min: float | None = attr.field(default=None)
|
|
26
|
-
max: float | None = attr.field(default=None)
|
|
25
|
+
min: float | None = attr.field(default=None) # noqa: A003
|
|
26
|
+
max: float | None = attr.field(default=None) # noqa: A003
|
|
27
27
|
quantiles: dict[str, float] | None = attr.field(factory=dict)
|
|
28
28
|
"""The property key is the quantile. Examples: 0.1 0.25 0.5 0.75 1"""
|
|
29
29
|
|
|
@@ -36,7 +36,7 @@ class DataQualityMetricsInputDatasetFacet(InputDatasetFacet):
|
|
|
36
36
|
rowCount: int | None = attr.field(default=None) # noqa: N815
|
|
37
37
|
"""The number of rows evaluated"""
|
|
38
38
|
|
|
39
|
-
bytes: int | None = attr.field(default=None)
|
|
39
|
+
bytes: int | None = attr.field(default=None) # noqa: A003
|
|
40
40
|
"""The size in bytes"""
|
|
41
41
|
|
|
42
42
|
fileCount: int | None = attr.field(default=None) # noqa: N815
|
|
@@ -12,6 +12,9 @@ class DocumentationDatasetFacet(DatasetFacet):
|
|
|
12
12
|
description: str
|
|
13
13
|
"""The description of the dataset."""
|
|
14
14
|
|
|
15
|
+
contentType: str | None = attr.field(default=None) # noqa: N815
|
|
16
|
+
"""MIME type of the description field content."""
|
|
17
|
+
|
|
15
18
|
@staticmethod
|
|
16
19
|
def _get_schema() -> str:
|
|
17
|
-
return "https://openlineage.io/spec/facets/1-0
|
|
20
|
+
return "https://openlineage.io/spec/facets/1-1-0/DocumentationDatasetFacet.json#/$defs/DocumentationDatasetFacet"
|
|
@@ -12,8 +12,11 @@ class DocumentationJobFacet(JobFacet):
|
|
|
12
12
|
description: str
|
|
13
13
|
"""The description of the job."""
|
|
14
14
|
|
|
15
|
+
contentType: str | None = attr.field(default=None) # noqa: N815
|
|
16
|
+
"""MIME type of the description field content."""
|
|
17
|
+
|
|
15
18
|
@staticmethod
|
|
16
19
|
def _get_schema() -> str:
|
|
17
20
|
return (
|
|
18
|
-
"https://openlineage.io/spec/facets/1-0
|
|
21
|
+
"https://openlineage.io/spec/facets/1-1-0/DocumentationJobFacet.json#/$defs/DocumentationJobFacet"
|
|
19
22
|
)
|
|
@@ -15,7 +15,7 @@ class Owner(RedactMixin):
|
|
|
15
15
|
the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example
|
|
16
16
|
application:foo, user:jdoe, team:data
|
|
17
17
|
"""
|
|
18
|
-
type: str | None = attr.field(default=None)
|
|
18
|
+
type: str | None = attr.field(default=None) # noqa: A003
|
|
19
19
|
"""The type of ownership (optional)"""
|
|
20
20
|
|
|
21
21
|
|
|
@@ -15,7 +15,7 @@ class Owner(RedactMixin):
|
|
|
15
15
|
the identifier of the owner of the Job. It is recommended to define this as a URN. For example
|
|
16
16
|
application:foo, user:jdoe, team:data
|
|
17
17
|
"""
|
|
18
|
-
type: str | None = attr.field(default=None)
|
|
18
|
+
type: str | None = attr.field(default=None) # noqa: A003
|
|
19
19
|
"""The type of ownership (optional)"""
|
|
20
20
|
|
|
21
21
|
|
{openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/parent_run.py
RENAMED
|
@@ -36,7 +36,7 @@ class ParentRunFacet(RunFacet):
|
|
|
36
36
|
return "https://openlineage.io/spec/facets/1-1-0/ParentRunFacet.json#/$defs/ParentRunFacet"
|
|
37
37
|
|
|
38
38
|
@classmethod
|
|
39
|
-
def create(cls, runId: str, namespace: str, name: str) -> ParentRunFacet: # noqa: N803
|
|
39
|
+
def create(cls, runId: str, namespace: str, name: str) -> "ParentRunFacet": # noqa: N803
|
|
40
40
|
import warnings
|
|
41
41
|
|
|
42
42
|
warnings.warn(
|
|
@@ -23,7 +23,7 @@ class SchemaDatasetFacetFields(RedactMixin):
|
|
|
23
23
|
name: str
|
|
24
24
|
"""The name of the field."""
|
|
25
25
|
|
|
26
|
-
type: str | None = attr.field(default=None)
|
|
26
|
+
type: str | None = attr.field(default=None) # noqa: A003
|
|
27
27
|
"""The type of the field."""
|
|
28
28
|
|
|
29
29
|
description: str | None = attr.field(default=None)
|
{openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/generated/sql_job.py
RENAMED
|
@@ -10,7 +10,8 @@ from openlineage.client.generated.base import JobFacet
|
|
|
10
10
|
@attr.define
|
|
11
11
|
class SQLJobFacet(JobFacet):
|
|
12
12
|
query: str
|
|
13
|
+
dialect: str | None = attr.field(default=None)
|
|
13
14
|
|
|
14
15
|
@staticmethod
|
|
15
16
|
def _get_schema() -> str:
|
|
16
|
-
return "https://openlineage.io/spec/facets/1-0
|
|
17
|
+
return "https://openlineage.io/spec/facets/1-1-0/SQLJobFacet.json#/$defs/SQLJobFacet"
|
|
@@ -6,7 +6,7 @@ import pathlib
|
|
|
6
6
|
import tempfile
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
|
-
import
|
|
9
|
+
import httpx
|
|
10
10
|
from openlineage.client.generator.base import (
|
|
11
11
|
camel_to_snake,
|
|
12
12
|
format_and_save_output,
|
|
@@ -19,7 +19,7 @@ BASE_SPEC_URL = "https://openlineage.io/spec/2-0-2/OpenLineage.json"
|
|
|
19
19
|
|
|
20
20
|
def get_base_spec() -> pathlib.Path:
|
|
21
21
|
"""Get the base spec from the OpenLineage repository."""
|
|
22
|
-
response =
|
|
22
|
+
response = httpx.get(BASE_SPEC_URL, timeout=10)
|
|
23
23
|
response.raise_for_status() # Raise an HTTPError for bad responses
|
|
24
24
|
tmpfile = pathlib.Path(tempfile.mkdtemp()) / "OpenLineage.json"
|
|
25
25
|
tmpfile.write_text(response.text)
|
|
@@ -157,7 +157,7 @@ class {{ class_name }}:
|
|
|
157
157
|
|
|
158
158
|
new_class = attr.make_class(
|
|
159
159
|
self.__class__.__name__,
|
|
160
|
-
{k: attr.field() for k in kwargs if k not in current_attrs},
|
|
160
|
+
{k: attr.field(default=None) for k in kwargs if k not in current_attrs},
|
|
161
161
|
bases=(self.__class__,),
|
|
162
162
|
)
|
|
163
163
|
new_class.__module__ = self.__class__.__module__
|
|
@@ -7,9 +7,13 @@ from typing import Any, ClassVar, Optional
|
|
|
7
7
|
|
|
8
8
|
import attr
|
|
9
9
|
from dateutil import parser
|
|
10
|
-
from openlineage.client.facet import NominalTimeRunFacet, ParentRunFacet
|
|
11
10
|
from openlineage.client.utils import RedactMixin
|
|
12
11
|
|
|
12
|
+
with warnings.catch_warnings():
|
|
13
|
+
warnings.simplefilter("ignore", DeprecationWarning)
|
|
14
|
+
from openlineage.client.facet import NominalTimeRunFacet, ParentRunFacet
|
|
15
|
+
|
|
16
|
+
|
|
13
17
|
warnings.warn(
|
|
14
18
|
"This module is deprecated. Please use `openlineage.client.event_v2`.", DeprecationWarning, stacklevel=2
|
|
15
19
|
)
|
|
@@ -47,7 +47,7 @@ class Serde:
|
|
|
47
47
|
def to_dict(cls, obj: Any) -> dict[Any, Any]:
|
|
48
48
|
if not isinstance(obj, dict):
|
|
49
49
|
obj = attr.asdict(obj)
|
|
50
|
-
return cast(dict[Any, Any], cls.remove_nulls_and_enums(obj))
|
|
50
|
+
return cast("dict[Any, Any]", cls.remove_nulls_and_enums(obj))
|
|
51
51
|
|
|
52
52
|
@classmethod
|
|
53
53
|
def to_json(cls, obj: Any) -> str:
|
|
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
|
|
|
11
11
|
from openlineage.client.generated.tags_run import TagsRunFacetFields
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
@attr.
|
|
14
|
+
@attr.define
|
|
15
15
|
class TagsConfig:
|
|
16
|
-
job: list[TagsJobFacetFields] = attr.
|
|
17
|
-
run: list[TagsRunFacetFields] = attr.
|
|
16
|
+
job: list[TagsJobFacetFields] = attr.field(factory=list)
|
|
17
|
+
run: list[TagsRunFacetFields] = attr.field(factory=list)
|
{openlineage_python-1.33.0 → openlineage_python-1.35.0}/openlineage/client/transport/__init__.py
RENAMED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, AmazonDataZoneTransport
|
|
6
|
+
from openlineage.client.transport.async_http import AsyncHttpConfig, AsyncHttpTransport
|
|
6
7
|
from openlineage.client.transport.composite import CompositeTransport
|
|
7
8
|
from openlineage.client.transport.console import ConsoleTransport
|
|
8
9
|
from openlineage.client.transport.factory import DefaultTransportFactory
|
|
@@ -20,6 +21,7 @@ from openlineage.client.transport.transport import Config, Transport, TransportF
|
|
|
20
21
|
_factory = DefaultTransportFactory()
|
|
21
22
|
_factory.register_transport(CompositeTransport.kind, CompositeTransport)
|
|
22
23
|
_factory.register_transport(HttpTransport.kind, HttpTransport)
|
|
24
|
+
_factory.register_transport(AsyncHttpTransport.kind, AsyncHttpTransport)
|
|
23
25
|
_factory.register_transport(KafkaTransport.kind, KafkaTransport)
|
|
24
26
|
_factory.register_transport(MSKIAMTransport.kind, MSKIAMTransport)
|
|
25
27
|
_factory.register_transport(ConsoleTransport.kind, ConsoleTransport)
|
|
@@ -41,14 +43,12 @@ def register_transport(clazz: type[Transport]) -> type[Transport]:
|
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
__all__ = [
|
|
44
|
-
"register_transport",
|
|
45
|
-
"get_default_factory",
|
|
46
|
-
"Config",
|
|
47
|
-
"Transport",
|
|
48
|
-
"TransportFactory",
|
|
49
46
|
"AmazonDataZoneConfig",
|
|
50
47
|
"AmazonDataZoneTransport",
|
|
48
|
+
"AsyncHttpConfig",
|
|
49
|
+
"AsyncHttpTransport",
|
|
51
50
|
"CompositeTransport",
|
|
51
|
+
"Config",
|
|
52
52
|
"ConsoleTransport",
|
|
53
53
|
"FileTransport",
|
|
54
54
|
"HttpConfig",
|
|
@@ -60,4 +60,8 @@ __all__ = [
|
|
|
60
60
|
"NoopTransport",
|
|
61
61
|
"TransformConfig",
|
|
62
62
|
"TransformTransport",
|
|
63
|
+
"Transport",
|
|
64
|
+
"TransportFactory",
|
|
65
|
+
"get_default_factory",
|
|
66
|
+
"register_transport",
|
|
63
67
|
]
|
|
@@ -57,10 +57,16 @@ class AmazonDataZoneTransport(Transport):
|
|
|
57
57
|
log.info(
|
|
58
58
|
"Successfully posted a LineageEvent: %s in Domain: %s", response["id"], response["domainId"]
|
|
59
59
|
)
|
|
60
|
-
except Exception as error:
|
|
60
|
+
except Exception as error:
|
|
61
61
|
msg = f"Failed to send lineage event to DataZone Domain {self.config.domain_id}: {event}"
|
|
62
62
|
raise RuntimeError(msg) from error
|
|
63
63
|
|
|
64
|
+
def close(self, timeout: float = -1) -> bool:
|
|
65
|
+
if timeout >= 0:
|
|
66
|
+
log.warning("AmazonDataZoneTransport does not support timeout")
|
|
67
|
+
self.datazone.close() # type: ignore[attr-defined]
|
|
68
|
+
return True
|
|
69
|
+
|
|
64
70
|
def _setup_datazone(self, endpoint_url: str | None = None) -> None:
|
|
65
71
|
try:
|
|
66
72
|
import boto3 # type: ignore[import-untyped]
|