openlineage-python 1.34.0__tar.gz → 1.35.0__tar.gz

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.34.0
+Version: 1.35.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage
@@ -14,10 +14,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.9
 Requires-Dist: attrs>=20.0
+Requires-Dist: build>=1.2.2
+Requires-Dist: httpx>=0.27.0
 Requires-Dist: packaging>=21.0
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: pyyaml>=5.4
-Requires-Dist: requests>=2.32.3
+Requires-Dist: requests>=2.32.4
 Provides-Extra: datazone
 Requires-Dist: boto3>=1.34.134; extra == 'datazone'
 Provides-Extra: docs
@@ -36,10 +38,12 @@ Requires-Dist: aws-msk-iam-sasl-signer-python>=1.0.1; extra == 'msk-iam'
 Requires-Dist: confluent-kafka>=2.1.1; extra == 'msk-iam'
 Provides-Extra: test
 Requires-Dist: covdefaults>=2.3; extra == 'test'
+Requires-Dist: fastapi>=0.100.0; extra == 'test'
 Requires-Dist: pytest-cov>=4.1; extra == 'test'
 Requires-Dist: pytest-mock>=3.10; extra == 'test'
 Requires-Dist: pytest>=7.3.1; extra == 'test'
 Requires-Dist: pyyaml>=5.4; extra == 'test'
+Requires-Dist: uvicorn>=0.20.0; extra == 'test'
 Description-Content-Type: text/markdown
 
 # OpenLineage Python Client

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/__init__.py

@@ -8,7 +8,7 @@ from openlineage.client.client import OpenLineageClient, OpenLineageClientOption
 
 with warnings.catch_warnings():
     warnings.simplefilter("ignore", DeprecationWarning)
-from openlineage.client.facet import set_producer as set_producer_v1
+    from openlineage.client.facet import set_producer as set_producer_v1
 from openlineage.client.facet_v2 import set_producer as set_producer_v2
 
 

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/client.py

@@ -28,7 +28,7 @@ from openlineage.client.transport import (
     TransportFactory,
     get_default_factory,
 )
-from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
+from openlineage.client.transport.http import HttpConfig, HttpTransport
 from openlineage.client.transport.noop import NoopConfig, NoopTransport
 from openlineage.client.utils import deep_merge_dicts
 
@@ -183,6 +183,15 @@ class OpenLineageClient:
         self.transport.emit(event)
         log.debug("OpenLineage event successfully emitted.")
 
+    def close(self, timeout: float = -1.0) -> bool:
+        """
+        Closes down the transport until all events are processed or timeout is reached.
+        Params:
+          timeout: Timeout in seconds. `-1` means to block until last event is processed, 0 means no timeout.
+
+        """
+        return self.transport.close(timeout)
+
     @property
     def config(self) -> OpenLineageConfig:
         """
@@ -238,7 +247,7 @@ class OpenLineageClient:
 
         # 2. Check if transport is provided explicitly
         if kwargs.get("transport"):
-            return cast(Transport, kwargs["transport"])
+            return cast("Transport", kwargs["transport"])
 
         # 3. Check if transport configuration is provided in YAML config file
         if self.config.transport and self.config.transport.get("type"):
@@ -292,28 +301,26 @@ class OpenLineageClient:
                     return path
                 if path and verbose:
                     log.debug("OpenLineage config file is missing or not readable: `%s`.", path)
-            except Exception:  # noqa: BLE001
+            except Exception:
                 # We can get different errors depending on system
                 if verbose:
                     log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
         return None
 
-    @staticmethod
-    def _http_transport_from_env_variables() -> HttpTransport:
-        config = HttpConfig(
-            url=os.environ["OPENLINEAGE_URL"],
-            auth=create_token_provider(
-                {
-                    "type": "api_key",
-                    "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
-                },
-            ),
-        )
-        endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
-        if endpoint is not None:
-            config.endpoint = endpoint
-
-        return HttpTransport(config)
+    def _http_transport_from_env_variables(self) -> HttpTransport:
+        """
+        Create HTTP transport from legacy environment variables
+        """
+        # Start with basic config from legacy environment variables
+        config_dict = {
+            "url": os.environ["OPENLINEAGE_URL"],
+            "auth": {
+                "type": "api_key",
+                "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
+            },
+            "endpoint": os.environ.get("OPENLINEAGE_ENDPOINT", "api/v1/lineage"),
+        }
+        return HttpTransport(HttpConfig.from_dict(config_dict))
 
     @staticmethod
     def _http_transport_from_url(
@@ -343,9 +350,13 @@ class OpenLineageClient:
                     default_transport_name,
                 )
                 return
+
+            api_key = os.environ.get("OPENLINEAGE_API_KEY")
+            endpoint = os.environ.get("OPENLINEAGE_ENDPOINT")
+
             os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__TYPE"] = "http"
             os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__URL"] = url
-            if api_key := os.environ.get("OPENLINEAGE_API_KEY"):
+            if api_key:
                 os.environ[
                     f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__AUTH"
                 ] = json.dumps(
@@ -354,11 +365,25 @@ class OpenLineageClient:
                         "apiKey": api_key,
                     }
                 )
-            if endpoint := os.environ.get("OPENLINEAGE_ENDPOINT"):
+            if endpoint:
                 os.environ[
                     f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__ENDPOINT"
                 ] = endpoint
 
+            if os.environ.get("OPENLINEAGE__TRANSPORT__TYPE") == "async_http":
+                # Special case - for seamless switch to async transport
+                if not os.getenv("OPENLINEAGE__TRANSPORT__URL"):
+                    os.environ["OPENLINEAGE__TRANSPORT__URL"] = url
+                if api_key and not os.getenv("OPENLINEAGE__TRANSPORT__AUTH"):
+                    os.environ["OPENLINEAGE__TRANSPORT__AUTH"] = json.dumps(
+                        {
+                            "type": "api_key",
+                            "apiKey": api_key,
+                        }
+                    )
+                if endpoint and not os.getenv("OPENLINEAGE__TRANSPORT__ENDPOINT"):
+                    os.environ["OPENLINEAGE__TRANSPORT__ENDPOINT"] = endpoint
+
     @classmethod
     def _load_config_from_env_variables(cls) -> dict[str, Any] | None:
         config: dict[str, Any] = {}

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.34.0"
+__version__ = "1.35.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/filter.py

@@ -5,10 +5,14 @@ from __future__ import annotations
 import logging
 import re
 import typing
+import warnings
 
 import attr
 from openlineage.client.event_v2 import RunEvent as RunEvent_v2
-from openlineage.client.run import RunEvent
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from openlineage.client.run import RunEvent
 
 log = logging.getLogger(__name__)
 RunEventType = typing.Union[RunEvent, RunEvent_v2]

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/base.py

@@ -23,7 +23,7 @@ class BaseEvent(RedactMixin):
     eventTime: str = attr.field()  # noqa: N815
     """the time the event occurred at"""
 
-    producer: str = attr.field(default="", kw_only=True)
+    producer: str = attr.field(default="", kw_only=True)  # noqa: N815
     schemaURL: str = attr.field(  # noqa: N815
         default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent", init=False
     )
@@ -70,7 +70,7 @@ class BaseEvent(RedactMixin):
 class BaseFacet(RedactMixin):
     """all fields of the base facet are prefixed with _ to avoid name conflicts in facets"""
 
-    _producer: str = attr.field(default="", kw_only=True)
+    _producer: str = attr.field(default="", kw_only=True)  # noqa: N815
     _schemaURL: str = attr.field(  # noqa: N815
         default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet", init=False
     )
@@ -86,7 +86,7 @@ class BaseFacet(RedactMixin):
     def skip_redact(self) -> list[str]:
         return self._base_skip_redact + self._additional_skip_redact
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> BaseFacet:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "BaseFacet":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/catalog_dataset.py

@@ -12,7 +12,7 @@ class CatalogDatasetFacet(DatasetFacet):
     framework: str
     """The storage framework for which the catalog is configured"""
 
-    type: str
+    type: str  # noqa: A003
     """Type of the catalog."""
 
     name: str

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/column_lineage_dataset.py

@@ -38,7 +38,7 @@ class Fields(RedactMixin):
     original data available (like a hash of PII for example)
     """
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> Fields:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Fields":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 
@@ -75,7 +75,7 @@ class InputField(RedactMixin):
     transformations: list[Transformation] | None = attr.field(factory=list)
     _skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> InputField:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "InputField":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 
@@ -102,7 +102,7 @@ class InputField(RedactMixin):
 
 @attr.define
 class Transformation(RedactMixin):
-    type: str
+    type: str  # noqa: A003
     """The type of the transformation. Allowed values are: DIRECT, INDIRECT"""
 
     subtype: str | None = attr.field(default=None)
@@ -116,7 +116,7 @@ class Transformation(RedactMixin):
 
     _skip_redact: ClassVar[list[str]] = ["type", "subtype", "masking"]
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> Transformation:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Transformation":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py

@@ -16,14 +16,14 @@ class ColumnMetrics(RedactMixin):
     distinctCount: int | None = attr.field(default=None)  # noqa: N815
     """The number of distinct values in this column for the rows evaluated"""
 
-    sum: float | None = attr.field(default=None)
+    sum: float | None = attr.field(default=None)  # noqa: A003
     """The total sum of values in this column for the rows evaluated"""
 
     count: float | None = attr.field(default=None)
     """The number of values in this column"""
 
-    min: float | None = attr.field(default=None)
-    max: float | None = attr.field(default=None)
+    min: float | None = attr.field(default=None)  # noqa: A003
+    max: float | None = attr.field(default=None)  # noqa: A003
     quantiles: dict[str, float] | None = attr.field(factory=dict)
     """The property key is the quantile. Examples: 0.1 0.25 0.5 0.75 1"""
 
@@ -36,7 +36,7 @@ class DataQualityMetricsInputDatasetFacet(InputDatasetFacet):
     rowCount: int | None = attr.field(default=None)  # noqa: N815
     """The number of rows evaluated"""
 
-    bytes: int | None = attr.field(default=None)
+    bytes: int | None = attr.field(default=None)  # noqa: A003
     """The size in bytes"""
 
     fileCount: int | None = attr.field(default=None)  # noqa: N815

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/ownership_dataset.py

@@ -15,7 +15,7 @@ class Owner(RedactMixin):
     the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example
     application:foo, user:jdoe, team:data
     """
-    type: str | None = attr.field(default=None)
+    type: str | None = attr.field(default=None)  # noqa: A003
     """The type of ownership (optional)"""
 
 

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/ownership_job.py

@@ -15,7 +15,7 @@ class Owner(RedactMixin):
     the identifier of the owner of the Job. It is recommended to define this as a URN. For example
     application:foo, user:jdoe, team:data
     """
-    type: str | None = attr.field(default=None)
+    type: str | None = attr.field(default=None)  # noqa: A003
     """The type of ownership (optional)"""
 
 

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/parent_run.py

@@ -36,7 +36,7 @@ class ParentRunFacet(RunFacet):
         return "https://openlineage.io/spec/facets/1-1-0/ParentRunFacet.json#/$defs/ParentRunFacet"
 
     @classmethod
-    def create(cls, runId: str, namespace: str, name: str) -> ParentRunFacet:  # noqa: N803
+    def create(cls, runId: str, namespace: str, name: str) -> "ParentRunFacet":  # noqa: N803
         import warnings
 
         warnings.warn(

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/schema_dataset.py

@@ -23,7 +23,7 @@ class SchemaDatasetFacetFields(RedactMixin):
     name: str
     """The name of the field."""
 
-    type: str | None = attr.field(default=None)
+    type: str | None = attr.field(default=None)  # noqa: A003
     """The type of the field."""
 
     description: str | None = attr.field(default=None)

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/source_code_location_job.py

@@ -11,7 +11,7 @@ from openlineage.client.generated.base import JobFacet
 
 @attr.define
 class SourceCodeLocationJobFacet(JobFacet):
-    type: str
+    type: str  # noqa: A003
     """the source control system"""
 
     url: str = attr.field()

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/sql_job.py

@@ -10,7 +10,8 @@ from openlineage.client.generated.base import JobFacet
 @attr.define
 class SQLJobFacet(JobFacet):
     query: str
+    dialect: str | None = attr.field(default=None)
 
     @staticmethod
     def _get_schema() -> str:
-        return "https://openlineage.io/spec/facets/1-0-1/SQLJobFacet.json#/$defs/SQLJobFacet"
+        return "https://openlineage.io/spec/facets/1-1-0/SQLJobFacet.json#/$defs/SQLJobFacet"

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generated/symlinks_dataset.py

@@ -16,7 +16,7 @@ class Identifier(RedactMixin):
     name: str
     """The dataset name"""
 
-    type: str
+    type: str  # noqa: A003
     """Identifier type"""
 
 

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/generator/cli.py

@@ -6,7 +6,7 @@ import pathlib
 import tempfile
 
 import click
-import requests
+import httpx
 from openlineage.client.generator.base import (
     camel_to_snake,
     format_and_save_output,
@@ -19,7 +19,7 @@ BASE_SPEC_URL = "https://openlineage.io/spec/2-0-2/OpenLineage.json"
 
 def get_base_spec() -> pathlib.Path:
     """Get the base spec from the OpenLineage repository."""
-    response = requests.get(BASE_SPEC_URL, timeout=10)
+    response = httpx.get(BASE_SPEC_URL, timeout=10)
     response.raise_for_status()  # Raise an HTTPError for bad responses
     tmpfile = pathlib.Path(tempfile.mkdtemp()) / "OpenLineage.json"
     tmpfile.write_text(response.text)

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/run.py

@@ -7,9 +7,13 @@ from typing import Any, ClassVar, Optional
 
 import attr
 from dateutil import parser
-from openlineage.client.facet import NominalTimeRunFacet, ParentRunFacet
 from openlineage.client.utils import RedactMixin
 
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from openlineage.client.facet import NominalTimeRunFacet, ParentRunFacet
+
+
 warnings.warn(
     "This module is deprecated. Please use `openlineage.client.event_v2`.", DeprecationWarning, stacklevel=2
 )

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/serde.py

@@ -47,7 +47,7 @@ class Serde:
     def to_dict(cls, obj: Any) -> dict[Any, Any]:
         if not isinstance(obj, dict):
             obj = attr.asdict(obj)
-        return cast(dict[Any, Any], cls.remove_nulls_and_enums(obj))
+        return cast("dict[Any, Any]", cls.remove_nulls_and_enums(obj))
 
     @classmethod
     def to_json(cls, obj: Any) -> str:

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/transport/__init__.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, AmazonDataZoneTransport
+from openlineage.client.transport.async_http import AsyncHttpConfig, AsyncHttpTransport
 from openlineage.client.transport.composite import CompositeTransport
 from openlineage.client.transport.console import ConsoleTransport
 from openlineage.client.transport.factory import DefaultTransportFactory
@@ -20,6 +21,7 @@ from openlineage.client.transport.transport import Config, Transport, TransportF
 _factory = DefaultTransportFactory()
 _factory.register_transport(CompositeTransport.kind, CompositeTransport)
 _factory.register_transport(HttpTransport.kind, HttpTransport)
+_factory.register_transport(AsyncHttpTransport.kind, AsyncHttpTransport)
 _factory.register_transport(KafkaTransport.kind, KafkaTransport)
 _factory.register_transport(MSKIAMTransport.kind, MSKIAMTransport)
 _factory.register_transport(ConsoleTransport.kind, ConsoleTransport)
@@ -41,14 +43,12 @@ def register_transport(clazz: type[Transport]) -> type[Transport]:
 
 
 __all__ = [
-    "register_transport",
-    "get_default_factory",
-    "Config",
-    "Transport",
-    "TransportFactory",
     "AmazonDataZoneConfig",
     "AmazonDataZoneTransport",
+    "AsyncHttpConfig",
+    "AsyncHttpTransport",
     "CompositeTransport",
+    "Config",
     "ConsoleTransport",
     "FileTransport",
     "HttpConfig",
@@ -60,4 +60,8 @@ __all__ = [
     "NoopTransport",
     "TransformConfig",
     "TransformTransport",
+    "Transport",
+    "TransportFactory",
+    "get_default_factory",
+    "register_transport",
 ]

{openlineage_python-1.34.0 → openlineage_python-1.35.0}/openlineage/client/transport/amazon_datazone.py

@@ -57,10 +57,16 @@ class AmazonDataZoneTransport(Transport):
             log.info(
                 "Successfully posted a LineageEvent: %s in Domain: %s", response["id"], response["domainId"]
             )
-        except Exception as error:  # noqa: BLE001
+        except Exception as error:
             msg = f"Failed to send lineage event to DataZone Domain {self.config.domain_id}: {event}"
             raise RuntimeError(msg) from error
 
+    def close(self, timeout: float = -1) -> bool:
+        if timeout >= 0:
+            log.warning("AmazonDataZoneTransport does not support timeout")
+        self.datazone.close()  # type: ignore[attr-defined]
+        return True
+
     def _setup_datazone(self, endpoint_url: str | None = None) -> None:
         try:
             import boto3  # type: ignore[import-untyped]