openlineage-python 1.34.0__tar.gz → 1.36.0__tar.gz

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.34.0
+Version: 1.36.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage
@@ -14,10 +14,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.9
 Requires-Dist: attrs>=20.0
+Requires-Dist: build>=1.2.2
+Requires-Dist: httpx>=0.27.0
 Requires-Dist: packaging>=21.0
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: pyyaml>=5.4
-Requires-Dist: requests>=2.32.3
+Requires-Dist: requests>=2.32.4
 Provides-Extra: datazone
 Requires-Dist: boto3>=1.34.134; extra == 'datazone'
 Provides-Extra: docs
@@ -36,10 +38,12 @@ Requires-Dist: aws-msk-iam-sasl-signer-python>=1.0.1; extra == 'msk-iam'
 Requires-Dist: confluent-kafka>=2.1.1; extra == 'msk-iam'
 Provides-Extra: test
 Requires-Dist: covdefaults>=2.3; extra == 'test'
+Requires-Dist: fastapi>=0.100.0; extra == 'test'
 Requires-Dist: pytest-cov>=4.1; extra == 'test'
 Requires-Dist: pytest-mock>=3.10; extra == 'test'
 Requires-Dist: pytest>=7.3.1; extra == 'test'
 Requires-Dist: pyyaml>=5.4; extra == 'test'
+Requires-Dist: uvicorn>=0.20.0; extra == 'test'
 Description-Content-Type: text/markdown
 
 # OpenLineage Python Client

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/__init__.py

@@ -8,7 +8,7 @@ from openlineage.client.client import OpenLineageClient, OpenLineageClientOption
 
 with warnings.catch_warnings():
     warnings.simplefilter("ignore", DeprecationWarning)
-from openlineage.client.facet import set_producer as set_producer_v1
+    from openlineage.client.facet import set_producer as set_producer_v1
 from openlineage.client.facet_v2 import set_producer as set_producer_v2
 
 

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/client.py

@@ -28,7 +28,7 @@ from openlineage.client.transport import (
     TransportFactory,
     get_default_factory,
 )
-from openlineage.client.transport.http import HttpConfig, HttpTransport, create_token_provider
+from openlineage.client.transport.http import HttpConfig, HttpTransport
 from openlineage.client.transport.noop import NoopConfig, NoopTransport
 from openlineage.client.utils import deep_merge_dicts
 
@@ -183,6 +183,15 @@ class OpenLineageClient:
         self.transport.emit(event)
         log.debug("OpenLineage event successfully emitted.")
 
+    def close(self, timeout: float = -1.0) -> bool:
+        """
+        Closes down the transport until all events are processed or timeout is reached.
+        Params:
+          timeout: Timeout in seconds. `-1` means to block until last event is processed, 0 means no timeout.
+
+        """
+        return self.transport.close(timeout)
+
     @property
     def config(self) -> OpenLineageConfig:
         """
@@ -223,9 +232,9 @@ class OpenLineageClient:
         This method determines the appropriate transport by executing a sequence of checks:
         1. Verifies if OpenLineage is disabled through environment variable.
         2. Looks for a transport object provided in the arguments.
-        3. Attempts to configure the transport from a YAML config file.
+        3. Attempts to configure the transport from user config, config file or env vars.
         4. Tries to initialize HTTP transport with an url argument (deprecated).
-        5. Tries to set up HTTP transport using environment variables.
+        5. Tries to set up HTTP transport using simple environment variables.
         6. If no configuration is found, defaults to a console transport and logs a warning message.
 
         Returns:
@@ -236,11 +245,11 @@ class OpenLineageClient:
             log.info("OpenLineage is disabled. No events will be emitted.")
             return NoopTransport(NoopConfig())
 
-        # 2. Check if transport is provided explicitly
+        # 2. Check if transport is provided explicitly as argument
         if kwargs.get("transport"):
-            return cast(Transport, kwargs["transport"])
+            return cast("Transport", kwargs["transport"])
 
-        # 3. Check if transport configuration is provided in YAML config file
+        # 3. Check if transport configuration is provided as config (explicit, file or env vars)
         if self.config.transport and self.config.transport.get("type"):
             factory = kwargs.get("factory") or get_default_factory()
             return factory.create(self.config.transport)
@@ -251,7 +260,7 @@ class OpenLineageClient:
                 url=kwargs["url"], options=kwargs.get("options"), session=kwargs.get("session")
             )
 
-        # 5. Check HTTP transport initialization with env variables
+        # 5. Check HTTP transport initialization with simple env variables
         if os.environ.get("OPENLINEAGE_URL"):
             return self._http_transport_from_env_variables()
 
@@ -292,7 +301,7 @@ class OpenLineageClient:
                     return path
                 if path and verbose:
                     log.debug("OpenLineage config file is missing or not readable: `%s`.", path)
-            except Exception:  # noqa: BLE001
+            except Exception:
                 # We can get different errors depending on system
                 if verbose:
                     log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
@@ -300,20 +309,19 @@ class OpenLineageClient:
 
     @staticmethod
     def _http_transport_from_env_variables() -> HttpTransport:
-        config = HttpConfig(
-            url=os.environ["OPENLINEAGE_URL"],
-            auth=create_token_provider(
-                {
-                    "type": "api_key",
-                    "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
-                },
-            ),
-        )
-        endpoint = os.environ.get("OPENLINEAGE_ENDPOINT", None)
-        if endpoint is not None:
-            config.endpoint = endpoint
-
-        return HttpTransport(config)
+        """
+        Create HTTP transport from legacy environment variables
+        """
+        # Start with basic config from legacy environment variables
+        config_dict = {
+            "url": os.environ["OPENLINEAGE_URL"],
+            "auth": {
+                "type": "api_key",
+                "apiKey": os.environ.get("OPENLINEAGE_API_KEY", ""),
+            },
+            "endpoint": os.environ.get("OPENLINEAGE_ENDPOINT", "api/v1/lineage"),
+        }
+        return HttpTransport(HttpConfig.from_dict(config_dict))
 
     @staticmethod
     def _http_transport_from_url(
@@ -338,14 +346,18 @@ class OpenLineageClient:
                 k.startswith(f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}")
                 for k in os.environ
             ):
-                log.warning(
+                log.info(
                     "%s already found in environment variables, skipping aliasing OPENLINEAGE_URL",
                     default_transport_name,
                 )
                 return
+
+            api_key = os.environ.get("OPENLINEAGE_API_KEY")
+            endpoint = os.environ.get("OPENLINEAGE_ENDPOINT")
+
             os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__TYPE"] = "http"
             os.environ[f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__URL"] = url
-            if api_key := os.environ.get("OPENLINEAGE_API_KEY"):
+            if api_key:
                 os.environ[
                     f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__AUTH"
                 ] = json.dumps(
@@ -354,16 +366,30 @@ class OpenLineageClient:
                         "apiKey": api_key,
                     }
                 )
-            if endpoint := os.environ.get("OPENLINEAGE_ENDPOINT"):
+            if endpoint:
                 os.environ[
                     f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}__ENDPOINT"
                 ] = endpoint
 
+            if os.environ.get("OPENLINEAGE__TRANSPORT__TYPE") == "async_http":
+                # Special case - for seamless switch to async transport
+                if not os.getenv("OPENLINEAGE__TRANSPORT__URL"):
+                    os.environ["OPENLINEAGE__TRANSPORT__URL"] = url
+                if api_key and not os.getenv("OPENLINEAGE__TRANSPORT__AUTH"):
+                    os.environ["OPENLINEAGE__TRANSPORT__AUTH"] = json.dumps(
+                        {
+                            "type": "api_key",
+                            "apiKey": api_key,
+                        }
+                    )
+                if endpoint and not os.getenv("OPENLINEAGE__TRANSPORT__ENDPOINT"):
+                    os.environ["OPENLINEAGE__TRANSPORT__ENDPOINT"] = endpoint
+
     @classmethod
     def _load_config_from_env_variables(cls) -> dict[str, Any] | None:
         config: dict[str, Any] = {}
 
-        # get os.environ.items only starting with OPENLINEAGE_ prefix and reverse sort
+        # get os.environ.items only starting with OPENLINEAGE__ prefix and reverse sort
         # to make sure that top-level keys have precedence
         env_vars = sorted(
             filter(lambda k: k[0].startswith(cls.DYNAMIC_ENV_VARS_PREFIX), os.environ.items()), reverse=True
@@ -442,8 +468,8 @@ class OpenLineageClient:
 
         return event
 
+    @staticmethod
     def _update_tag_facet(
-        self,
         tags_facet: TagsJobFacet | TagsRunFacet,
         user_tags: list[TagsJobFacetFields | TagsRunFacetFields],
     ) -> TagsJobFacet | TagsRunFacet:

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.34.0"
+__version__ = "1.36.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/filter.py

@@ -5,10 +5,14 @@ from __future__ import annotations
 import logging
 import re
 import typing
+import warnings
 
 import attr
 from openlineage.client.event_v2 import RunEvent as RunEvent_v2
-from openlineage.client.run import RunEvent
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from openlineage.client.run import RunEvent
 
 log = logging.getLogger(__name__)
 RunEventType = typing.Union[RunEvent, RunEvent_v2]

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/base.py

@@ -23,7 +23,7 @@ class BaseEvent(RedactMixin):
     eventTime: str = attr.field()  # noqa: N815
     """the time the event occurred at"""
 
-    producer: str = attr.field(default="", kw_only=True)
+    producer: str = attr.field(default="", kw_only=True)  # noqa: N815
     schemaURL: str = attr.field(  # noqa: N815
         default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent", init=False
     )
@@ -70,7 +70,7 @@ class BaseEvent(RedactMixin):
 class BaseFacet(RedactMixin):
     """all fields of the base facet are prefixed with _ to avoid name conflicts in facets"""
 
-    _producer: str = attr.field(default="", kw_only=True)
+    _producer: str = attr.field(default="", kw_only=True)  # noqa: N815
     _schemaURL: str = attr.field(  # noqa: N815
         default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet", init=False
     )
@@ -86,7 +86,7 @@ class BaseFacet(RedactMixin):
     def skip_redact(self) -> list[str]:
         return self._base_skip_redact + self._additional_skip_redact
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> BaseFacet:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "BaseFacet":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/catalog_dataset.py

@@ -12,7 +12,7 @@ class CatalogDatasetFacet(DatasetFacet):
     framework: str
     """The storage framework for which the catalog is configured"""
 
-    type: str
+    type: str  # noqa: A003
     """Type of the catalog."""
 
     name: str

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/column_lineage_dataset.py

@@ -38,7 +38,7 @@ class Fields(RedactMixin):
     original data available (like a hash of PII for example)
     """
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> Fields:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Fields":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 
@@ -75,7 +75,7 @@ class InputField(RedactMixin):
     transformations: list[Transformation] | None = attr.field(factory=list)
     _skip_redact: ClassVar[list[str]] = ["namespace", "name", "field"]
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> InputField:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "InputField":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 
@@ -102,7 +102,7 @@ class InputField(RedactMixin):
 
 @attr.define
 class Transformation(RedactMixin):
-    type: str
+    type: str  # noqa: A003
     """The type of the transformation. Allowed values are: DIRECT, INDIRECT"""
 
     subtype: str | None = attr.field(default=None)
@@ -116,7 +116,7 @@ class Transformation(RedactMixin):
 
     _skip_redact: ClassVar[list[str]] = ["type", "subtype", "masking"]
 
-    def with_additional_properties(self, **kwargs: dict[str, Any]) -> Transformation:
+    def with_additional_properties(self, **kwargs: dict[str, Any]) -> "Transformation":
         """Add additional properties to updated class instance."""
         current_attrs = [a.name for a in attr.fields(self.__class__)]
 

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/data_quality_metrics_input_dataset.py

@@ -16,14 +16,14 @@ class ColumnMetrics(RedactMixin):
     distinctCount: int | None = attr.field(default=None)  # noqa: N815
     """The number of distinct values in this column for the rows evaluated"""
 
-    sum: float | None = attr.field(default=None)
+    sum: float | None = attr.field(default=None)  # noqa: A003
     """The total sum of values in this column for the rows evaluated"""
 
     count: float | None = attr.field(default=None)
     """The number of values in this column"""
 
-    min: float | None = attr.field(default=None)
-    max: float | None = attr.field(default=None)
+    min: float | None = attr.field(default=None)  # noqa: A003
+    max: float | None = attr.field(default=None)  # noqa: A003
     quantiles: dict[str, float] | None = attr.field(factory=dict)
     """The property key is the quantile. Examples: 0.1 0.25 0.5 0.75 1"""
 
@@ -36,7 +36,7 @@ class DataQualityMetricsInputDatasetFacet(InputDatasetFacet):
     rowCount: int | None = attr.field(default=None)  # noqa: N815
     """The number of rows evaluated"""
 
-    bytes: int | None = attr.field(default=None)
+    bytes: int | None = attr.field(default=None)  # noqa: A003
     """The size in bytes"""
 
     fileCount: int | None = attr.field(default=None)  # noqa: N815

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/ownership_dataset.py

@@ -15,7 +15,7 @@ class Owner(RedactMixin):
     the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example
     application:foo, user:jdoe, team:data
     """
-    type: str | None = attr.field(default=None)
+    type: str | None = attr.field(default=None)  # noqa: A003
     """The type of ownership (optional)"""
 
 

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/ownership_job.py

@@ -15,7 +15,7 @@ class Owner(RedactMixin):
     the identifier of the owner of the Job. It is recommended to define this as a URN. For example
     application:foo, user:jdoe, team:data
     """
-    type: str | None = attr.field(default=None)
+    type: str | None = attr.field(default=None)  # noqa: A003
     """The type of ownership (optional)"""
 
 

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/parent_run.py

@@ -36,7 +36,7 @@ class ParentRunFacet(RunFacet):
         return "https://openlineage.io/spec/facets/1-1-0/ParentRunFacet.json#/$defs/ParentRunFacet"
 
     @classmethod
-    def create(cls, runId: str, namespace: str, name: str) -> ParentRunFacet:  # noqa: N803
+    def create(cls, runId: str, namespace: str, name: str) -> "ParentRunFacet":  # noqa: N803
         import warnings
 
         warnings.warn(

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/schema_dataset.py

@@ -23,7 +23,7 @@ class SchemaDatasetFacetFields(RedactMixin):
     name: str
     """The name of the field."""
 
-    type: str | None = attr.field(default=None)
+    type: str | None = attr.field(default=None)  # noqa: A003
     """The type of the field."""
 
     description: str | None = attr.field(default=None)

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/source_code_location_job.py

@@ -11,7 +11,7 @@ from openlineage.client.generated.base import JobFacet
 
 @attr.define
 class SourceCodeLocationJobFacet(JobFacet):
-    type: str
+    type: str  # noqa: A003
     """the source control system"""
 
     url: str = attr.field()

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/sql_job.py

@@ -10,7 +10,8 @@ from openlineage.client.generated.base import JobFacet
 @attr.define
 class SQLJobFacet(JobFacet):
     query: str
+    dialect: str | None = attr.field(default=None)
 
     @staticmethod
     def _get_schema() -> str:
-        return "https://openlineage.io/spec/facets/1-0-1/SQLJobFacet.json#/$defs/SQLJobFacet"
+        return "https://openlineage.io/spec/facets/1-1-0/SQLJobFacet.json#/$defs/SQLJobFacet"

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generated/symlinks_dataset.py

@@ -16,7 +16,7 @@ class Identifier(RedactMixin):
     name: str
     """The dataset name"""
 
-    type: str
+    type: str  # noqa: A003
     """Identifier type"""
 
 

{openlineage_python-1.34.0 → openlineage_python-1.36.0}/openlineage/client/generator/cli.py

@@ -6,7 +6,7 @@ import pathlib
 import tempfile
 
 import click
-import requests
+import httpx
 from openlineage.client.generator.base import (
     camel_to_snake,
     format_and_save_output,
@@ -19,7 +19,7 @@ BASE_SPEC_URL = "https://openlineage.io/spec/2-0-2/OpenLineage.json"
 
 def get_base_spec() -> pathlib.Path:
     """Get the base spec from the OpenLineage repository."""
-    response = requests.get(BASE_SPEC_URL, timeout=10)
+    response = httpx.get(BASE_SPEC_URL, timeout=10)
     response.raise_for_status()  # Raise an HTTPError for bad responses
     tmpfile = pathlib.Path(tempfile.mkdtemp()) / "OpenLineage.json"
     tmpfile.write_text(response.text)