openlineage-python 1.11.3__tar.gz → 1.13.1__tar.gz

{openlineage_python-1.11.3 → openlineage_python-1.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: openlineage-python
-Version: 1.11.3
+Version: 1.13.1
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage
@@ -14,6 +14,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Requires-Dist: attrs>=20.0
+Requires-Dist: packaging>=21.0
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: pyyaml>=5.4
 Requires-Dist: requests>=2.20.0

openlineage_python-1.13.1/openlineage/client/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from openlineage.client.client import OpenLineageClient, OpenLineageClientOptions
+from openlineage.client.facet import set_producer as set_producer_v1
+from openlineage.client.facet_v2 import set_producer as set_producer_v2
+
+
+def set_producer(producer: str) -> None:
+    set_producer_v1(producer)
+    set_producer_v2(producer)
+
+
+__all__ = ["OpenLineageClient", "OpenLineageClientOptions", "set_producer"]

{openlineage_python-1.11.3 → openlineage_python-1.13.1}/openlineage/client/client.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import logging
 import os
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar, Union
 
 import attr
 from openlineage.client.filter import Filter, create_filter
@@ -15,10 +15,15 @@ if TYPE_CHECKING:
     from requests import Session
     from requests.adapters import HTTPAdapter
 
+from openlineage.client import event_v2
 from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
 from openlineage.client.transport import Transport, TransportFactory, get_default_factory
 from openlineage.client.transport.http import HttpConfig, HttpTransport
 
+Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
+Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
+Event = Union[Event_v1, Event_v2]
+
 
 @attr.s
 class OpenLineageClientOptions:
@@ -62,7 +67,7 @@ class OpenLineageClient:
         elif transport:
             self.transport = transport
         else:
-            transport_config = None if "transport" not in self.config else self.config["transport"]
+            transport_config = self.config.get("transport", None)
             self.transport = factory.create(transport_config)
 
         self._filters: list[Filter] = []
@@ -86,10 +91,12 @@ class OpenLineageClient:
             ),
         )
 
-    def emit(self, event: RunEvent | DatasetEvent | JobEvent) -> None:
-        if not (isinstance(event, (RunEvent, DatasetEvent, JobEvent))):
+    def emit(self, event: Event) -> None:
+        from typing import get_args
+
+        if type(event) not in get_args(Event):
             msg = "`emit` only accepts RunEvent, DatasetEvent, JobEvent classes"
-            raise ValueError(msg)  # noqa: TRY004
+            raise ValueError(msg)
         if self._filters and self.filter_event(event) is None:
             return
         if not self.transport:
@@ -112,8 +119,8 @@ class OpenLineageClient:
 
     def filter_event(
         self,
-        event: RunEvent | DatasetEvent | JobEvent,
-    ) -> RunEvent | DatasetEvent | JobEvent | None:
+        event: Event,
+    ) -> Event | None:
         """Filters jobs according to config-defined events"""
         for _filter in self._filters:
             if isinstance(event, RunEvent) and _filter.filter_event(event) is None:

{openlineage_python-1.11.3 → openlineage_python-1.13.1}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.11.3"
+__version__ = "1.13.1"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

openlineage_python-1.13.1/openlineage/client/event_v2.py

@@ -0,0 +1,35 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+
+from openlineage.client.generated.base import (
+    PRODUCER,
+    BaseEvent,
+    Dataset,
+    DatasetEvent,
+    InputDataset,
+    Job,
+    JobEvent,
+    OutputDataset,
+    Run,
+    RunEvent,
+    set_producer,
+)
+from openlineage.client.generated.base import (
+    EventType as RunState,
+)
+
+__all__ = [
+    "BaseEvent",
+    "RunEvent",
+    "JobEvent",
+    "DatasetEvent",
+    "RunState",
+    "Dataset",
+    "InputDataset",
+    "OutputDataset",
+    "Run",
+    "Job",
+    "PRODUCER",
+    "set_producer",
+]

{openlineage_python-1.11.3 → openlineage_python-1.13.1}/openlineage/client/facet.py

@@ -1,5 +1,6 @@
 # Copyright 2018-2024 contributors to the OpenLineage project
 # SPDX-License-Identifier: Apache-2.0
+import warnings
 from enum import Enum
 from typing import Any, ClassVar, Dict, List, Optional
 
@@ -7,6 +8,10 @@ import attr
 from openlineage.client.constants import DEFAULT_PRODUCER
 from openlineage.client.utils import RedactMixin
 
+warnings.warn(
+    "This module is deprecated. Please use `openlineage.client.facet_v2`.", DeprecationWarning, stacklevel=2
+)
+
 SCHEMA_URI = "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json"
 
 PRODUCER = DEFAULT_PRODUCER
@@ -85,7 +90,7 @@ class DocumentationJobFacet(BaseFacet):
 
 @attr.s
 class SourceCodeLocationJobFacet(BaseFacet):
-    type: str = attr.ib()  # noqa: A003
+    type: str = attr.ib()
     url: str = attr.ib()
 
     _additional_skip_redact: ClassVar[List[str]] = ["type", "url"]
@@ -116,7 +121,7 @@ class DocumentationDatasetFacet(BaseFacet):
 @attr.s
 class SchemaField(RedactMixin):
     name: str = attr.ib()
-    type: str = attr.ib()  # noqa: A003
+    type: str = attr.ib()
     description: Optional[str] = attr.ib(default=None)
 
     _do_not_redact: ClassVar[List[str]] = ["name", "type"]
@@ -160,17 +165,17 @@ class OutputStatisticsOutputDatasetFacet(BaseFacet):
 class ColumnMetric:
     nullCount: Optional[int] = attr.ib(default=None)  # noqa: N815
     distinctCount: Optional[int] = attr.ib(default=None)  # noqa: N815
-    sum: Optional[int] = attr.ib(default=None)  # noqa: A003
+    sum: Optional[int] = attr.ib(default=None)
     count: Optional[int] = attr.ib(default=None)
-    min: Optional[float] = attr.ib(default=None)  # noqa: A003
-    max: Optional[float] = attr.ib(default=None)  # noqa: A003
+    min: Optional[float] = attr.ib(default=None)
+    max: Optional[float] = attr.ib(default=None)
     quantiles: Optional[Dict[str, float]] = attr.ib(default=None)
 
 
 @attr.s
 class DataQualityMetricsInputDatasetFacet(BaseFacet):
     rowCount: Optional[int] = attr.ib(default=None)  # noqa: N815
-    bytes: Optional[int] = attr.ib(default=None)  # noqa: A003
+    bytes: Optional[int] = attr.ib(default=None)
     fileCount: Optional[int] = attr.ib(default=None)  # noqa: N815
     columnMetrics: Dict[str, ColumnMetric] = attr.ib(factory=dict)  # noqa: N815
 
@@ -241,7 +246,7 @@ class ErrorMessageRunFacet(BaseFacet):
 class SymlinksDatasetFacetIdentifiers:
     namespace: str = attr.ib()
     name: str = attr.ib()
-    type: str = attr.ib()  # noqa: A003
+    type: str = attr.ib()
 
 
 @attr.s
@@ -272,7 +277,7 @@ class StorageDatasetFacet(BaseFacet):
 @attr.s
 class OwnershipJobFacetOwners:
     name: str = attr.ib()
-    type: Optional[str] = attr.ib(default=None)  # noqa: A003
+    type: Optional[str] = attr.ib(default=None)
 
 
 @attr.s
@@ -344,7 +349,7 @@ class LifecycleStateChangeDatasetFacet(BaseFacet):
 @attr.s
 class OwnershipDatasetFacetOwners:
     name: str = attr.ib()
-    type: str = attr.ib()  # noqa: A003
+    type: str = attr.ib()
 
 
 @attr.s

openlineage_python-1.13.1/openlineage/client/facet_v2.py

@@ -0,0 +1,74 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from openlineage.client.generated import (
+    column_lineage_dataset,
+    data_quality_assertions_dataset,
+    data_quality_metrics_input_dataset,
+    dataset_version_dataset,
+    datasource_dataset,
+    documentation_dataset,
+    documentation_job,
+    error_message_run,
+    external_query_run,
+    extraction_error_run,
+    job_type_job,
+    lifecycle_state_change_dataset,
+    nominal_time_run,
+    output_statistics_output_dataset,
+    ownership_dataset,
+    ownership_job,
+    parent_run,
+    processing_engine_run,
+    schema_dataset,
+    source_code_job,
+    source_code_location_job,
+    sql_job,
+    storage_dataset,
+    symlinks_dataset,
+)
+from openlineage.client.generated.base import (
+    PRODUCER,
+    BaseFacet,
+    DatasetFacet,
+    InputDatasetFacet,
+    JobFacet,
+    OutputDatasetFacet,
+    RunFacet,
+    set_producer,
+)
+
+__all__ = [
+    "PRODUCER",
+    "BaseFacet",
+    "DatasetFacet",
+    "InputDatasetFacet",
+    "JobFacet",
+    "OutputDatasetFacet",
+    "RunFacet",
+    "set_producer",
+    "column_lineage_dataset",
+    "data_quality_assertions_dataset",
+    "data_quality_metrics_input_dataset",
+    "dataset_version_dataset",
+    "datasource_dataset",
+    "documentation_dataset",
+    "documentation_job",
+    "error_message_run",
+    "external_query_run",
+    "extraction_error_run",
+    "job_type_job",
+    "lifecycle_state_change_dataset",
+    "nominal_time_run",
+    "output_statistics_output_dataset",
+    "ownership_dataset",
+    "ownership_job",
+    "parent_run",
+    "processing_engine_run",
+    "schema_dataset",
+    "source_code_job",
+    "source_code_location_job",
+    "sql_job",
+    "storage_dataset",
+    "symlinks_dataset",
+]

{openlineage_python-1.11.3 → openlineage_python-1.13.1}/openlineage/client/filter.py

@@ -5,12 +5,14 @@ from __future__ import annotations
 import re
 import typing
 
-if typing.TYPE_CHECKING:
-    from openlineage.client.run import RunEvent
+from openlineage.client.event_v2 import RunEvent as RunEvent_v2
+from openlineage.client.run import RunEvent
+
+RunEventType = typing.Union[RunEvent, RunEvent_v2]
 
 
 class Filter:
-    def filter_event(self, event: RunEvent) -> RunEvent | None:
+    def filter_event(self, event: RunEventType) -> RunEventType | None:
         ...
 
 
@@ -18,7 +20,7 @@ class ExactMatchFilter(Filter):
     def __init__(self, match: str) -> None:
         self.match = match
 
-    def filter_event(self, event: RunEvent) -> RunEvent | None:
+    def filter_event(self, event: RunEventType) -> RunEventType | None:
         if self.match == event.job.name:
             return None
         return event
@@ -28,7 +30,7 @@ class RegexFilter(Filter):
     def __init__(self, regex: str) -> None:
         self.pattern = re.compile(regex)
 
-    def filter_event(self, event: RunEvent) -> RunEvent | None:
+    def filter_event(self, event: RunEventType) -> RunEventType | None:
         if self.pattern.match(event.job.name):
             return None
         return event

openlineage_python-1.13.1/openlineage/client/generated/base.py

@@ -0,0 +1,321 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import ClassVar
+
+from attr import define, field
+from openlineage.client.constants import DEFAULT_PRODUCER
+from openlineage.client.utils import RedactMixin
+
+PRODUCER = DEFAULT_PRODUCER
+
+
+def set_producer(producer: str) -> None:
+    global PRODUCER  # noqa: PLW0603
+    PRODUCER = producer
+
+
+@define(kw_only=True)
+class BaseEvent(RedactMixin):
+    eventTime: str = field()  # noqa: N815
+    """the time the event occurred at"""
+
+    producer: str = field(default="", kw_only=True)
+    schemaURL: str = field(  # noqa: N815
+        default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent", init=False
+    )
+    _base_skip_redact: ClassVar[list[str]] = ["producer", "schemaURL"]
+    _additional_skip_redact: ClassVar[list[str]] = []
+
+    def __attrs_post_init__(self) -> None:
+        if not self.producer:
+            self.producer = PRODUCER
+        self.schemaURL = self._get_schema()
+
+    @property
+    def skip_redact(self) -> list[str]:
+        return self._base_skip_redact + self._additional_skip_redact
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseEvent"
+
+    @eventTime.validator
+    def eventtime_check(self, attribute: str, value: str) -> None:  # noqa: ARG002
+        from dateutil import parser
+
+        parser.isoparse(value)
+        if "t" not in value.lower():
+            # make sure date-time contains time
+            msg = f"Parsed date-time has to contain time: {value}"
+            raise ValueError(msg)
+
+    @producer.validator
+    def producer_check(self, attribute: str, value: str) -> None:  # noqa: ARG002
+        from urllib.parse import urlparse
+
+        result = urlparse(value)
+        if value and not all([result.scheme, result.netloc]):
+            msg = "producer is not a valid URI"
+            raise ValueError(msg)
+
+    @schemaURL.validator
+    def schemaurl_check(self, attribute: str, value: str) -> None:  # noqa: ARG002
+        from urllib.parse import urlparse
+
+        result = urlparse(value)
+        if value and not all([result.scheme, result.netloc]):
+            msg = "schemaURL is not a valid URI"
+            raise ValueError(msg)
+
+
+@define
+class BaseFacet(RedactMixin):
+    """all fields of the base facet are prefixed with _ to avoid name conflicts in facets"""
+
+    _producer: str = field(default="", kw_only=True)
+    _schemaURL: str = field(  # noqa: N815
+        default="https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet", init=False
+    )
+    _base_skip_redact: ClassVar[list[str]] = ["_producer", "_schemaURL"]
+    _additional_skip_redact: ClassVar[list[str]] = []
+
+    def __attrs_post_init__(self) -> None:
+        if not self._producer:
+            self._producer = PRODUCER
+        self._schemaURL = self._get_schema()
+
+    @property
+    def skip_redact(self) -> list[str]:
+        return self._base_skip_redact + self._additional_skip_redact
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/BaseFacet"
+
+    @_producer.validator
+    def _producer_check(self, attribute: str, value: str) -> None:  # noqa: ARG002
+        from urllib.parse import urlparse
+
+        result = urlparse(value)
+        if value and not all([result.scheme, result.netloc]):
+            msg = "_producer is not a valid URI"
+            raise ValueError(msg)
+
+    @_schemaURL.validator
+    def _schemaurl_check(self, attribute: str, value: str) -> None:  # noqa: ARG002
+        from urllib.parse import urlparse
+
+        result = urlparse(value)
+        if value and not all([result.scheme, result.netloc]):
+            msg = "_schemaURL is not a valid URI"
+            raise ValueError(msg)
+
+
+@define
+class Dataset(RedactMixin):
+    namespace: str
+    """The namespace containing that dataset"""
+
+    name: str
+    """The unique name for that dataset within that namespace"""
+
+    facets: dict[str, DatasetFacet] | None = field(factory=dict, kw_only=True)  # type: ignore[assignment]
+    """The facets for this dataset"""
+
+    _skip_redact: ClassVar[list[str]] = ["namespace", "name"]
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/Dataset"
+
+
+@define(kw_only=True)
+class DatasetEvent(BaseEvent):
+    dataset: StaticDataset
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetEvent"
+
+
+@define
+class DatasetFacet(BaseFacet):
+    """A Dataset Facet"""
+
+    _deleted: bool | None = field(default=None, kw_only=True)
+    """set to true to delete a facet"""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+
+
+class EventType(Enum):
+    """
+    the current transition of the run state. It is required to issue 1 START event and 1 of [ COMPLETE,
+    ABORT, FAIL ] event per run. Additional events with OTHER eventType can be added to the same run.
+    For example to send additional metadata after the run is complete
+    """
+
+    START = "START"
+    RUNNING = "RUNNING"
+    COMPLETE = "COMPLETE"
+    ABORT = "ABORT"
+    FAIL = "FAIL"
+    OTHER = "OTHER"
+
+
+@define
+class InputDataset(Dataset):
+    """An input dataset"""
+
+    inputFacets: dict[str, InputDatasetFacet] | None = field(factory=dict)  # type: ignore[assignment]# noqa: N815
+    """The input facets for this dataset."""
+
+    _additional_skip_redact: ClassVar[list[str]] = ["namespace", "name"]
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/InputDataset"
+
+
+@define
+class InputDatasetFacet(BaseFacet):
+    """An Input Dataset Facet"""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/InputDatasetFacet"
+
+
+@define
+class Job(RedactMixin):
+    namespace: str
+    """The namespace containing that job"""
+
+    name: str
+    """The unique name for that job within that namespace"""
+
+    facets: dict[str, JobFacet] | None = field(factory=dict)  # type: ignore[assignment]
+    """The job facets."""
+
+    _skip_redact: ClassVar[list[str]] = ["namespace", "name"]
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/Job"
+
+
+@define(kw_only=True)
+class JobEvent(BaseEvent):
+    job: Job
+    inputs: list[InputDataset] | None = field(factory=list)  # type: ignore[assignment]
+    """The set of **input** datasets."""
+
+    outputs: list[OutputDataset] | None = field(factory=list)  # type: ignore[assignment]
+    """The set of **output** datasets."""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobEvent"
+
+
+@define
+class JobFacet(BaseFacet):
+    """A Job Facet"""
+
+    _deleted: bool | None = field(default=None, kw_only=True)
+    """set to true to delete a facet"""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+
+
+@define
+class OutputDataset(Dataset):
+    """An output dataset"""
+
+    outputFacets: dict[str, OutputDatasetFacet] | None = field(factory=dict)  # type: ignore[assignment]# noqa: N815
+    """The output facets for this dataset"""
+
+    _additional_skip_redact: ClassVar[list[str]] = ["namespace", "name"]
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/OutputDataset"
+
+
+@define
+class OutputDatasetFacet(BaseFacet):
+    """An Output Dataset Facet"""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/OutputDatasetFacet"
+
+
+@define
+class Run(RedactMixin):
+    runId: str = field()  # noqa: N815
+    """The globally unique ID of the run associated with the job."""
+
+    facets: dict[str, RunFacet] | None = field(factory=dict)  # type: ignore[assignment]
+    """The run facets."""
+
+    _skip_redact: ClassVar[list[str]] = ["runId"]
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/Run"
+
+    @runId.validator
+    def runid_check(self, attribute: str, value: str) -> None:  # noqa: ARG002
+        from uuid import UUID
+
+        UUID(value)
+
+
+@define(kw_only=True)
+class RunEvent(BaseEvent):
+    run: Run
+    job: Job
+    eventType: EventType | None = field(default=None)  # noqa: N815
+    """
+    the current transition of the run state. It is required to issue 1 START event and 1 of [ COMPLETE,
+    ABORT, FAIL ] event per run. Additional events with OTHER eventType can be added to the same run.
+    For example to send additional metadata after the run is complete
+    """
+    inputs: list[InputDataset] | None = field(factory=list)  # type: ignore[assignment]
+    """The set of **input** datasets."""
+
+    outputs: list[OutputDataset] | None = field(factory=list)  # type: ignore[assignment]
+    """The set of **output** datasets."""
+
+    _additional_skip_redact: ClassVar[list[str]] = ["eventType", "eventTime"]
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent"
+
+
+@define
+class RunFacet(BaseFacet):
+    """A Run Facet"""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+
+
+@define
+class StaticDataset(Dataset):
+    """A Dataset sent within static metadata events"""
+
+    @staticmethod
+    def _get_schema() -> str:
+        return "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/StaticDataset"