openlineage-python 1.35.0__tar.gz → 1.37.0__tar.gz

{openlineage_python-1.35.0 → openlineage_python-1.37.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openlineage-python
-Version: 1.35.0
+Version: 1.37.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage

{openlineage_python-1.35.0 → openlineage_python-1.37.0}/openlineage/client/client.py

@@ -67,11 +67,9 @@ class OpenLineageConfig:
         if "filters" in params:
             config.filters = [FilterConfig(**filter_config) for filter_config in params["filters"]]
         if "tags" in params:
-            job_tags = params["tags"].get("job", {})
-            run_tags = params["tags"].get("run", {})
             config.tags = TagsConfig(
-                job=[TagsJobFacetFields(key, value, "USER") for (key, value) in job_tags.items()],
-                run=[TagsRunFacetFields(key, value, "USER") for (key, value) in run_tags.items()],
+                job=params["tags"].get("job", {}),
+                run=params["tags"].get("run", {}),
             )
         return config
 
@@ -232,9 +230,9 @@ class OpenLineageClient:
         This method determines the appropriate transport by executing a sequence of checks:
         1. Verifies if OpenLineage is disabled through environment variable.
         2. Looks for a transport object provided in the arguments.
-        3. Attempts to configure the transport from a YAML config file.
+        3. Attempts to configure the transport from user config, config file or env vars.
         4. Tries to initialize HTTP transport with an url argument (deprecated).
-        5. Tries to set up HTTP transport using environment variables.
+        5. Tries to set up HTTP transport using simple environment variables.
         6. If no configuration is found, defaults to a console transport and logs a warning message.
 
         Returns:
@@ -245,11 +243,11 @@ class OpenLineageClient:
             log.info("OpenLineage is disabled. No events will be emitted.")
             return NoopTransport(NoopConfig())
 
-        # 2. Check if transport is provided explicitly
+        # 2. Check if transport is provided explicitly as argument
         if kwargs.get("transport"):
             return cast("Transport", kwargs["transport"])
 
-        # 3. Check if transport configuration is provided in YAML config file
+        # 3. Check if transport configuration is provided as config (explicit, file or env vars)
         if self.config.transport and self.config.transport.get("type"):
             factory = kwargs.get("factory") or get_default_factory()
             return factory.create(self.config.transport)
@@ -260,7 +258,7 @@ class OpenLineageClient:
                 url=kwargs["url"], options=kwargs.get("options"), session=kwargs.get("session")
             )
 
-        # 5. Check HTTP transport initialization with env variables
+        # 5. Check HTTP transport initialization with simple env variables
         if os.environ.get("OPENLINEAGE_URL"):
             return self._http_transport_from_env_variables()
 
@@ -307,7 +305,8 @@ class OpenLineageClient:
                     log.exception("Couldn't check if OpenLineage config file is readable: `%s`", path)
         return None
 
-    def _http_transport_from_env_variables(self) -> HttpTransport:
+    @staticmethod
+    def _http_transport_from_env_variables() -> HttpTransport:
         """
         Create HTTP transport from legacy environment variables
         """
@@ -345,7 +344,7 @@ class OpenLineageClient:
                 k.startswith(f"OPENLINEAGE__TRANSPORT__TRANSPORTS__{default_transport_name}")
                 for k in os.environ
             ):
-                log.warning(
+                log.info(
                     "%s already found in environment variables, skipping aliasing OPENLINEAGE_URL",
                     default_transport_name,
                 )
@@ -388,7 +387,7 @@ class OpenLineageClient:
     def _load_config_from_env_variables(cls) -> dict[str, Any] | None:
         config: dict[str, Any] = {}
 
-        # get os.environ.items only starting with OPENLINEAGE_ prefix and reverse sort
+        # get os.environ.items only starting with OPENLINEAGE__ prefix and reverse sort
         # to make sure that top-level keys have precedence
         env_vars = sorted(
             filter(lambda k: k[0].startswith(cls.DYNAMIC_ENV_VARS_PREFIX), os.environ.items()), reverse=True
@@ -451,14 +450,16 @@ class OpenLineageClient:
         """
         run_event_types = (RunEvent, event_v2.RunEvent)
         run_and_job_event_types = (RunEvent, event_v2.RunEvent, JobEvent, event_v2.JobEvent)
-        tags_job = self.config.tags.job
+        # tags_job = self.config.tags.job
+        tags_job = [TagsJobFacetFields(key, value, "USER") for (key, value) in self.config.tags.job.items()]
         if isinstance(event, run_and_job_event_types) and tags_job:
             # Ensure facets exists
             event.job.facets = {} if not event.job.facets else event.job.facets
             tags_facet = event.job.facets.get("tags", TagsJobFacet())
             event.job.facets["tags"] = self._update_tag_facet(tags_facet, tags_job)  # type: ignore [arg-type, assignment]
 
-        tags_run = self.config.tags.run
+        # tags_run = self.config.tags.run
+        tags_run = [TagsRunFacetFields(key, value, "USER") for (key, value) in self.config.tags.run.items()]
         if isinstance(event, run_event_types) and tags_run:
             # Ensure facets exists
             event.run.facets = {} if not event.run.facets else event.run.facets
@@ -467,8 +468,8 @@ class OpenLineageClient:
 
         return event
 
+    @staticmethod
     def _update_tag_facet(
-        self,
         tags_facet: TagsJobFacet | TagsRunFacet,
         user_tags: list[TagsJobFacetFields | TagsRunFacetFields],
     ) -> TagsJobFacet | TagsRunFacet:

{openlineage_python-1.35.0 → openlineage_python-1.37.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.35.0"
+__version__ = "1.37.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

openlineage_python-1.37.0/openlineage/client/naming/dataset.py

@@ -0,0 +1,512 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Utility module containing dataset naming helpers based on OpenLineage's dataset naming
+specification.
+
+Each class represents naming logic for a specific platform (e.g., BigQuery, AWS Glue),
+and provides methods to construct valid dataset names and namespaces.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Protocol
+
+import attr
+
+
+def _check_not_empty(instance: object, attribute: attr.Attribute[str], value: str) -> str:  # noqa: ARG001
+    """Validator to check if a string is not None and not empty."""
+    if value is None:
+        msg = f"{attribute.name} cannot be None"
+        raise ValueError(msg)
+    if not value.strip():
+        msg = f"{attribute.name} cannot be empty"
+        raise ValueError(msg)
+    return value
+
+
+def _check_enum_not_none(
+    _: object, attribute: attr.Attribute[PubSubResourceType], value: PubSubResourceType
+) -> PubSubResourceType:
+    """Validator to check if an enum value is not None."""
+    if value is None:
+        msg = f"{attribute.name} cannot be None"
+        raise ValueError(msg)
+    return PubSubResourceType(value)
+
+
+class DatasetNaming(Protocol):
+    """Interface representing dataset naming logic for a specific platform."""
+
+    def get_namespace(self) -> str:
+        """
+        Returns the dataset namespace (e.g., URI or ARN identifying the data source).
+
+        Returns:
+            The namespace string
+        """
+
+    def get_name(self) -> str:
+        """
+        Returns the dataset name specific to the platform format.
+
+        Returns:
+            The dataset name string
+        """
+
+
+@attr.define
+class Athena(DatasetNaming):
+    """Naming implementation for Athena."""
+
+    region_name: str = attr.field(validator=_check_not_empty)
+    catalog: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"awsathena://athena.{self.region_name}.amazonaws.com"
+
+    def get_name(self) -> str:
+        return f"{self.catalog}.{self.database}.{self.table}"
+
+
+@attr.define
+class AWSGlue(DatasetNaming):
+    """Naming implementation for AWS Glue."""
+
+    region_name: str = attr.field(validator=_check_not_empty)
+    account_id: str = attr.field(validator=_check_not_empty)
+    database_name: str = attr.field(validator=_check_not_empty)
+    table_name: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"arn:aws:glue:{self.region_name}:{self.account_id}"
+
+    def get_name(self) -> str:
+        return f"table/{self.database_name}/{self.table_name}"
+
+
+@attr.define
+class AzureCosmosDB(DatasetNaming):
+    """Naming implementation for Azure Cosmos DB."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"azurecosmos://{self.host}/dbs/{self.database}"
+
+    def get_name(self) -> str:
+        return f"colls/{self.table}"
+
+
+@attr.define
+class AzureDataExplorer(DatasetNaming):
+    """Naming implementation for Azure Data Explorer."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"azurekusto://{self.host}.kusto.windows.net"
+
+    def get_name(self) -> str:
+        return f"{self.database}/{self.table}"
+
+
+@attr.define
+class AzureSynapse(DatasetNaming):
+    """Naming implementation for AzureSynapse."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"sqlserver://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.schema}.{self.table}"
+
+
+@attr.define
+class BigQuery(DatasetNaming):
+    """
+    Naming implementation for BigQuery.
+
+    Namespace is always "bigquery".
+    """
+
+    project_id: str = attr.field(validator=_check_not_empty)
+    dataset_name: str = attr.field(validator=_check_not_empty)
+    table_name: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return "bigquery"
+
+    def get_name(self) -> str:
+        return f"{self.project_id}.{self.dataset_name}.{self.table_name}"
+
+
+@attr.define
+class Cassandra(DatasetNaming):
+    """Naming implementation for Cassandra."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    keyspace: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"cassandra://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.keyspace}.{self.table}"
+
+
+@attr.define
+class MySQL(DatasetNaming):
+    """Naming implementation for MySQL."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"mysql://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.table}"
+
+
+@attr.define
+class CrateDB(DatasetNaming):
+    """Naming implementation for CrateDB."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"crate://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.schema}.{self.table}"
+
+
+@attr.define
+class DB2(DatasetNaming):
+    """Naming implementation for DB2."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"db2://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.schema}.{self.table}"
+
+
+@attr.define
+class OceanBase(DatasetNaming):
+    """Naming implementation for OceanBase."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"oceanbase://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.table}"
+
+
+@attr.define
+class Oracle(DatasetNaming):
+    """Naming implementation for Oracle."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    service_name: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"oracle://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.service_name}.{self.schema}.{self.table}"
+
+
+@attr.define
+class Postgres(DatasetNaming):
+    """Naming implementation for Postgres."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"postgres://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.schema}.{self.table}"
+
+
+@attr.define
+class Teradata(DatasetNaming):
+    """Naming implementation for Teradata."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"teradata://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.table}"
+
+
+@attr.define
+class Redshift(DatasetNaming):
+    """Naming implementation for Redshift."""
+
+    cluster_identifier: str = attr.field(validator=_check_not_empty)
+    region: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"redshift://{self.cluster_identifier}.{self.region}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.schema}.{self.table}"
+
+
+@attr.define
+class Snowflake(DatasetNaming):
+    """Naming implementation for Snowflake."""
+
+    organization_name: str = attr.field(validator=_check_not_empty)
+    account_name: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"snowflake://{self.organization_name}-{self.account_name}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.schema}.{self.table}"
+
+
+@attr.define
+class Trino(DatasetNaming):
+    """Naming implementation for Trino."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    catalog: str = attr.field(validator=_check_not_empty)
+    schema: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"trino://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.catalog}.{self.schema}.{self.table}"
+
+
+@attr.define
+class ABFSS(DatasetNaming):
+    """Naming implementation for ABFSS."""
+
+    container: str = attr.field(validator=_check_not_empty)
+    service: str = attr.field(validator=_check_not_empty)
+    path: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"abfss://{self.container}@{self.service}.dfs.core.windows.net"
+
+    def get_name(self) -> str:
+        return self.path
+
+
+@attr.define
+class DBFS(DatasetNaming):
+    """Naming implementation for DBFS."""
+
+    workspace: str = attr.field(validator=_check_not_empty)
+    path: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"dbfs://{self.workspace}"
+
+    def get_name(self) -> str:
+        return self.path
+
+
+@attr.define
+class GCS(DatasetNaming):
+    """Naming implementation for GCS."""
+
+    bucket_name: str = attr.field(validator=_check_not_empty)
+    object_key: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"gs://{self.bucket_name}"
+
+    def get_name(self) -> str:
+        return self.object_key
+
+
+@attr.define
+class HDFS(DatasetNaming):
+    """Naming implementation for HDFS."""
+
+    namenode_host: str = attr.field(validator=_check_not_empty)
+    namenode_port: str = attr.field(validator=_check_not_empty)
+    path: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"hdfs://{self.namenode_host}:{self.namenode_port}"
+
+    def get_name(self) -> str:
+        return self.path
+
+
+@attr.define
+class Hive(DatasetNaming):
+    """Naming implementation for Hive."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    database: str = attr.field(validator=_check_not_empty)
+    table: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"hive://{self.host}:{self.port}"
+
+    def get_name(self) -> str:
+        return f"{self.database}.{self.table}"
+
+
+@attr.define
+class Kafka(DatasetNaming):
+    """Naming implementation for Kafka."""
+
+    bootstrap_server_host: str = attr.field(validator=_check_not_empty)
+    port: str = attr.field(validator=_check_not_empty)
+    topic: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"kafka://{self.bootstrap_server_host}:{self.port}"
+
+    def get_name(self) -> str:
+        return self.topic
+
+
+@attr.define
+class LocalFileSystem(DatasetNaming):
+    """Naming implementation for Local File System."""
+
+    path: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return "file"
+
+    def get_name(self) -> str:
+        return self.path
+
+
+@attr.define
+class RemoteFileSystem(DatasetNaming):
+    """Naming implementation for Remote File System."""
+
+    host: str = attr.field(validator=_check_not_empty)
+    path: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"file://{self.host}"
+
+    def get_name(self) -> str:
+        return self.path
+
+
+@attr.define
+class S3(DatasetNaming):
+    """Naming implementation for S3."""
+
+    bucket_name: str = attr.field(validator=_check_not_empty)
+    object_key: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"s3://{self.bucket_name}"
+
+    def get_name(self) -> str:
+        return self.object_key
+
+
+@attr.define
+class WASBS(DatasetNaming):
+    """Naming implementation for WASBS."""
+
+    container: str = attr.field(validator=_check_not_empty)
+    service_name: str = attr.field(validator=_check_not_empty)
+    object_key: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return f"wasbs://{self.container}@{self.service_name}.dfs.core.windows.net"
+
+    def get_name(self) -> str:
+        return self.object_key
+
+
+class PubSubResourceType(Enum):
+    """Enum representing supported Pub/Sub resource types."""
+
+    TOPIC = "topic"
+    SUBSCRIPTION = "subscription"
+
+
+@attr.define
+class PubSubNaming(DatasetNaming):
+    """
+    Naming implementation for Google Pub/Sub.
+
+    Namespace is fixed as "pubsub". Name format depends on the resource type:
+    - topic:{project_id}:{topic_id}
+    - subscription:{project_id}:{subscription_id}
+    """
+
+    resource_type: PubSubResourceType = attr.field(validator=_check_enum_not_none)
+    project_id: str = attr.field(validator=_check_not_empty)
+    resource_id: str = attr.field(validator=_check_not_empty)
+
+    def get_namespace(self) -> str:
+        return "pubsub"
+
+    def get_name(self) -> str:
+        return f"{self.resource_type.value}:{self.project_id}:{self.resource_id}"

openlineage_python-1.37.0/openlineage/client/tags.py

@@ -0,0 +1,11 @@
+# Copyright 2018-2025 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import attr
+
+
+@attr.define
+class TagsConfig:
+    job: dict[str, str] = attr.field(factory=dict)
+    run: dict[str, str] = attr.field(factory=dict)

{openlineage_python-1.35.0 → openlineage_python-1.37.0}/openlineage/client/transport/__init__.py

@@ -6,6 +6,7 @@ from openlineage.client.transport.amazon_datazone import AmazonDataZoneConfig, A
 from openlineage.client.transport.async_http import AsyncHttpConfig, AsyncHttpTransport
 from openlineage.client.transport.composite import CompositeTransport
 from openlineage.client.transport.console import ConsoleTransport
+from openlineage.client.transport.datadog import DatadogConfig, DatadogTransport
 from openlineage.client.transport.factory import DefaultTransportFactory
 from openlineage.client.transport.file import FileTransport
 from openlineage.client.transport.http import HttpConfig, HttpTransport
@@ -29,6 +30,7 @@ _factory.register_transport(NoopTransport.kind, NoopTransport)
 _factory.register_transport(FileTransport.kind, FileTransport)
 _factory.register_transport(TransformTransport.kind, TransformTransport)
 _factory.register_transport(AmazonDataZoneTransport.kind, AmazonDataZoneTransport)
+_factory.register_transport(DatadogTransport.kind, DatadogTransport)
 
 
 def get_default_factory() -> DefaultTransportFactory:
@@ -50,6 +52,8 @@ __all__ = [
     "CompositeTransport",
     "Config",
     "ConsoleTransport",
+    "DatadogConfig",
+    "DatadogTransport",
     "FileTransport",
     "HttpConfig",
     "HttpTransport",