openlineage-python 1.8.0__tar.gz → 1.9.0__tar.gz

openlineage_python-1.9.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.1
+Name: openlineage-python
+Version: 1.9.0
+Summary: OpenLineage Python Client
+Author-email: OpenLineage <info@openlineage.io>
+Keywords: openlineage
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Requires-Dist: attrs>=20.0
+Requires-Dist: python-dateutil>=2.8.2
+Requires-Dist: pyyaml>=5.4
+Requires-Dist: requests>=2.20.0
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka>=2.1.1; extra == 'kafka'
+Provides-Extra: test
+Requires-Dist: covdefaults>=2.3; extra == 'test'
+Requires-Dist: pytest-cov>=4.1; extra == 'test'
+Requires-Dist: pytest-mock>=3.10; extra == 'test'
+Requires-Dist: pytest>=7.3.1; extra == 'test'
+Requires-Dist: pyyaml>=5.4; extra == 'test'
+Description-Content-Type: text/markdown
+
+# OpenLineage Python Client
+
+Python client for [OpenLineage](https://openlineage.io).
+
+Refer to the [documentation](https://openlineage.io/docs/client/python) for details.

openlineage_python-1.9.0/README.md

@@ -0,0 +1,5 @@
+# OpenLineage Python Client
+
+Python client for [OpenLineage](https://openlineage.io).
+
+Refer to the [documentation](https://openlineage.io/docs/client/python) for details.

{openlineage_python-1.8.0 → openlineage_python-1.9.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.8.0"
+__version__ = "1.9.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"

{openlineage_python-1.8.0 → openlineage_python-1.9.0}/openlineage/client/transport/file.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import io
 from dataclasses import dataclass
 from datetime import datetime
 from typing import TYPE_CHECKING, Any
@@ -25,14 +26,7 @@ class FileConfig(Config):
             msg = "`log_file_path` key not passed to FileConfig"
             raise RuntimeError(msg)
 
-        log_file_path = params["log_file_path"]
-
-        with open(log_file_path, "a" if cls.append else "w") as log_file_handle:
-            if not log_file_handle.writable():
-                msg = f"Log file {log_file_path} is not writeable"
-                raise RuntimeError(msg)
-
-        return cls(log_file_path)
+        return cls(log_file_path=params["log_file_path"], append=params.get("append", False))
 
 
 class FileTransport(Transport):
@@ -46,8 +40,13 @@ class FileTransport(Transport):
         if self.config.append:
             log_file_path = self.config.log_file_path
         else:
-            timestr = datetime.now().strftime("%Y%m%d-%H%M%S.%f")
-            log_file_path = f"{self.config.log_file_path}-{timestr}"
-
-        with open(log_file_path, "a" if self.config.append else "w") as log_file_handle:
-            log_file_handle.write(Serde.to_json(event) + "\n")
+            time_str = datetime.now().strftime("%Y%m%d-%H%M%S.%f")
+            log_file_path = f"{self.config.log_file_path}-{time_str}"
+
+        try:
+            with open(log_file_path, "a" if self.config.append else "w") as log_file_handle:
+                log_file_handle.write(Serde.to_json(event) + "\n")
+        except (PermissionError, io.UnsupportedOperation) as error:
+            # If we lack write permissions or file is opened in wrong mode
+            msg = f"Log file `{log_file_path}` is not writeable"
+            raise RuntimeError(msg) from error

{openlineage_python-1.8.0 → openlineage_python-1.9.0}/pyproject.toml

@@ -6,7 +6,7 @@ requires = [
 
 [project]
 name = "openlineage-python"
-version = "1.8.0"
+version = "1.9.0"
 description = "OpenLineage Python Client"
 readme = "README.md"
 keywords = [

{openlineage_python-1.8.0 → openlineage_python-1.9.0}/tests/test_file.py

@@ -2,13 +2,16 @@
 # SPDX-License-Identifier: Apache-2.0
 import datetime
 import json
+import os
 import tempfile
 import time
 import uuid
 from os import listdir
-from os.path import isfile, join
+from os.path import exists, isfile, join
 from typing import Any, Dict, List
+from unittest import mock
 
+import pytest
 from openlineage.client import OpenLineageClient
 from openlineage.client.run import Job, Run, RunEvent, RunState
 from openlineage.client.transport.file import FileConfig, FileTransport
@@ -90,3 +93,102 @@ def test_client_with_file_transport_write_emits() -> None:
             test_event = test_event_set[i]
 
             assert_test_events(log_line, test_event)
+
+
+@pytest.mark.parametrize(
+    "append",
+    [
+        True,
+        False,
+    ],
+)
+@mock.patch("openlineage.client.transport.file.datetime")
+def test_file_transport_raises_error_with_non_writeable_file(mock_dt, append) -> None:
+    event = RunEvent(
+        eventType=RunState.START,
+        eventTime=datetime.datetime.now().isoformat(),
+        run=Run(runId=str(uuid.uuid4())),
+        job=Job(namespace="file", name="test"),
+        producer="prod",
+    )
+    mock_dt.now().strftime.return_value = "timestr"
+
+    with tempfile.TemporaryDirectory() as log_dir:
+        log_base_file_path = join(log_dir, "logtest")
+
+        config = FileConfig(log_file_path=log_base_file_path, append=append)
+        transport = FileTransport(config)
+
+        log_file_path = log_base_file_path if append else f"{log_base_file_path}-timestr"
+
+        # Create the file and make it read-only
+        with open(log_file_path, "w") as f:
+            f.write("This is a test file.")
+        os.chmod(log_file_path, 0o444)
+
+        with pytest.raises(RuntimeError, match=f"Log file `{log_file_path}` is not writeable"):
+            transport.emit(event)
+
+        os.remove(log_file_path)
+
+
+@pytest.mark.parametrize(
+    "append",
+    [
+        True,
+        False,
+    ],
+)
+def test_file_config_from_dict(append) -> None:
+    log_dir = tempfile.TemporaryDirectory()
+    file_path = join(log_dir.name, "logtest")
+
+    config = FileConfig.from_dict(params={"log_file_path": file_path, "append": append})
+    assert config.append is append
+    assert config.log_file_path == file_path
+
+
+def test_file_config_from_dict_no_file() -> None:
+    with pytest.raises(RuntimeError):
+        FileConfig.from_dict({})
+
+
+@pytest.mark.parametrize(
+    "append",
+    [
+        True,
+        False,
+    ],
+)
+def test_file_config_does_not_create_file_if_missing(append) -> None:
+    log_dir = tempfile.TemporaryDirectory()
+    file_path = join(log_dir.name, "test")
+
+    assert not exists(file_path)
+    FileConfig.from_dict(params={"log_file_path": file_path, "append": append})
+    assert not exists(file_path)
+
+
+@pytest.mark.parametrize(
+    "append",
+    [
+        True,
+        False,
+    ],
+)
+def test_file_config_append_mode_does_not_modify_file_if_exists(append) -> None:
+    log_dir = tempfile.TemporaryDirectory()
+    file_path = join(log_dir.name, "test-file")
+
+    with open(file_path, "w") as file:
+        file.write("test content")
+
+    assert exists(file_path)
+    FileConfig.from_dict(params={"log_file_path": file_path, "append": append})
+    assert exists(file_path)
+
+    with open(file_path) as file:
+        file_content = file.read()
+    assert file_content == "test content"
+
+    os.remove(file_path)

openlineage_python-1.8.0/PKG-INFO

@@ -1,109 +0,0 @@
-Metadata-Version: 2.1
-Name: openlineage-python
-Version: 1.8.0
-Summary: OpenLineage Python Client
-Author-email: OpenLineage <info@openlineage.io>
-Keywords: openlineage
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8
-Requires-Dist: attrs>=20.0
-Requires-Dist: python-dateutil>=2.8.2
-Requires-Dist: pyyaml>=5.4
-Requires-Dist: requests>=2.20.0
-Provides-Extra: kafka
-Requires-Dist: confluent-kafka>=2.1.1; extra == 'kafka'
-Provides-Extra: test
-Requires-Dist: covdefaults>=2.3; extra == 'test'
-Requires-Dist: pytest-cov>=4.1; extra == 'test'
-Requires-Dist: pytest-mock>=3.10; extra == 'test'
-Requires-Dist: pytest>=7.3.1; extra == 'test'
-Requires-Dist: pyyaml>=5.4; extra == 'test'
-Description-Content-Type: text/markdown
-
-# OpenLineage-python
-
-To install from source, run:
-
-```bash
-$ python -m pip install .
-```
-
-## Configuration
-### Config file
-
-The most common way to configure the OpenLineage Client is by `.yaml` file, which contains all the
-details of how to connect to your OpenLineage backend.
-
-The config file is located by:
-1) looking at the `OPENLINEAGE_CONFIG` environment variable
-2) looking for the `openlineage.yml` file in the current working directory
-3) looking for the `openlineage.yml` file in the `$HOME/.openlineage` directory.
-
-Different ways of connecting to OpenLineage backend are supported
-by the standardized `Transport` interface.
-This is an example config for specifying `http` transport:
-
-```yaml
-transport:
-  type: "http"
-  url: "https://backend:5000"
-  auth:
-    type: "api_key"
-    api_key: "f048521b-dfe8-47cd-9c65-0cb07d57591e"
-```
-
-The `type` property is required. It can be one of the built-in transports or a custom one.
-There are four built-in transports, `http`, `kafka`, `console` and `file`.
-Custom transports `type` is a fully qualified class name that can be imported.
-
-The rest of the properties are defined by the particular transport.
-Specifications for the built-in options are below.
-
-#### HTTP
-
-* `url` - required string parameter.
-* `endpoint` - optional string parameter specifying the endpoint to which events are sent. By default `api/v1/lineage`.
-* `timeout` - optional float parameter specifying the timeout when sending events. By default 5 seconds.
-* `verify` optional boolean attribute specifying if the client should verify TLS certificates of the backend. By default true.
-* `auth` - optional dictionary specifying authentication options. The type property is required.
-    * `type`: required property if an auth dictionary is set. Set to `api_key` or to the fully qualified class name of your TokenProvider.
-    * `api_key`: if `api_key` type is set, it sets value at `Authentication` HTTP header as `Bearer`.
-
-#### Kafka
-
-For KafkaTransport, `confluent-kafka` needs to be installed.
-You can also install `pip install openlineage-python[kafka]`
-
-* `config` - required string parameter. A dictionary that contains a Kafka producer config as in [Kafka producer config](https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration).
-* `topic` - required string parameter. The topic on what events will be sent.
-* `flush` - optional boolean parameter. If set to True, Kafka will flush after each event. By default true.
-
-#### File
-
-* `log_file_path` - required string parameter specifying the path of the file (if `append` is true a file path is expected, otherwise a file prefix is expected).
-* `append` - optional boolean parameter. If set to True, each event will be appended to a single file (log_file_path), otherwise, all event would be written separately in distinct files suffixed by a timestring. By default false.
-
-### Custom transport
-
-To implement a custom transport, follow the instructions in the `openlineage/client/transport/transport.py` file.
-
-### Config as env vars
-
-If there is no config file found, the OpenLineage client looks at environment variables.
-This way of configuring the client supports only `http` transport, and only a subset of its config.
-
-* `OPENLINEAGE_URL` - point to the service that will consume OpenLineage events.
-* `OPENLINEAGE_ENDPOINT` - specify the endpoint to send the OpenLineage events.
-* `OPENLINEAGE_API_KEY` - set if the consumer of OpenLineage events requires a `Bearer` authentication key.
-
-`OPENLINEAGE_URL`, `OPENLINEAGE_ENDPOINT`, and `OPENLINEAGE_API_KEY` can also be set up manually when creating a client instance.
-
-#### Logging level
-In addition to conventional logging approaches, the OpenLineage client library provides an alternative way of configuring its logging behavior. By setting the `OPENLINEAGE_CLIENT_LOGGING` environment variable, you can establish the logging level for the `openlineage.client` and its child modules.

openlineage_python-1.8.0/README.md

@@ -1,80 +0,0 @@
-# OpenLineage-python
-
-To install from source, run:
-
-```bash
-$ python -m pip install .
-```
-
-## Configuration
-### Config file
-
-The most common way to configure the OpenLineage Client is by `.yaml` file, which contains all the
-details of how to connect to your OpenLineage backend.
-
-The config file is located by:
-1) looking at the `OPENLINEAGE_CONFIG` environment variable
-2) looking for the `openlineage.yml` file in the current working directory
-3) looking for the `openlineage.yml` file in the `$HOME/.openlineage` directory.
-
-Different ways of connecting to OpenLineage backend are supported
-by the standardized `Transport` interface.
-This is an example config for specifying `http` transport:
-
-```yaml
-transport:
-  type: "http"
-  url: "https://backend:5000"
-  auth:
-    type: "api_key"
-    api_key: "f048521b-dfe8-47cd-9c65-0cb07d57591e"
-```
-
-The `type` property is required. It can be one of the built-in transports or a custom one.
-There are four built-in transports, `http`, `kafka`, `console` and `file`.
-Custom transports `type` is a fully qualified class name that can be imported.
-
-The rest of the properties are defined by the particular transport.
-Specifications for the built-in options are below.
-
-#### HTTP
-
-* `url` - required string parameter.
-* `endpoint` - optional string parameter specifying the endpoint to which events are sent. By default `api/v1/lineage`.
-* `timeout` - optional float parameter specifying the timeout when sending events. By default 5 seconds.
-* `verify` optional boolean attribute specifying if the client should verify TLS certificates of the backend. By default true.
-* `auth` - optional dictionary specifying authentication options. The type property is required.
-    * `type`: required property if an auth dictionary is set. Set to `api_key` or to the fully qualified class name of your TokenProvider.
-    * `api_key`: if `api_key` type is set, it sets value at `Authentication` HTTP header as `Bearer`.
-
-#### Kafka
-
-For KafkaTransport, `confluent-kafka` needs to be installed.
-You can also install `pip install openlineage-python[kafka]`
-
-* `config` - required string parameter. A dictionary that contains a Kafka producer config as in [Kafka producer config](https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration).
-* `topic` - required string parameter. The topic on what events will be sent.
-* `flush` - optional boolean parameter. If set to True, Kafka will flush after each event. By default true.
-
-#### File
-
-* `log_file_path` - required string parameter specifying the path of the file (if `append` is true a file path is expected, otherwise a file prefix is expected).
-* `append` - optional boolean parameter. If set to True, each event will be appended to a single file (log_file_path), otherwise, all event would be written separately in distinct files suffixed by a timestring. By default false.
-
-### Custom transport
-
-To implement a custom transport, follow the instructions in the `openlineage/client/transport/transport.py` file.
-
-### Config as env vars
-
-If there is no config file found, the OpenLineage client looks at environment variables.
-This way of configuring the client supports only `http` transport, and only a subset of its config.
-
-* `OPENLINEAGE_URL` - point to the service that will consume OpenLineage events.
-* `OPENLINEAGE_ENDPOINT` - specify the endpoint to send the OpenLineage events.
-* `OPENLINEAGE_API_KEY` - set if the consumer of OpenLineage events requires a `Bearer` authentication key.
-
-`OPENLINEAGE_URL`, `OPENLINEAGE_ENDPOINT`, and `OPENLINEAGE_API_KEY` can also be set up manually when creating a client instance.
-
-#### Logging level
-In addition to conventional logging approaches, the OpenLineage client library provides an alternative way of configuring its logging behavior. By setting the `OPENLINEAGE_CLIENT_LOGGING` environment variable, you can establish the logging level for the `openlineage.client` and its child modules.