openlineage-python 1.12.0__tar.gz → 1.14.0__tar.gz

openlineage_python-1.14.0/.gitignore

@@ -0,0 +1,2 @@
+_build
+converted

{openlineage_python-1.12.0 → openlineage_python-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: openlineage-python
-Version: 1.12.0
+Version: 1.14.0
 Summary: OpenLineage Python Client
 Author-email: OpenLineage <info@openlineage.io>
 Keywords: openlineage
@@ -18,6 +18,11 @@ Requires-Dist: packaging>=21.0
 Requires-Dist: python-dateutil>=2.8.2
 Requires-Dist: pyyaml>=5.4
 Requires-Dist: requests>=2.20.0
+Provides-Extra: docs
+Requires-Dist: beautifulsoup4>=4.12.0; extra == 'docs'
+Requires-Dist: pydata-sphinx-theme>=0.13; extra == 'docs'
+Requires-Dist: sphinx-autodoc-typehints>=2.0.0; extra == 'docs'
+Requires-Dist: sphinx>=7.0; extra == 'docs'
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka>=2.1.1; extra == 'kafka'
 Provides-Extra: msk-iam

openlineage_python-1.14.0/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

openlineage_python-1.14.0/docs/conf.py

@@ -0,0 +1,61 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+import builtins
+
+builtins.__sphinx_build__ = True
+
+project = "OpenLineage Python Client"
+copyright = "2024, OpenLineage"  # noqa: A001
+author = "OpenLineage"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ["sphinx.ext.autodoc"]
+
+templates_path = ["_templates"]
+exclude_patterns = []
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "pydata_sphinx_theme"
+html_static_path = ["_static"]
+
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx_autodoc_typehints"]
+
+napoleon_use_param = True
+
+autodoc_typehints = "description"
+always_document_param_types = True
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "venv"]
+source_suffix = [".rst"]
+
+autodoc_default_options = {
+    "members": True,
+    "member-order": "bysource",
+    "undoc-members": True,
+    "show-inheritance": True,
+}
+
+autodoc_mock_imports = [
+    "confluent_kafka",
+]
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "pydata_sphinx_theme"
+
+html_static_path = ["_static"]

openlineage_python-1.14.0/docs/convert.py

@@ -0,0 +1,76 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import pathlib
+
+from bs4 import BeautifulSoup
+
+SPHINX_ROOT_DIR = pathlib.Path(__file__).parent.resolve()
+SPHINX_BUILD_DIR = SPHINX_ROOT_DIR / pathlib.Path("_build/html/source")
+DOCS_OUTPUT_DIR = SPHINX_ROOT_DIR / "converted"
+
+
+def html_to_mdx(html: str) -> str:
+    # Because the HTML uses `class` and has `{}` in it, it isn't valid
+    # MDX. As such, we use React's dangerouslySetInnerHTML.
+    return f"""
+
+<div dangerouslySetInnerHTML={{{{__html: {json.dumps(html)}}}}}></div>
+
+"""
+
+
+def bs4_to_mdx(soup: BeautifulSoup, sidebar: BeautifulSoup) -> str:
+    return html_to_mdx(str(sidebar) + str(soup))
+
+
+def convert_html_to_md(html_file: pathlib.Path) -> str:
+    html = html_file.read_text()
+    soup = BeautifulSoup(html, "html.parser")
+
+    body = soup.find("main").find("div", {"class": "bd-article-container"})
+    sidebar = soup.find("main").find("div", {"class": "bd-sidebar-secondary"})
+    article = body.find("article")
+
+    # Remove all the "permalink to this heading" links.
+    for link in article.find_all("a", {"class": "headerlink"}):
+        link.decompose()
+
+    # Remove the trailing " – " from arguments that are missing  # noqa: RUF003
+    # a description.
+    for item in article.select("dl.field-list dd p"):
+        # Note - that's U+2013, not a normal hyphen.
+        if str(item).endswith(" – </p>"):  # noqa: RUF001
+            parent = item.parent
+            new_item = BeautifulSoup(str(item)[:-7] + "</p>", "html.parser")
+            parent.p.replace_with(new_item)
+
+    # Extract title from the h1.
+    title_element = article.find("h1")
+    title = title_element.text
+    print(title)
+    title_element.decompose()
+
+    md_meta = f"""---
+title: {title}
+---\n\n"""
+
+    return md_meta + bs4_to_mdx(article, sidebar)
+
+
+def main():
+    DOCS_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    for doc in SPHINX_BUILD_DIR.glob("**/*.html"):
+        md = convert_html_to_md(doc)
+
+        outfile = DOCS_OUTPUT_DIR / doc.relative_to(SPHINX_BUILD_DIR).with_suffix(".md")
+        outfile.parent.mkdir(parents=True, exist_ok=True)
+        outfile.write_text(md)
+
+        print(f"Generated {outfile}")
+
+
+if __name__ == "__main__":
+    main()

openlineage_python-1.14.0/docs/index.rst

@@ -0,0 +1,13 @@
+.. OpenLineage Python Client documentation master file, created by
+   sphinx-quickstart on Fri Mar 15 21:16:12 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to OpenLineage Python Client's documentation!
+=====================================================
+
+.. toctree::
+   :maxdepth: 1
+   :titlesonly:
+
+   source/openlineage.client

openlineage_python-1.14.0/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

openlineage_python-1.14.0/docs/release-api-reference.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# Copyright 2018-2023 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+function git-website() {
+  command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@
+}
+
+git config --global user.email "openlineage-bot-key@openlineage.io"
+git config --global user.name "OpenLineage deploy bot"
+
+WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website}
+REFERENCE_DIR=/docs/development/developing/python/api-reference
+REPO="git@github.com:OpenLineage/docs"
+DEFAULT_BRANCH=main
+BRANCH_NAME=${BRANCH_NAME:-$DEFAULT_BRANCH}
+
+if [[ -d $WEBSITE_DIR ]]; then
+  # Check if we're in git repository and the repository points at website
+  if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then
+    # Make sure we're at the head of the main branch
+    git checkout main
+    git reset --hard origin/master
+  else
+    echo "$WEBSITE_DIR is not empty - failing"
+    exit 1
+  fi
+else
+  git clone -b $BRANCH_NAME --depth 1 $REPO $WEBSITE_DIR
+fi
+
+# check if there are any changes in javadoc in the latest commit
+if [[ $(diff -qr --exclude _category_.json $WEBSITE_DIR$REFERENCE_DIR './converted' | wc -l) -eq 0 ]]; then
+  echo "no changes in python client reference detected, skipping publishing"
+  exit 0
+fi
+
+echo "Changes detected, updating python client reference..."
+mv ./converted/* $WEBSITE_DIR$REFERENCE_DIR/
+
+# commit new spec and push
+git-website add -A $WEBSITE_DIR$REFERENCE_DIR
+git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -s -m "openlineage python client reference update"
+git-website push

openlineage_python-1.14.0/docs/source/openlineage.client.rst

@@ -0,0 +1,330 @@
+Python Client
+=============
+
+openlineage.client.client module
+-----------------------------------
+
+.. automodule:: openlineage.client.client
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.event\_v2 module
+-----------------------------------
+
+.. automodule:: openlineage.client.event_v2
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.facet module
+-------------------------------
+
+.. automodule:: openlineage.client.facet
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.facet\_v2 module
+-----------------------------------
+
+.. automodule:: openlineage.client.facet_v2
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.filter module
+--------------------------------
+
+.. automodule:: openlineage.client.filter
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.run module
+-----------------------------
+
+.. automodule:: openlineage.client.run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.serde module
+-------------------------------
+
+.. automodule:: openlineage.client.serde
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.utils module
+-------------------------------
+
+.. automodule:: openlineage.client.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.base module
+----------------------------------------
+
+.. automodule:: openlineage.client.generated.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.column\_lineage\_dataset module
+------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.column_lineage_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.data\_quality\_assertions\_dataset module
+----------------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.data_quality_assertions_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.data\_quality\_metrics\_input\_dataset module
+--------------------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.data_quality_metrics_input_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.dataset\_version\_dataset module
+-------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.dataset_version_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.datasource\_dataset module
+-------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.datasource_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.documentation\_dataset module
+----------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.documentation_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.documentation\_job module
+------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.documentation_job
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.error\_message\_run module
+-------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.error_message_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.external\_query\_run module
+--------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.external_query_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.extraction\_error\_run module
+----------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.extraction_error_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.job\_type\_job module
+--------------------------------------------------
+
+.. automodule:: openlineage.client.generated.job_type_job
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.lifecycle\_state\_change\_dataset module
+---------------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.lifecycle_state_change_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.nominal\_time\_run module
+------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.nominal_time_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.output\_statistics\_output\_dataset module
+-----------------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.output_statistics_output_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.ownership\_dataset module
+------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.ownership_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.ownership\_job module
+--------------------------------------------------
+
+.. automodule:: openlineage.client.generated.ownership_job
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.parent\_run module
+-----------------------------------------------
+
+.. automodule:: openlineage.client.generated.parent_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.processing\_engine\_run module
+-----------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.processing_engine_run
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.schema\_dataset module
+---------------------------------------------------
+
+.. automodule:: openlineage.client.generated.schema_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.source\_code\_job module
+-----------------------------------------------------
+
+.. automodule:: openlineage.client.generated.source_code_job
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.source\_code\_location\_job module
+---------------------------------------------------------------
+
+.. automodule:: openlineage.client.generated.source_code_location_job
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.sql\_job module
+--------------------------------------------
+
+.. automodule:: openlineage.client.generated.sql_job
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.storage\_dataset module
+----------------------------------------------------
+
+.. automodule:: openlineage.client.generated.storage_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.generated.symlinks\_dataset module
+-----------------------------------------------------
+
+.. automodule:: openlineage.client.generated.symlinks_dataset
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.console module
+-------------------------------------------
+
+.. automodule:: openlineage.client.transport.console
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.factory module
+-------------------------------------------
+
+.. automodule:: openlineage.client.transport.factory
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.file module
+----------------------------------------
+
+.. automodule:: openlineage.client.transport.file
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.http module
+----------------------------------------
+
+.. automodule:: openlineage.client.transport.http
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.kafka module
+-----------------------------------------
+
+.. automodule:: openlineage.client.transport.kafka
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.msk\_iam module
+--------------------------------------------
+
+.. automodule:: openlineage.client.transport.msk_iam
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.noop module
+----------------------------------------
+
+.. automodule:: openlineage.client.transport.noop
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+openlineage.client.transport.transport module
+---------------------------------------------
+
+.. automodule:: openlineage.client.transport.transport
+   :members:
+   :undoc-members:
+   :show-inheritance:

openlineage_python-1.14.0/openlineage/client/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from openlineage.client.client import OpenLineageClient, OpenLineageClientOptions
+from openlineage.client.facet import set_producer as set_producer_v1
+from openlineage.client.facet_v2 import set_producer as set_producer_v2
+
+
+def set_producer(producer: str) -> None:
+    set_producer_v1(producer)
+    set_producer_v2(producer)
+
+
+__all__ = ["OpenLineageClient", "OpenLineageClientOptions", "set_producer"]

{openlineage_python-1.12.0 → openlineage_python-1.14.0}/openlineage/client/client.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import logging
 import os
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar, Union
 
 import attr
 from openlineage.client.filter import Filter, create_filter
@@ -15,10 +15,15 @@ if TYPE_CHECKING:
     from requests import Session
     from requests.adapters import HTTPAdapter
 
+from openlineage.client import event_v2
 from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
 from openlineage.client.transport import Transport, TransportFactory, get_default_factory
 from openlineage.client.transport.http import HttpConfig, HttpTransport
 
+Event_v1 = Union[RunEvent, DatasetEvent, JobEvent]
+Event_v2 = Union[event_v2.RunEvent, event_v2.DatasetEvent, event_v2.JobEvent]
+Event = Union[Event_v1, Event_v2]
+
 
 @attr.s
 class OpenLineageClientOptions:
@@ -62,7 +67,7 @@ class OpenLineageClient:
         elif transport:
             self.transport = transport
         else:
-            transport_config = None if "transport" not in self.config else self.config["transport"]
+            transport_config = self.config.get("transport", None)
             self.transport = factory.create(transport_config)
 
         self._filters: list[Filter] = []
@@ -86,10 +91,12 @@ class OpenLineageClient:
             ),
         )
 
-    def emit(self, event: RunEvent | DatasetEvent | JobEvent) -> None:
-        if not (isinstance(event, (RunEvent, DatasetEvent, JobEvent))):
+    def emit(self, event: Event) -> None:
+        from typing import get_args
+
+        if type(event) not in get_args(Event):
             msg = "`emit` only accepts RunEvent, DatasetEvent, JobEvent classes"
-            raise ValueError(msg)  # noqa: TRY004
+            raise ValueError(msg)
         if self._filters and self.filter_event(event) is None:
             return
         if not self.transport:
@@ -112,8 +119,8 @@ class OpenLineageClient:
 
     def filter_event(
         self,
-        event: RunEvent | DatasetEvent | JobEvent,
-    ) -> RunEvent | DatasetEvent | JobEvent | None:
+        event: Event,
+    ) -> Event | None:
         """Filters jobs according to config-defined events"""
         for _filter in self._filters:
             if isinstance(event, RunEvent) and _filter.filter_event(event) is None:

{openlineage_python-1.12.0 → openlineage_python-1.14.0}/openlineage/client/constants.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-__version__ = "1.12.0"
+__version__ = "1.14.0"
 
 DEFAULT_TIMEOUT_MS = 5000
 DEFAULT_NAMESPACE_NAME = "default"