omniload 0.0.0.dev0__tar.gz → 0.2.0__tar.gz

{omniload-0.0.0.dev0 → omniload-0.2.0}/LICENSE.Apache-2.0

@@ -174,28 +174,3 @@
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2023 The Contributors
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

omniload-0.2.0/MANIFEST.in

@@ -0,0 +1,9 @@
+include *.md *.rst LICENSE*
+recursive-include doc *.md *.rst
+
+global-exclude *.db
+prune */.idea
+prune */__pycache__
+prune release
+prune tests
+prune tmp

omniload-0.2.0/PKG-INFO

@@ -0,0 +1,250 @@
+Metadata-Version: 2.4
+Name: omniload
+Version: 0.2.0
+Summary: omniload is a polyglot data loader based on dlt. Copy data between any source and any destination.
+Author: Mustafa Ersan, Sanju Shah Gupta, Saravjeet 'Aman' Singh
+Author-email: Burak Karakan <burak.karakan@getbruin.com>
+Maintainer-email: Andreas Motl <andreas.motl@panodata.org>
+License-Expression: MIT
+Project-URL: Changelog, https://omniload.readthedocs.io/changelog.html
+Project-URL: Documentation, https://omniload.readthedocs.io/
+Project-URL: Issues, https://github.com/panodata/omniload/issues
+Project-URL: Repository, https://github.com/panodata/omniload
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Customer Service
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Manufacturing
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: System Administrators
+Classifier: Intended Audience :: Telecommunications Industry
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: OS Independent
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Unix
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Adaptive Technologies
+Classifier: Topic :: Communications
+Classifier: Topic :: Database
+Classifier: Topic :: Documentation
+Classifier: Topic :: Education
+Classifier: Topic :: Internet
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Office/Business
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Object Brokering
+Classifier: Topic :: Software Development :: Pre-processors
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Software Development :: Version Control
+Classifier: Topic :: System :: Archiving
+Classifier: Topic :: System :: Clustering
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Topic :: System :: Logging
+Classifier: Topic :: System :: Monitoring
+Classifier: Topic :: System :: Networking
+Classifier: Topic :: System :: Systems Administration
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: LICENSE.Apache-2.0
+License-File: NOTICE
+Requires-Dist: aiohttp<4
+Requires-Dist: asana<3.3
+Requires-Dist: click<8.5
+Requires-Dist: clickhouse-connect<1,>=0.8
+Requires-Dist: clickhouse-driver<0.3
+Requires-Dist: clickhouse-sqlalchemy<0.4
+Requires-Dist: confluent-kafka<2.15
+Requires-Dist: couchbase<5,>=4
+Requires-Dist: databricks-sql-connector<5
+Requires-Dist: databricks-sqlalchemy<3
+Requires-Dist: dataclasses-json<0.7
+Requires-Dist: dlt<1.22
+Requires-Dist: dlt-cratedb<0.2
+Requires-Dist: duckdb<1.6
+Requires-Dist: duckdb-engine<0.18
+Requires-Dist: elasticsearch<10
+Requires-Dist: facebook-business<26
+Requires-Dist: flatten-json<0.2
+Requires-Dist: gcsfs
+Requires-Dist: google-ads<32
+Requires-Dist: google-analytics-data<0.24
+Requires-Dist: google-api-python-client<3
+Requires-Dist: google-auth<3
+Requires-Dist: google-cloud-bigquery-storage<3
+Requires-Dist: google-cloud-spanner<4
+Requires-Dist: influxdb-client<2
+Requires-Dist: mysql-connector-python<10
+Requires-Dist: oracledb<4
+Requires-Dist: orjson<4
+Requires-Dist: pendulum<4,>=3
+Requires-Dist: psycopg2-binary<3
+Requires-Dist: py-machineid<2
+Requires-Dist: pyairtable<4
+Requires-Dist: pyarrow<25,>=18
+Requires-Dist: pyathena<4
+Requires-Dist: pymongo<5
+Requires-Dist: pymongoarrow<2
+Requires-Dist: pymysql<2
+Requires-Dist: pyrate-limiter<4
+Requires-Dist: python-quickbooks<0.10
+Requires-Dist: requests<3
+Requires-Dist: rudder-sdk-python<2.2
+Requires-Dist: s3fs
+Requires-Dist: simple-salesforce<2
+Requires-Dist: smartsheet-python-sdk<5
+Requires-Dist: snowflake-connector-python<5,>=3.18
+Requires-Dist: snowflake-sqlalchemy<2
+Requires-Dist: sqlalchemy<2.1
+Requires-Dist: sqlalchemy-bigquery<2
+Requires-Dist: sqlalchemy-cratedb<1
+Requires-Dist: sqlalchemy-hana<5
+Requires-Dist: sqlalchemy-redshift<2
+Requires-Dist: sqlalchemy-spanner<2
+Requires-Dist: stripe<16
+Requires-Dist: tldextract<6
+Requires-Dist: toolz<2
+Requires-Dist: tqdm<5
+Requires-Dist: trino[sqlalchemy]==0.337.0
+Requires-Dist: typer<0.27
+Requires-Dist: urllib3<2.8
+Provides-Extra: develop
+Requires-Dist: pkginfo<2; extra == "develop"
+Requires-Dist: poethepoet<1; extra == "develop"
+Requires-Dist: pyproject-fmt<3; extra == "develop"
+Requires-Dist: pytest-repeat<1; extra == "develop"
+Requires-Dist: ruff<0.16; extra == "develop"
+Requires-Dist: ty==0.0.51; extra == "develop"
+Requires-Dist: validate-pyproject<1; extra == "develop"
+Provides-Extra: full
+Requires-Dist: omniload[ibm-db,odbc,oracle]; extra == "full"
+Provides-Extra: ibm-db
+Requires-Dist: ibm-db<4; platform_machine != "aarch64" and extra == "ibm-db"
+Requires-Dist: ibm-db-sa<0.5; platform_machine != "aarch64" and extra == "ibm-db"
+Provides-Extra: odbc
+Requires-Dist: pyodbc<5.4; extra == "odbc"
+Provides-Extra: oracle
+Requires-Dist: oracledb>=2.0.0; extra == "oracle"
+Provides-Extra: release
+Requires-Dist: build<2; extra == "release"
+Requires-Dist: twine<7; extra == "release"
+Provides-Extra: test
+Requires-Dist: pendulum[test]<4,>=3; extra == "test"
+Requires-Dist: pytest<10; extra == "test"
+Requires-Dist: pytest-asyncio<2; extra == "test"
+Requires-Dist: pytest-cov<8; extra == "test"
+Requires-Dist: pytest-xdist[psutil]<4; extra == "test"
+Requires-Dist: testcontainers[mysql,postgres]<4.15,>=4.9.1; extra == "test"
+Requires-Dist: types-requests<3; extra == "test"
+Requires-Dist: verlib2; extra == "test"
+Dynamic: license-file
+
+<div align="center">
+
+# omniload
+<p>Copy data between any source and any destination.</p>
+<img src="https://github.com/panodata/omniload/blob/main/resources/demo.gif?raw=true" width="750" />
+
+[![License](https://img.shields.io/github/license/panodata/omniload.svg)](https://github.com/panodata/omniload/blob/main/LICENSE)
+[![Downloads / month](https://pepy.tech/badge/omniload/month)](https://pepy.tech/project/omniload/)
+[![Python versions](https://img.shields.io/pypi/pyversions/omniload.svg)](https://pypi.org/project/omniload/)
+
+</div>
+
+## About
+
+omniload is a polyglot data loader framework based on dlt.
+It allows you to load data from any source into any destination,
+either using a concise CLI from your shell,
+or the Python API from your own applications.
+
+omniload provides the same efficient incremental data loading modes
+inherited by [dlt]: `append`, `merge`, and `delete+insert`.
+
+## Install
+We recommend using [uv] to install or run `omniload`.
+
+```shell
+pip install uv
+uvx omniload
+```
+
+Alternatively, if you'd like to install it globally:
+```shell
+uv pip install --system omniload
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
+
+## Synopsis
+
+The next command instructs omniload to read the table `public.some_data` from
+your PostgreSQL instance, and to write the data to your BigQuery warehouse
+under the schema `omniload` and table `some_data`.
+
+```shell
+omniload ingest \
+    --source-uri 'postgresql://admin:admin@localhost:8837/web?sslmode=disable' \
+    --source-table 'public.some_data' \
+    --dest-uri 'bigquery://<your-project-name>?credentials_path=/path/to/service/account.json' \
+    --dest-table 'omniload.some_data'
+```
+
+## Handbook
+
+Please visit the [quickstart documentation], or inspect the
+list of supported [sources and destinations].
+
+## Project
+
+### Contribute
+
+Contributions are very much welcome. Please visit the [sandbox documentation]
+to learn how to spin up a development environment on your workstation and submit
+patches, or create a [ticket][Issues] to report a bug or propose a feature.
+
+### Status
+
+Breaking changes should be expected until a 1.0 release, so version pinning is
+strongly recommended, especially when using this software as a library.
+For example:
+```shell
+pip install 'omniload[full]==0.0.42'
+```
+
+### License
+
+The project is licensed under the MIT License, see the [LICENSE] file for details.
+Some components are licensed under the Apache 2.0 license, see the [NOTICE] file for details.
+
+### Acknowledgements
+
+This project would not have been possible without the amazing work by the
+authors and contributors to [SQLAlchemy], [dlt], and [ingestr], turtles all
+the way down. Kudos.
+
+
+[dlt]: https://github.com/dlt-hub/dlt
+[quickstart documentation]: https://omniload.readthedocs.io/getting-started/quickstart.html
+[ingestr]: https://bruin-data.github.io/ingestr/
+[Issues]: https://github.com/panodata/omniload/issues
+[LICENSE]: https://github.com/panodata/omniload/blob/main/LICENSE
+[NOTICE]: https://github.com/panodata/omniload/blob/main/NOTICE
+[sandbox documentation]: https://omniload.readthedocs.io/sandbox.html
+[sources and destinations]: https://omniload.readthedocs.io/supported-sources/
+[SQLAlchemy]: https://www.sqlalchemy.org/
+[uv]: https://docs.astral.sh/uv/

omniload-0.2.0/README.md

@@ -0,0 +1,95 @@
+<div align="center">
+
+# omniload
+<p>Copy data between any source and any destination.</p>
+<img src="https://github.com/panodata/omniload/blob/main/resources/demo.gif?raw=true" width="750" />
+
+[![License](https://img.shields.io/github/license/panodata/omniload.svg)](https://github.com/panodata/omniload/blob/main/LICENSE)
+[![Downloads / month](https://pepy.tech/badge/omniload/month)](https://pepy.tech/project/omniload/)
+[![Python versions](https://img.shields.io/pypi/pyversions/omniload.svg)](https://pypi.org/project/omniload/)
+
+</div>
+
+## About
+
+omniload is a polyglot data loader framework based on dlt.
+It allows you to load data from any source into any destination,
+either using a concise CLI from your shell,
+or the Python API from your own applications.
+
+omniload provides the same efficient incremental data loading modes
+inherited by [dlt]: `append`, `merge`, and `delete+insert`.
+
+## Install
+We recommend using [uv] to install or run `omniload`.
+
+```shell
+pip install uv
+uvx omniload
+```
+
+Alternatively, if you'd like to install it globally:
+```shell
+uv pip install --system omniload
+```
+
+While installation with vanilla `pip` is possible, it's an order of magnitude slower.
+
+## Synopsis
+
+The next command instructs omniload to read the table `public.some_data` from
+your PostgreSQL instance, and to write the data to your BigQuery warehouse
+under the schema `omniload` and table `some_data`.
+
+```shell
+omniload ingest \
+    --source-uri 'postgresql://admin:admin@localhost:8837/web?sslmode=disable' \
+    --source-table 'public.some_data' \
+    --dest-uri 'bigquery://<your-project-name>?credentials_path=/path/to/service/account.json' \
+    --dest-table 'omniload.some_data'
+```
+
+## Handbook
+
+Please visit the [quickstart documentation], or inspect the
+list of supported [sources and destinations].
+
+## Project
+
+### Contribute
+
+Contributions are very much welcome. Please visit the [sandbox documentation]
+to learn how to spin up a development environment on your workstation and submit
+patches, or create a [ticket][Issues] to report a bug or propose a feature.
+
+### Status
+
+Breaking changes should be expected until a 1.0 release, so version pinning is
+strongly recommended, especially when using this software as a library.
+For example:
+```shell
+pip install 'omniload[full]==0.0.42'
+```
+
+### License
+
+The project is licensed under the MIT License, see the [LICENSE] file for details.
+Some components are licensed under the Apache 2.0 license, see the [NOTICE] file for details.
+
+### Acknowledgements
+
+This project would not have been possible without the amazing work by the
+authors and contributors to [SQLAlchemy], [dlt], and [ingestr], turtles all
+the way down. Kudos.
+
+
+[dlt]: https://github.com/dlt-hub/dlt
+[quickstart documentation]: https://omniload.readthedocs.io/getting-started/quickstart.html
+[ingestr]: https://bruin-data.github.io/ingestr/
+[Issues]: https://github.com/panodata/omniload/issues
+[LICENSE]: https://github.com/panodata/omniload/blob/main/LICENSE
+[NOTICE]: https://github.com/panodata/omniload/blob/main/NOTICE
+[sandbox documentation]: https://omniload.readthedocs.io/sandbox.html
+[sources and destinations]: https://omniload.readthedocs.io/supported-sources/
+[SQLAlchemy]: https://www.sqlalchemy.org/
+[uv]: https://docs.astral.sh/uv/

omniload-0.2.0/docs/conf.py

@@ -0,0 +1,164 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "omniload"
+copyright = "2024-2026, The omniload developers"  # noqa: A001
+author = "The omniload developers"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "myst_parser",
+    "sphinx_copybutton",
+    "sphinx_design",
+    "sphinx_sitemap",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.ifconfig",
+    "sphinxcontrib.mermaid",
+    "sphinxext.opengraph",
+]
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+suppress_warnings = [
+    "myst.header",
+]
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+# Where the documentation is published to. Needed for sphinx-sitemap.
+html_baseurl = "https://omniload.readthedocs.io/"
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "furo"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+
+html_title = "omniload"
+
+html_theme_options = {
+    "sidebar_hide_name": False,
+    # https://github.com/pradyunsg/furo/blob/main/src/furo/assets/styles/variables/_colors.scss
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+html_show_sourcelink = True
+
+
+# -- Intersphinx ----------------------------------------------------------
+
+intersphinx_mapping = {
+    "crash": ("https://cratedb.com/docs/crate/crash/en/latest/", None),
+    "cloud": ("https://cratedb.com/docs/cloud/en/latest/", None),
+    "croud": ("https://cratedb.com/docs/cloud/cli/en/latest/", None),
+    "guide": ("https://cratedb.com/docs/guide/", None),
+    "influxio": ("https://influxio.readthedocs.io/", None),
+}
+linkcheck_ignore = [
+    r"https://pulse.internetsociety.org/",
+    r"https://www.g2.com/",
+    r"https://www.trustpilot.com/",
+    r"https://developer.eu.surveymonkey.com/",
+    r"https://developer.salesforce.com/",
+    r"https://developers.facebook.com/",
+    r"https://docs.indeed.com/",
+    r"https://quickbooks.intuit.com/",
+    r"https://app.asana.com/",
+    r"https://developer.ca.surveymonkey.com/",
+    r"https://docs.customer.io/",
+    r"https://personio.de/",
+    r"https://www.adjust.com/",
+    r"https://support.appsflyer.com/",
+    r"https://help.docebo.com/",
+    r"https://www.reddit.com/",
+]
+linkcheck_anchors_ignore_for_url = [
+    r"https://developers.zoom.us/",
+    r"https://docs.customer.io/",
+    r"https://github.com/",
+    r"https://support.appsflyer.com/",
+    r"https://support.axon.ai/",
+    r"https://developers.zoom.us/",
+    r"https://docs.customer.io/",
+]
+
+# Disable caching remote inventories completely.
+# http://www.sphinx-doc.org/en/stable/ext/intersphinx.html#confval-intersphinx_cache_limit
+# intersphinx_cache_limit = 0
+
+
+# -- Extension configuration -------------------------------------------------
+
+sphinx_tabs_valid_builders = ["linkcheck"]
+todo_include_todos = True
+
+# Configure sphinx-copybutton
+copybutton_remove_prompts = True
+copybutton_line_continuation_character = "\\"
+copybutton_prompt_text = (
+    r">>> |\.\.\. |\$ |sh\$ |PS> |cr> |mysql> |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: "
+)
+copybutton_prompt_is_regexp = True
+
+# Configure sphinxext-opengraph
+ogp_site_url = "https://omniload.readthedocs.io/"
+ogp_enable_meta_description = True
+# ogp_image = "http://example.org/image.png"
+# ogp_description_length = 300
+
+# Configure sphinx-sitemap
+sitemap_url_scheme = "{link}"
+sitemap_excludes = [
+    "search.html",
+    "genindex.html",
+    "_snippet/*",
+]
+
+# -- Options for MyST -------------------------------------------------
+
+myst_heading_anchors = 3
+myst_enable_extensions = [
+    "attrs_block",
+    "attrs_inline",
+    "colon_fence",
+    "deflist",
+    "fieldlist",
+    "html_admonition",
+    "html_image",
+    "linkify",
+    "replacements",
+    "strikethrough",
+    "substitution",
+    "tasklist",
+]
+myst_substitutions = {}
+myst_html_meta = {
+    "google-site-verification": "kDH2Ws762oLwf5V3l-fGrYR9-lgLF6S_Bpd4Dairh7A",
+    "msvalidate.01": "E9B1983DD903904FDB7F4E5612962EE7",
+}

omniload-0.2.0/omniload/__init__.py

@@ -0,0 +1,13 @@
+from importlib.metadata import PackageNotFoundError, version
+from unittest import mock
+
+__appname__ = "omniload"
+
+
+try:
+    __version__ = version(__appname__)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0-dev"
+
+# Turn off dlt's telemetry to tame three more requests per invocation.
+mock.patch("dlt.common.runtime.telemetry._TELEMETRY_STARTED", True)

{omniload-0.0.0.dev0 → omniload-0.2.0}/omniload/conftest.py

@@ -3,7 +3,8 @@ import tempfile
 from concurrent.futures import ThreadPoolExecutor
 
 import pytest
-from main_test import DESTINATIONS, SOURCES  # type: ignore
+
+from .main_test import DESTINATIONS, SOURCES
 
 
 def pytest_configure(config):
@@ -37,7 +38,7 @@ def start_containers(config):
 
     unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
     for container in unique_containers:
-        container.container_lock_dir = config.shared_directory
+        container.container_lock_dir = config.shared_directory  # ty: ignore[invalid-assignment]
 
     with ThreadPoolExecutor() as executor:
         for container in unique_containers: