omniload 0.1.0__tar.gz → 0.2.0__tar.gz

{omniload-0.1.0 → omniload-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omniload
-Version: 0.1.0
+Version: 0.2.0
 Summary: omniload is a polyglot data loader based on dlt. Copy data between any source and any destination.
 Author: Mustafa Ersan, Sanju Shah Gupta, Saravjeet 'Aman' Singh
 Author-email: Burak Karakan <burak.karakan@getbruin.com>
@@ -63,12 +63,12 @@ License-File: LICENSE
 License-File: LICENSE.Apache-2.0
 License-File: NOTICE
 Requires-Dist: aiohttp<4
-Requires-Dist: asana<4
-Requires-Dist: click<8.2
+Requires-Dist: asana<3.3
+Requires-Dist: click<8.5
 Requires-Dist: clickhouse-connect<1,>=0.8
 Requires-Dist: clickhouse-driver<0.3
 Requires-Dist: clickhouse-sqlalchemy<0.4
-Requires-Dist: confluent-kafka<2.13
+Requires-Dist: confluent-kafka<2.15
 Requires-Dist: couchbase<5,>=4
 Requires-Dist: databricks-sql-connector<5
 Requires-Dist: databricks-sqlalchemy<3
@@ -77,12 +77,12 @@ Requires-Dist: dlt<1.22
 Requires-Dist: dlt-cratedb<0.2
 Requires-Dist: duckdb<1.6
 Requires-Dist: duckdb-engine<0.18
-Requires-Dist: elasticsearch<9
-Requires-Dist: facebook-business<24
+Requires-Dist: elasticsearch<10
+Requires-Dist: facebook-business<26
 Requires-Dist: flatten-json<0.2
 Requires-Dist: gcsfs
-Requires-Dist: google-ads<29
-Requires-Dist: google-analytics-data<0.19
+Requires-Dist: google-ads<32
+Requires-Dist: google-analytics-data<0.24
 Requires-Dist: google-api-python-client<3
 Requires-Dist: google-auth<3
 Requires-Dist: google-cloud-bigquery-storage<3
@@ -90,13 +90,15 @@ Requires-Dist: google-cloud-spanner<4
 Requires-Dist: influxdb-client<2
 Requires-Dist: mysql-connector-python<10
 Requires-Dist: oracledb<4
+Requires-Dist: orjson<4
 Requires-Dist: pendulum<4,>=3
 Requires-Dist: psycopg2-binary<3
 Requires-Dist: py-machineid<2
-Requires-Dist: pyairtable<3
+Requires-Dist: pyairtable<4
 Requires-Dist: pyarrow<25,>=18
 Requires-Dist: pyathena<4
 Requires-Dist: pymongo<5
+Requires-Dist: pymongoarrow<2
 Requires-Dist: pymysql<2
 Requires-Dist: pyrate-limiter<4
 Requires-Dist: python-quickbooks<0.10
@@ -104,7 +106,7 @@ Requires-Dist: requests<3
 Requires-Dist: rudder-sdk-python<2.2
 Requires-Dist: s3fs
 Requires-Dist: simple-salesforce<2
-Requires-Dist: smartsheet-python-sdk<4
+Requires-Dist: smartsheet-python-sdk<5
 Requires-Dist: snowflake-connector-python<5,>=3.18
 Requires-Dist: snowflake-sqlalchemy<2
 Requires-Dist: sqlalchemy<2.1
@@ -113,19 +115,20 @@ Requires-Dist: sqlalchemy-cratedb<1
 Requires-Dist: sqlalchemy-hana<5
 Requires-Dist: sqlalchemy-redshift<2
 Requires-Dist: sqlalchemy-spanner<2
-Requires-Dist: stripe<11
+Requires-Dist: stripe<16
 Requires-Dist: tldextract<6
+Requires-Dist: toolz<2
 Requires-Dist: tqdm<5
 Requires-Dist: trino[sqlalchemy]==0.337.0
-Requires-Dist: typer<0.14
+Requires-Dist: typer<0.27
 Requires-Dist: urllib3<2.8
 Provides-Extra: develop
-Requires-Dist: mypy<1.16; extra == "develop"
 Requires-Dist: pkginfo<2; extra == "develop"
 Requires-Dist: poethepoet<1; extra == "develop"
 Requires-Dist: pyproject-fmt<3; extra == "develop"
 Requires-Dist: pytest-repeat<1; extra == "develop"
 Requires-Dist: ruff<0.16; extra == "develop"
+Requires-Dist: ty==0.0.51; extra == "develop"
 Requires-Dist: validate-pyproject<1; extra == "develop"
 Provides-Extra: full
 Requires-Dist: omniload[ibm-db,odbc,oracle]; extra == "full"
@@ -145,7 +148,7 @@ Requires-Dist: pytest<10; extra == "test"
 Requires-Dist: pytest-asyncio<2; extra == "test"
 Requires-Dist: pytest-cov<8; extra == "test"
 Requires-Dist: pytest-xdist[psutil]<4; extra == "test"
-Requires-Dist: testcontainers[mysql,postgres]<4.9; extra == "test"
+Requires-Dist: testcontainers[mysql,postgres]<4.15,>=4.9.1; extra == "test"
 Requires-Dist: types-requests<3; extra == "test"
 Requires-Dist: verlib2; extra == "test"
 Dynamic: license-file

{omniload-0.1.0 → omniload-0.2.0}/omniload/conftest.py

@@ -4,7 +4,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 import pytest
 
-from .main_test import DESTINATIONS, SOURCES  # type: ignore
+from .main_test import DESTINATIONS, SOURCES
 
 
 def pytest_configure(config):
@@ -38,7 +38,7 @@ def start_containers(config):
 
     unique_containers = set(SOURCES.values()) | set(DESTINATIONS.values())
     for container in unique_containers:
-        container.container_lock_dir = config.shared_directory
+        container.container_lock_dir = config.shared_directory  # ty: ignore[invalid-assignment]
 
     with ThreadPoolExecutor() as executor:
         for container in unique_containers:

{omniload-0.1.0 → omniload-0.2.0}/omniload/main.py

@@ -1,7 +1,7 @@
 import warnings
 from datetime import datetime
 from enum import Enum
-from typing import Optional
+from typing import Any, Dict, Optional
 
 import typer
 from typing_extensions import Annotated
@@ -92,42 +92,42 @@ def ingest(
             help="The URI of the [green]source[/green]",
             envvar=["SOURCE_URI", "OMNILOAD_SOURCE_URI"],
         ),
-    ],  # type: ignore
+    ],
     dest_uri: Annotated[
         str,
         typer.Option(
             help="The URI of the [cyan]destination[/cyan]",
             envvar=["DESTINATION_URI", "OMNILOAD_DESTINATION_URI"],
         ),
-    ],  # type: ignore
+    ],
     source_table: Annotated[
         str,
         typer.Option(
             help="The table name in the [green]source[/green] to fetch",
             envvar=["SOURCE_TABLE", "OMNILOAD_SOURCE_TABLE"],
         ),
-    ],  # type: ignore
-    dest_table: Annotated[
+    ],
+    dest_table: Annotated[  # ty: ignore[invalid-parameter-default]
         str,
         typer.Option(
             help="The table in the [cyan]destination[/cyan] to save the data into",
             envvar=["DESTINATION_TABLE", "OMNILOAD_DESTINATION_TABLE"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     incremental_key: Annotated[
         Optional[str],
         typer.Option(
             help="The incremental key from the table to be used for incremental strategies",
             envvar=["INCREMENTAL_KEY", "OMNILOAD_INCREMENTAL_KEY"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     incremental_strategy: Annotated[
         IncrementalStrategy,
         typer.Option(
             help="The incremental strategy to use",
             envvar=["INCREMENTAL_STRATEGY", "OMNILOAD_INCREMENTAL_STRATEGY"],
         ),
-    ] = IncrementalStrategy.create_replace,  # type: ignore
+    ] = IncrementalStrategy.create_replace,
     interval_start: Annotated[
         Optional[datetime],
         typer.Option(
@@ -135,7 +135,7 @@ def ingest(
             formats=DATE_FORMATS,
             envvar=["INTERVAL_START", "OMNILOAD_INTERVAL_START"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     interval_end: Annotated[
         Optional[datetime],
         typer.Option(
@@ -143,147 +143,147 @@ def ingest(
             formats=DATE_FORMATS,
             envvar=["INTERVAL_END", "OMNILOAD_INTERVAL_END"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     primary_key: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The key that will be used to deduplicate the resulting table",
             envvar=["PRIMARY_KEY", "OMNILOAD_PRIMARY_KEY"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     partition_by: Annotated[
         Optional[str],
         typer.Option(
             help="The partition key to be used for partitioning the destination table",
             envvar=["PARTITION_BY", "OMNILOAD_PARTITION_BY"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     cluster_by: Annotated[
         Optional[str],
         typer.Option(
             help="The clustering key to be used for clustering the destination table, not every destination supports clustering.",
             envvar=["CLUSTER_BY", "OMNILOAD_CLUSTER_BY"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     dry_run: Annotated[
         Optional[bool],
         typer.Option(
             help="Display data transfer plan but don't invoke it",
             envvar=["DRY_RUN", "OMNILOAD_DRY_RUN"],
         ),
-    ] = False,  # type: ignore
+    ] = False,
     full_refresh: Annotated[
         bool,
         typer.Option(
             help="Ignore the state and refresh the destination table completely",
             envvar=["FULL_REFRESH", "OMNILOAD_FULL_REFRESH"],
         ),
-    ] = False,  # type: ignore
+    ] = False,
     progress: Annotated[
         Progress,
         typer.Option(
             help="The progress display type, must be one of 'interactive', 'log'",
             envvar=["PROGRESS", "OMNILOAD_PROGRESS"],
         ),
-    ] = Progress.interactive,  # type: ignore
+    ] = Progress.interactive,
     sql_backend: Annotated[
         SqlBackend,
         typer.Option(
             help="The SQL backend to use",
             envvar=["SQL_BACKEND", "OMNILOAD_SQL_BACKEND"],
         ),
-    ] = SqlBackend.default,  # type: ignore
+    ] = SqlBackend.default,
     loader_file_format: Annotated[
         Optional[LoaderFileFormat],
         typer.Option(
             help="The file format to use when loading data",
             envvar=["LOADER_FILE_FORMAT", "OMNILOAD_LOADER_FILE_FORMAT"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     page_size: Annotated[
         Optional[int],
         typer.Option(
             help="The page size to be used when fetching data from SQL sources",
             envvar=["PAGE_SIZE", "OMNILOAD_PAGE_SIZE"],
         ),
-    ] = 50000,  # type: ignore
+    ] = 50000,
     loader_file_size: Annotated[
         Optional[int],
         typer.Option(
             help="The file size to be used by the loader to split the data into multiple files. This can be set independent of the page size, since page size is used for fetching the data from the sources whereas this is used for the processing/loading part.",
             envvar=["LOADER_FILE_SIZE", "OMNILOAD_LOADER_FILE_SIZE"],
         ),
-    ] = 100000,  # type: ignore
+    ] = 100000,
     schema_naming: Annotated[
         SchemaNaming,
         typer.Option(
             help="The naming convention to use when moving the tables from source to destination. The default behavior is explained here: https://dlthub.com/docs/general-usage/schema#naming-convention",
             envvar=["SCHEMA_NAMING", "OMNILOAD_SCHEMA_NAMING"],
         ),
-    ] = SchemaNaming.default,  # type: ignore
+    ] = SchemaNaming.default,
     pipelines_dir: Annotated[
         Optional[str],
         typer.Option(
             help="The path to store dlt-related pipeline metadata. By default, omniload will create a temporary directory and delete it after the execution is done in order to make retries stateless.",
             envvar=["PIPELINES_DIR", "OMNILOAD_PIPELINES_DIR"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     extract_parallelism: Annotated[
         Optional[int],
         typer.Option(
             help="The number of parallel jobs to run for extracting data from the source, only applicable for certain sources",
             envvar=["EXTRACT_PARALLELISM", "OMNILOAD_EXTRACT_PARALLELISM"],
         ),
-    ] = 5,  # type: ignore
+    ] = 5,
     sql_reflection_level: Annotated[
         SqlReflectionLevel,
         typer.Option(
             help="The reflection level to use when reflecting the table schema from the source",
             envvar=["SQL_REFLECTION_LEVEL", "OMNILOAD_SQL_REFLECTION_LEVEL"],
         ),
-    ] = SqlReflectionLevel.full,  # type: ignore
+    ] = SqlReflectionLevel.full,
     sql_limit: Annotated[
         Optional[int],
         typer.Option(
             help="The limit to use when fetching data from the source",
             envvar=["SQL_LIMIT", "OMNILOAD_SQL_LIMIT"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     sql_exclude_columns: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The columns to exclude from the source table",
             envvar=["SQL_EXCLUDE_COLUMNS", "OMNILOAD_SQL_EXCLUDE_COLUMNS"],
         ),
-    ] = [],  # type: ignore
+    ] = [],
     columns: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The column types to be used for the destination table in the format of 'column_name:column_type'",
             envvar=["OMNILOAD_COLUMNS"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     yield_limit: Annotated[
         Optional[int],
         typer.Option(
             help="Limit the number of pages yielded from the source",
             envvar=["YIELD_LIMIT", "OMNILOAD_YIELD_LIMIT"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     staging_bucket: Annotated[
         Optional[str],
         typer.Option(
             help="The staging bucket to be used for the ingestion, must be prefixed with 'gs://' or 's3://'",
             envvar=["STAGING_BUCKET", "OMNILOAD_STAGING_BUCKET"],
         ),
-    ] = None,  # type: ignore
+    ] = None,
     mask: Annotated[
         Optional[list[str]],
         typer.Option(
             help="Column masking configuration in format 'column:algorithm[:param]'. Can be specified multiple times.",
             envvar=["MASK", "OMNILOAD_MASK"],
         ),
-    ] = [],  # type: ignore
+    ] = [],
 ):
     import hashlib
     import tempfile
@@ -471,12 +471,12 @@ def ingest(
 
                 column_hints[key]["primary_key"] = True
 
-        pipeline = dlt.pipeline(  # type: ignore
+        pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),
             destination=dlt_dest,
             progress=progressInstance,
             pipelines_dir=pipelines_dir,
-            refresh="drop_resources" if full_refresh else None,
+            refresh="drop_resources" if full_refresh else None,  # ty: ignore[invalid-argument-type]
         )
 
         if source.handles_incrementality():
@@ -532,9 +532,9 @@ def ingest(
             # In order to eliminate this, we have introduced a new option to omniload, --columns, which allows the user to specify the column types for the destination table.
             # This way, omniload will know the data type of the incremental field, and will be able to convert the start and end dates to the correct data type before running the ingestion.
             if interval_start:
-                interval_start = interval_start.date()  # type: ignore
+                interval_start = interval_start.date()  # ty: ignore[invalid-assignment]
             if interval_end:
-                interval_end = interval_end.date()  # type: ignore
+                interval_end = interval_end.date()  # ty: ignore[invalid-assignment]
 
         if factory.source_scheme.startswith("spanner"):
             # we tend to use the 'pyarrow' backend in general, however, it has issues with JSON objects, so we override it to 'sqlalchemy' for Spanner.
@@ -608,7 +608,7 @@ def ingest(
             engine_type = ClickhouseDestination.engine_type(dest_uri)
 
             def apply_clickhouse_adapter(x):
-                kwargs = {"settings": settings}
+                kwargs: Dict[str, Any] = {"settings": settings}
                 if engine_type:
                     kwargs["table_engine_type"] = engine_type
                 clickhouse_adapter(x, **kwargs)
@@ -656,13 +656,13 @@ def ingest(
                     table=dest_table,
                     staging_bucket=staging_bucket,
                 ),
-                write_disposition=write_disposition,  # type: ignore
+                write_disposition=write_disposition,
                 primary_key=(
                     primary_key if primary_key and len(primary_key) > 0 else None
-                ),  # type: ignore
+                ),
                 loader_file_format=(
-                    loader_file_format.value if loader_file_format is not None else None  # type: ignore
-                ),  # type: ignore
+                    loader_file_format.value if loader_file_format is not None else None
+                ),
             )
 
         # Databricks concurrency error patterns that are safe to retry
@@ -795,7 +795,7 @@ def example_uris():
 
 @app.command()
 def version():
-    from omniload import __version__  # type: ignore
+    from omniload import __version__
 
     print(f"v{__version__}")