omniload 0.3.0__tar.gz → 0.4.0__tar.gz

omniload-0.4.0/NOTICE

@@ -0,0 +1,35 @@
+NOTICE
+
+This project contains code from the following Apache 2.0 licensed project:
+
+dlt-hub/verified-sources
+https://github.com/dlt-hub/verified-sources
+Copyright 2022-2025 ScaleVector
+Licensed under the Apache License, Version 2.0
+
+The following directories contain code from dlt-hub:
+- omniload/source/airtable/
+- omniload/source/asana/
+- omniload/source/chess/
+- omniload/source/facebook_ads/
+- omniload/source/filesystem/
+- omniload/source/freshdesk/
+- omniload/source/github/
+- omniload/source/google_ads/
+- omniload/source/google_analytics/
+- omniload/source/google_sheets/
+- omniload/source/hubspot/
+- omniload/source/jira/
+- omniload/source/kafka/
+- omniload/source/kinesis/
+- omniload/source/mongodb/
+- omniload/source/notion/
+- omniload/source/personio/
+- omniload/source/pipedrive/
+- omniload/source/salesforce/
+- omniload/source/shopify/
+- omniload/source/slack/
+- omniload/source/stripe/
+- omniload/source/zendesk/
+
+The Apache License 2.0 can be found in the LICENSE.Apache-2.0 file.

{omniload-0.3.0 → omniload-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omniload
-Version: 0.3.0
+Version: 0.4.0
 Summary: omniload is a polyglot data loader based on dlt. Copy data between any source and any destination.
 Author: Mustafa Ersan, Sanju Shah Gupta, Saravjeet 'Aman' Singh
 Author-email: Burak Karakan <burak.karakan@getbruin.com>
@@ -128,7 +128,7 @@ Requires-Dist: poethepoet<1; extra == "develop"
 Requires-Dist: pyproject-fmt<3; extra == "develop"
 Requires-Dist: pytest-repeat<1; extra == "develop"
 Requires-Dist: ruff<0.16; extra == "develop"
-Requires-Dist: ty==0.0.52; extra == "develop"
+Requires-Dist: ty==0.0.53; extra == "develop"
 Requires-Dist: validate-pyproject<1; extra == "develop"
 Provides-Extra: full
 Requires-Dist: omniload[ibm-db,odbc,oracle]; extra == "full"

{omniload-0.3.0 → omniload-0.4.0}/omniload/__init__.py

@@ -10,7 +10,7 @@ from omniload.api import (
     SqlReflectionLevel,
     run_ingest,
 )
-from omniload.src.errors import IngestJobError, ValidationError
+from omniload.error import IngestJobError, ValidationError
 
 __appname__ = "omniload"
 
@@ -21,7 +21,7 @@ except PackageNotFoundError:  # pragma: no cover
     __version__ = "0.0.0-dev"
 
 # Turn off dlt's telemetry to tame three more requests per invocation.
-mock.patch("dlt.common.runtime.telemetry._TELEMETRY_STARTED", True)
+mock.patch("dlt.common.runtime.telemetry._TELEMETRY_STARTED", True).start()
 
 __all__ = [
     "IncrementalStrategy",

{omniload-0.3.0 → omniload-0.4.0}/omniload/api.py

@@ -16,7 +16,7 @@ from datetime import datetime
 from enum import Enum
 from typing import TYPE_CHECKING
 
-from omniload.src.errors import IngestJobError, ValidationError
+from omniload.error import IngestJobError, ValidationError
 
 if TYPE_CHECKING:
     from dlt.common.pipeline import LoadInfo
@@ -145,18 +145,19 @@ def run_ingest(
     from dlt.common.schema.typing import TColumnSchema
     from dlt.pipeline.exceptions import PipelineStepFailed
 
-    import omniload.src.partition as partition
-    import omniload.src.resource as resource
-    from omniload.src.collector.spinner import SpinnerCollector
-    from omniload.src.destinations import AthenaDestination, ClickhouseDestination
-    from omniload.src.factory import SourceDestinationFactory
-    from omniload.src.filters import (
+    import omniload.core.resource as resource
+    from omniload.codec import hint
+    from omniload.codec.filter import (
         cast_set_to_list,
         cast_spanner_types,
         create_masking_filter,
         handle_mysql_empty_dates,
     )
-    from omniload.src.sources import MongoDbSource
+    from omniload.core.factory import SourceDestinationFactory
+    from omniload.source.mongodb.api import MongoDbSource
+    from omniload.target.athena import AthenaDestination
+    from omniload.target.clickhouse import ClickhouseDestination
+    from omniload.util.spinner import SpinnerCollector
 
     incremental_strategy = _coerce(incremental_strategy, IncrementalStrategy)
     progress = _coerce(progress, Progress)
@@ -428,7 +429,7 @@ def run_ingest(
         # https://github.com/dlt-hub/dlt/issues/2248
         # TODO(turtledev): only apply for write dispositions that actually cause an exception.
         # TODO(turtledev): make batch size configurable
-        import omniload.src.arrow as arrow
+        import omniload.source.arrow.adapter as arrow
 
         resource.for_each(dlt_source, lambda x: x.add_map(arrow.as_list))
 
@@ -440,7 +441,7 @@ def run_ingest(
         resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
 
     if isinstance(source, MongoDbSource):
-        from omniload.src.resource import TypeHintMap
+        from omniload.core.resource import TypeHintMap
 
         resource.for_each(dlt_source, lambda x: x.add_map(TypeHintMap().type_hint_map))
 
@@ -451,7 +452,7 @@ def run_ingest(
     resource.for_each(dlt_source, col_h)
 
     if isinstance(destination, AthenaDestination) and partition_by:
-        partition.apply_athena_hints(dlt_source, partition_by, column_hints)
+        hint.apply_athena_hints(dlt_source, partition_by, column_hints)
 
     if isinstance(destination, ClickhouseDestination):
         from dlt.destinations.adapters import clickhouse_adapter

omniload-0.3.0/omniload/src/filters.py → omniload-0.4.0/omniload/codec/filter.py

@@ -54,7 +54,7 @@ def table_adapter_exclude_columns(cols: list[str]):
 
 
 def create_masking_filter(mask_configs: list[str]):
-    from omniload.src.masking import create_masking_mapper
+    from omniload.codec.masking import create_masking_mapper
 
     if not mask_configs:
         return lambda x: x

omniload-0.3.0/omniload/src/partition.py → omniload-0.4.0/omniload/codec/hint.py

@@ -3,7 +3,7 @@ from typing import Dict
 from dlt.common.schema.typing import TColumnSchema
 from dlt.sources import DltResource, DltSource
 
-import omniload.src.resource as resource
+import omniload.core.resource as resource
 
 
 def apply_athena_hints(

omniload-0.4.0/omniload/core/factory.py

@@ -0,0 +1,51 @@
+from urllib.parse import urlparse
+
+from omniload.core.model import DestinationProtocol, LazyRegistry, SourceProtocol
+from omniload.core.registry import (
+    SQL_SOURCE_SCHEMES,
+    destinations,
+    sources,
+)
+from omniload.core.router import SqlSourceRouter
+
+
+def parse_scheme_from_uri(uri: str) -> str:
+    parsed = urlparse(uri)
+    if parsed.scheme != "":
+        return parsed.scheme
+
+    uri_parts = uri.split("://")
+    if len(uri_parts) > 1:
+        return uri_parts[0]
+
+    raise ValueError(f"Could not parse scheme from uri: {uri}")
+
+
+class SourceDestinationFactory:
+    source_scheme: str
+    destination_scheme: str
+    sources: LazyRegistry = sources
+    destinations: LazyRegistry = destinations
+
+    def __init__(self, source_uri: str, destination_uri: str):
+        self.source_uri = source_uri
+        self.source_scheme = parse_scheme_from_uri(source_uri)
+
+        self.destination_uri = destination_uri
+        self.destination_scheme = parse_scheme_from_uri(destination_uri)
+
+    def get_source(self) -> SourceProtocol:
+        if self.source_scheme in SQL_SOURCE_SCHEMES:
+            return SqlSourceRouter()
+        elif self.source_scheme in self.sources:
+            return self.sources[self.source_scheme]()
+        else:
+            raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
+
+    def get_destination(self) -> DestinationProtocol:
+        if self.destination_scheme in self.destinations:
+            return self.destinations[self.destination_scheme]()
+        else:
+            raise ValueError(
+                f"Unsupported destination scheme: {self.destination_scheme}"
+            )

omniload-0.4.0/omniload/core/model.py

@@ -0,0 +1,68 @@
+import importlib
+import typing
+from dataclasses import dataclass
+from typing import Any, Iterator, Mapping, Protocol
+
+if typing.TYPE_CHECKING:
+    from dlt.common.destination import Destination
+
+
+class SourceProtocol(Protocol):
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        pass
+
+    def handles_incrementality(self) -> bool:
+        pass
+
+
+class DestinationProtocol(Protocol):
+    def dlt_dest(self, uri: str, **kwargs) -> "Destination":
+        pass
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        pass
+
+    def post_load(self) -> None:
+        pass
+
+
+@dataclass
+class TableDefinition:
+    dataset: str
+    table: str
+
+
+def table_string_to_dataclass(table: str) -> TableDefinition:
+    table_fields = table.split(".", 1)
+    if len(table_fields) != 2:
+        raise ValueError("Table name must be in the format <schema>.<table>")
+
+    return TableDefinition(dataset=table_fields[0], table=table_fields[1])
+
+
+class LazyRegistry(Mapping):
+    """Mapping that imports a connector class only when its scheme is first accessed."""
+
+    def __init__(self, paths: dict[str, str]) -> None:
+        self._paths = paths
+        self._cache: dict[str, Any] = {}
+
+    def _load(self, dotted_path: str) -> Any:
+        module_path, cls_name = dotted_path.rsplit(":", 1)
+        return getattr(importlib.import_module(module_path), cls_name)
+
+    def __getitem__(self, key: str) -> Any:
+        if key not in self._cache:
+            if key not in self._paths:
+                raise KeyError(key)
+            self._cache[key] = self._load(self._paths[key])
+        return self._cache[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._paths)
+
+    def __len__(self) -> int:
+        return len(self._paths)
+
+    def __contains__(self, key: object) -> bool:  # O(1), no import triggered
+        return key in self._paths

omniload-0.4.0/omniload/core/registry.py

@@ -0,0 +1,147 @@
+from omniload.core.model import LazyRegistry
+
+SQL_SOURCE_SCHEMES = [
+    "bigquery",
+    "crate",
+    "cratedb",
+    "duckdb",
+    "mssql",
+    "mssql+pyodbc",
+    "mysql",
+    "mysql+pymysql",
+    "mysql+mysqlconnector",
+    "md",
+    "motherduck",
+    "postgres",
+    "postgresql",
+    "postgresql+psycopg2",
+    "redshift",
+    "redshift+psycopg2",
+    "snowflake",
+    "sqlite",
+    "oracle",
+    "oracle+cx_oracle",
+    "oracle+oracledb",
+    "hana",
+    "clickhouse",
+    "databricks",
+    "db2",
+    "spanner",
+    "trino",
+]
+
+sources: LazyRegistry = LazyRegistry(
+    {
+        "adjust": "omniload.source.adjust.api:AdjustSource",
+        "airtable": "omniload.source.airtable.api:AirtableSource",
+        "allium": "omniload.source.allium.api:AlliumSource",
+        "anthropic": "omniload.source.anthropic.api:AnthropicSource",
+        "applovin": "omniload.source.applovin.api:AppLovinSource",
+        "applovinmax": "omniload.source.applovin_max.api:ApplovinMaxSource",
+        "appsflyer": "omniload.source.appsflyer.api:AppsflyerSource",
+        "appstore": "omniload.source.appstore.api:AppleAppStoreSource",
+        "asana": "omniload.source.asana.api:AsanaSource",
+        "attio": "omniload.source.attio.api:AttioSource",
+        "bruin": "omniload.source.bruin.api:BruinSource",
+        "chess": "omniload.source.chess.api:ChessSource",
+        "clickup": "omniload.source.clickup.api:ClickupSource",
+        "couchbase": "omniload.source.couchbase.api:CouchbaseSource",
+        "csv": "omniload.source.csv.api:LocalCsvSource",
+        "cursor": "omniload.source.cursor.api:CursorSource",
+        "customerio": "omniload.source.customer_io.api:CustomerIoSource",
+        "docebo": "omniload.source.docebo.api:DoceboSource",
+        "dune": "omniload.source.dune.api:DuneSource",
+        "dynamodb": "omniload.source.dynamodb.api:DynamoDBSource",
+        "elasticsearch": "omniload.source.elasticsearch.api:ElasticsearchSource",
+        "facebookads": "omniload.source.facebook_ads.api:FacebookAdsSource",
+        "fireflies": "omniload.source.fireflies.api:FirefliesSource",
+        "fluxx": "omniload.source.fluxx.api:FluxxSource",
+        "frankfurter": "omniload.source.frankfurter.api:FrankfurterSource",
+        "freshdesk": "omniload.source.freshdesk.api:FreshdeskSource",
+        "fundraiseup": "omniload.source.fundraiseup.api:FundraiseupSource",
+        "github": "omniload.source.github.api:GitHubSource",
+        "googleads": "omniload.source.google_ads.api:GoogleAdsSource",
+        "googleanalytics": "omniload.source.google_analytics.api:GoogleAnalyticsSource",
+        "gorgias": "omniload.source.gorgias.api:GorgiasSource",
+        "gs": "omniload.source.blobstorage.api:GCSSource",
+        "gsheets": "omniload.source.google_sheets.api:GoogleSheetsSource",
+        "hostaway": "omniload.source.hostaway.api:HostawaySource",
+        "http": "omniload.source.http.api:HttpSource",
+        "https": "omniload.source.http.api:HttpSource",
+        "hubspot": "omniload.source.hubspot.api:HubspotSource",
+        "indeed": "omniload.source.indeed.api:IndeedSource",
+        "influxdb": "omniload.source.influxdb.api:InfluxDBSource",
+        "intercom": "omniload.source.intercom.api:IntercomSource",
+        "isoc-pulse": "omniload.source.isoc_pulse.api:IsocPulseSource",
+        "jira": "omniload.source.jira.api:JiraSource",
+        "kafka": "omniload.source.kafka.api:KafkaSource",
+        "kinesis": "omniload.source.kinesis.api:KinesisSource",
+        "klaviyo": "omniload.source.klaviyo.api:KlaviyoSource",
+        "linear": "omniload.source.linear.api:LinearSource",
+        "linkedinads": "omniload.source.linkedin_ads.api:LinkedInAdsSource",
+        "mailchimp": "omniload.source.mailchimp.api:MailchimpSource",
+        "mixpanel": "omniload.source.mixpanel.api:MixpanelSource",
+        "mmap": "omniload.source.arrow.api:ArrowMemoryMappedSource",
+        "monday": "omniload.source.monday.api:MondaySource",
+        "mongodb": "omniload.source.mongodb.api:MongoDbSource",
+        "mongodb+srv": "omniload.source.mongodb.api:MongoDbSource",
+        "notion": "omniload.source.notion.api:NotionSource",
+        "personio": "omniload.source.personio.api:PersonioSource",
+        "phantombuster": "omniload.source.phantombuster.api:PhantombusterSource",
+        "pinterest": "omniload.source.pinterest.api:PinterestSource",
+        "pipedrive": "omniload.source.pipedrive.api:PipedriveSource",
+        "plusvibeai": "omniload.source.plusvibeai.api:PlusVibeAISource",
+        "primer": "omniload.source.primer.api:PrimerSource",
+        "quickbooks": "omniload.source.quickbooks.api:QuickBooksSource",
+        "redditads": "omniload.source.reddit_ads.api:RedditAdsSource",
+        "revenuecat": "omniload.source.revenuecat.api:RevenueCatSource",
+        "s3": "omniload.source.blobstorage.api:S3Source",
+        "salesforce": "omniload.source.salesforce.api:SalesforceSource",
+        "sftp": "omniload.source.sftp.api:SFTPSource",
+        "shopify": "omniload.source.shopify.api:ShopifySource",
+        "slack": "omniload.source.slack.api:SlackSource",
+        "smartsheet": "omniload.source.smartsheets.api:SmartsheetSource",
+        "snapchatads": "omniload.source.snapchat_ads.api:SnapchatAdsSource",
+        "socrata": "omniload.source.socrata.api:SocrataSource",
+        "solidgate": "omniload.source.solidgate.api:SolidgateSource",
+        "stripe": "omniload.source.stripe.api:StripeAnalyticsSource",
+        "tiktok": "omniload.source.tiktok_ads.api:TikTokSource",
+        "trustpilot": "omniload.source.trustpilot.api:TrustpilotSource",
+        "wise": "omniload.source.wise.api:WiseSource",
+        "zendesk": "omniload.source.zendesk.api:ZendeskSource",
+        "zoom": "omniload.source.zoom.api:ZoomSource",
+    }
+)
+
+destinations: LazyRegistry = LazyRegistry(
+    {
+        "athena": "omniload.target.athena:AthenaDestination",
+        "bigquery": "omniload.target.bigquery:BigQueryDestination",
+        "clickhouse": "omniload.target.clickhouse:ClickhouseDestination",
+        "clickhouse+native": "omniload.target.clickhouse:ClickhouseDestination",
+        "cratedb": "omniload.target.cratedb:CrateDBDestination",
+        "csv": "omniload.target.csv:CsvDestination",
+        "databricks": "omniload.target.databricks:DatabricksDestination",
+        "duckdb": "omniload.target.duckdb:DuckDBDestination",
+        "elasticsearch": "omniload.target.elasticsearch.api:ElasticsearchDestination",
+        "gs": "omniload.target.blobstorage:GCSDestination",
+        "md": "omniload.target.motherduck:MotherduckDestination",
+        "mongodb": "omniload.target.mongodb:MongoDBDestination",
+        "mongodb+srv": "omniload.target.mongodb:MongoDBDestination",
+        "motherduck": "omniload.target.motherduck:MotherduckDestination",
+        "mssql": "omniload.target.mssql:MsSQLDestination",
+        "mysql": "omniload.target.mysql:MySqlDestination",
+        "mysql+pymysql": "omniload.target.mysql:MySqlDestination",
+        "postgres": "omniload.target.postgresql:PostgresDestination",
+        "postgresql": "omniload.target.postgresql:PostgresDestination",
+        "postgresql+psycopg2": "omniload.target.postgresql:PostgresDestination",
+        "redshift": "omniload.target.redshift:RedshiftDestination",
+        "redshift+psycopg2": "omniload.target.redshift:RedshiftDestination",
+        "redshift+redshift_connector": "omniload.target.redshift:RedshiftDestination",
+        "s3": "omniload.target.blobstorage:S3Destination",
+        "snowflake": "omniload.target.snowflake:SnowflakeDestination",
+        "sqlite": "omniload.target.sqlite:SqliteDestination",
+        "synapse": "omniload.target.synapse:SynapseDestination",
+        "trino": "omniload.target.trino:TrinoDestination",
+    }
+)