ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/factory.py

@@ -7,54 +7,108 @@ from ingestr.src.destinations import (
     AthenaDestination,
     BigQueryDestination,
     ClickhouseDestination,
+    CrateDBDestination,
     CsvDestination,
     DatabricksDestination,
     DuckDBDestination,
+    ElasticsearchDestination,
+    GCSDestination,
+    MongoDBDestination,
+    MotherduckDestination,
     MsSQLDestination,
+    MySqlDestination,
     PostgresDestination,
     RedshiftDestination,
+    S3Destination,
     SnowflakeDestination,
+    SqliteDestination,
     SynapseDestination,
+    TrinoDestination,
 )
 from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
+    AlliumSource,
+    AnthropicSource,
     AppleAppStoreSource,
+    ApplovinMaxSource,
+    AppLovinSource,
     AppsflyerSource,
     ArrowMemoryMappedSource,
     AsanaSource,
+    AttioSource,
     ChessSource,
+    ClickupSource,
+    CouchbaseSource,
+    CursorSource,
+    DoceboSource,
     DynamoDBSource,
+    ElasticsearchSource,
     FacebookAdsSource,
+    FluxxSource,
+    FrankfurterSource,
+    FreshdeskSource,
+    FundraiseupSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HostawaySource,
+    HttpSource,
     HubspotSource,
+    InfluxDBSource,
+    IntercomSource,
+    IsocPulseSource,
+    JiraSource,
     KafkaSource,
+    KinesisSource,
     KlaviyoSource,
+    LinearSource,
     LinkedInAdsSource,
     LocalCsvSource,
+    MailchimpSource,
+    MixpanelSource,
+    MondaySource,
     MongoDbSource,
     NotionSource,
+    PersonioSource,
+    PhantombusterSource,
+    PinterestSource,
+    PipedriveSource,
+    PlusVibeAISource,
+    QuickBooksSource,
+    RevenueCatSource,
     S3Source,
+    SalesforceSource,
+    SFTPSource,
     ShopifySource,
     SlackSource,
+    SmartsheetSource,
+    SnapchatAdsSource,
+    SocrataSource,
+    SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
+    TrustpilotSource,
+    WiseSource,
     ZendeskSource,
+    ZoomSource,
 )
 
 SQL_SOURCE_SCHEMES = [
     "bigquery",
+    "crate",
     "duckdb",
     "mssql",
+    "mssql+pyodbc",
     "mysql",
     "mysql+pymysql",
     "mysql+mysqlconnector",
+    "md",
+    "motherduck",
     "postgres",
     "postgresql",
     "postgresql+psycopg2",
@@ -64,7 +118,13 @@ SQL_SOURCE_SCHEMES = [
     "sqlite",
     "oracle",
     "oracle+cx_oracle",
+    "oracle+oracledb",
     "hana",
+    "clickhouse",
+    "databricks",
+    "db2",
+    "spanner",
+    "trino",
 ]
 
 
@@ -103,8 +163,16 @@ class SourceDestinationFactory:
     source_scheme: str
     destination_scheme: str
     sources: Dict[str, Type[SourceProtocol]] = {
+        "allium": AlliumSource,
+        "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
+        "couchbase": CouchbaseSource,
+        "cursor": CursorSource,
+        "docebo": DoceboSource,
+        "http": HttpSource,
+        "https": HttpSource,
         "mongodb": MongoDbSource,
+        "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,
@@ -113,10 +181,15 @@ class SourceDestinationFactory:
         "chess": ChessSource,
         "stripe": StripeAnalyticsSource,
         "facebookads": FacebookAdsSource,
+        "fluxx": FluxxSource,
         "slack": SlackSource,
+        "hostaway": HostawaySource,
         "hubspot": HubspotSource,
+        "intercom": IntercomSource,
+        "jira": JiraSource,
         "airtable": AirtableSource,
         "klaviyo": KlaviyoSource,
+        "mixpanel": MixpanelSource,
         "appsflyer": AppsflyerSource,
         "kafka": KafkaSource,
         "adjust": AdjustSource,
@@ -131,11 +204,44 @@ class SourceDestinationFactory:
         "appstore": AppleAppStoreSource,
         "gs": GCSSource,
         "linkedinads": LinkedInAdsSource,
+        "linear": LinearSource,
+        "applovin": AppLovinSource,
+        "applovinmax": ApplovinMaxSource,
+        "salesforce": SalesforceSource,
+        "personio": PersonioSource,
+        "kinesis": KinesisSource,
+        "pipedrive": PipedriveSource,
+        "frankfurter": FrankfurterSource,
+        "freshdesk": FreshdeskSource,
+        "fundraiseup": FundraiseupSource,
+        "trustpilot": TrustpilotSource,
+        "phantombuster": PhantombusterSource,
+        "elasticsearch": ElasticsearchSource,
+        "attio": AttioSource,
+        "solidgate": SolidgateSource,
+        "quickbooks": QuickBooksSource,
+        "isoc-pulse": IsocPulseSource,
+        "smartsheet": SmartsheetSource,
+        "sftp": SFTPSource,
+        "pinterest": PinterestSource,
+        "revenuecat": RevenueCatSource,
+        "socrata": SocrataSource,
+        "snapchatads": SnapchatAdsSource,
+        "zoom": ZoomSource,
+        "clickup": ClickupSource,
+        "influxdb": InfluxDBSource,
+        "wise": WiseSource,
+        "plusvibeai": PlusVibeAISource,
+        "monday": MondaySource,
+        "mailchimp": MailchimpSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
+        "cratedb": CrateDBDestination,
         "databricks": DatabricksDestination,
         "duckdb": DuckDBDestination,
+        "motherduck": MotherduckDestination,
+        "md": MotherduckDestination,
         "mssql": MsSQLDestination,
         "postgres": PostgresDestination,
         "postgresql": PostgresDestination,
@@ -149,12 +255,20 @@ class SourceDestinationFactory:
         "athena": AthenaDestination,
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
+        "elasticsearch": ElasticsearchDestination,
+        "mongodb": MongoDBDestination,
+        "mongodb+srv": MongoDBDestination,
+        "s3": S3Destination,
+        "gs": GCSDestination,
+        "sqlite": SqliteDestination,
+        "mysql": MySqlDestination,
+        "mysql+pymysql": MySqlDestination,
+        "trino": TrinoDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):
         self.source_uri = source_uri
-        source_fields = urlparse(source_uri)
-        self.source_scheme = source_fields.scheme
+        self.source_scheme = parse_scheme_from_uri(source_uri)
 
         self.destination_uri = destination_uri
         self.destination_scheme = parse_scheme_from_uri(destination_uri)

ingestr/src/filesystem/__init__.py

@@ -37,9 +37,14 @@ def readers(
         file_glob (str, optional): The filter to apply to the files in glob format. by default lists all files in bucket_url non-recursively
     """
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
-    filesystem_resource.apply_hints(
-        incremental=dlt.sources.incremental("modification_date"),
-    )
+
+    # NOTE: incremental support is disabled until we can figure out
+    #       how to support incremental loads per matching file, rather
+    #       than a blanket threshold.
+    #
+    # filesystem_resource.apply_hints(
+    #     incremental=dlt.sources.incremental("modification_date"),
+    # )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),
         filesystem_resource | dlt.transformer(name="read_jsonl")(_read_jsonl),

ingestr/src/filters.py

@@ -1,6 +1,3 @@
-from dlt.common.libs.sql_alchemy import Table
-
-
 def cast_set_to_list(row):
     # this handles just the sqlalchemy backend for now
     if isinstance(row, dict):
@@ -10,10 +7,56 @@ def cast_set_to_list(row):
     return row
 
 
+def cast_spanner_types(row):
+    if not isinstance(row, dict):
+        return row
+
+    from google.cloud.spanner_v1.data_types import JsonObject
+
+    for key in row.keys():
+        if isinstance(row[key], JsonObject):
+            import json
+
+            row[key] = json.loads(row[key].serialize())
+    return row
+
+
+def handle_mysql_empty_dates(row):
+    # MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
+    if not isinstance(row, dict):
+        return row
+
+    for key in row.keys():
+        if not isinstance(row[key], str):
+            continue
+
+        if row[key] == "0000-00-00":
+            from datetime import date
+
+            row[key] = date(1970, 1, 1)
+
+        elif row[key] == "0000-00-00 00:00:00":
+            from datetime import datetime
+
+            row[key] = datetime(1970, 1, 1, 0, 0, 0)
+    return row
+
+
 def table_adapter_exclude_columns(cols: list[str]):
+    from dlt.common.libs.sql_alchemy import Table
+
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:
             table._columns.remove(col)  # type: ignore
 
     return excluder
+
+
+def create_masking_filter(mask_configs: list[str]):
+    from ingestr.src.masking import create_masking_mapper
+
+    if not mask_configs:
+        return lambda x: x
+
+    return create_masking_mapper(mask_configs)