ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/facebook_ads/__init__.py

@@ -22,13 +22,8 @@ from .settings import (
     DEFAULT_ADCREATIVE_FIELDS,
     DEFAULT_ADSET_FIELDS,
     DEFAULT_CAMPAIGN_FIELDS,
-    DEFAULT_INSIGHT_FIELDS,
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
-    INSIGHTS_BREAKDOWNS_OPTIONS,
-    INSIGHTS_PRIMARY_KEY,
-    INVALID_INSIGHTS_FIELDS,
-    TInsightsBreakdownOptions,
     TInsightsLevels,
 )
 
@@ -106,10 +101,9 @@ def facebook_insights_source(
     account_id: str = dlt.config.value,
     access_token: str = dlt.secrets.value,
     initial_load_past_days: int = 1,
-    fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
-    attribution_window_days_lag: int = 7,
+    dimensions: Sequence[str] = None,
+    fields: Sequence[str] = None,
     time_increment_days: int = 1,
-    breakdowns: TInsightsBreakdownOptions = "ads_insights",
     action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
     level: TInsightsLevels = "ad",
     action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
@@ -118,6 +112,9 @@ def facebook_insights_source(
     app_api_version: str = None,
     start_date: pendulum.DateTime | None = None,
     end_date: pendulum.DateTime | None = None,
+    insights_max_wait_to_finish_seconds: int = 60 * 60 * 4,
+    insights_max_wait_to_start_seconds: int = 60 * 30,
+    insights_max_async_sleep_seconds: int = 20,
 ) -> DltResource:
     """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
 
@@ -153,14 +150,19 @@ def facebook_insights_source(
     if start_date is None:
         start_date = pendulum.today().subtract(days=initial_load_past_days)
 
+    if dimensions is None:
+        dimensions = []
+    if fields is None:
+        fields = []
+
     columns = {}
     for field in fields:
         if field in INSIGHT_FIELDS_TYPES:
             columns[field] = INSIGHT_FIELDS_TYPES[field]
 
     @dlt.resource(
-        primary_key=INSIGHTS_PRIMARY_KEY,
         write_disposition="merge",
+        merge_key="date_start",
         columns=columns,
     )
     def facebook_insights(
@@ -175,25 +177,24 @@ def facebook_insights_source(
         ),
     ) -> Iterator[TDataItems]:
         start_date = date_start.last_value
-        end_date = (
-            pendulum.instance(date_start.end_value)
-            if date_start.end_value
-            else pendulum.now()
-        )
+        if date_start.end_value:
+            end_date_val = pendulum.instance(date_start.end_value)
+
+            end_date = (
+                end_date_val
+                if isinstance(end_date_val, pendulum.Date)
+                else end_date_val.date()
+            )
+        else:
+            end_date = pendulum.now().date()
 
         while start_date <= end_date:
             query = {
                 "level": level,
                 "action_breakdowns": list(action_breakdowns),
-                "breakdowns": list(
-                    INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
-                ),
+                "breakdowns": dimensions,
                 "limit": batch_size,
-                "fields": list(
-                    set(fields)
-                    .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
-                    .difference(INVALID_INSIGHTS_FIELDS)
-                ),
+                "fields": fields,
                 "time_increment": time_increment_days,
                 "action_attribution_windows": list(action_attribution_windows),
                 "time_ranges": [
@@ -207,7 +208,9 @@ def facebook_insights_source(
             }
             job = execute_job(
                 account.get_insights(params=query, is_async=True),
-                insights_max_async_sleep_seconds=20,
+                insights_max_async_sleep_seconds=insights_max_async_sleep_seconds,
+                insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds,
+                insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds,
             )
             output = list(map(process_report_item, job.get_result()))
             yield output

ingestr/src/facebook_ads/helpers.py

@@ -144,7 +144,7 @@ def execute_job(
             raise InsightsJobTimeout(
                 "facebook_insights",
                 pretty_error_message.format(
-                    job_id, insights_max_wait_to_finish_seconds // 60
+                    job_id, insights_max_wait_to_finish_seconds
                 ),
             )
 
@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
             logger.error(
                 f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
             )
+
+
+def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
+    import typing
+
+    from ingestr.src.facebook_ads.settings import (
+        INSIGHTS_BREAKDOWNS_OPTIONS,
+        TInsightsBreakdownOptions,
+        TInsightsLevels,
+    )
+
+    parts = table.split(":")
+
+    source_kwargs = {}
+
+    breakdown_type = parts[1]
+
+    valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
+    if breakdown_type in valid_breakdowns:
+        dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
+        fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
+        source_kwargs["dimensions"] = dimensions
+        source_kwargs["fields"] = fields
+    else:
+        dimensions = breakdown_type.split(",")
+        valid_levels = list(typing.get_args(TInsightsLevels))
+        level = None
+        for valid_level in reversed(valid_levels):
+            if valid_level in dimensions:
+                level = valid_level
+                dimensions.remove(valid_level)
+                break
+
+        source_kwargs["level"] = level
+        source_kwargs["dimensions"] = dimensions
+
+    # If custom metrics are provided, parse them
+    if len(parts) == 3:
+        fields = [f.strip() for f in parts[2].split(",") if f.strip()]
+        if not fields:
+            raise ValueError(
+                "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
+            )
+        source_kwargs["fields"] = fields
+
+    return source_kwargs

ingestr/src/factory.py

@@ -11,7 +11,10 @@ from ingestr.src.destinations import (
     CsvDestination,
     DatabricksDestination,
     DuckDBDestination,
+    ElasticsearchDestination,
     GCSDestination,
+    MongoDBDestination,
+    MotherduckDestination,
     MsSQLDestination,
     MySqlDestination,
     PostgresDestination,
@@ -20,10 +23,13 @@ from ingestr.src.destinations import (
     SnowflakeDestination,
     SqliteDestination,
     SynapseDestination,
+    TrinoDestination,
 )
 from ingestr.src.sources import (
     AdjustSource,
     AirtableSource,
+    AlliumSource,
+    AnthropicSource,
     AppleAppStoreSource,
     ApplovinMaxSource,
     AppLovinSource,
@@ -33,45 +39,60 @@ from ingestr.src.sources import (
     AttioSource,
     ChessSource,
     ClickupSource,
+    CouchbaseSource,
+    CursorSource,
+    DoceboSource,
     DynamoDBSource,
     ElasticsearchSource,
     FacebookAdsSource,
+    FluxxSource,
     FrankfurterSource,
     FreshdeskSource,
+    FundraiseupSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HostawaySource,
+    HttpSource,
     HubspotSource,
     InfluxDBSource,
+    IntercomSource,
     IsocPulseSource,
+    JiraSource,
     KafkaSource,
     KinesisSource,
     KlaviyoSource,
     LinearSource,
     LinkedInAdsSource,
     LocalCsvSource,
+    MailchimpSource,
     MixpanelSource,
+    MondaySource,
     MongoDbSource,
     NotionSource,
     PersonioSource,
     PhantombusterSource,
     PinterestSource,
     PipedriveSource,
+    PlusVibeAISource,
     QuickBooksSource,
+    RevenueCatSource,
     S3Source,
     SalesforceSource,
     SFTPSource,
     ShopifySource,
     SlackSource,
     SmartsheetSource,
+    SocrataSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
     TrustpilotSource,
+    WiseSource,
     ZendeskSource,
     ZoomSource,
 )
@@ -85,6 +106,8 @@ SQL_SOURCE_SCHEMES = [
     "mysql",
     "mysql+pymysql",
     "mysql+mysqlconnector",
+    "md",
+    "motherduck",
     "postgres",
     "postgresql",
     "postgresql+psycopg2",
@@ -99,6 +122,7 @@ SQL_SOURCE_SCHEMES = [
     "databricks",
     "db2",
     "spanner",
+    "trino",
 ]
 
 
@@ -137,7 +161,14 @@ class SourceDestinationFactory:
     source_scheme: str
     destination_scheme: str
     sources: Dict[str, Type[SourceProtocol]] = {
+        "allium": AlliumSource,
+        "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
+        "couchbase": CouchbaseSource,
+        "cursor": CursorSource,
+        "docebo": DoceboSource,
+        "http": HttpSource,
+        "https": HttpSource,
         "mongodb": MongoDbSource,
         "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
@@ -148,8 +179,12 @@ class SourceDestinationFactory:
         "chess": ChessSource,
         "stripe": StripeAnalyticsSource,
         "facebookads": FacebookAdsSource,
+        "fluxx": FluxxSource,
         "slack": SlackSource,
+        "hostaway": HostawaySource,
         "hubspot": HubspotSource,
+        "intercom": IntercomSource,
+        "jira": JiraSource,
         "airtable": AirtableSource,
         "klaviyo": KlaviyoSource,
         "mixpanel": MixpanelSource,
@@ -176,6 +211,7 @@ class SourceDestinationFactory:
         "pipedrive": PipedriveSource,
         "frankfurter": FrankfurterSource,
         "freshdesk": FreshdeskSource,
+        "fundraiseup": FundraiseupSource,
         "trustpilot": TrustpilotSource,
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
@@ -186,15 +222,23 @@ class SourceDestinationFactory:
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
+        "revenuecat": RevenueCatSource,
+        "socrata": SocrataSource,
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
+        "wise": WiseSource,
+        "plusvibeai": PlusVibeAISource,
+        "monday": MondaySource,
+        "mailchimp": MailchimpSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,
         "cratedb": CrateDBDestination,
         "databricks": DatabricksDestination,
         "duckdb": DuckDBDestination,
+        "motherduck": MotherduckDestination,
+        "md": MotherduckDestination,
         "mssql": MsSQLDestination,
         "postgres": PostgresDestination,
         "postgresql": PostgresDestination,
@@ -208,11 +252,15 @@ class SourceDestinationFactory:
         "athena": AthenaDestination,
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
+        "elasticsearch": ElasticsearchDestination,
+        "mongodb": MongoDBDestination,
+        "mongodb+srv": MongoDBDestination,
         "s3": S3Destination,
         "gs": GCSDestination,
         "sqlite": SqliteDestination,
         "mysql": MySqlDestination,
         "mysql+pymysql": MySqlDestination,
+        "trino": TrinoDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

ingestr/src/filesystem/__init__.py

@@ -37,9 +37,14 @@ def readers(
         file_glob (str, optional): The filter to apply to the files in glob format. by default lists all files in bucket_url non-recursively
     """
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
-    filesystem_resource.apply_hints(
-        incremental=dlt.sources.incremental("modification_date"),
-    )
+
+    # NOTE: incremental support is disabled until we can figure out
+    #       how to support incremental loads per matching file, rather
+    #       than a blanket threshold.
+    #
+    # filesystem_resource.apply_hints(
+    #     incremental=dlt.sources.incremental("modification_date"),
+    # )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),
         filesystem_resource | dlt.transformer(name="read_jsonl")(_read_jsonl),

ingestr/src/filters.py

@@ -51,3 +51,12 @@ def table_adapter_exclude_columns(cols: list[str]):
             table._columns.remove(col)  # type: ignore
 
     return excluder
+
+
+def create_masking_filter(mask_configs: list[str]):
+    from ingestr.src.masking import create_masking_mapper
+
+    if not mask_configs:
+        return lambda x: x
+
+    return create_masking_mapper(mask_configs)