ingestr 0.13.32__py3-none-any.whl → 0.13.34__py3-none-any.whl

ingestr/main.py

@@ -264,6 +264,13 @@ def ingest(
             envvar=["YIELD_LIMIT", "INGESTR_YIELD_LIMIT"],
         ),
     ] = None,  # type: ignore
+    staging_bucket: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The staging bucket to be used for the ingestion, must be prefixed with 'gs://' or 's3://'",
+            envvar=["STAGING_BUCKET", "INGESTR_STAGING_BUCKET"],
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -272,8 +279,6 @@ def ingest(
     import dlt
     import humanize
     import typer
-    from dlt.common.data_types import TDataType
-    from dlt.common.destination import Destination
     from dlt.common.pipeline import LoadInfo
     from dlt.common.runtime.collector import Collector, LogCollector
     from dlt.common.schema.typing import TColumnSchema
@@ -318,7 +323,7 @@ def ingest(
         return (source_table, dest_table)
 
     def validate_loader_file_format(
-        dlt_dest: Destination, loader_file_format: Optional[LoaderFileFormat]
+        dlt_dest, loader_file_format: Optional[LoaderFileFormat]
     ):
         if (
             loader_file_format
@@ -330,9 +335,11 @@ def ingest(
             )
             raise typer.Abort()
 
-    def parse_columns(columns: list[str]) -> dict[str, TDataType]:
+    def parse_columns(columns: list[str]) -> dict:
         from typing import cast, get_args
 
+        from dlt.common.data_types import TDataType
+
         possible_types = get_args(TDataType)
 
         types: dict[str, TDataType] = {}
@@ -417,7 +424,9 @@ def ingest(
             pipelines_dir = tempfile.mkdtemp()
             is_pipelines_dir_temp = True
 
-        dlt_dest = destination.dlt_dest(uri=dest_uri, dest_table=dest_table)
+        dlt_dest = destination.dlt_dest(
+            uri=dest_uri, dest_table=dest_table, staging_bucket=staging_bucket
+        )
         validate_loader_file_format(dlt_dest, loader_file_format)
 
         if partition_by:
@@ -566,6 +575,7 @@ def ingest(
             **destination.dlt_run_params(
                 uri=dest_uri,
                 table=dest_table,
+                staging_bucket=staging_bucket,
             ),
             write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore

ingestr/src/airtable/__init__.py

@@ -55,7 +55,7 @@ def airtable_resource(
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [primary_key_field["name"]]
+    primary_key: List[str] = [f"fields__{primary_key_field['name']}"]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.32"
+version = "v0.13.34"

ingestr/src/destinations.py

@@ -60,6 +60,22 @@ class BigQueryDestination:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            if not staging_bucket.startswith("gs://"):
+                raise ValueError("Staging bucket must start with gs://")
+
+            os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = staging_bucket
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID"] = (
+                credentials.get("project_id", None)
+            )
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY"] = (
+                credentials.get("private_key", None)
+            )
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL"] = (
+                credentials.get("client_email", None)
+            )
+
         project_id = None
         if source_fields.hostname:
             project_id = source_fields.hostname
@@ -83,6 +99,10 @@ class BigQueryDestination:
             "table_name": table_fields[-1],
         }
 
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            res["staging"] = "filesystem"
+
         return res
 
     def post_load(self):

ingestr/src/filters.py

@@ -1,6 +1,3 @@
-from dlt.common.libs.sql_alchemy import Table
-
-
 def cast_set_to_list(row):
     # this handles just the sqlalchemy backend for now
     if isinstance(row, dict):
@@ -32,6 +29,8 @@ def handle_mysql_empty_dates(row):
 
 
 def table_adapter_exclude_columns(cols: list[str]):
+    from dlt.common.libs.sql_alchemy import Table
+
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:

ingestr/src/hubspot/__init__.py

@@ -197,11 +197,18 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-        get_custom_object = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
+        custom_objects = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
         object_type_id = None
+        associations = None
+        if ":" in custom_object_name:
+            fields = custom_object_name.split(":")
+            if len(fields) == 2:
+                custom_object_name = fields[0]
+                associations = fields[1]
 
         custom_object_lowercase = custom_object_name.lower()
-        for custom_object in get_custom_object["results"]:
+
+        for custom_object in custom_objects["results"]:
             if custom_object["name"].lower() == custom_object_lowercase:
                 object_type_id = custom_object["objectTypeId"]
                 break
@@ -223,6 +230,8 @@ def hubspot(
         props = ",".join(sorted(list(set(props))))
 
         custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+        if associations:
+            custom_object_endpoint += f"&associations={associations}"
 
         """Hubspot custom object details resource"""
         yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")

ingestr/src/hubspot/helpers.py

@@ -148,10 +148,6 @@ def fetch_data(
                             "updatedAt": _result.get("updatedAt", ""),
                         }
                     )
-                elif resource_name == "custom":
-                    _objects.append(
-                        _result.get("properties", ""),
-                    )
                 else:
                     _obj = _result.get("properties", _result)
                     if "id" not in _obj and "id" in _result:

ingestr/src/partition.py

@@ -1,7 +1,6 @@
 from typing import Dict
 
 from dlt.common.schema.typing import TColumnSchema
-from dlt.destinations.adapters import athena_adapter, athena_partition
 from dlt.sources import DltResource, DltSource
 
 import ingestr.src.resource as resource
@@ -12,6 +11,8 @@ def apply_athena_hints(
     partition_column: str,
     additional_hints: Dict[str, TColumnSchema] = {},
 ) -> None:
+    from dlt.destinations.adapters import athena_adapter, athena_partition
+
     def _apply_partition_hint(resource: DltResource) -> None:
         columns = resource.columns if resource.columns else {}
 

ingestr/src/sources.py

@@ -13,91 +13,37 @@ from typing import (
     List,
     Literal,
     Optional,
+    TypeAlias,
     Union,
 )
 from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
 
-import dlt
 import pendulum
-from dlt.common.configuration.specs import (
-    AwsCredentials,
-)
-from dlt.common.libs.sql_alchemy import (
-    Engine,
-    MetaData,
-)
 from dlt.common.time import ensure_pendulum_datetime
-from dlt.common.typing import TDataItem, TSecretStrValue
 from dlt.extract import Incremental
+from dlt.sources import incremental as dlt_incremental
 from dlt.sources.credentials import (
     ConnectionStringCredentials,
 )
-from dlt.sources.sql_database import sql_table
-from dlt.sources.sql_database.helpers import TableLoader
-from dlt.sources.sql_database.schema_types import (
-    ReflectionLevel,
-    SelectAny,
-    Table,
-    TTypeAdapter,
-)
-from sqlalchemy import Column
-from sqlalchemy import types as sa
 
 from ingestr.src import blob
-from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
-from ingestr.src.adjust.adjust_helpers import parse_filters
-from ingestr.src.applovin import applovin_source
-from ingestr.src.applovin_max import applovin_max_source
-from ingestr.src.arrow import memory_mapped_arrow
-from ingestr.src.chess import source
 from ingestr.src.errors import (
     InvalidBlobTableError,
     MissingValueError,
     UnsupportedResourceError,
 )
-from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
-from ingestr.src.filesystem import readers
-from ingestr.src.filters import table_adapter_exclude_columns
-from ingestr.src.frankfurter import frankfurter_source
-from ingestr.src.frankfurter.helpers import validate_dates
-from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
-from ingestr.src.gorgias import gorgias_source
-from ingestr.src.hubspot import hubspot
-from ingestr.src.kafka import kafka_consumer
-from ingestr.src.kafka.helpers import KafkaCredentials
-from ingestr.src.klaviyo._init_ import klaviyo_source
-from ingestr.src.linkedin_ads import linked_in_ads_source
-from ingestr.src.linkedin_ads.dimension_time_enum import (
-    Dimension,
-    TimeGranularity,
-)
-from ingestr.src.notion import notion_databases
-from ingestr.src.personio import personio_source
-from ingestr.src.shopify import shopify_source
-from ingestr.src.slack import slack_source
-from ingestr.src.sql_database.callbacks import (
-    chained_query_adapter_callback,
-    custom_query_variable_subsitution,
-    limit_callback,
-    type_adapter_callback,
-)
 from ingestr.src.table_definition import TableDefinition, table_string_to_dataclass
-from ingestr.src.tiktok_ads import tiktok_source
-from ingestr.src.time import isotime
-from ingestr.src.zendesk import zendesk_chat, zendesk_support, zendesk_talk
-from ingestr.src.zendesk.helpers.credentials import (
-    ZendeskCredentialsOAuth,
-    ZendeskCredentialsToken,
-)
-
-TableBackend = Literal["sqlalchemy", "pyarrow", "pandas", "connectorx"]
-TQueryAdapter = Callable[[SelectAny, Table], SelectAny]
 
 
 class SqlSource:
     table_builder: Callable
 
-    def __init__(self, table_builder=sql_table) -> None:
+    def __init__(self, table_builder=None) -> None:
+        if table_builder is None:
+            from dlt.sources.sql_database import sql_table
+
+            table_builder = sql_table
+
         self.table_builder = table_builder
 
     def handles_incrementality(self) -> bool:
@@ -112,7 +58,7 @@ class SqlSource:
         if kwargs.get("incremental_key"):
             start_value = kwargs.get("interval_start")
             end_value = kwargs.get("interval_end")
-            incremental = dlt.sources.incremental(
+            incremental = dlt_incremental(
                 kwargs.get("incremental_key", ""),
                 initial_value=start_value,
                 end_value=end_value,
@@ -167,6 +113,27 @@ class SqlSource:
         if uri.startswith("db2://"):
             uri = uri.replace("db2://", "db2+ibm_db://")
 
+        from dlt.common.libs.sql_alchemy import (
+            Engine,
+            MetaData,
+        )
+        from dlt.sources.sql_database.schema_types import (
+            ReflectionLevel,
+            SelectAny,
+            Table,
+            TTypeAdapter,
+        )
+        from sqlalchemy import Column
+        from sqlalchemy import types as sa
+
+        from ingestr.src.filters import table_adapter_exclude_columns
+        from ingestr.src.sql_database.callbacks import (
+            chained_query_adapter_callback,
+            custom_query_variable_subsitution,
+            limit_callback,
+            type_adapter_callback,
+        )
+
         query_adapters = []
         if kwargs.get("sql_limit"):
             query_adapters.append(
@@ -185,6 +152,13 @@ class SqlSource:
             defer_table_reflect = True
             query_value = table.split(":", 1)[1]
 
+            TableBackend: TypeAlias = Literal[
+                "sqlalchemy", "pyarrow", "pandas", "connectorx"
+            ]
+            TQueryAdapter: TypeAlias = Callable[[SelectAny, Table], SelectAny]
+            import dlt
+            from dlt.common.typing import TDataItem
+
             # this is a very hacky version of the table_rows function. it is built this way to go around the dlt's table loader.
             # I didn't want to write a full fledged sqlalchemy source for now, and wanted to benefit from the existing stuff to begin with.
             # this is by no means a production ready solution, but it works for now.
@@ -235,6 +209,8 @@ class SqlSource:
                     *cols,
                 )
 
+                from dlt.sources.sql_database.helpers import TableLoader
+
                 loader = TableLoader(
                     engine,
                     backend,
@@ -277,7 +253,12 @@ class SqlSource:
 class ArrowMemoryMappedSource:
     table_builder: Callable
 
-    def __init__(self, table_builder=memory_mapped_arrow) -> None:
+    def __init__(self, table_builder=None) -> None:
+        if table_builder is None:
+            from ingestr.src.arrow import memory_mapped_arrow
+
+            table_builder = memory_mapped_arrow
+
         self.table_builder = table_builder
 
     def handles_incrementality(self) -> bool:
@@ -289,7 +270,7 @@ class ArrowMemoryMappedSource:
             start_value = kwargs.get("interval_start")
             end_value = kwargs.get("interval_end")
 
-            incremental = dlt.sources.incremental(
+            incremental = dlt_incremental(
                 kwargs.get("incremental_key", ""),
                 initial_value=start_value,
                 end_value=end_value,
@@ -341,7 +322,7 @@ class MongoDbSource:
             start_value = kwargs.get("interval_start")
             end_value = kwargs.get("interval_end")
 
-            incremental = dlt.sources.incremental(
+            incremental = dlt_incremental(
                 kwargs.get("incremental_key", ""),
                 initial_value=start_value,
                 end_value=end_value,
@@ -366,7 +347,7 @@ class LocalCsvSource:
 
     def dlt_source(self, uri: str, table: str, **kwargs):
         def csv_file(
-            incremental: Optional[dlt.sources.incremental[Any]] = None,
+            incremental: Optional[dlt_incremental[Any]] = None,
         ):
             file_path = uri.split("://")[1]
             myFile = open(file_path, "r")
@@ -408,11 +389,13 @@ class LocalCsvSource:
             if page:
                 yield page
 
-        return dlt.resource(
+        from dlt import resource
+
+        return resource(
             csv_file,
             merge_key=kwargs.get("merge_key"),  # type: ignore
         )(
-            incremental=dlt.sources.incremental(
+            incremental=dlt_incremental(
                 kwargs.get("incremental_key", ""),
                 initial_value=kwargs.get("interval_start"),
                 end_value=kwargs.get("interval_end"),
@@ -428,7 +411,12 @@ class LocalCsvSource:
 class NotionSource:
     table_builder: Callable
 
-    def __init__(self, table_builder=notion_databases) -> None:
+    def __init__(self, table_builder=None) -> None:
+        if table_builder is None:
+            from ingestr.src.notion import notion_databases
+
+            table_builder = notion_databases
+
         self.table_builder = table_builder
 
     def handles_incrementality(self) -> bool:
@@ -488,6 +476,8 @@ class ShopifySource:
                 f"Table name '{table}' is not supported for Shopify source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
+        from ingestr.src.shopify import shopify_source
+
         return shopify_source(
             private_app_password=api_key[0],
             shop_url=f"https://{source_fields.netloc}",
@@ -532,6 +522,8 @@ class GorgiasSource:
         if kwargs.get("interval_end"):
             date_args["end_date"] = kwargs.get("interval_end")
 
+        from ingestr.src.gorgias import gorgias_source
+
         return gorgias_source(
             domain=source_fields.netloc,
             email=email[0],
@@ -629,6 +621,8 @@ class ChessSource:
                 f"Resource '{table}' is not supported for Chess source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
+        from ingestr.src.chess import source
+
         return source(players=list_players, **date_args).with_resources(
             table_mapping[table]
         )
@@ -713,6 +707,11 @@ class FacebookAdsSource:
                 "access_token and accound_id are required to connect to Facebook Ads."
             )
 
+        from ingestr.src.facebook_ads import (
+            facebook_ads_source,
+            facebook_insights_source,
+        )
+
         endpoint = None
         if table in ["campaigns", "ad_sets", "ad_creatives", "ads", "leads"]:
             endpoint = table
@@ -723,7 +722,7 @@ class FacebookAdsSource:
             ).with_resources("facebook_insights")
         else:
             raise ValueError(
-                "fResource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                f"Resource '{table}' is not supported for Facebook Ads source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
         return facebook_ads_source(
@@ -770,6 +769,8 @@ class SlackSource:
         if kwargs.get("interval_end"):
             date_args["end_date"] = kwargs.get("interval_end")
 
+        from ingestr.src.slack import slack_source
+
         return slack_source(
             access_token=api_key[0],
             table_per_channel=False,
@@ -799,13 +800,20 @@ class HubspotSource:
 
         endpoint = None
 
+        from ingestr.src.hubspot import hubspot
+
         if table.startswith("custom:"):
             fields = table.split(":", 2)
-            if len(fields) != 2:
+            if len(fields) != 2 and len(fields) != 3:
                 raise ValueError(
-                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type>"
+                    "Invalid Hubspot custom table format. Expected format: custom:<custom_object_type> or custom:<custom_object_type>:<associations>"
                 )
-            endpoint = fields[1]
+
+            if len(fields) == 2:
+                endpoint = fields[1]
+            else:
+                endpoint = f"{fields[1]}:{fields[2]}"
+
             return hubspot(
                 api_key=api_key[0],
                 custom_object=endpoint,
@@ -905,6 +913,9 @@ class KlaviyoSource:
             )
 
         start_date = kwargs.get("interval_start") or "2000-01-01"
+
+        from ingestr.src.klaviyo import klaviyo_source
+
         return klaviyo_source(
             api_key=api_key[0],
             start_date=start_date,
@@ -938,6 +949,9 @@ class KafkaSource:
             raise ValueError("group_id in the URI is required to connect to kafka")
 
         start_date = kwargs.get("interval_start")
+        from ingestr.src.kafka import kafka_consumer
+        from ingestr.src.kafka.helpers import KafkaCredentials
+
         return kafka_consumer(
             topics=[table],
             credentials=KafkaCredentials(
@@ -993,6 +1007,9 @@ class AdjustSource:
         if kwargs.get("interval_end"):
             end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
 
+        from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
+        from ingestr.src.adjust.adjust_helpers import parse_filters
+
         dimensions = None
         metrics = None
         filters = []
@@ -1099,6 +1116,12 @@ class ZendeskSource:
         if not subdomain:
             raise ValueError("Subdomain is required to connect with Zendesk")
 
+        from ingestr.src.zendesk import zendesk_chat, zendesk_support, zendesk_talk
+        from ingestr.src.zendesk.helpers.credentials import (
+            ZendeskCredentialsOAuth,
+            ZendeskCredentialsToken,
+        )
+
         if not source_fields.username and source_fields.password:
             oauth_token = source_fields.password
             if not oauth_token:
@@ -1157,7 +1180,7 @@ class ZendeskSource:
             ).with_resources(table)
         else:
             raise ValueError(
-                "fResource '{table}' is not supported for Zendesk source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                f"Resource '{table}' is not supported for Zendesk source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
 
@@ -1206,6 +1229,8 @@ class S3Source:
                 "S3 Source only supports specific formats files: csv, jsonl, parquet"
             )
 
+        from ingestr.src.filesystem import readers
+
         return readers(bucket_url, fs, path_to_file).with_resources(endpoint)
 
 
@@ -1300,6 +1325,8 @@ class TikTokSource:
                 filter_name = list(filters.keys())[0]
                 filter_value = list(map(int, filters[list(filters.keys())[0]]))
 
+        from ingestr.src.tiktok_ads import tiktok_source
+
         return tiktok_source(
             start_date=start_date,
             end_date=end_date,
@@ -1348,9 +1375,12 @@ class AsanaSource:
                 f"Resource '{table}' is not supported for Asana source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
             )
 
-        dlt.secrets["sources.asana_source.access_token"] = access_token[0]
+        import dlt
+
         from ingestr.src.asana_source import asana_source
 
+        dlt.secrets["sources.asana_source.access_token"] = access_token[0]
+
         src = asana_source()
         src.workspaces.add_filter(lambda w: w["gid"] == workspace)
         return src.with_resources(table)
@@ -1396,6 +1426,9 @@ class DynamoDBSource:
         if not secret_key:
             raise ValueError("secret_access_key is required to connect to Dynamodb")
 
+        from dlt.common.configuration.specs import AwsCredentials
+        from dlt.common.typing import TSecretStrValue
+
         creds = AwsCredentials(
             aws_access_key_id=access_key[0],
             aws_secret_access_key=TSecretStrValue(secret_key[0]),
@@ -1406,8 +1439,11 @@ class DynamoDBSource:
         incremental = None
         incremental_key = kwargs.get("incremental_key")
 
+        from ingestr.src.dynamodb import dynamodb
+        from ingestr.src.time import isotime
+
         if incremental_key:
-            incremental = dlt.sources.incremental(
+            incremental = dlt_incremental(
                 incremental_key.strip(),
                 initial_value=isotime(kwargs.get("interval_start")),
                 end_value=isotime(kwargs.get("interval_end")),
@@ -1415,8 +1451,6 @@ class DynamoDBSource:
                 range_start="closed",
             )
 
-        from ingestr.src.dynamodb import dynamodb
-
         # bug: we never validate table.
         return dynamodb(table, creds, incremental)
 
@@ -1517,6 +1551,12 @@ class GitHubSource:
 
         access_token = source_fields.get("access_token", [""])[0]
 
+        from ingestr.src.github import (
+            github_reactions,
+            github_repo_events,
+            github_stargazers,
+        )
+
         if table in ["issues", "pull_requests"]:
             return github_reactions(
                 owner=owner, name=repo, access_token=access_token
@@ -1667,6 +1707,8 @@ class GCSSource:
                 "GCS Source only supports specific formats files: csv, jsonl, parquet"
             )
 
+        from ingestr.src.filesystem import readers
+
         return readers(bucket_url, fs, path_to_file).with_resources(endpoint)
 
 
@@ -1818,6 +1860,12 @@ class LinkedInAdsSource:
                 "'date' or 'month' is required to connect to LinkedIn Ads, please provide at least one of these dimensions."
             )
 
+        from ingestr.src.linkedin_ads import linked_in_ads_source
+        from ingestr.src.linkedin_ads.dimension_time_enum import (
+            Dimension,
+            TimeGranularity,
+        )
+
         if "date" in dimensions:
             time_granularity = TimeGranularity.daily
             dimensions.remove("date")
@@ -1876,6 +1924,8 @@ class AppLovinSource:
             custom_report = table
             table = "custom_report"
 
+        from ingestr.src.applovin import applovin_source
+
         src = applovin_source(
             api_key[0],
             start_date.strftime("%Y-%m-%d"),
@@ -1943,6 +1993,8 @@ class ApplovinMaxSource:
 
         end_date = interval_end.date() if interval_end is not None else None
 
+        from ingestr.src.applovin_max import applovin_max_source
+
         return applovin_max_source(
             start_date=start_date,
             end_date=end_date,
@@ -2020,6 +2072,8 @@ class PersonioSource:
         ]:
             raise UnsupportedResourceError(table, "Personio")
 
+        from ingestr.src.personio import personio_source
+
         return personio_source(
             client_id=client_id[0],
             client_secret=client_secret[0],
@@ -2054,14 +2108,17 @@ class KinesisSource:
         if start_date is not None:
             # the resource will read all messages after this timestamp.
             start_date = ensure_pendulum_datetime(start_date)
+
+        from dlt.common.configuration.specs import AwsCredentials
+
+        from ingestr.src.kinesis import kinesis_stream
+
         credentials = AwsCredentials(
             aws_access_key_id=aws_access_key_id[0],
             aws_secret_access_key=aws_secret_access_key[0],
             region_name=region_name[0],
         )
 
-        from ingestr.src.kinesis import kinesis_stream
-
         return kinesis_stream(
             stream_name=table, credentials=credentials, initial_at_timestamp=start_date
         )
@@ -2109,6 +2166,9 @@ class FrankfurterSource:
     def dlt_source(self, uri: str, table: str, **kwargs):
         # start and end dates only assigned and validated for exchange_rates table
         # Note: if an end date but no start date is provided, start date and end date will be set to current date
+        from ingestr.src.frankfurter import frankfurter_source
+        from ingestr.src.frankfurter.helpers import validate_dates
+
         if table == "exchange_rates":
             if kwargs.get("interval_start"):
                 start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))

ingestr/src/telemetry/event.py

@@ -1,13 +1,4 @@
 import os
-import platform
-
-import machineid
-import rudderstack.analytics as rudder_analytics  # type: ignore
-
-from ingestr.src.version import __version__  # type: ignore
-
-rudder_analytics.write_key = "2cUr13DDQcX2x2kAfMEfdrKvrQa"
-rudder_analytics.dataPlaneUrl = "https://getbruinbumlky.dataplane.rudderstack.com"
 
 
 def track(event_name, event_properties: dict):
@@ -16,6 +7,16 @@ def track(event_name, event_properties: dict):
     ):
         return
 
+    import platform
+
+    import machineid
+    import rudderstack.analytics as rudder_analytics  # type: ignore
+
+    from ingestr.src.version import __version__  # type: ignore
+
+    rudder_analytics.write_key = "2cUr13DDQcX2x2kAfMEfdrKvrQa"
+    rudder_analytics.dataPlaneUrl = "https://getbruinbumlky.dataplane.rudderstack.com"
+
     try:
         if not event_properties:
             event_properties = {}

{ingestr-0.13.32.dist-info → ingestr-0.13.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.32
+Version: 0.13.34
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.32.dist-info → ingestr-0.13.34.dist-info}/RECORD

@@ -1,22 +1,22 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=yPEn1FvfjCdWZe0PZqgKB79GQiwUy_qH3QelET13RCE,24870
+ingestr/main.py,sha256=mRlGSqi2sHcZ2AKlwn5MqoMvFxXlSjcZxmPJr76rmRk,25187
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=mvCtlkSChRqpUCkS3LXuEwmkyyBr5oK4zK-YUERhKf8,21
-ingestr/src/destinations.py,sha256=_jRB_0rc-HNb5uvy30ZuFSm_NpK_8I9OJHmwbt0jxCM,12057
+ingestr/src/buildinfo.py,sha256=i3Tz80qXUH6VzMC8jzlySZd05zyoaaBcvoyLd2q-wKg,21
+ingestr/src/destinations.py,sha256=0fEwLY78SQDXbHcX4iz4Xc7H8FXN-QhVJL9uoUTZOs4,12924
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
-ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
+ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
-ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
+ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=gWWYlj3cyscernMrCePDR8Q_zJF5wXFwGM6x0LVRbV4,74921
+ingestr/src/sources.py,sha256=uRERygJ41y0MNXF3-FJvHr4btxlEM93ZeWr_Liz3N2M,76181
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
 ingestr/src/adjust/adjust_helpers.py,sha256=IHSS94A7enOWkZ8cP5iW3RdYt0Xl3qZGAmDc1Xy4qkI,3802
-ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
+ingestr/src/airtable/__init__.py,sha256=mdzeaq0g12HR8gbhtVR_aS_5GVWPZn6XD-zHUE5FunI,2788
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
 ingestr/src/applovin_max/__init__.py,sha256=ZrxOUSirGxkGDmM9wsQO3anwNVzqtoCwN_OuCXfPkXE,3285
 ingestr/src/appsflyer/__init__.py,sha256=QoK-B3cYYMD3bqzQaLWNH6FkJyjRbzRkBF2n6urxubs,8071
@@ -62,14 +62,14 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=QheZb_F2TEBdzd29SEgkU3AMdIIF7Gpj-t27EXnSIZ4,11448
-ingestr/src/hubspot/helpers.py,sha256=4aVOSzIsQV3RemqRJEJLu7BWMdcOuubwNvrjuMu87rg,8045
+ingestr/src/hubspot/__init__.py,sha256=wqHefhc_YRI5dNFCcpvH-UUilNThE49sbGouSBiHYsw,11776
+ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
 ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
 ingestr/src/kinesis/helpers.py,sha256=aF0GCDKSectaaW8XPdERY_6bUs0ky19dcBs24ZFn-o0,2473
-ingestr/src/klaviyo/_init_.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
+ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
@@ -104,7 +104,7 @@ ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG
 ingestr/src/stripe_analytics/__init__.py,sha256=0HCL0qsrh_si1RR3a4k9XS94VWQ4v9aG7CqXF-V-57M,4593
 ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
-ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
+ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=jmWHvZzN1Vt_PWrJkgq5a2wIwon-OBEzXoZx0jEy-74,3905
@@ -122,8 +122,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.32.dist-info/METADATA,sha256=RK2X2A37bphtuSAtyffInwt0ev8kKgx7VeZMDILrwsM,13574
-ingestr-0.13.32.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.32.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.32.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.32.dist-info/RECORD,,
+ingestr-0.13.34.dist-info/METADATA,sha256=84NPfN9LSTGrw79p3116CXH9BZGjnXgEvglsXpVhEY0,13574
+ingestr-0.13.34.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.34.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.34.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.34.dist-info/RECORD,,