ingestr 0.13.31__tar.gz → 0.13.32__tar.gz

{ingestr-0.13.31 → ingestr-0.13.32}/.gitignore

@@ -21,4 +21,5 @@ node_modules
 *.db.wal
 *.sql
 ingest.sh
-test.env
+test.env
+*.log

{ingestr-0.13.31 → ingestr-0.13.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.31
+Version: 0.13.32
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -419,11 +419,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Google Analytics</td>
         <td>✅</td>
         <td>-</td>
-    </tr>
-    <tr>
-        <td>Intercom</td>
-        <td>✅</td>
-        <td>-</td>
     </tr>
      <tr>
         <td>Klaviyo</td>

{ingestr-0.13.31 → ingestr-0.13.32}/README.md

@@ -222,11 +222,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Google Analytics</td>
         <td>✅</td>
         <td>-</td>
-    </tr>
-    <tr>
-        <td>Intercom</td>
-        <td>✅</td>
-        <td>-</td>
     </tr>
      <tr>
         <td>Klaviyo</td>

{ingestr-0.13.31 → ingestr-0.13.32}/docs/supported-sources/athena.md

@@ -18,9 +18,17 @@ athena://?bucket=<your-destination-bucket> \
 URI parameters:
 - `bucket` (required): The name of the bucket where the data will be stored, containing the Parquet files that Athena will work with, e.g. `your_bucket_name` or `s3://your_bucket_name`.
 - `access_key_id` and `secret_access_key` (required): These are AWS credentials that will be used to authenticate with AWS services like S3 and Athena.
-- `region_name` (required): The AWS region of the Athena service and S3 buckets, e.g. `eu-central-1`
+- `session_token` (optional): The session token for temporary credentials.
+- `region_name` (required if there's no local profile found): The AWS region of the Athena service and S3 buckets, e.g. `eu-central-1`
 - `query_results_path` (optional): The query location path where the results of Athena queries will be saved, e.g. `dest_path` or `s3://dest_path`. If not provided, it will default to the bucket specified in the `bucket` parameter.
 - `workgroup` (optional): The name of the Athena workgroup, e.g. `my_group`
+- `profile` (optional): The name of the AWS profile to use, e.g. `my_profile`
+
+You have two ways of providing credentials:
+1. Provide `access_key_id` and `secret_access_key` directly in the URI.
+2. Provide the name of the AWS profile to use in the `profile` parameter.
+
+If there's no access key and secret key provided, ingestr will try to find the credentials in the local AWS credentials file.
 
 ## Setting up an Athena Integration
 Athena requires a `bucket`, `access_key_id`, `secret_access_key` and `region_name` to access the S3 bucket. Please follow the guide on dltHub to obtain [credentials](https://dlthub.com/docs/dlt-ecosystem/destinations/athena#2-setup-bucket-storage-and-athena-credentials). Once you've completed the guide, you should have all the above-mentioned credentials.

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/main.py

@@ -3,15 +3,9 @@ from enum import Enum
 from typing import Optional
 
 import typer
-from dlt.common.runtime.collector import Collector
 from rich.console import Console
-from rich.status import Status
 from typing_extensions import Annotated
 
-import ingestr.src.partition as partition
-import ingestr.src.resource as resource
-from ingestr.src.destinations import AthenaDestination
-from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -46,45 +40,6 @@ PARQUET_SUPPORTED_DESTINATIONS = [
 JSON_RETURNING_SOURCES = ["notion"]
 
 
-class SpinnerCollector(Collector):
-    status: Status
-    current_step: str
-    started: bool
-
-    def __init__(self) -> None:
-        self.status = Status("Ingesting data...", spinner="dots")
-        self.started = False
-
-    def update(
-        self,
-        name: str,
-        inc: int = 1,
-        total: Optional[int] = None,
-        message: Optional[str] = None,  # type: ignore
-        label: str = "",
-        **kwargs,
-    ) -> None:
-        self.status.update(self.current_step)
-
-    def _start(self, step: str) -> None:
-        self.current_step = self.__step_to_label(step)
-        self.status.start()
-
-    def __step_to_label(self, step: str) -> str:
-        verb = step.split(" ")[0].lower()
-        if verb.startswith("normalize"):
-            return "Normalizing the data"
-        elif verb.startswith("load"):
-            return "Loading the data to the destination"
-        elif verb.startswith("extract"):
-            return "Extracting the data from the source"
-
-        return f"{verb.capitalize()} the data"
-
-    def _stop(self) -> None:
-        self.status.stop()
-
-
 class IncrementalStrategy(str, Enum):
     create_replace = "replace"
     append = "append"
@@ -323,8 +278,12 @@ def ingest(
     from dlt.common.runtime.collector import Collector, LogCollector
     from dlt.common.schema.typing import TColumnSchema
 
+    import ingestr.src.partition as partition
+    import ingestr.src.resource as resource
+    from ingestr.src.collector.spinner import SpinnerCollector
+    from ingestr.src.destinations import AthenaDestination
     from ingestr.src.factory import SourceDestinationFactory
-    from ingestr.src.telemetry.event import track
+    from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
 
     def report_errors(run_info: LoadInfo):
         for load_package in run_info.load_packages:
@@ -406,6 +365,7 @@ def ingest(
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
     dlt.config["extract.max_parallel_items"] = extract_parallelism
+    dlt.config["load.raise_on_max_retries"] = 15
     if schema_naming != SchemaNaming.default:
         dlt.config["schema.naming"] = schema_naming.value
 

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/adjust/adjust_helpers.py

@@ -36,7 +36,7 @@ class AdjustAPI:
     def __init__(self, api_key):
         self.api_key = api_key
         self.request_client = Client(
-            request_timeout=8.0,
+            request_timeout=1000,  # Adjust support recommends 1000 seconds of read timeout.
             raise_for_status=False,
             retry_condition=retry_on_limit,
             request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/applovin_max/__init__.py

@@ -68,7 +68,6 @@ def applovin_max_source(
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/appsflyer/client.py

@@ -46,7 +46,6 @@ class AppsflyerClient:
             )
 
         request_client = Client(
-            request_timeout=10.0,
             raise_for_status=False,
             retry_condition=retry_on_limit,
             request_max_attempts=12,

ingestr-0.13.32/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.32"

ingestr-0.13.32/ingestr/src/collector/spinner.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from dlt.common.runtime.collector import Collector
+from rich.status import Status
+
+
+class SpinnerCollector(Collector):
+    status: Status
+    current_step: str
+    started: bool
+
+    def __init__(self) -> None:
+        self.status = Status("Ingesting data...", spinner="dots")
+        self.started = False
+
+    def update(
+        self,
+        name: str,
+        inc: int = 1,
+        total: Optional[int] = None,
+        message: Optional[str] = None,  # type: ignore
+        label: str = "",
+        **kwargs,
+    ) -> None:
+        self.status.update(self.current_step)
+
+    def _start(self, step: str) -> None:
+        self.current_step = self.__step_to_label(step)
+        self.status.start()
+
+    def __step_to_label(self, step: str) -> str:
+        verb = step.split(" ")[0].lower()
+        if verb.startswith("normalize"):
+            return "Normalizing the data"
+        elif verb.startswith("load"):
+            return "Loading the data to the destination"
+        elif verb.startswith("extract"):
+            return "Extracting the data from the source"
+
+        return f"{verb.capitalize()} the data"
+
+    def _stop(self) -> None:
+        self.status.stop()

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/destinations.py

@@ -223,34 +223,47 @@ class AthenaDestination:
             query_result_path = bucket
 
         access_key_id = source_params.get("access_key_id", [None])[0]
-        if not access_key_id:
-            raise ValueError("The AWS access_key_id is required to connect to Athena.")
-
         secret_access_key = source_params.get("secret_access_key", [None])[0]
-        if not secret_access_key:
-            raise ValueError("The AWS secret_access_key is required to connect Athena")
+        session_token = source_params.get("session_token", [None])[0]
+        profile_name = source_params.get("profile", ["default"])[0]
+        region_name = source_params.get("region_name", [None])[0]
 
-        work_group = source_params.get("workgroup", [None])[0]
+        if not access_key_id and not secret_access_key:
+            import botocore.session  # type: ignore
+
+            session = botocore.session.Session(profile=profile_name)
+            default = session.get_credentials()
+            if not profile_name:
+                raise ValueError(
+                    "You have to either provide access_key_id and secret_access_key pair or a valid AWS profile name."
+                )
+            access_key_id = default.access_key
+            secret_access_key = default.secret_key
+            session_token = default.token
+            if region_name is None:
+                region_name = session.get_config_variable("region")
 
-        region_name = source_params.get("region_name", [None])[0]
         if not region_name:
             raise ValueError("The region_name is required to connect to Athena.")
 
         os.environ["DESTINATION__BUCKET_URL"] = bucket
-        os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
-        os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
-            secret_access_key
-        )
+        if access_key_id and secret_access_key:
+            os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+            os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+                secret_access_key
+            )
+        if session_token:
+            os.environ["DESTINATION__CREDENTIALS__AWS_SESSION_TOKEN"] = session_token
 
-        credentials = AwsCredentials(
-            aws_access_key_id=access_key_id,
-            aws_secret_access_key=secret_access_key,
-            region_name=region_name,
-        )
         return dlt.destinations.athena(
             query_result_bucket=query_result_path,
-            athena_work_group=work_group,
-            credentials=credentials,
+            athena_work_group=source_params.get("workgroup", [None])[0],
+            credentials=AwsCredentials(
+                aws_access_key_id=access_key_id,  # type: ignore
+                aws_secret_access_key=secret_access_key,  # type: ignore
+                aws_session_token=session_token,
+                region_name=region_name,
+            ),
             destination_name=bucket,
         )
 

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/klaviyo/_init_.py

@@ -18,7 +18,6 @@ def retry_on_limit(response: requests.Response, exception: BaseException) -> boo
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/linkedin_ads/helpers.py

@@ -18,7 +18,6 @@ def retry_on_limit(
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/sources.py

@@ -18,9 +18,7 @@ from typing import (
 from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
 
 import dlt
-import gcsfs  # type: ignore
 import pendulum
-import s3fs  # type: ignore
 from dlt.common.configuration.specs import (
     AwsCredentials,
 )
@@ -42,22 +40,16 @@ from dlt.sources.sql_database.schema_types import (
     Table,
     TTypeAdapter,
 )
-from google.ads.googleads.client import GoogleAdsClient  # type: ignore
 from sqlalchemy import Column
 from sqlalchemy import types as sa
 
 from ingestr.src import blob
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
-from ingestr.src.airtable import airtable_source
 from ingestr.src.applovin import applovin_source
 from ingestr.src.applovin_max import applovin_max_source
-from ingestr.src.appstore import app_store
-from ingestr.src.appstore.client import AppStoreConnectClient
 from ingestr.src.arrow import memory_mapped_arrow
-from ingestr.src.asana_source import asana_source
 from ingestr.src.chess import source
-from ingestr.src.dynamodb import dynamodb
 from ingestr.src.errors import (
     InvalidBlobTableError,
     MissingValueError,
@@ -69,25 +61,18 @@ from ingestr.src.filters import table_adapter_exclude_columns
 from ingestr.src.frankfurter import frankfurter_source
 from ingestr.src.frankfurter.helpers import validate_dates
 from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
-from ingestr.src.google_ads import google_ads
-from ingestr.src.google_analytics import google_analytics
-from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.gorgias import gorgias_source
 from ingestr.src.hubspot import hubspot
 from ingestr.src.kafka import kafka_consumer
 from ingestr.src.kafka.helpers import KafkaCredentials
-from ingestr.src.kinesis import kinesis_stream
 from ingestr.src.klaviyo._init_ import klaviyo_source
 from ingestr.src.linkedin_ads import linked_in_ads_source
 from ingestr.src.linkedin_ads.dimension_time_enum import (
     Dimension,
     TimeGranularity,
 )
-from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.personio import personio_source
-from ingestr.src.pipedrive import pipedrive_source
-from ingestr.src.salesforce import salesforce_source
 from ingestr.src.shopify import shopify_source
 from ingestr.src.slack import slack_source
 from ingestr.src.sql_database.callbacks import (
@@ -96,7 +81,6 @@ from ingestr.src.sql_database.callbacks import (
     limit_callback,
     type_adapter_callback,
 )
-from ingestr.src.stripe_analytics import stripe_source
 from ingestr.src.table_definition import TableDefinition, table_string_to_dataclass
 from ingestr.src.tiktok_ads import tiktok_source
 from ingestr.src.time import isotime
@@ -338,7 +322,12 @@ class ArrowMemoryMappedSource:
 class MongoDbSource:
     table_builder: Callable
 
-    def __init__(self, table_builder=mongodb_collection) -> None:
+    def __init__(self, table_builder=None) -> None:
+        if table_builder is None:
+            from ingestr.src.mongodb import mongodb_collection
+
+            table_builder = mongodb_collection
+
         self.table_builder = table_builder
 
     def handles_incrementality(self) -> bool:
@@ -554,7 +543,12 @@ class GorgiasSource:
 class GoogleSheetsSource:
     table_builder: Callable
 
-    def __init__(self, table_builder=google_spreadsheet) -> None:
+    def __init__(self, table_builder=None) -> None:
+        if table_builder is None:
+            from ingestr.src.google_sheets import google_spreadsheet
+
+            table_builder = google_spreadsheet
+
         self.table_builder = table_builder
 
     def handles_incrementality(self) -> bool:
@@ -685,6 +679,8 @@ class StripeAnalyticsSource:
         if kwargs.get("interval_end"):
             date_args["end_date"] = kwargs.get("interval_end")
 
+        from ingestr.src.stripe_analytics import stripe_source
+
         return stripe_source(
             endpoints=[
                 endpoint,
@@ -860,6 +856,8 @@ class AirtableSource:
                 "base_id and access_token in the URI are required to connect to Airtable"
             )
 
+        from ingestr.src.airtable import airtable_source
+
         return airtable_source(
             base_id=base_id[0], table_names=tables, access_token=access_token[0]
         )
@@ -1189,6 +1187,8 @@ class S3Source:
 
         bucket_url = f"s3://{bucket_name}/"
 
+        import s3fs  # type: ignore
+
         fs = s3fs.S3FileSystem(
             key=access_key_id[0],
             secret=secret_access_key[0],
@@ -1349,6 +1349,8 @@ class AsanaSource:
             )
 
         dlt.secrets["sources.asana_source.access_token"] = access_token[0]
+        from ingestr.src.asana_source import asana_source
+
         src = asana_source()
         src.workspaces.add_filter(lambda w: w["gid"] == workspace)
         return src.with_resources(table)
@@ -1413,6 +1415,8 @@ class DynamoDBSource:
                 range_start="closed",
             )
 
+        from ingestr.src.dynamodb import dynamodb
+
         # bug: we never validate table.
         return dynamodb(table, creds, incremental)
 
@@ -1474,6 +1478,8 @@ class GoogleAnalyticsSource:
         if kwargs.get("interval_end") is not None:
             end_date = pendulum.instance(kwargs.get("interval_end"))  # type: ignore
 
+        from ingestr.src.google_analytics import google_analytics
+
         return google_analytics(
             property_id=property_id[0],
             start_date=start_date,
@@ -1543,6 +1549,8 @@ class AppleAppStoreSource:
         else:
             key = base64.b64decode(key_base64[0]).decode()  # type: ignore
 
+        from ingestr.src.appstore.client import AppStoreConnectClient
+
         return AppStoreConnectClient(key.encode(), key_id, issuer_id)
 
     def dlt_source(self, uri: str, table: str, **kwargs):
@@ -1583,6 +1591,8 @@ class AppleAppStoreSource:
         if app_ids is None:
             raise MissingValueError("app_id", "App Store")
 
+        from ingestr.src.appstore import app_store
+
         src = app_store(
             client,
             app_ids,
@@ -1639,6 +1649,8 @@ class GCSSource:
         # (The RECOMMENDED way of passing service account credentials)
         # directly with gcsfs. As a workaround, we construct the GCSFileSystem
         # and pass it directly to filesystem.readers.
+        import gcsfs  # type: ignore
+
         fs = gcsfs.GCSFileSystem(
             token=credentials,
         )
@@ -1662,7 +1674,9 @@ class GoogleAdsSource:
     def handles_incrementality(self) -> bool:
         return True
 
-    def init_client(self, params: Dict[str, List[str]]) -> GoogleAdsClient:
+    def init_client(self, params: Dict[str, List[str]]):
+        from google.ads.googleads.client import GoogleAdsClient  # type: ignore
+
         dev_token = params.get("dev_token")
         if dev_token is None or len(dev_token) == 0:
             raise MissingValueError("dev_token", "Google Ads")
@@ -1716,6 +1730,7 @@ class GoogleAdsSource:
             raise MissingValueError("customer_id", "Google Ads")
 
         params = parse_qs(parsed_uri.query)
+
         client = self.init_client(params)
 
         start_date = kwargs.get("interval_start") or datetime.now(
@@ -1737,6 +1752,8 @@ class GoogleAdsSource:
             report_spec = table
             table = "daily_report"
 
+        from ingestr.src.google_ads import google_ads
+
         src = google_ads(
             client,
             customer_id,
@@ -1954,6 +1971,8 @@ class SalesforceSource:
             if v is None:
                 raise MissingValueError(k, "Salesforce")
 
+        from ingestr.src.salesforce import salesforce_source
+
         src = salesforce_source(**creds)  # type: ignore
 
         if table not in src.resources:
@@ -2040,6 +2059,9 @@ class KinesisSource:
             aws_secret_access_key=aws_secret_access_key[0],
             region_name=region_name[0],
         )
+
+        from ingestr.src.kinesis import kinesis_stream
+
         return kinesis_stream(
             stream_name=table, credentials=credentials, initial_at_timestamp=start_date
         )
@@ -2073,6 +2095,8 @@ class PipedriveSource:
         ]:
             raise UnsupportedResourceError(table, "Pipedrive")
 
+        from ingestr.src.pipedrive import pipedrive_source
+
         return pipedrive_source(
             pipedrive_api_key=api_key, since_timestamp=start_date
         ).with_resources(table)

{ingestr-0.13.31 → ingestr-0.13.32}/ingestr/src/tiktok_ads/tiktok_helpers.py

@@ -17,7 +17,6 @@ def retry_on_limit(
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

ingestr-0.13.31/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.31"