ingestr 0.13.31__tar.gz → 0.13.33__tar.gz

{ingestr-0.13.31 → ingestr-0.13.33}/.gitignore

@@ -21,4 +21,5 @@ node_modules
 *.db.wal
 *.sql
 ingest.sh
-test.env
+test.env
+*.log

ingestr-0.13.33/.gitleaksignore

@@ -0,0 +1,3 @@
+ingestr/src/telemetry/event.py:generic-api-key:17
+ingestr/src/testdata/fakebqcredentials.json:private-key:5
+docs/supported-sources/shopify.md:generic-api-key:26

{ingestr-0.13.31 → ingestr-0.13.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.31
+Version: 0.13.33
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -419,11 +419,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Google Analytics</td>
         <td>✅</td>
         <td>-</td>
-    </tr>
-    <tr>
-        <td>Intercom</td>
-        <td>✅</td>
-        <td>-</td>
     </tr>
      <tr>
         <td>Klaviyo</td>

{ingestr-0.13.31 → ingestr-0.13.33}/README.md

@@ -222,11 +222,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Google Analytics</td>
         <td>✅</td>
         <td>-</td>
-    </tr>
-    <tr>
-        <td>Intercom</td>
-        <td>✅</td>
-        <td>-</td>
     </tr>
      <tr>
         <td>Klaviyo</td>

{ingestr-0.13.31 → ingestr-0.13.33}/docs/supported-sources/athena.md

@@ -18,9 +18,17 @@ athena://?bucket=<your-destination-bucket> \
 URI parameters:
 - `bucket` (required): The name of the bucket where the data will be stored, containing the Parquet files that Athena will work with, e.g. `your_bucket_name` or `s3://your_bucket_name`.
 - `access_key_id` and `secret_access_key` (required): These are AWS credentials that will be used to authenticate with AWS services like S3 and Athena.
-- `region_name` (required): The AWS region of the Athena service and S3 buckets, e.g. `eu-central-1`
+- `session_token` (optional): The session token for temporary credentials.
+- `region_name` (required if there's no local profile found): The AWS region of the Athena service and S3 buckets, e.g. `eu-central-1`
 - `query_results_path` (optional): The query location path where the results of Athena queries will be saved, e.g. `dest_path` or `s3://dest_path`. If not provided, it will default to the bucket specified in the `bucket` parameter.
 - `workgroup` (optional): The name of the Athena workgroup, e.g. `my_group`
+- `profile` (optional): The name of the AWS profile to use, e.g. `my_profile`
+
+You have two ways of providing credentials:
+1. Provide `access_key_id` and `secret_access_key` directly in the URI.
+2. Provide the name of the AWS profile to use in the `profile` parameter.
+
+If there's no access key and secret key provided, ingestr will try to find the credentials in the local AWS credentials file.
 
 ## Setting up an Athena Integration
 Athena requires a `bucket`, `access_key_id`, `secret_access_key` and `region_name` to access the S3 bucket. Please follow the guide on dltHub to obtain [credentials](https://dlthub.com/docs/dlt-ecosystem/destinations/athena#2-setup-bucket-storage-and-athena-credentials). Once you've completed the guide, you should have all the above-mentioned credentials.

{ingestr-0.13.31 → ingestr-0.13.33}/docs/supported-sources/bigquery.md

@@ -17,4 +17,16 @@ URI parameters:
 
 The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's BigQuery dialect [here](https://github.com/googleapis/python-bigquery-sqlalchemy?tab=readme-ov-file#connection-string-parameters).
 
+### Using GCS as a staging area
+
+ingestr can use GCS as a staging area for BigQuery. To do this, you need to set the `--staging-bucket` flag when you are running the command.
+
+```bash
+ingestr ingest 
+    --source-uri $SOURCE_URI
+    --dest-uri $BIGQUERY_URI
+    --source-table raw.input 
+    --dest-table raw.output
+    --staging-bucket "gs://your-bucket-name" # [!code focus]
+```
 

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/main.py

@@ -3,15 +3,9 @@ from enum import Enum
 from typing import Optional
 
 import typer
-from dlt.common.runtime.collector import Collector
 from rich.console import Console
-from rich.status import Status
 from typing_extensions import Annotated
 
-import ingestr.src.partition as partition
-import ingestr.src.resource as resource
-from ingestr.src.destinations import AthenaDestination
-from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
 from ingestr.src.telemetry.event import track
 
 app = typer.Typer(
@@ -46,45 +40,6 @@ PARQUET_SUPPORTED_DESTINATIONS = [
 JSON_RETURNING_SOURCES = ["notion"]
 
 
-class SpinnerCollector(Collector):
-    status: Status
-    current_step: str
-    started: bool
-
-    def __init__(self) -> None:
-        self.status = Status("Ingesting data...", spinner="dots")
-        self.started = False
-
-    def update(
-        self,
-        name: str,
-        inc: int = 1,
-        total: Optional[int] = None,
-        message: Optional[str] = None,  # type: ignore
-        label: str = "",
-        **kwargs,
-    ) -> None:
-        self.status.update(self.current_step)
-
-    def _start(self, step: str) -> None:
-        self.current_step = self.__step_to_label(step)
-        self.status.start()
-
-    def __step_to_label(self, step: str) -> str:
-        verb = step.split(" ")[0].lower()
-        if verb.startswith("normalize"):
-            return "Normalizing the data"
-        elif verb.startswith("load"):
-            return "Loading the data to the destination"
-        elif verb.startswith("extract"):
-            return "Extracting the data from the source"
-
-        return f"{verb.capitalize()} the data"
-
-    def _stop(self) -> None:
-        self.status.stop()
-
-
 class IncrementalStrategy(str, Enum):
     create_replace = "replace"
     append = "append"
@@ -309,6 +264,13 @@ def ingest(
             envvar=["YIELD_LIMIT", "INGESTR_YIELD_LIMIT"],
         ),
     ] = None,  # type: ignore
+    staging_bucket: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The staging bucket to be used for the ingestion, must be prefixed with 'gs://' or 's3://'",
+            envvar=["STAGING_BUCKET", "INGESTR_STAGING_BUCKET"],
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -317,14 +279,16 @@ def ingest(
     import dlt
     import humanize
     import typer
-    from dlt.common.data_types import TDataType
-    from dlt.common.destination import Destination
     from dlt.common.pipeline import LoadInfo
     from dlt.common.runtime.collector import Collector, LogCollector
     from dlt.common.schema.typing import TColumnSchema
 
+    import ingestr.src.partition as partition
+    import ingestr.src.resource as resource
+    from ingestr.src.collector.spinner import SpinnerCollector
+    from ingestr.src.destinations import AthenaDestination
     from ingestr.src.factory import SourceDestinationFactory
-    from ingestr.src.telemetry.event import track
+    from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
 
     def report_errors(run_info: LoadInfo):
         for load_package in run_info.load_packages:
@@ -359,7 +323,7 @@ def ingest(
         return (source_table, dest_table)
 
     def validate_loader_file_format(
-        dlt_dest: Destination, loader_file_format: Optional[LoaderFileFormat]
+        dlt_dest, loader_file_format: Optional[LoaderFileFormat]
     ):
         if (
             loader_file_format
@@ -371,9 +335,11 @@ def ingest(
             )
             raise typer.Abort()
 
-    def parse_columns(columns: list[str]) -> dict[str, TDataType]:
+    def parse_columns(columns: list[str]) -> dict:
         from typing import cast, get_args
 
+        from dlt.common.data_types import TDataType
+
         possible_types = get_args(TDataType)
 
         types: dict[str, TDataType] = {}
@@ -406,6 +372,7 @@ def ingest(
     dlt.config["data_writer.file_max_items"] = loader_file_size
     dlt.config["extract.workers"] = extract_parallelism
     dlt.config["extract.max_parallel_items"] = extract_parallelism
+    dlt.config["load.raise_on_max_retries"] = 15
     if schema_naming != SchemaNaming.default:
         dlt.config["schema.naming"] = schema_naming.value
 
@@ -457,7 +424,9 @@ def ingest(
             pipelines_dir = tempfile.mkdtemp()
             is_pipelines_dir_temp = True
 
-        dlt_dest = destination.dlt_dest(uri=dest_uri, dest_table=dest_table)
+        dlt_dest = destination.dlt_dest(
+            uri=dest_uri, dest_table=dest_table, staging_bucket=staging_bucket
+        )
         validate_loader_file_format(dlt_dest, loader_file_format)
 
         if partition_by:
@@ -606,6 +575,7 @@ def ingest(
             **destination.dlt_run_params(
                 uri=dest_uri,
                 table=dest_table,
+                staging_bucket=staging_bucket,
             ),
             write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/adjust/adjust_helpers.py

@@ -36,7 +36,7 @@ class AdjustAPI:
     def __init__(self, api_key):
         self.api_key = api_key
         self.request_client = Client(
-            request_timeout=8.0,
+            request_timeout=1000,  # Adjust support recommends 1000 seconds of read timeout.
             raise_for_status=False,
             retry_condition=retry_on_limit,
             request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/airtable/__init__.py

@@ -55,7 +55,7 @@ def airtable_resource(
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [primary_key_field["name"]]
+    primary_key: List[str] = [f"fields__{primary_key_field['name']}"]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/applovin_max/__init__.py

@@ -68,7 +68,6 @@ def applovin_max_source(
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/appsflyer/client.py

@@ -46,7 +46,6 @@ class AppsflyerClient:
             )
 
         request_client = Client(
-            request_timeout=10.0,
             raise_for_status=False,
             retry_condition=retry_on_limit,
             request_max_attempts=12,

ingestr-0.13.33/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.33"

ingestr-0.13.33/ingestr/src/collector/spinner.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from dlt.common.runtime.collector import Collector
+from rich.status import Status
+
+
+class SpinnerCollector(Collector):
+    status: Status
+    current_step: str
+    started: bool
+
+    def __init__(self) -> None:
+        self.status = Status("Ingesting data...", spinner="dots")
+        self.started = False
+
+    def update(
+        self,
+        name: str,
+        inc: int = 1,
+        total: Optional[int] = None,
+        message: Optional[str] = None,  # type: ignore
+        label: str = "",
+        **kwargs,
+    ) -> None:
+        self.status.update(self.current_step)
+
+    def _start(self, step: str) -> None:
+        self.current_step = self.__step_to_label(step)
+        self.status.start()
+
+    def __step_to_label(self, step: str) -> str:
+        verb = step.split(" ")[0].lower()
+        if verb.startswith("normalize"):
+            return "Normalizing the data"
+        elif verb.startswith("load"):
+            return "Loading the data to the destination"
+        elif verb.startswith("extract"):
+            return "Extracting the data from the source"
+
+        return f"{verb.capitalize()} the data"
+
+    def _stop(self) -> None:
+        self.status.stop()

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/destinations.py

@@ -60,6 +60,22 @@ class BigQueryDestination:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            if not staging_bucket.startswith("gs://"):
+                raise ValueError("Staging bucket must start with gs://")
+
+            os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = staging_bucket
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID"] = (
+                credentials.get("project_id", None)
+            )
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY"] = (
+                credentials.get("private_key", None)
+            )
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL"] = (
+                credentials.get("client_email", None)
+            )
+
         project_id = None
         if source_fields.hostname:
             project_id = source_fields.hostname
@@ -83,6 +99,10 @@ class BigQueryDestination:
             "table_name": table_fields[-1],
         }
 
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            res["staging"] = "filesystem"
+
         return res
 
     def post_load(self):
@@ -223,34 +243,47 @@ class AthenaDestination:
             query_result_path = bucket
 
         access_key_id = source_params.get("access_key_id", [None])[0]
-        if not access_key_id:
-            raise ValueError("The AWS access_key_id is required to connect to Athena.")
-
         secret_access_key = source_params.get("secret_access_key", [None])[0]
-        if not secret_access_key:
-            raise ValueError("The AWS secret_access_key is required to connect Athena")
+        session_token = source_params.get("session_token", [None])[0]
+        profile_name = source_params.get("profile", ["default"])[0]
+        region_name = source_params.get("region_name", [None])[0]
 
-        work_group = source_params.get("workgroup", [None])[0]
+        if not access_key_id and not secret_access_key:
+            import botocore.session  # type: ignore
+
+            session = botocore.session.Session(profile=profile_name)
+            default = session.get_credentials()
+            if not profile_name:
+                raise ValueError(
+                    "You have to either provide access_key_id and secret_access_key pair or a valid AWS profile name."
+                )
+            access_key_id = default.access_key
+            secret_access_key = default.secret_key
+            session_token = default.token
+            if region_name is None:
+                region_name = session.get_config_variable("region")
 
-        region_name = source_params.get("region_name", [None])[0]
         if not region_name:
             raise ValueError("The region_name is required to connect to Athena.")
 
         os.environ["DESTINATION__BUCKET_URL"] = bucket
-        os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
-        os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
-            secret_access_key
-        )
+        if access_key_id and secret_access_key:
+            os.environ["DESTINATION__CREDENTIALS__AWS_ACCESS_KEY_ID"] = access_key_id
+            os.environ["DESTINATION__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = (
+                secret_access_key
+            )
+        if session_token:
+            os.environ["DESTINATION__CREDENTIALS__AWS_SESSION_TOKEN"] = session_token
 
-        credentials = AwsCredentials(
-            aws_access_key_id=access_key_id,
-            aws_secret_access_key=secret_access_key,
-            region_name=region_name,
-        )
         return dlt.destinations.athena(
             query_result_bucket=query_result_path,
-            athena_work_group=work_group,
-            credentials=credentials,
+            athena_work_group=source_params.get("workgroup", [None])[0],
+            credentials=AwsCredentials(
+                aws_access_key_id=access_key_id,  # type: ignore
+                aws_secret_access_key=secret_access_key,  # type: ignore
+                aws_session_token=session_token,
+                region_name=region_name,
+            ),
             destination_name=bucket,
         )
 

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/filters.py

@@ -1,6 +1,3 @@
-from dlt.common.libs.sql_alchemy import Table
-
-
 def cast_set_to_list(row):
     # this handles just the sqlalchemy backend for now
     if isinstance(row, dict):
@@ -32,6 +29,8 @@ def handle_mysql_empty_dates(row):
 
 
 def table_adapter_exclude_columns(cols: list[str]):
+    from dlt.common.libs.sql_alchemy import Table
+
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:

ingestr-0.13.31/ingestr/src/klaviyo/_init_.py → ingestr-0.13.33/ingestr/src/klaviyo/__init__.py

@@ -18,7 +18,6 @@ def retry_on_limit(response: requests.Response, exception: BaseException) -> boo
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/linkedin_ads/helpers.py

@@ -18,7 +18,6 @@ def retry_on_limit(
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

{ingestr-0.13.31 → ingestr-0.13.33}/ingestr/src/partition.py

@@ -1,7 +1,6 @@
 from typing import Dict
 
 from dlt.common.schema.typing import TColumnSchema
-from dlt.destinations.adapters import athena_adapter, athena_partition
 from dlt.sources import DltResource, DltSource
 
 import ingestr.src.resource as resource
@@ -12,6 +11,8 @@ def apply_athena_hints(
     partition_column: str,
     additional_hints: Dict[str, TColumnSchema] = {},
 ) -> None:
+    from dlt.destinations.adapters import athena_adapter, athena_partition
+
     def _apply_partition_hint(resource: DltResource) -> None:
         columns = resource.columns if resource.columns else {}