ingestr 0.13.32__tar.gz → 0.13.34__tar.gz

ingestr-0.13.34/.gitleaksignore

@@ -0,0 +1,3 @@
+ingestr/src/telemetry/event.py:generic-api-key:17
+ingestr/src/testdata/fakebqcredentials.json:private-key:5
+docs/supported-sources/shopify.md:generic-api-key:26

{ingestr-0.13.32 → ingestr-0.13.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.32
+Version: 0.13.34
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.32 → ingestr-0.13.34}/docs/supported-sources/bigquery.md

@@ -17,4 +17,16 @@ URI parameters:
 
 The same URI structure can be used both for sources and destinations. You can read more about SQLAlchemy's BigQuery dialect [here](https://github.com/googleapis/python-bigquery-sqlalchemy?tab=readme-ov-file#connection-string-parameters).
 
+### Using GCS as a staging area
+
+ingestr can use GCS as a staging area for BigQuery. To do this, you need to set the `--staging-bucket` flag when you are running the command.
+
+```bash
+ingestr ingest 
+    --source-uri $SOURCE_URI
+    --dest-uri $BIGQUERY_URI
+    --source-table raw.input 
+    --dest-table raw.output
+    --staging-bucket "gs://your-bucket-name" # [!code focus]
+```
 

{ingestr-0.13.32 → ingestr-0.13.34}/docs/supported-sources/hubspot.md

@@ -42,3 +42,23 @@ HubSpot source allows ingesting the following sources into separate tables:
 - `contacts`: Retrieves information about visitors, potential customers, and leads.
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+## Custom Objects
+
+HubSpot custom objects are supported by using the `custom` table.
+
+The format for the custom object is as follows:
+
+```plaintext
+custom:<custom_object_name>:<optional associations>
+```
+
+For example, to ingest the `course` custom object with the `contacts` and `companies` associations, the format would be:
+
+```plaintext
+custom:course:contacts,companies
+```
+
+This would pull all the data for the `course` custom object and include the `contacts` and `companies` associations in the resulting table.
+
+

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/main.py

@@ -264,6 +264,13 @@ def ingest(
             envvar=["YIELD_LIMIT", "INGESTR_YIELD_LIMIT"],
         ),
     ] = None,  # type: ignore
+    staging_bucket: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The staging bucket to be used for the ingestion, must be prefixed with 'gs://' or 's3://'",
+            envvar=["STAGING_BUCKET", "INGESTR_STAGING_BUCKET"],
+        ),
+    ] = None,  # type: ignore
 ):
     import hashlib
     import tempfile
@@ -272,8 +279,6 @@ def ingest(
     import dlt
     import humanize
     import typer
-    from dlt.common.data_types import TDataType
-    from dlt.common.destination import Destination
     from dlt.common.pipeline import LoadInfo
     from dlt.common.runtime.collector import Collector, LogCollector
     from dlt.common.schema.typing import TColumnSchema
@@ -318,7 +323,7 @@ def ingest(
         return (source_table, dest_table)
 
     def validate_loader_file_format(
-        dlt_dest: Destination, loader_file_format: Optional[LoaderFileFormat]
+        dlt_dest, loader_file_format: Optional[LoaderFileFormat]
     ):
         if (
             loader_file_format
@@ -330,9 +335,11 @@ def ingest(
             )
             raise typer.Abort()
 
-    def parse_columns(columns: list[str]) -> dict[str, TDataType]:
+    def parse_columns(columns: list[str]) -> dict:
         from typing import cast, get_args
 
+        from dlt.common.data_types import TDataType
+
         possible_types = get_args(TDataType)
 
         types: dict[str, TDataType] = {}
@@ -417,7 +424,9 @@ def ingest(
             pipelines_dir = tempfile.mkdtemp()
             is_pipelines_dir_temp = True
 
-        dlt_dest = destination.dlt_dest(uri=dest_uri, dest_table=dest_table)
+        dlt_dest = destination.dlt_dest(
+            uri=dest_uri, dest_table=dest_table, staging_bucket=staging_bucket
+        )
         validate_loader_file_format(dlt_dest, loader_file_format)
 
         if partition_by:
@@ -566,6 +575,7 @@ def ingest(
             **destination.dlt_run_params(
                 uri=dest_uri,
                 table=dest_table,
+                staging_bucket=staging_bucket,
             ),
             write_disposition=write_disposition,  # type: ignore
             primary_key=(primary_key if primary_key and len(primary_key) > 0 else None),  # type: ignore

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/src/airtable/__init__.py

@@ -55,7 +55,7 @@ def airtable_resource(
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [primary_key_field["name"]]
+    primary_key: List[str] = [f"fields__{primary_key_field['name']}"]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

ingestr-0.13.34/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.34"

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/src/destinations.py

@@ -60,6 +60,22 @@ class BigQueryDestination:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            if not staging_bucket.startswith("gs://"):
+                raise ValueError("Staging bucket must start with gs://")
+
+            os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = staging_bucket
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID"] = (
+                credentials.get("project_id", None)
+            )
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__PRIVATE_KEY"] = (
+                credentials.get("private_key", None)
+            )
+            os.environ["DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL"] = (
+                credentials.get("client_email", None)
+            )
+
         project_id = None
         if source_fields.hostname:
             project_id = source_fields.hostname
@@ -83,6 +99,10 @@ class BigQueryDestination:
             "table_name": table_fields[-1],
         }
 
+        staging_bucket = kwargs.get("staging_bucket", None)
+        if staging_bucket:
+            res["staging"] = "filesystem"
+
         return res
 
     def post_load(self):

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/src/filters.py

@@ -1,6 +1,3 @@
-from dlt.common.libs.sql_alchemy import Table
-
-
 def cast_set_to_list(row):
     # this handles just the sqlalchemy backend for now
     if isinstance(row, dict):
@@ -32,6 +29,8 @@ def handle_mysql_empty_dates(row):
 
 
 def table_adapter_exclude_columns(cols: list[str]):
+    from dlt.common.libs.sql_alchemy import Table
+
     def excluder(table: Table):
         cols_to_remove = [col for col in table._columns if col.name in cols]  # type: ignore
         for col in cols_to_remove:

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/src/hubspot/__init__.py

@@ -197,11 +197,18 @@ def hubspot(
         api_key: str = api_key,
         custom_object_name: str = custom_object,
     ) -> Iterator[TDataItems]:
-        get_custom_object = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
+        custom_objects = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
         object_type_id = None
+        associations = None
+        if ":" in custom_object_name:
+            fields = custom_object_name.split(":")
+            if len(fields) == 2:
+                custom_object_name = fields[0]
+                associations = fields[1]
 
         custom_object_lowercase = custom_object_name.lower()
-        for custom_object in get_custom_object["results"]:
+
+        for custom_object in custom_objects["results"]:
             if custom_object["name"].lower() == custom_object_lowercase:
                 object_type_id = custom_object["objectTypeId"]
                 break
@@ -223,6 +230,8 @@ def hubspot(
         props = ",".join(sorted(list(set(props))))
 
         custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
+        if associations:
+            custom_object_endpoint += f"&associations={associations}"
 
         """Hubspot custom object details resource"""
         yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/src/hubspot/helpers.py

@@ -148,10 +148,6 @@ def fetch_data(
                             "updatedAt": _result.get("updatedAt", ""),
                         }
                     )
-                elif resource_name == "custom":
-                    _objects.append(
-                        _result.get("properties", ""),
-                    )
                 else:
                     _obj = _result.get("properties", _result)
                     if "id" not in _obj and "id" in _result:

{ingestr-0.13.32 → ingestr-0.13.34}/ingestr/src/partition.py

@@ -1,7 +1,6 @@
 from typing import Dict
 
 from dlt.common.schema.typing import TColumnSchema
-from dlt.destinations.adapters import athena_adapter, athena_partition
 from dlt.sources import DltResource, DltSource
 
 import ingestr.src.resource as resource
@@ -12,6 +11,8 @@ def apply_athena_hints(
     partition_column: str,
     additional_hints: Dict[str, TColumnSchema] = {},
 ) -> None:
+    from dlt.destinations.adapters import athena_adapter, athena_partition
+
     def _apply_partition_hint(resource: DltResource) -> None:
         columns = resource.columns if resource.columns else {}