ingestr 0.13.17__tar.gz → 0.13.19__tar.gz

{ingestr-0.13.17 → ingestr-0.13.19}/Makefile

@@ -11,7 +11,10 @@ venv/touchfile: requirements-dev.txt requirements.txt
 	. venv/bin/activate; pip install uv; $(MAKE) deps
 	touch venv/touchfile
 
-deps:
+lock-deps:
+	@uv pip compile requirements.in --quiet -o requirements.txt 
+
+deps: lock-deps
 	uv pip install -r requirements-dev.txt
 
 deps-ci:
@@ -20,17 +23,17 @@ deps-ci:
 test-ci:
 	TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto -x -rP -vv --tb=short --durations=10 --cov=ingestr --no-cov-on-fail
 
-test: venv
+test : venv lock-deps
 	. venv/bin/activate; $(MAKE) test-ci
 
-test-specific: venv
+test-specific: venv lock-deps
 	. venv/bin/activate; pytest -rP -vv --tb=short --capture=no -k $(test)
 
 lint-ci:
 	ruff format ingestr && ruff check ingestr --fix
 	mypy --config-file pyproject.toml --explicit-package-bases ingestr
 
-lint: venv
+lint: venv lock-deps
 	. venv/bin/activate; $(MAKE) lint-ci
 
 lint-docs:
@@ -38,7 +41,7 @@ lint-docs:
 
 tl: test lint
 
-build:
+build: lock-deps
 	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
 	rm -rf dist && python3 -m build
 	rm -f ${BUILDINFO}

{ingestr-0.13.17 → ingestr-0.13.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.17
+Version: 0.13.19
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,50 +14,177 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: aiobotocore==2.21.1
+Requires-Dist: aiohappyeyeballs==2.4.8
+Requires-Dist: aiohttp==3.11.13
+Requires-Dist: aioitertools==0.12.0
+Requires-Dist: aiosignal==1.3.2
+Requires-Dist: alembic==1.15.1
+Requires-Dist: annotated-types==0.7.0
 Requires-Dist: asana==3.2.3
+Requires-Dist: asn1crypto==1.5.1
+Requires-Dist: asynch==0.2.4
+Requires-Dist: attrs==25.1.0
+Requires-Dist: backoff==2.2.1
+Requires-Dist: beautifulsoup4==4.13.3
+Requires-Dist: boto3==1.37.1
+Requires-Dist: botocore==1.37.1
+Requires-Dist: cachetools==5.5.2
+Requires-Dist: certifi==2025.1.31
+Requires-Dist: cffi==1.17.1
+Requires-Dist: charset-normalizer==3.4.1
+Requires-Dist: ciso8601==2.3.2
+Requires-Dist: click==8.1.8
 Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: cryptography==44.0.2
+Requires-Dist: curlify==2.2.1
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
+Requires-Dist: decorator==5.2.1
+Requires-Dist: deprecation==2.1.0
 Requires-Dist: dlt==1.6.1
+Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.15.0
 Requires-Dist: duckdb==1.2.0
+Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
+Requires-Dist: frozenlist==1.5.0
+Requires-Dist: fsspec==2024.10.0
 Requires-Dist: gcsfs==2024.10.0
+Requires-Dist: gitdb==4.0.12
+Requires-Dist: gitpython==3.1.44
+Requires-Dist: giturlparse==0.12.0
 Requires-Dist: google-ads==25.1.0
 Requires-Dist: google-analytics-data==0.18.17
+Requires-Dist: google-api-core==2.24.1
 Requires-Dist: google-api-python-client==2.130.0
+Requires-Dist: google-auth-httplib2==0.2.0
+Requires-Dist: google-auth-oauthlib==1.2.1
+Requires-Dist: google-auth==2.38.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
+Requires-Dist: google-cloud-bigquery==3.30.0
+Requires-Dist: google-cloud-core==2.4.2
+Requires-Dist: google-cloud-storage==3.1.0
+Requires-Dist: google-crc32c==1.6.0
+Requires-Dist: google-resumable-media==2.7.2
+Requires-Dist: googleapis-common-protos==1.69.0
+Requires-Dist: greenlet==3.1.1
+Requires-Dist: grpcio-status==1.62.3
+Requires-Dist: grpcio==1.70.0
+Requires-Dist: hdbcli==2.23.27
+Requires-Dist: hexbytes==1.3.0
+Requires-Dist: httplib2==0.22.0
+Requires-Dist: humanize==4.12.1
+Requires-Dist: idna==3.10
+Requires-Dist: inflection==0.5.1
+Requires-Dist: isodate==0.7.2
+Requires-Dist: jmespath==1.0.1
+Requires-Dist: jsonpath-ng==1.7.0
+Requires-Dist: leb128==1.0.8
+Requires-Dist: lxml==5.3.1
+Requires-Dist: lz4==4.4.3
+Requires-Dist: makefun==1.15.6
+Requires-Dist: mako==1.3.9
+Requires-Dist: markdown-it-py==3.0.0
+Requires-Dist: markupsafe==3.0.2
+Requires-Dist: marshmallow==3.26.1
+Requires-Dist: mdurl==0.1.2
+Requires-Dist: monotonic==1.6
+Requires-Dist: more-itertools==10.6.0
+Requires-Dist: multidict==6.1.0
+Requires-Dist: mypy-extensions==1.0.0
 Requires-Dist: mysql-connector-python==9.2.0
+Requires-Dist: numpy==2.2.3
+Requires-Dist: oauthlib==3.2.2
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: orjson==3.10.15
+Requires-Dist: packaging==24.2
+Requires-Dist: pandas==2.2.3
+Requires-Dist: pathvalidate==3.2.3
 Requires-Dist: pendulum==3.0.0
+Requires-Dist: platformdirs==4.3.6
+Requires-Dist: pluggy==1.5.0
+Requires-Dist: ply==3.11
+Requires-Dist: propcache==0.3.0
+Requires-Dist: proto-plus==1.26.0
+Requires-Dist: protobuf==4.25.6
 Requires-Dist: psutil==6.1.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3
 Requires-Dist: pyarrow==18.1.0
+Requires-Dist: pyasn1-modules==0.4.1
+Requires-Dist: pyasn1==0.6.1
 Requires-Dist: pyathena==3.12.2
+Requires-Dist: pycountry==24.6.1
+Requires-Dist: pycparser==2.22
+Requires-Dist: pydantic-core==2.27.2
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: pygments==2.19.1
+Requires-Dist: pyjwt==2.10.1
 Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
+Requires-Dist: pyopenssl==25.0.0
+Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
+Requires-Dist: python-dateutil==2.9.0.post0
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: pytz==2025.1
+Requires-Dist: pyyaml==6.0.2
 Requires-Dist: redshift-connector==2.1.5
+Requires-Dist: requests-file==2.1.0
+Requires-Dist: requests-oauthlib==1.3.1
+Requires-Dist: requests-toolbelt==1.0.0
+Requires-Dist: requests==2.32.3
+Requires-Dist: requirements-parser==0.11.0
+Requires-Dist: rich-argparse==1.7.0
 Requires-Dist: rich==13.9.4
+Requires-Dist: rsa==4.9
 Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2024.10.0
+Requires-Dist: s3transfer==0.11.3
+Requires-Dist: scramp==1.4.5
+Requires-Dist: semver==3.0.4
+Requires-Dist: setuptools==75.8.2
+Requires-Dist: shellingham==1.5.4
 Requires-Dist: simple-salesforce==1.12.6
+Requires-Dist: simplejson==3.20.1
+Requires-Dist: six==1.17.0
+Requires-Dist: smmap==5.0.2
+Requires-Dist: snowflake-connector-python==3.14.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
+Requires-Dist: sortedcontainers==2.4.0
+Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
 Requires-Dist: stripe==10.7.0
+Requires-Dist: tenacity==9.0.0
+Requires-Dist: thrift==0.16.0
+Requires-Dist: time-machine==2.16.0
+Requires-Dist: tomlkit==0.13.2
 Requires-Dist: tqdm==4.67.1
 Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
+Requires-Dist: types-setuptools==75.8.2.20250305
+Requires-Dist: typing-extensions==4.12.2
+Requires-Dist: typing-inspect==0.9.0
+Requires-Dist: tzdata==2025.1
+Requires-Dist: tzlocal==5.3
+Requires-Dist: uritemplate==4.1.1
+Requires-Dist: urllib3==2.3.0
+Requires-Dist: wrapt==1.17.2
+Requires-Dist: yarl==1.18.3
+Requires-Dist: zeep==4.3.1
+Requires-Dist: zstandard==0.23.0
 Requires-Dist: zstd==1.5.6.5
 Provides-Extra: odbc
 Requires-Dist: pyodbc==5.1.0; extra == 'odbc'

{ingestr-0.13.17 → ingestr-0.13.19}/docs/.vitepress/config.mjs

@@ -40,6 +40,7 @@ export default defineConfig({
             text: "Incremental Loading",
             link: "/getting-started/incremental-loading.md",
           },
+          { text: "Load Kinesis Data to BigQuery", link: "/tutorials/load-kinesis-bigquery.md" },
           { text: "Telemetry", link: "/getting-started/telemetry.md" },
         ],
       },

{ingestr-0.13.17 → ingestr-0.13.19}/docs/supported-sources/hubspot.md

@@ -39,7 +39,6 @@ HubSpot source allows ingesting the following sources into separate tables:
 - `products`: Retrieves pricing information of products.
 - `tickets`: Handles requests for help from customers or users.
 - `quotes`: Retrieves price proposals that salespeople can create and send to their contacts.
-- `hubspot_events_for_objects`: Retrieves web analytics events for a given object type and object IDs.
 - `contacts`: Retrieves information about visitors, potential customers, and leads.
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.19/docs/tutorials/load-kinesis-bigquery.md

@@ -0,0 +1,67 @@
+# Load Kinesis Data to BigQuery
+
+Welcome! 👋 This tutorial will guide you through loading data from `Amazon Kinesis` into `Google BigQuery` using `ingestr`, a command-line tool that enables data ingestion between any source and destination using simple flags, no coding required.
+
+Amazon Kinesis is a cloud-based service for real-time data streaming and analytics that processes large data streams in real-time. You often need to store this data in a data warehouse like BigQuery for analysis and reporting. This is where ingestr simplifies the process.
+
+## Prerequisites
+- Install ingestr by following the instructions [here](../getting-started/quickstart.md#Installation)
+- AWS credentials - Access key and Secret key
+- BigQuery service account
+
+## Configuration Steps 
+### Source Configuration - Kinesis
+
+#### `--source-uri`
+This flag connects to your Kinesis stream. The URI format is:
+
+```bash
+kinesis://?aws_access_key_id=$KEY_ID&aws_secret_access_key=$SECRET_KEY&region_name=eu-central-1
+```
+
+Required parameters:
+- `aws_access_key_id`: Your AWS access key
+- `aws_secret_access_key`: Your AWS secret key
+- `region_name`: AWS region of your Kinesis stream.
+
+#### `--source-table`
+This flag specifies which Kinesis stream to read from:
+```bash
+--source-table 'kinesis_stream_name'
+```
+
+### Destination Configuration - BigQuery
+#### `--dest-uri`
+
+This flag connects to BigQuery. The URI format is:
+```bash
+bigquery://project-name?credentials_path=/path/to/service/account.json&location=<location>
+```
+Required parameters:
+- `project-name`: Your BigQuery project name
+- `credentials_path`: Path to service account JSON file
+- `location`: (Optional) Dataset location
+
+#### `--dest-table`
+This flag specifies where to save the data:
+
+```bash
+--dest-table 'schema.table_name'
+```
+
+Now that we've configured all our flags, we can run a single command to connect to Kinesis, read from our specified stream, and load the data into our BigQuery target table.
+
+```bash
+ingestr ingest \
+    --source-uri 'kinesis://?aws_access_key_id=id_123&aws_secret_access_key=secret_123&region_name=eu-central-1' \
+    --source-table 'stream_name_1' \
+    --dest-uri 'bigquery://test-playground?credentials_path=/Users/abc.json' \
+    --dest-table 'dest.results'
+```
+
+After running this command, your Kinesis data will be loaded into BigQuery. Here's what the data looks like in the destination:
+
+<img alt="kinesis_bigquery" src="../media/kinesis.bigquery.png" />
+
+🎉 Congratulations!
+You've successfully loaded data from Amazon Kinesis to your desired destination.

{ingestr-0.13.17 → ingestr-0.13.19}/ingestr/main.py

@@ -8,6 +8,9 @@ from rich.console import Console
 from rich.status import Status
 from typing_extensions import Annotated
 
+import ingestr.src.partition as partition
+import ingestr.src.resource as resource
+from ingestr.src.destinations import AthenaDestination
 from ingestr.src.filters import cast_set_to_list
 from ingestr.src.telemetry.event import track
 
@@ -357,14 +360,6 @@ def ingest(
             )
             raise typer.Abort()
 
-    def run_on_resource(source, executable):
-        if hasattr(source, "selected_resources") and source.selected_resources:
-            resource_names = list(source.selected_resources.keys())
-            for res in resource_names:
-                executable(source.resources[res])
-        else:
-            executable(source)
-
     def parse_columns(columns: list[str]) -> dict[str, TDataType]:
         from typing import cast, get_args
 
@@ -553,20 +548,23 @@ def ingest(
             sql_exclude_columns=sql_exclude_columns,
         )
 
-        run_on_resource(dlt_source, lambda x: x.add_map(cast_set_to_list))
+        resource.for_each(dlt_source, lambda x: x.add_map(cast_set_to_list))
 
         def col_h(x):
             if column_hints:
                 x.apply_hints(columns=column_hints)
 
-        run_on_resource(dlt_source, col_h)
+        resource.for_each(dlt_source, col_h)
+
+        if isinstance(destination, AthenaDestination) and partition_by:
+            partition.apply_athena_hints(dlt_source, partition_by, column_hints)
 
         if original_incremental_strategy == IncrementalStrategy.delete_insert:
 
             def set_primary_key(x):
                 x.incremental.primary_key = ()
 
-            run_on_resource(dlt_source, set_primary_key)
+            resource.for_each(dlt_source, set_primary_key)
 
         if (
             factory.destination_scheme in PARQUET_SUPPORTED_DESTINATIONS

{ingestr-0.13.17 → ingestr-0.13.19}/ingestr/src/adjust/adjust_helpers.py

@@ -82,7 +82,7 @@ class AdjustAPI:
             items = result.get("rows", [])
             yield items
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}")
+            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
 
     def fetch_events(self):
         headers = {"Authorization": f"Bearer {self.api_key}"}
@@ -93,7 +93,7 @@ class AdjustAPI:
             result = response.json()
             yield result
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}")
+            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
 
 
 def parse_filters(filters_raw: str) -> dict:

{ingestr-0.13.17 → ingestr-0.13.19}/ingestr/src/blob.py

@@ -1,6 +1,6 @@
 import warnings
 from typing import Tuple, TypeAlias
-from urllib.parse import ParseResult
+from urllib.parse import ParseResult, urlparse
 
 BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
@@ -14,13 +14,16 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     Supports the following Forms:
     - uri: "gs://"
       table: "bucket-name/file-glob"
+    - uri: "gs://uri-bucket-name" (uri-bucket-name is preferred)
+      table: "gs://table-bucket-name/file-glob"
+    - uri: "gs://"
+      table: "gs://bucket-name/file-glob"
     - uri: gs://bucket-name/file-glob
       table: None
     - uri: "gs://bucket-name"
       table: "file-glob"
 
-    The first form is the prefered method. Other forms are supported
-    for backward compatibility, but discouraged.
+    The first form is the prefered method. Other forms are supported but discouraged.
     """
 
     table = table.strip()
@@ -34,15 +37,15 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         )
         return host, uri.path.lstrip("/")
 
+    table_uri = urlparse(table)
+
     if host != "":
-        warnings.warn(
-            f"Using the form '{uri.scheme}://bucket-name' is deprecated and will be removed in future versions.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return host, table.lstrip("/")
+        return host, table_uri.path.lstrip("/")
+
+    if table_uri.hostname:
+        return table_uri.hostname, table_uri.path.lstrip("/")
 
-    parts = table.lstrip("/").split("/", maxsplit=1)
+    parts = table_uri.path.lstrip("/").split("/", maxsplit=1)
     if len(parts) != 2:
         return "", parts[0]
 

ingestr-0.13.19/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.19"

{ingestr-0.13.17 → ingestr-0.13.19}/ingestr/src/hubspot/__init__.py

@@ -199,14 +199,6 @@ def crm_objects(
 
     props = ",".join(sorted(list(set(props))))
 
-    if len(props) > 2000:
-        raise ValueError(
-            "Your request to Hubspot is too long to process. "
-            "Maximum allowed query length is 2000 symbols, while "
-            f"your list of properties `{props[:200]}`... is {len(props)} "
-            "symbols long. Use the `props` argument of the resource to "
-            "set the list of properties to extract from the endpoint."
-        )
 
     params = {"properties": props, "limit": 100}
 

{ingestr-0.13.17 → ingestr-0.13.19}/ingestr/src/kinesis/__init__.py

@@ -16,6 +16,7 @@ from .helpers import get_shard_iterator, max_sequence_by_shard
     name=lambda args: args["stream_name"],
     primary_key="kinesis_msg_id",
     standalone=True,
+    max_table_nesting=0
 )
 def kinesis_stream(
     stream_name: str,

ingestr-0.13.19/ingestr/src/partition.py

@@ -0,0 +1,31 @@
+from typing import Dict
+
+from dlt.common.schema.typing import TColumnSchema
+from dlt.destinations.adapters import athena_adapter, athena_partition
+from dlt.sources import DltResource, DltSource
+
+import ingestr.src.resource as resource
+
+
+def apply_athena_hints(
+    source: DltSource | DltResource,
+    partition_column: str,
+    additional_hints: Dict[str, TColumnSchema] = {},
+) -> None:
+    def _apply_partition_hint(resource: DltResource) -> None:
+
+        columns = resource.columns if resource.columns else {}
+
+        partition_hint = (
+            columns.get(partition_column)  # type: ignore
+            or additional_hints.get(partition_column)
+        )
+
+        athena_adapter(
+            resource,
+            athena_partition.day(partition_column)
+            if partition_hint and partition_hint.get("data_type") in ("timestamp", "date")
+            else partition_column,
+        )
+
+    resource.for_each(source, _apply_partition_hint)

ingestr-0.13.19/ingestr/src/resource.py

@@ -0,0 +1,17 @@
+from typing import Callable
+
+from dlt.sources import DltResource, DltSource
+
+
+def for_each(
+    source: DltSource | DltResource, ex: Callable[[DltResource], None | DltResource]
+):
+    """
+    Apply a function to each resource in a source.
+    """
+    if hasattr(source, "selected_resources") and source.selected_resources:
+        resource_names = list(source.selected_resources.keys())
+        for res in resource_names:
+            ex(source.resources[res])  # type: ignore[union-attr]
+    else:
+        ex(source)  # type: ignore[arg-type]

ingestr-0.13.17/requirements.txt → ingestr-0.13.19/requirements.in

@@ -47,4 +47,3 @@ clickhouse-connect==0.8.14
 clickhouse-driver==0.2.9
 clickhouse-sqlalchemy==0.2.7
 zstd==1.5.6.5
-