ingestr 0.13.20__tar.gz → 0.13.22__tar.gz

{ingestr-0.13.20 → ingestr-0.13.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.20
+Version: 0.13.22
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: aiobotocore==2.21.1
 Requires-Dist: aiohappyeyeballs==2.4.8
-Requires-Dist: aiohttp==3.11.13
+Requires-Dist: aiohttp==3.11.15
 Requires-Dist: aioitertools==0.12.0
 Requires-Dist: aiosignal==1.3.2
 Requires-Dist: alembic==1.15.1
@@ -55,8 +55,8 @@ Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
-Requires-Dist: fsspec==2024.10.0
-Requires-Dist: gcsfs==2024.10.0
+Requires-Dist: fsspec==2025.3.2
+Requires-Dist: gcsfs==2025.3.2
 Requires-Dist: gitdb==4.0.12
 Requires-Dist: gitpython==3.1.44
 Requires-Dist: giturlparse==0.12.0
@@ -149,7 +149,7 @@ Requires-Dist: rich-argparse==1.7.0
 Requires-Dist: rich==13.9.4
 Requires-Dist: rsa==4.9
 Requires-Dist: rudder-sdk-python==2.1.4
-Requires-Dist: s3fs==2024.10.0
+Requires-Dist: s3fs==2025.3.2
 Requires-Dist: s3transfer==0.11.3
 Requires-Dist: scramp==1.4.5
 Requires-Dist: semver==3.0.4

{ingestr-0.13.20 → ingestr-0.13.22}/docs/.vitepress/config.mjs

@@ -126,6 +126,7 @@ export default defineConfig({
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Personio", link: "/supported-sources/personio.md" },
+              { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
               { text: "S3", link: "/supported-sources/s3.md" },
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },

ingestr-0.13.22/docs/supported-sources/pipedrive.md

@@ -0,0 +1,43 @@
+# Pipedrive
+[Pipedrive](https://www.pipedrive.com/) is a cloud-based sales Customer Relationship Management (CRM) tool designed to help businesses manage leads and deals, track communication, and automate sales processes.
+
+ingestr supports pipedrive as a source.
+
+## URI format
+
+The URI format for pipedrive is as follows:
+
+```plaintext
+pipedrive://?api_token=<api_token>
+```
+
+URI parameters:
+- api_token: token used for authentication with the Pipedrive API
+
+## Setting up a pipedrive Integration
+
+To grab pipedrive credentials, please follow the guide [here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/pipedrive#grab-api-token).
+
+Once you complete the guide, you should have a `api_token`. Let's say your `api_token` is token_123, here's a sample command that will copy the data from pipedriveinto a DuckDB database:
+
+```bash
+ingestr ingest \
+--source-uri 'pipedrive://?api_token=token' \
+--source-table 'users' \
+--dest-uri duckdb:///pipedrive.duckdb \
+--dest-table 'dest.users'
+```
+
+<img alt="pipedrive_img" src="../media/pipedrive.png"/>
+
+pipedrive source allows ingesting the following resources into separate tables:
+
+- `activities`: Refers to scheduled events or tasks associated with deals, contacts, or organizations
+- `organizations`: Refers to company or entity with which you have potential or existing business dealings.
+- `products`: Refers to items or services offered for sale that can be associated with deals
+- `deals`: Refers to potential sale or transaction that you can track through various stages
+- `users`: Refers to Individual with a unique login credential who can access and use the platform
+- `persons`: Refers individual contacts or leads that can be linked to sales deals
+
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.22/docs/tutorials/load-kinesis-bigquery.md

@@ -0,0 +1,130 @@
+# Load Data from Amazon Kinesis to Google BigQuery
+
+Welcome! 👋  
+ This beginner-friendly guide will help you load data from `Amazon Kinesis` into `Google BigQuery` using `ingestr` — a simple yet powerful command-line tool. No prior experience is needed, and best of all, no coding required!
+
+By the end of this guide, you'll have your Kinesis data securely stored in BigQuery. But before we dive in, let’s take a quick look at `ingestr`.
+
+## Overview of ingestr
+
+`ingestr` is a command-line tool that simplifies data ingestion by allowing users to load data from a source to a destination using simple command-line flags.
+
+### ingestr Command
+
+```bash
+ingestr ingest \   
+   --source-uri '<your-source-uri-here>' \   
+   --source-table '<your-schema>.<your-table>' \   
+   --dest-uri '<your-destination-uri-here>' \   
+   --dest-table '<your-schema>.<your-table>'
+```
+
+- `ingestr ingest`: Executes the data ingestion process.
+- `--source-uri TEXT`: Specifies the URI of the data source.
+- `--dest-uri TEXT`: Specifies the URI of the destination.
+- `--source-table TEXT`: Defines the table to fetch data from.
+- `--dest-table TEXT`: Specifies the destination table. If not provided, it defaults to `--source-table`.
+
+With this command, we connect to the source, retrieve the specified data, and load it into the destination database.
+
+## Let's Load Data from Kinesis to BigQuery Together!
+
+Amazon Kinesis is a cloud-based service for real-time data streaming and analytics that processes large data streams. To analyze this data, you may need to load it into a data warehouse like Google BigQuery. `ingestr` makes this process simple.
+
+### Step 1: Install ingestr
+
+Ensure `ingestr` is installed. If not, follow the installation guide [here](../getting-started/quickstart.md#Installation).
+
+### Step 2: Get AWS Credentials
+Kinesis will be our data source. To access it, you need AWS credentials.
+
+1. Log in to your AWS account.
+2. Navigate to `IAM` (Identity and Access Management).
+3. Create a new IAM user or select an existing one.
+4. Assign necessary permissions (e.g., `AmazonKinesisReadOnlyAccess`).
+5. Generate and copy the `Access Key ID` and `Secret Access Key`.
+
+For more details, read [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html).
+
+### Step 3: Configure Kinesis as Source
+
+#### `--source-uri`
+This flag connects to your Kinesis stream. The URI format is:
+
+```bash
+kinesis://?aws_access_key_id=<YOUR_KEY_ID>&aws_secret_access_key=<YOUR_SECRET_KEY>&region_name=<YOUR_REGION>
+```
+
+Required parameters:
+- `aws_access_key_id`: Your AWS access key
+- `aws_secret_access_key`: Your AWS secret key
+- `region_name`: AWS region of your Kinesis stream
+
+#### `--source-table`
+This flag specifies which Kinesis stream to read from:
+
+```bash
+--source-table 'kinesis_stream_name'
+```
+This flag specifies which Kinesis stream to read from:
+### Step 4: Configure BigQuery as Destination
+
+#### `--dest-uri`
+This flag connects to BigQuery. The URI format is:
+
+```bash
+bigquery://<project-name>?credentials_path=/path/to/service/account.json&location=<location>
+```
+
+Required parameters:
+- `project-name`: Your BigQuery project name
+- `credentials_path`: Path to the service account JSON file
+- `location`: (Optional) Dataset location
+
+#### `--dest-table`
+This flag specifies where to save the data:
+
+```bash
+--dest-table 'dataset.table_name'
+```
+
+### Step 5: Run the ingestr Command
+
+Execute the following command to load data from Kinesis to BigQuery:
+
+```bash
+ingestr ingest \   
+    --source-uri 'kinesis://?aws_access_key_id=<YOUR_KEY_ID>&aws_secret_access_key=<YOUR_SECRET_KEY>&region_name=eu-central-1' \   
+    --source-table 'kinesis_stream_name' \   
+    --dest-uri 'bigquery://project-name?credentials_path=/Users/abc.json' \   
+    --dest-table 'dataset.results'
+```
+
+### Step 6: Verify Data in BigQuery
+Once the command runs successfully, your Kinesis data will be available in BigQuery. Follow these steps to verify the data:
+
+1. Open the [BigQuery Console](https://console.cloud.google.com/bigquery) and select your project.
+
+2. In the left-hand side panel:
+    - Expand your project.
+    - Navigate to the appropriate dataset and click on the table name.
+
+3. Select the "Preview" tab to view a sample of the ingested data.
+    - Confirm that rows are present and fields appear as expected.
+
+5. Go to the "Query" tab and run a basic query to inspect your data more closely. For example:
+```sql
+SELECT * FROM `project-name.dataset.results` LIMIT 100;
+
+```
+
+Ensure that the retrieved data matches what was expected from the Kinesis stream.
+
+### Example Output
+
+After running the ingestion process, your Kinesis data will be available in BigQuery. Here's an example of what the data might look like:
+
+<img alt="kinesis_bigquery" src="../media/kinesis.bigquery.png" />
+
+## 🎉 Congratulations!
+You have successfully loaded data from Amazon Kinesis to BigQuery using `ingestr`.

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/main.py

@@ -127,40 +127,44 @@ class SqlReflectionLevel(str, Enum):
 def ingest(
     source_uri: Annotated[
         str,
-        typer.Option(help="The URI of the [green]source[/green]", envvar="SOURCE_URI"),
+        typer.Option(
+            help="The URI of the [green]source[/green]",
+            envvar=["SOURCE_URI", "INGESTR_SOURCE_URI"],
+        ),
     ],  # type: ignore
     dest_uri: Annotated[
         str,
         typer.Option(
-            help="The URI of the [cyan]destination[/cyan]", envvar="DESTINATION_URI"
+            help="The URI of the [cyan]destination[/cyan]",
+            envvar=["DESTINATION_URI", "INGESTR_DESTINATION_URI"],
         ),
     ],  # type: ignore
     source_table: Annotated[
         str,
         typer.Option(
             help="The table name in the [green]source[/green] to fetch",
-            envvar="SOURCE_TABLE",
+            envvar=["SOURCE_TABLE", "INGESTR_SOURCE_TABLE"],
         ),
     ],  # type: ignore
     dest_table: Annotated[
         str,
         typer.Option(
             help="The table in the [cyan]destination[/cyan] to save the data into",
-            envvar="DESTINATION_TABLE",
+            envvar=["DESTINATION_TABLE", "INGESTR_DESTINATION_TABLE"],
         ),
     ] = None,  # type: ignore
     incremental_key: Annotated[
         Optional[str],
         typer.Option(
             help="The incremental key from the table to be used for incremental strategies",
-            envvar="INCREMENTAL_KEY",
+            envvar=["INCREMENTAL_KEY", "INGESTR_INCREMENTAL_KEY"],
         ),
     ] = None,  # type: ignore
     incremental_strategy: Annotated[
         IncrementalStrategy,
         typer.Option(
             help="The incremental strategy to use",
-            envvar="INCREMENTAL_STRATEGY",
+            envvar=["INCREMENTAL_STRATEGY", "INGESTR_INCREMENTAL_STRATEGY"],
         ),
     ] = IncrementalStrategy.create_replace,  # type: ignore
     interval_start: Annotated[
@@ -168,7 +172,7 @@ def ingest(
         typer.Option(
             help="The start of the interval the incremental key will cover",
             formats=DATE_FORMATS,
-            envvar="INTERVAL_START",
+            envvar=["INTERVAL_START", "INGESTR_INTERVAL_START"],
         ),
     ] = None,  # type: ignore
     interval_end: Annotated[
@@ -176,126 +180,126 @@ def ingest(
         typer.Option(
             help="The end of the interval the incremental key will cover",
             formats=DATE_FORMATS,
-            envvar="INTERVAL_END",
+            envvar=["INTERVAL_END", "INGESTR_INTERVAL_END"],
         ),
     ] = None,  # type: ignore
     primary_key: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The key that will be used to deduplicate the resulting table",
-            envvar="PRIMARY_KEY",
+            envvar=["PRIMARY_KEY", "INGESTR_PRIMARY_KEY"],
         ),
     ] = None,  # type: ignore
     partition_by: Annotated[
         Optional[str],
         typer.Option(
             help="The partition key to be used for partitioning the destination table",
-            envvar="PARTITION_BY",
+            envvar=["PARTITION_BY", "INGESTR_PARTITION_BY"],
         ),
     ] = None,  # type: ignore
     cluster_by: Annotated[
         Optional[str],
         typer.Option(
             help="The clustering key to be used for clustering the destination table, not every destination supports clustering.",
-            envvar="CLUSTER_BY",
+            envvar=["CLUSTER_BY", "INGESTR_CLUSTER_BY"],
         ),
     ] = None,  # type: ignore
     yes: Annotated[
         Optional[bool],
         typer.Option(
             help="Skip the confirmation prompt and ingest right away",
-            envvar="SKIP_CONFIRMATION",
+            envvar=["SKIP_CONFIRMATION", "INGESTR_SKIP_CONFIRMATION"],
         ),
     ] = False,  # type: ignore
     full_refresh: Annotated[
         bool,
         typer.Option(
             help="Ignore the state and refresh the destination table completely",
-            envvar="FULL_REFRESH",
+            envvar=["FULL_REFRESH", "INGESTR_FULL_REFRESH"],
         ),
     ] = False,  # type: ignore
     progress: Annotated[
         Progress,
         typer.Option(
             help="The progress display type, must be one of 'interactive', 'log'",
-            envvar="PROGRESS",
+            envvar=["PROGRESS", "INGESTR_PROGRESS"],
         ),
     ] = Progress.interactive,  # type: ignore
     sql_backend: Annotated[
         SqlBackend,
         typer.Option(
             help="The SQL backend to use",
-            envvar="SQL_BACKEND",
+            envvar=["SQL_BACKEND", "INGESTR_SQL_BACKEND"],
         ),
     ] = SqlBackend.pyarrow,  # type: ignore
     loader_file_format: Annotated[
         Optional[LoaderFileFormat],
         typer.Option(
             help="The file format to use when loading data",
-            envvar="LOADER_FILE_FORMAT",
+            envvar=["LOADER_FILE_FORMAT", "INGESTR_LOADER_FILE_FORMAT"],
         ),
     ] = None,  # type: ignore
     page_size: Annotated[
         Optional[int],
         typer.Option(
             help="The page size to be used when fetching data from SQL sources",
-            envvar="PAGE_SIZE",
+            envvar=["PAGE_SIZE", "INGESTR_PAGE_SIZE"],
         ),
     ] = 50000,  # type: ignore
     loader_file_size: Annotated[
         Optional[int],
         typer.Option(
             help="The file size to be used by the loader to split the data into multiple files. This can be set independent of the page size, since page size is used for fetching the data from the sources whereas this is used for the processing/loading part.",
-            envvar="LOADER_FILE_SIZE",
+            envvar=["LOADER_FILE_SIZE", "INGESTR_LOADER_FILE_SIZE"],
         ),
     ] = 100000,  # type: ignore
     schema_naming: Annotated[
         SchemaNaming,
         typer.Option(
             help="The naming convention to use when moving the tables from source to destination. The default behavior is explained here: https://dlthub.com/docs/general-usage/schema#naming-convention",
-            envvar="SCHEMA_NAMING",
+            envvar=["SCHEMA_NAMING", "INGESTR_SCHEMA_NAMING"],
         ),
     ] = SchemaNaming.default,  # type: ignore
     pipelines_dir: Annotated[
         Optional[str],
         typer.Option(
             help="The path to store dlt-related pipeline metadata. By default, ingestr will create a temporary directory and delete it after the execution is done in order to make retries stateless.",
-            envvar="PIPELINES_DIR",
+            envvar=["PIPELINES_DIR", "INGESTR_PIPELINES_DIR"],
         ),
     ] = None,  # type: ignore
     extract_parallelism: Annotated[
         Optional[int],
         typer.Option(
             help="The number of parallel jobs to run for extracting data from the source, only applicable for certain sources",
-            envvar="EXTRACT_PARALLELISM",
+            envvar=["EXTRACT_PARALLELISM", "INGESTR_EXTRACT_PARALLELISM"],
         ),
     ] = 5,  # type: ignore
     sql_reflection_level: Annotated[
         SqlReflectionLevel,
         typer.Option(
             help="The reflection level to use when reflecting the table schema from the source",
-            envvar="SQL_REFLECTION_LEVEL",
+            envvar=["SQL_REFLECTION_LEVEL", "INGESTR_SQL_REFLECTION_LEVEL"],
         ),
     ] = SqlReflectionLevel.full,  # type: ignore
     sql_limit: Annotated[
         Optional[int],
         typer.Option(
             help="The limit to use when fetching data from the source",
-            envvar="SQL_LIMIT",
+            envvar=["SQL_LIMIT", "INGESTR_SQL_LIMIT"],
         ),
     ] = None,  # type: ignore
     sql_exclude_columns: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The columns to exclude from the source table",
-            envvar="SQL_EXCLUDE_COLUMNS",
+            envvar=["SQL_EXCLUDE_COLUMNS", "INGESTR_SQL_EXCLUDE_COLUMNS"],
         ),
     ] = [],  # type: ignore
     columns: Annotated[
         Optional[list[str]],
         typer.Option(
             help="The column types to be used for the destination table in the format of 'column_name:column_type'",
-            envvar="COLUMNS",
+            envvar=["COLUMNS", "INGESTR_COLUMNS"],
         ),
     ] = None,  # type: ignore
 ):

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/src/adjust/adjust_helpers.py

@@ -82,7 +82,9 @@ class AdjustAPI:
             items = result.get("rows", [])
             yield items
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
+            raise HTTPError(
+                f"Request failed with status code: {response.status_code}, {response.text}."
+            )
 
     def fetch_events(self):
         headers = {"Authorization": f"Bearer {self.api_key}"}
@@ -93,7 +95,9 @@ class AdjustAPI:
             result = response.json()
             yield result
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
+            raise HTTPError(
+                f"Request failed with status code: {response.status_code}, {response.text}."
+            )
 
 
 def parse_filters(filters_raw: str) -> dict:

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/src/applovin_max/__init__.py

@@ -105,11 +105,13 @@ def get_data(
         if response.status_code == 404:
             if "No Mediation App Id found for platform" in response.text:
                 return None
-        error_message = f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        error_message = (
+            f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        )
         raise requests.HTTPError(error_message)
-    
+
     response_url = response.json().get("ad_revenue_report_url")
     df = pd.read_csv(response_url)
     df["Date"] = pd.to_datetime(df["Date"])
     df["partition_date"] = df["Date"].dt.date
-    return df
+    return df

ingestr-0.13.22/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.22"

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/src/factory.py

@@ -43,6 +43,7 @@ from ingestr.src.sources import (
     MongoDbSource,
     NotionSource,
     PersonioSource,
+    PipedriveSource,
     S3Source,
     SalesforceSource,
     ShopifySource,
@@ -144,6 +145,7 @@ class SourceDestinationFactory:
         "salesforce": SalesforceSource,
         "personio": PersonioSource,
         "kinesis": KinesisSource,
+        "pipedrive": PipedriveSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/src/hubspot/__init__.py

@@ -199,7 +199,6 @@ def crm_objects(
 
     props = ",".join(sorted(list(set(props))))
 
-
     params = {"properties": props, "limit": 100}
 
     yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, params=params)

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/src/kinesis/__init__.py

@@ -16,7 +16,7 @@ from .helpers import get_shard_iterator, max_sequence_by_shard
     name=lambda args: args["stream_name"],
     primary_key="kinesis_msg_id",
     standalone=True,
-    max_table_nesting=0
+    max_table_nesting=0,
 )
 def kinesis_stream(
     stream_name: str,
@@ -75,7 +75,6 @@ def kinesis_stream(
 
     # get next shard to fetch messages from
     while shard_id := shard_ids.pop(0) if shard_ids else None:
-     
         shard_iterator, _ = get_shard_iterator(
             kinesis_client,
             stream_name,
@@ -83,14 +82,14 @@ def kinesis_stream(
             last_msg,  # type: ignore
             initial_at_datetime,  # type: ignore
         )
-       
+
         while shard_iterator:
             records = []
             records_response = kinesis_client.get_records(
                 ShardIterator=shard_iterator,
                 Limit=chunk_size,  # The size of data can be up to 1 MB, it must be controlled by the user
             )
-            
+
             for record in records_response["Records"]:
                 sequence_number = record["SequenceNumber"]
                 content = record["Data"]

{ingestr-0.13.20 → ingestr-0.13.22}/ingestr/src/partition.py

@@ -13,7 +13,6 @@ def apply_athena_hints(
     additional_hints: Dict[str, TColumnSchema] = {},
 ) -> None:
     def _apply_partition_hint(resource: DltResource) -> None:
-
         columns = resource.columns if resource.columns else {}
 
         partition_hint = (
@@ -24,7 +23,8 @@ def apply_athena_hints(
         athena_adapter(
             resource,
             athena_partition.day(partition_column)
-            if partition_hint and partition_hint.get("data_type") in ("timestamp", "date")
+            if partition_hint
+            and partition_hint.get("data_type") in ("timestamp", "date")
             else partition_column,
         )