ingestr 0.13.78__tar.gz → 0.13.79__tar.gz

{ingestr-0.13.78 → ingestr-0.13.79}/.gitignore

@@ -24,4 +24,5 @@ node_modules
 ingest.sh
 test.env
 *.log
-.claude
+.claude
+helper_scripts

{ingestr-0.13.78 → ingestr-0.13.79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.78
+Version: 0.13.79
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -355,6 +355,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>MotherDuck</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>MySQL</td>
         <td>✅</td>

{ingestr-0.13.78 → ingestr-0.13.79}/README.md

@@ -137,6 +137,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>MotherDuck</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>MySQL</td>
         <td>✅</td>

{ingestr-0.13.78 → ingestr-0.13.79}/docs/.vitepress/config.mjs

@@ -90,6 +90,7 @@ export default defineConfig({
                 link: "/supported-sources/mssql.md",
               },
               { text: "MongoDB", link: "/supported-sources/mongodb.md" },
+              { text: "MotherDuck", link: "/supported-sources/motherduck.md" },
               { text: "MySQL", link: "/supported-sources/mysql.md" },
               { text: "Oracle", link: "/supported-sources/oracle.md" },
               { text: "Postgres", link: "/supported-sources/postgres.md" },

{ingestr-0.13.78 → ingestr-0.13.79}/docs/supported-sources/attio.md

@@ -39,4 +39,7 @@ Attio source supports ingesting the following sources into separate tables:
 - `list_entries:{list_id}`: Lists all items in a specific list. For example: `list_entries:8abc-123-456-789d-123`
 - `all_list_entries:{object_api_slug}`: Fetches all the lists for an object, and then fetches all the entries from that list. For eg:  Fetches all lists for an object, and then fetches all entries from those lists. For example: `all_list_entries:companies`
 
-Use this as `--source-table` parameter in the `ingestr ingest` command.
+Use this as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!WARNING]
+> Attio does not support incremental loading, which means ingestr will do a full-refresh.

{ingestr-0.13.78 → ingestr-0.13.79}/docs/supported-sources/hubspot.md

@@ -43,22 +43,7 @@ HubSpot source allows ingesting the following sources into separate tables:
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.
 
-## Custom Objects
-
-HubSpot custom objects are supported by using the `custom` table.
-
-The format for the custom object is as follows:
-
-```plaintext
-custom:<custom_object_name>:<optional associations>
-```
-
-For example, to ingest the `course` custom object with the `contacts` and `companies` associations, the format would be:
-
-```plaintext
-custom:course:contacts,companies
-```
-
-This would pull all the data for the `course` custom object and include the `contacts` and `companies` associations in the resulting table.
+> [!WARNING]
+> Hubspot does not support incremental loading, which means ingestr will do a full-refresh.
 
 

ingestr-0.13.79/docs/supported-sources/mongodb.md

@@ -0,0 +1,150 @@
+# MongoDB
+MongoDB is a popular, open source NoSQL database known for its flexibility, scalability, and wide adoption in a variety of applications.
+
+ingestr supports MongoDB as a source.
+
+## URI format
+The URI format for MongoDB is as follows:
+
+```plaintext
+mongodb://user:password@host:port
+```
+
+URI parameters:
+- `user`: the user name to connect to the database
+- `password`: the password for the user
+- `host`: the host address of the database server
+- `port`: the port number the database server is listening on, default is 27017 for MongoDB
+
+
+> [!CAUTION]
+> Do not put the database name at the end of the URI for MongoDB, instead make it a part of `--source-table` option as `database.collection` format.
+
+
+You can read more about MongoDB's connection string format [here](https://docs.mongodb.com/manual/reference/connection-string/).
+
+## Source table format
+
+The `--source-table` option for MongoDB supports two formats:
+
+### Basic format
+```plaintext
+database.collection
+```
+
+This performs a simple collection scan, equivalent to `db.collection.find()`.
+
+### Custom aggregation format
+```plaintext
+database.collection:[aggregation_pipeline]
+```
+
+This allows you to specify a custom MongoDB aggregation pipeline as a JSON array.
+
+## Custom aggregations
+
+ingestr supports custom MongoDB aggregation pipelines, similar to how SQL sources support custom queries. This allows you to perform complex data transformations, filtering, and projections directly in MongoDB before the data is ingested.
+
+### Basic syntax
+
+Use the following format for custom aggregations:
+
+```bash
+ingestr ingest \
+  --source-uri "mongodb://user:password@host:port" \
+  --source-table 'database.collection:[{"$match": {...}}, {"$project": {...}}]'
+```
+
+### Examples
+
+#### Simple filtering
+```bash
+ingestr ingest \
+  --source-uri "mongodb://localhost:27017" \
+  --source-table 'mydb.users:[{"$match": {"status": "active"}}]'
+```
+
+#### Complex aggregation with grouping
+```bash
+ingestr ingest \
+  --source-uri "mongodb://localhost:27017" \
+  --source-table 'mydb.orders:[
+    {"$match": {"status": "completed"}},
+    {"$group": {
+      "_id": "$customer_id",
+      "total_orders": {"$sum": 1},
+      "total_amount": {"$sum": "$amount"}
+    }}
+  ]'
+```
+
+#### Projection and transformation
+```bash
+ingestr ingest \
+  --source-uri "mongodb://localhost:27017" \
+  --source-table 'mydb.products:[
+    {"$project": {
+      "name": 1,
+      "price": 1,
+      "category": 1,
+      "price_usd": {"$multiply": ["$price", 1.1]}
+    }}
+  ]'
+```
+
+### Incremental loads with custom aggregations
+
+Custom aggregations support incremental loading when combined with the `--incremental-key` option. The incremental key must be included in the projected fields of your aggregation pipeline.
+
+#### Using interval placeholders
+
+You can use `:interval_start` and `:interval_end` placeholders in your aggregation pipeline, which will be automatically replaced with the actual datetime values during incremental loads:
+
+```bash
+ingestr ingest \
+  --source-uri "mongodb://localhost:27017" \
+  --source-table 'mydb.events:[
+    {"$match": {
+      "created_at": {
+        "$gte": ":interval_start",
+        "$lt": ":interval_end"
+      }
+    }},
+    {"$project": {
+      "_id": 1,
+      "event_type": 1,
+      "user_id": 1,
+      "created_at": 1
+    }}
+  ]' \
+  --incremental-key "created_at"
+```
+
+#### Requirements for incremental loads
+
+When using incremental loads with custom aggregations:
+
+1. **Incremental key projection**: The field specified in `--incremental-key` must be included in your projection
+2. **Datetime type**: The incremental key should be a datetime field
+3. **Pipeline validation**: ingestr validates that your aggregation pipeline properly projects the incremental key
+
+### Validation and error handling
+
+ingestr performs several validations on custom aggregation pipelines:
+
+- **JSON validation**: Ensures the aggregation pipeline is valid JSON
+- **Array format**: Aggregation pipelines must be JSON arrays
+- **Incremental key validation**: When using `--incremental-key`, validates that the key is projected in the pipeline
+- **Clear error messages**: Provides specific error messages for common issues
+
+### Limitations
+
+- **Parallel loading**: Custom aggregations don't support parallel loading due to MongoDB cursor limitations. The loader automatically falls back to sequential processing.
+- **Arrow format**: When using Arrow data format with custom aggregations, data is converted to Arrow format after loading rather than using native MongoDB Arrow integration.
+
+### Performance considerations
+
+- Use `$match` stages early in your pipeline to filter data as soon as possible
+- Add appropriate indexes to support your aggregation pipeline
+- Consider using `$limit` to restrict the number of documents processed
+- For large datasets, MongoDB's `allowDiskUse: true` option is automatically enabled for aggregation pipelines

ingestr-0.13.79/docs/supported-sources/motherduck.md

@@ -0,0 +1,46 @@
+# MotherDuck
+MotherDuck is a managed cloud service built on DuckDB, designed for fast analytics and data processing in the cloud.
+
+ingestr supports MotherDuck as both a source and destination.
+
+## URI format
+The URI format for MotherDuck is as follows:
+
+```plaintext
+motherduck://<database-name>?token=<your-token>
+```
+
+Alternatively, you can use the `md://` scheme:
+```plaintext
+md://<database-name>?token=<your-token>
+```
+
+URI parameters:
+- `database-name`: the name of your MotherDuck database (optional, can be omitted for default connection)
+- `token`: your MotherDuck authentication token
+
+## Authentication
+
+### Using Token in URI
+Include the token directly in the URI:
+```plaintext
+md://<database-name>?token=<your-token>
+```
+
+### Connection without Database Name
+If you want to connect without specifying a specific database:
+```plaintext
+md://?token=<your-token>
+```
+
+## Getting Your Token
+
+1. Go to the MotherDuck UI
+2. Click on your organization name in the top left and select "Settings"
+3. Click "+ Create token"
+4. Specify a name for the token
+5. Choose between Read/Write or Read Scaling token type
+6. Set expiration if desired and click "Create token"
+7. Copy the generated token
+
+The same URI structure can be used both for sources and destinations. You can read more about MotherDuck's connection options in their [official documentation](https://motherduck.com/docs/key-tasks/authenticating-and-connecting-to-motherduck/). 

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/main.py

@@ -1,3 +1,4 @@
+import warnings
 from datetime import datetime
 from enum import Enum
 from typing import Optional
@@ -8,6 +9,14 @@ from typing_extensions import Annotated
 
 from ingestr.src.telemetry.event import track
 
+try:
+    from duckdb_engine import DuckDBEngineWarning
+
+    warnings.filterwarnings("ignore", category=DuckDBEngineWarning)
+except ImportError:
+    # duckdb-engine not installed
+    pass
+
 app = typer.Typer(
     name="ingestr",
     help="ingestr is the CLI tool to ingest data from one source to another",
@@ -506,7 +515,6 @@ def ingest(
 
         if factory.source_scheme == "sqlite":
             source_table = "main." + source_table.split(".")[-1]
-    
 
         if (
             incremental_key
@@ -600,10 +608,9 @@ def ingest(
         if factory.source_scheme == "influxdb":
             if primary_key:
                 write_disposition = "merge"
-            
 
         start_time = datetime.now()
-     
+
         run_info: LoadInfo = pipeline.run(
             dlt_source,
             **destination.dlt_run_params(

ingestr-0.13.79/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.79"

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/destinations.py

@@ -147,6 +147,24 @@ class DuckDBDestination(GenericSqlDestination):
         return dlt.destinations.duckdb(uri, **kwargs)
 
 
+class MotherduckDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        from urllib.parse import parse_qs, urlparse
+
+        parsed = urlparse(uri)
+        query = parse_qs(parsed.query)
+        token = query.get("token", [None])[0]
+        from dlt.destinations.impl.motherduck.configuration import MotherDuckCredentials
+
+        creds = {
+            "password": token,
+        }
+        if parsed.path.lstrip("/"):
+            creds["database"] = parsed.path.lstrip("/")
+
+        return dlt.destinations.motherduck(MotherDuckCredentials(creds), **kwargs)
+
+
 def handle_datetimeoffset(dto_value: bytes) -> datetime.datetime:
     # ref: https://github.com/mkleehammer/pyodbc/issues/134#issuecomment-281739794
     tup = struct.unpack(

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/facebook_ads/__init__.py

@@ -26,7 +26,6 @@ from .settings import (
     DEFAULT_LEAD_FIELDS,
     INSIGHT_FIELDS_TYPES,
     INSIGHTS_BREAKDOWNS_OPTIONS,
-    INSIGHTS_PRIMARY_KEY,
     INVALID_INSIGHTS_FIELDS,
     TInsightsBreakdownOptions,
     TInsightsLevels,

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/factory.py

@@ -12,6 +12,7 @@ from ingestr.src.destinations import (
     DatabricksDestination,
     DuckDBDestination,
     GCSDestination,
+    MotherduckDestination,
     MsSQLDestination,
     MySqlDestination,
     PostgresDestination,
@@ -85,6 +86,8 @@ SQL_SOURCE_SCHEMES = [
     "mysql",
     "mysql+pymysql",
     "mysql+mysqlconnector",
+    "md",
+    "motherduck",
     "postgres",
     "postgresql",
     "postgresql+psycopg2",
@@ -195,6 +198,8 @@ class SourceDestinationFactory:
         "cratedb": CrateDBDestination,
         "databricks": DatabricksDestination,
         "duckdb": DuckDBDestination,
+        "motherduck": MotherduckDestination,
+        "md": MotherduckDestination,
         "mssql": MsSQLDestination,
         "postgres": PostgresDestination,
         "postgresql": PostgresDestination,

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/freshdesk/__init__.py

@@ -4,6 +4,8 @@ etc. to the database"""
 from typing import Any, Dict, Generator, Iterable, List, Optional
 
 import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
 from dlt.sources import DltResource
 
 from .freshdesk_client import FreshdeskClient
@@ -12,10 +14,12 @@ from .settings import DEFAULT_ENDPOINTS
 
 @dlt.source()
 def freshdesk_source(
-    endpoints: Optional[List[str]] = None,
+    domain: str,
+    api_secret_key: str,
+    start_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
     per_page: int = 100,
-    domain: str = dlt.secrets.value,
-    api_secret_key: str = dlt.secrets.value,
+    endpoints: Optional[List[str]] = None,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from specified Freshdesk API endpoints.
@@ -39,7 +43,11 @@ def freshdesk_source(
     def incremental_resource(
         endpoint: str,
         updated_at: Optional[Any] = dlt.sources.incremental(
-            "updated_at", initial_value="2022-01-01T00:00:00Z"
+            "updated_at",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
         ),
     ) -> Generator[Dict[Any, Any], Any, None]:
         """
@@ -48,15 +56,22 @@ def freshdesk_source(
         to ensure incremental loading.
         """
 
-        # Retrieve the last updated timestamp to fetch only new or updated records.
-        if updated_at is not None:
-            updated_at = updated_at.last_value
+        if updated_at.last_value is not None:
+            start_date = ensure_pendulum_datetime(updated_at.last_value)
+        else:
+            start_date = start_date
+
+        if updated_at.end_value is not None:
+            end_date = ensure_pendulum_datetime(updated_at.end_value)
+        else:
+            end_date = pendulum.now(tz="UTC")
 
         # Use the FreshdeskClient instance to fetch paginated responses
         yield from freshdesk.paginated_response(
             endpoint=endpoint,
             per_page=per_page,
-            updated_at=updated_at,
+            start_date=start_date,
+            end_date=end_date,
         )
 
     # Set default endpoints if not provided

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/freshdesk/freshdesk_client.py

@@ -2,8 +2,9 @@
 
 import logging
 import time
-from typing import Any, Dict, Iterable, Optional
+from typing import Any, Dict, Iterable
 
+import pendulum
 from dlt.common.typing import TDataItem
 from dlt.sources.helpers import requests
 
@@ -67,7 +68,8 @@ class FreshdeskClient:
         self,
         endpoint: str,
         per_page: int,
-        updated_at: Optional[str] = None,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
     ) -> Iterable[TDataItem]:
         """
         Fetches a paginated response from a specified endpoint.
@@ -88,8 +90,8 @@ class FreshdeskClient:
                 param_key = (
                     "updated_since" if endpoint == "tickets" else "_updated_since"
                 )
-                if updated_at:
-                    params[param_key] = updated_at
+
+                params[param_key] = start_date.to_iso8601_string()
 
             # Handle requests with rate-limiting
             # A maximum of 300 pages (30000 tickets) will be returned.
@@ -98,5 +100,14 @@ class FreshdeskClient:
 
             if not data:
                 break  # Stop if no data or max page limit reached
-            yield data
+
+            filtered_data = [
+                item
+                for item in data
+                if "updated_at" in item
+                and pendulum.parse(item["updated_at"]) <= end_date
+            ]
+            if not filtered_data:
+                break
+            yield filtered_data
             page += 1

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/github/__init__.py

@@ -91,7 +91,9 @@ def github_repo_events(
     """
 
     # use naming function in table name to generate separate tables for each event
-    @dlt.resource(primary_key= "id", table_name=lambda i: i["type"], write_disposition="merge")
+    @dlt.resource(
+        primary_key="id", table_name=lambda i: i["type"], write_disposition="merge"
+    )
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "created_at",
@@ -105,7 +107,7 @@ def github_repo_events(
         repos_path = (
             f"/repos/{urllib.parse.quote(owner)}/{urllib.parse.quote(name)}/events"
         )
-        
+
         # Get the date range from the incremental state
         start_filter = pendulum.parse(
             last_created_at.last_value or last_created_at.initial_value
@@ -115,7 +117,7 @@ def github_repo_events(
             if last_created_at.end_value
             else pendulum.now()
         )
-        
+
         for page in get_rest_pages(access_token, repos_path + "?per_page=100"):
             # Filter events by date range
             filtered_events = []

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/github/helpers.py

@@ -61,6 +61,7 @@ def get_stargazers(
             page_items,
         )
 
+
 def get_reactions_data(
     node_type: str,
     owner: str,

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/influxdb/__init__.py

@@ -7,6 +7,7 @@ from dlt.sources import DltResource
 
 from .client import InfluxClient
 
+
 @dlt.source(max_table_nesting=0)
 def influxdb_source(
     measurement: str,

{ingestr-0.13.78 → ingestr-0.13.79}/ingestr/src/mongodb/__init__.py

@@ -106,6 +106,7 @@ def mongodb_collection(
     filter_: Optional[Dict[str, Any]] = None,
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
     pymongoarrow_schema: Optional[Any] = None,
+    custom_query: Optional[List[Dict[str, Any]]] = None,
 ) -> Any:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.
@@ -132,6 +133,7 @@ def mongodb_collection(
                 exclude (dict) - {"released": False, "runtime": False}
             Note: Can't mix include and exclude statements '{"title": True, "released": False}`
         pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types to convert BSON to Arrow
+        custom_query (Optional[List[Dict[str, Any]]]): Custom MongoDB aggregation pipeline to execute instead of find()
 
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
@@ -161,4 +163,5 @@ def mongodb_collection(
         filter_=filter_ or {},
         projection=projection,
         pymongoarrow_schema=pymongoarrow_schema,
+        custom_query=custom_query,
     )