ingestr 0.13.41__tar.gz → 0.13.43__tar.gz

{ingestr-0.13.41 → ingestr-0.13.43}/Makefile

@@ -8,7 +8,7 @@ venv: venv/touchfile
 
 venv/touchfile: requirements-dev.txt requirements.txt
 	test -d venv || python3 -m venv venv
-	. venv/bin/activate; pip install uv; $(MAKE) deps
+	. venv/bin/activate; pip install --disable-pip-version-check uv; $(MAKE) deps
 	touch venv/touchfile
 
 lock-deps:

{ingestr-0.13.41 → ingestr-0.13.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.41
+Version: 0.13.43
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.41 → ingestr-0.13.43}/docs/.vitepress/config.mjs

@@ -114,6 +114,7 @@ export default defineConfig({
               { text: "Applovin", link: "/supported-sources/applovin.md"},
               { text: "Applovin Max", link: "/supported-sources/applovin_max.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
+              { text: "Attio", link: "/supported-sources/attio.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               {
                 text: "Facebook Ads",

ingestr-0.13.43/docs/supported-sources/attio.md

@@ -0,0 +1,42 @@
+# Attio
+[Attio](https://attio.com/) is an AI-native CRM platform that helps companies build, scale, and grow their business.
+
+ingestr supports Attio as a source.
+
+## URI format
+
+The URI format for Attio is as follows:
+
+```plaintext
+attio://?api_key=<api_key>
+```
+
+URI parameters:
+- `api_key`: the API key used for authentication with the Attio API
+
+## Setting up a Attio Integration
+
+You can find your Attio API key by following the guide [here](https://attio.com/help/apps/other-apps/generating-an-api-key).
+
+Let's say your `api_key` is key_123, here's a sample command that will copy the data from Attio into a DuckDB database:
+
+
+```bash
+ingestr ingest \
+--source-uri 'Attio://?api_key=key_123' \
+--source-table 'objects' \
+--dest-uri duckdb:///attio.duckdb \
+--dest-table 'dest.objects'
+```
+
+## Tables
+
+Attio source supports ingesting the following sources into separate tables:
+
+- `objects`: Objects are the data types used to store facts about your customers. Fetches all objects.
+- `records:{object_api_slug}`: Fetches all records of an object. For example: `records:companies`
+- `lists`: Fetches all lists
+- `list_entries:{list_id}`: Lists all items in a specific list. For example: `list_entries:8abc-123-456-789d-123`
+- `all_list_entries:{object_api_slug}`: Fetches all the lists for an object, and then fetches all the entries from that list. For eg:  Fetches all lists for an object, and then fetches all entries from those lists. For example: `all_list_entries:companies`
+
+Use this as `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.43/ingestr/src/attio/__init__.py

@@ -0,0 +1,100 @@
+from typing import Iterable, Iterator
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import AttioClient
+
+
+@dlt.source(max_table_nesting=0)
+def attio_source(
+    api_key: str,
+    params: list[str],
+) -> Iterable[DltResource]:
+    
+    attio_client = AttioClient(api_key)
+
+    @dlt.resource(
+        name="objects",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_objects() -> Iterator[dict]:
+        if len(params) != 0:
+            raise ValueError("Objects table must be in the format `objects`")
+
+        path = "objects"
+        yield attio_client.fetch_data(path, "get")
+
+    @dlt.resource(
+        name="records",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_records() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "Records table must be in the format `records:{object_api_slug}`"
+            )
+
+        object_id = params[0]
+        path = f"objects/{object_id}/records/query"
+
+        yield attio_client.fetch_data(path, "post")
+
+    @dlt.resource(
+        name="lists",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_lists() -> Iterator[dict]:
+        path = "lists"
+        yield attio_client.fetch_data(path, "get")
+
+    @dlt.resource(
+        name="list_entries",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_list_entries() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "List entries table must be in the format `list_entries:{list_id}`"
+            )
+        path = f"lists/{params[0]}/entries/query"
+
+        yield attio_client.fetch_data(path, "post")
+
+    @dlt.resource(
+        name="all_list_entries",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_all_list_entries() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "All list entries table must be in the format `all_list_entries:{object_api_slug}`"
+            )
+        path = "lists"
+        for lst in attio_client.fetch_data(path, "get"):
+            if params[0] in lst["parent_object"]:
+                path = f"lists/{lst['id']['list_id']}/entries/query"
+                yield from attio_client.fetch_data(path, "post")
+
+    return (
+        fetch_objects,
+        fetch_records,
+        fetch_lists,
+        fetch_list_entries,
+        fetch_all_list_entries,
+    )

ingestr-0.13.43/ingestr/src/attio/helpers.py

@@ -0,0 +1,54 @@
+from ingestr.src.http_client import create_client
+
+
+class AttioClient:
+    def __init__(self, api_key: str):
+        self.base_url = "https://api.attio.com/v2"
+        self.headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+        self.client = create_client()
+
+    def fetch_data(self, path: str, method: str, limit: int = 1000, params=None):
+        url = f"{self.base_url}/{path}"
+        if params is None:
+            params = {}
+        offset = 0
+        while True:
+            query_params = {**params, "limit": limit, "offset": offset}
+            if method == "get":
+                response = self.client.get(
+                    url, headers=self.headers, params=query_params
+                )
+            else:
+                response = self.client.post(
+                    url, headers=self.headers, params=query_params
+                )
+
+            if response.status_code != 200:
+                raise Exception(f"HTTP {response.status_code} error: {response.text}")
+
+            response_data = response.json()
+            if "data" not in response_data:
+                print(f"API Response: {response_data}")
+                raise Exception(
+                    "Attio API returned a response without the expected data"
+                )
+
+            data = response_data["data"]
+
+            for item in data:
+                flat_item = flatten_item(item)
+                yield flat_item
+
+            if len(data) < limit:
+                break
+            offset += limit
+
+
+def flatten_item(item: dict) -> dict:
+    if "id" in item:
+        for key, value in item["id"].items():
+            item[key] = value
+    return item

ingestr-0.13.43/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.43"

{ingestr-0.13.41 → ingestr-0.13.43}/ingestr/src/factory.py

@@ -26,6 +26,7 @@ from ingestr.src.sources import (
     AppsflyerSource,
     ArrowMemoryMappedSource,
     AsanaSource,
+    AttioSource,
     ChessSource,
     DynamoDBSource,
     ElasticsearchSource,
@@ -120,6 +121,7 @@ class SourceDestinationFactory:
     sources: Dict[str, Type[SourceProtocol]] = {
         "csv": LocalCsvSource,
         "mongodb": MongoDbSource,
+        "mongodb+srv": MongoDbSource,
         "notion": NotionSource,
         "gsheets": GoogleSheetsSource,
         "shopify": ShopifySource,
@@ -156,6 +158,7 @@ class SourceDestinationFactory:
         "freshdesk": FreshdeskSource,
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
+        "attio": AttioSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.13.43/ingestr/src/http_client.py

@@ -0,0 +1,17 @@
+import requests
+from dlt.sources.helpers.requests import Client
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 502

{ingestr-0.13.41 → ingestr-0.13.43}/ingestr/src/mongodb/__init__.py

@@ -1,8 +1,9 @@
 """Source that loads collections form any a mongo database, supports incremental loads."""
 
-from typing import Any, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Union
 
 import dlt
+from dlt.common.data_writers import TDataItemFormat
 from dlt.sources import DltResource
 
 from .helpers import (
@@ -21,6 +22,10 @@ def mongodb(
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = dlt.config.value,
+    limit: Optional[int] = None,
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = None,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Iterable[DltResource]:
     """
     A DLT source which loads data from a mongo database using PyMongo.
@@ -34,6 +39,18 @@ def mongodb(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]):
+            The maximum number of documents to load. The limit is
+            applied to each requested collection separately.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields of a collection
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types of a collection to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -58,19 +75,36 @@ def mongodb(
             primary_key="_id",
             write_disposition=write_disposition,
             spec=MongoDbCollectionConfiguration,
-        )(client, collection, incremental=incremental, parallel=parallel)
+        )(
+            client,
+            collection,
+            incremental=incremental,
+            parallel=parallel,
+            limit=limit,
+            filter_=filter_ or {},
+            projection=projection,
+            pymongoarrow_schema=pymongoarrow_schema,
+        )
 
 
-@dlt.common.configuration.with_config(
-    sections=("sources", "mongodb"), spec=MongoDbCollectionResourceConfiguration
+@dlt.resource(
+    name=lambda args: args["collection"],
+    standalone=True,
+    spec=MongoDbCollectionResourceConfiguration,
 )
 def mongodb_collection(
-    connection_url: str = dlt.config.value,
+    connection_url: str = dlt.secrets.value,
     database: Optional[str] = dlt.config.value,
     collection: str = dlt.config.value,
     incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = False,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 10000,
+    data_item_format: Optional[TDataItemFormat] = "object",
+    filter_: Optional[Dict[str, Any]] = None,
+    projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
+    pymongoarrow_schema: Optional[Any] = None,
 ) -> Any:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.
@@ -83,6 +117,21 @@ def mongodb_collection(
             E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
         write_disposition (str): Write disposition of the resource.
         parallel (Optional[bool]): Option to enable parallel loading for the collection. Default is False.
+        limit (Optional[int]): The number of documents load.
+        chunk_size (Optional[int]): The number of documents load in each batch.
+        data_item_format (Optional[TDataItemFormat]): The data format to use for loading.
+            Supported formats:
+                object - Python objects (dicts, lists).
+                arrow - Apache Arrow tables.
+        filter_ (Optional[Dict[str, Any]]): The filter to apply to the collection.
+        projection: (Optional[Union[Mapping[str, Any], Iterable[str]]]): The projection to select fields
+            when loading the collection. Supported inputs:
+                include (list) - ["year", "title"]
+                include (dict) - {"year": True, "title": True}
+                exclude (dict) - {"released": False, "runtime": False}
+            Note: Can't mix include and exclude statements '{"title": True, "released": False}`
+        pymongoarrow_schema (pymongoarrow.schema.Schema): Mapping of expected field types to convert BSON to Arrow
+
     Returns:
         Iterable[DltResource]: A list of DLT resources for each collection to be loaded.
     """
@@ -100,4 +149,15 @@ def mongodb_collection(
         name=collection_obj.name,
         primary_key="_id",
         write_disposition=write_disposition,
-    )(client, collection_obj, incremental=incremental, parallel=parallel)
+    )(
+        client,
+        collection_obj,
+        incremental=incremental,
+        parallel=parallel,
+        limit=limit,
+        chunk_size=chunk_size,
+        data_item_format=data_item_format,
+        filter_=filter_ or {},
+        projection=projection,
+        pymongoarrow_schema=pymongoarrow_schema,
+    )