ingestr 0.13.42__tar.gz → 0.13.44__tar.gz

{ingestr-0.13.42 → ingestr-0.13.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.42
+Version: 0.13.44
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.42 → ingestr-0.13.44}/docs/.vitepress/config.mjs

@@ -114,6 +114,7 @@ export default defineConfig({
               { text: "Applovin", link: "/supported-sources/applovin.md"},
               { text: "Applovin Max", link: "/supported-sources/applovin_max.md"},
               { text: "Asana", link: "/supported-sources/asana.md" },
+              { text: "Attio", link: "/supported-sources/attio.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
               {
                 text: "Facebook Ads",

ingestr-0.13.44/docs/supported-sources/attio.md

@@ -0,0 +1,42 @@
+# Attio
+[Attio](https://attio.com/) is an AI-native CRM platform that helps companies build, scale, and grow their business.
+
+ingestr supports Attio as a source.
+
+## URI format
+
+The URI format for Attio is as follows:
+
+```plaintext
+attio://?api_key=<api_key>
+```
+
+URI parameters:
+- `api_key`: the API key used for authentication with the Attio API
+
+## Setting up a Attio Integration
+
+You can find your Attio API key by following the guide [here](https://attio.com/help/apps/other-apps/generating-an-api-key).
+
+Let's say your `api_key` is key_123, here's a sample command that will copy the data from Attio into a DuckDB database:
+
+
+```bash
+ingestr ingest \
+--source-uri 'Attio://?api_key=key_123' \
+--source-table 'objects' \
+--dest-uri duckdb:///attio.duckdb \
+--dest-table 'dest.objects'
+```
+
+## Tables
+
+Attio source supports ingesting the following sources into separate tables:
+
+- `objects`: Objects are the data types used to store facts about your customers. Fetches all objects.
+- `records:{object_api_slug}`: Fetches all records of an object. For example: `records:companies`
+- `lists`: Fetches all lists
+- `list_entries:{list_id}`: Lists all items in a specific list. For example: `list_entries:8abc-123-456-789d-123`
+- `all_list_entries:{object_api_slug}`: Fetches all the lists for an object, and then fetches all the entries from that list. For eg:  Fetches all lists for an object, and then fetches all entries from those lists. For example: `all_list_entries:companies`
+
+Use this as `--source-table` parameter in the `ingestr ingest` command.

ingestr-0.13.44/ingestr/src/attio/__init__.py

@@ -0,0 +1,100 @@
+from typing import Iterable, Iterator
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import AttioClient
+
+
+@dlt.source(max_table_nesting=0)
+def attio_source(
+    api_key: str,
+    params: list[str],
+) -> Iterable[DltResource]:
+    
+    attio_client = AttioClient(api_key)
+
+    @dlt.resource(
+        name="objects",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_objects() -> Iterator[dict]:
+        if len(params) != 0:
+            raise ValueError("Objects table must be in the format `objects`")
+
+        path = "objects"
+        yield attio_client.fetch_data(path, "get")
+
+    @dlt.resource(
+        name="records",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_records() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "Records table must be in the format `records:{object_api_slug}`"
+            )
+
+        object_id = params[0]
+        path = f"objects/{object_id}/records/query"
+
+        yield attio_client.fetch_data(path, "post")
+
+    @dlt.resource(
+        name="lists",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_lists() -> Iterator[dict]:
+        path = "lists"
+        yield attio_client.fetch_data(path, "get")
+
+    @dlt.resource(
+        name="list_entries",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_list_entries() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "List entries table must be in the format `list_entries:{list_id}`"
+            )
+        path = f"lists/{params[0]}/entries/query"
+
+        yield attio_client.fetch_data(path, "post")
+
+    @dlt.resource(
+        name="all_list_entries",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_all_list_entries() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "All list entries table must be in the format `all_list_entries:{object_api_slug}`"
+            )
+        path = "lists"
+        for lst in attio_client.fetch_data(path, "get"):
+            if params[0] in lst["parent_object"]:
+                path = f"lists/{lst['id']['list_id']}/entries/query"
+                yield from attio_client.fetch_data(path, "post")
+
+    return (
+        fetch_objects,
+        fetch_records,
+        fetch_lists,
+        fetch_list_entries,
+        fetch_all_list_entries,
+    )

ingestr-0.13.44/ingestr/src/attio/helpers.py

@@ -0,0 +1,54 @@
+from ingestr.src.http_client import create_client
+
+
+class AttioClient:
+    def __init__(self, api_key: str):
+        self.base_url = "https://api.attio.com/v2"
+        self.headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+        self.client = create_client()
+
+    def fetch_data(self, path: str, method: str, limit: int = 1000, params=None):
+        url = f"{self.base_url}/{path}"
+        if params is None:
+            params = {}
+        offset = 0
+        while True:
+            query_params = {**params, "limit": limit, "offset": offset}
+            if method == "get":
+                response = self.client.get(
+                    url, headers=self.headers, params=query_params
+                )
+            else:
+                response = self.client.post(
+                    url, headers=self.headers, params=query_params
+                )
+
+            if response.status_code != 200:
+                raise Exception(f"HTTP {response.status_code} error: {response.text}")
+
+            response_data = response.json()
+            if "data" not in response_data:
+                print(f"API Response: {response_data}")
+                raise Exception(
+                    "Attio API returned a response without the expected data"
+                )
+
+            data = response_data["data"]
+
+            for item in data:
+                flat_item = flatten_item(item)
+                yield flat_item
+
+            if len(data) < limit:
+                break
+            offset += limit
+
+
+def flatten_item(item: dict) -> dict:
+    if "id" in item:
+        for key, value in item["id"].items():
+            item[key] = value
+    return item

ingestr-0.13.44/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.44"

{ingestr-0.13.42 → ingestr-0.13.44}/ingestr/src/destinations.py

@@ -413,6 +413,10 @@ class S3Destination:
             raise MissingValueError("secret_access_key", "S3")
 
         endpoint_url = params.get("endpoint_url", [None])[0]
+        if endpoint_url is not None:
+            parsed_endpoint = urlparse(endpoint_url)
+            if not parsed_endpoint.scheme or not parsed_endpoint.netloc:
+                raise ValueError("Invalid endpoint_url. Must be a valid URL.")
 
         creds = AwsCredentials(
             aws_access_key_id=access_key_id,
@@ -420,8 +424,21 @@ class S3Destination:
             endpoint_url=endpoint_url,
         )
 
-        dest_table = self.validate_table(kwargs["dest_table"])
+        dest_table = kwargs["dest_table"]
+
+        # only validate if dest_table is not a full URI
+        if not parsed_uri.netloc:
+            dest_table = self.validate_table(dest_table)
+
         table_parts = dest_table.split("/")
+
+        if parsed_uri.path.strip("/"):
+            path_parts = parsed_uri.path.strip("/ ").split("/")
+            table_parts = path_parts + table_parts
+
+        if parsed_uri.netloc:
+            table_parts.insert(0, parsed_uri.netloc.strip())
+            
         base_path = "/".join(table_parts[:-1])
 
         opts = {
@@ -444,10 +461,9 @@ class S3Destination:
         return table
 
     def dlt_run_params(self, uri: str, table: str, **kwargs):
-        table = self.validate_table(table)
         table_parts = table.split("/")
         return {
-            "table_name": table_parts[-1],
+            "table_name": table_parts[-1].strip(),
         }
 
     def post_load(self) -> None:

{ingestr-0.13.42 → ingestr-0.13.44}/ingestr/src/factory.py

@@ -26,6 +26,7 @@ from ingestr.src.sources import (
     AppsflyerSource,
     ArrowMemoryMappedSource,
     AsanaSource,
+    AttioSource,
     ChessSource,
     DynamoDBSource,
     ElasticsearchSource,
@@ -157,6 +158,7 @@ class SourceDestinationFactory:
         "freshdesk": FreshdeskSource,
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
+        "attio": AttioSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.13.44/ingestr/src/http_client.py

@@ -0,0 +1,17 @@
+import requests
+from dlt.sources.helpers.requests import Client
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 502

{ingestr-0.13.42 → ingestr-0.13.44}/ingestr/src/sources.py

@@ -21,6 +21,7 @@ from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
 import pendulum
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.extract import Incremental
+from dlt.extract.exceptions import ResourcesNotFoundError
 from dlt.sources import incremental as dlt_incremental
 from dlt.sources.credentials import (
     ConnectionStringCredentials,
@@ -2392,3 +2393,29 @@ class ElasticsearchSource:
             verify_certs=verify_certs,
             incremental=incremental,
         ).with_resources(table)
+
+
+class AttioSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        api_key = query_params.get("api_key")
+
+        if api_key is None:
+            raise MissingValueError("api_key", "Attio")
+
+        parts = table.replace(" ", "").split(":")
+        table_name = parts[0]
+        params = parts[1:]
+
+        from ingestr.src.attio import attio_source
+
+        try:
+            return attio_source(api_key=api_key[0], params=params).with_resources(
+                table_name
+            )
+        except ResourcesNotFoundError:
+            raise UnsupportedResourceError(table_name, "Attio")

ingestr-0.13.42/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.42"