ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/appsflyer/client.py

@@ -4,19 +4,6 @@ import requests
 from dlt.sources.helpers.requests import Client
 from requests.exceptions import HTTPError
 
-DEFAULT_GROUPING = ["c", "geo", "app_id", "install_time"]
-DEFAULT_KPIS = [
-    "impressions",
-    "clicks",
-    "installs",
-    "cost",
-    "revenue",
-    "average_ecpi",
-    "loyal_users",
-    "uninstalls",
-    "roi",
-]
-
 
 class AppsflyerClient:
     def __init__(self, api_key: str):
@@ -33,15 +20,20 @@ class AppsflyerClient:
         self,
         from_date: str,
         to_date: str,
+        dimensions: list[str],
+        metrics: list[str],
         maximum_rows=1000000,
-        dimensions=DEFAULT_GROUPING,
-        metrics=DEFAULT_KPIS,
     ):
+        excluded_metrics = exclude_metrics_for_date_range(metrics, from_date, to_date)
+        included_metrics = [
+            metric for metric in metrics if metric not in excluded_metrics
+        ]
+
         params = {
             "from": from_date,
             "to": to_date,
             "groupings": ",".join(dimensions),
-            "kpis": ",".join(metrics),
+            "kpis": ",".join(included_metrics),
             "format": "json",
             "maximum_rows": maximum_rows,
         }
@@ -54,7 +46,6 @@ class AppsflyerClient:
             )
 
         request_client = Client(
-            request_timeout=10.0,
             raise_for_status=False,
             retry_condition=retry_on_limit,
             request_max_attempts=12,
@@ -68,39 +59,52 @@ class AppsflyerClient:
 
             if response.status_code == 200:
                 result = response.json()
-                yield result
+                yield standardize_keys(result, excluded_metrics)
             else:
                 raise HTTPError(
-                    f"Request failed with status code: {response.status_code}"
+                    f"Request failed with status code: {response.status_code}: {response.text}"
                 )
 
         except requests.RequestException as e:
             raise HTTPError(f"Request failed: {e}")
 
-    def fetch_campaigns(
-        self,
-        start_date: str,
-        end_date: str,
-    ):
-        metrics = DEFAULT_KPIS + [
-            "cohort_day_1_revenue_per_user",
-            "cohort_day_1_total_revenue_per_user",
-            "cohort_day_3_revenue_per_user",
-            "cohort_day_3_total_revenue_per_user",
-            "cohort_day_7_total_revenue_per_user",
-            "cohort_day_7_revenue_per_user",
-            "cohort_day_14_total_revenue_per_user",
-            "cohort_day_14_revenue_per_user",
-            "cohort_day_21_total_revenue_per_user",
-            "cohort_day_21_revenue_per_user",
-            "retention_day_7",
-        ]
-        return self._fetch_data(start_date, end_date, metrics=metrics)
 
-    def fetch_creatives(
-        self,
-        start_date: str,
-        end_date: str,
-    ):
-        dimensions = DEFAULT_GROUPING + ["af_adset_id", "af_adset", "af_ad_id"]
-        return self._fetch_data(start_date, end_date, dimensions=dimensions)
+def standardize_keys(data: list[dict], excluded_metrics: list[str]) -> list[dict]:
+    def fix_key(key: str) -> str:
+        return key.lower().replace("-", "").replace("  ", "_").replace(" ", "_")
+
+    standardized = []
+    for item in data:
+        standardized_item = {}
+        for key, value in item.items():
+            standardized_item[fix_key(key)] = value
+
+        for metric in excluded_metrics:
+            if metric not in standardized_item:
+                standardized_item[fix_key(metric)] = None
+
+        standardized.append(standardized_item)
+
+    return standardized
+
+
+def exclude_metrics_for_date_range(
+    metrics: list[str], from_date: str, to_date: str
+) -> list[str]:
+    """
+    Some of the cohort metrics are not available if there hasn't been enough time to have data for that cohort.
+    This means if you request data for yesterday with cohort day 7 metrics, you will get an error because 7 days hasn't passed yet.
+    One would expect the API to handle this gracefully, but it doesn't.
+
+    This function will exclude the metrics that are not available for the given date range.
+    """
+    import pendulum
+
+    excluded_metrics = []
+    days_between_today_and_end = (pendulum.now() - pendulum.parse(to_date)).days  # type: ignore
+    for metric in metrics:
+        if "cohort_day_" in metric:
+            day_count = int(metric.split("_")[2])
+            if days_between_today_and_end <= day_count:
+                excluded_metrics.append(metric)
+    return excluded_metrics

ingestr/src/appstore/__init__.py

@@ -38,6 +38,7 @@ def app_store(
             name=resource.name,
             primary_key=resource.primary_key,
             columns=resource.columns,
+            write_disposition="merge",
         )(client, app_ids, resource.report_name, start_date, end_date)
 
 

ingestr/src/arrow/__init__.py

@@ -1,6 +1,7 @@
 from typing import Any, Optional
 
 import dlt
+import pyarrow as pa  # type: ignore
 from dlt.common.schema.typing import TColumnNames, TTableSchemaColumns
 from dlt.extract.items import TTableHintTemplate
 
@@ -21,7 +22,6 @@ def memory_mapped_arrow(
     def arrow_mmap(
         incremental: Optional[dlt.sources.incremental[Any]] = incremental,
     ):
-        import pyarrow as pa  # type: ignore
         import pyarrow.ipc as ipc  # type: ignore
 
         with pa.memory_map(path, "rb") as mmap:
@@ -71,3 +71,11 @@ def memory_mapped_arrow(
         yield table
 
     return arrow_mmap
+
+
+BATCH_SIZE = 1000
+
+
+def as_list(table: pa.Table):
+    for batch in table.to_batches(BATCH_SIZE):
+        yield from batch.to_pylist()

ingestr/src/asana_source/__init__.py

@@ -182,7 +182,7 @@ def tasks(
 
 @dlt.transformer(
     data_from=tasks,
-    write_disposition="append",
+    write_disposition="replace",
 )
 @dlt.defer
 def stories(

ingestr/src/attio/__init__.py

@@ -0,0 +1,102 @@
+from typing import Iterable, Iterator
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import AttioClient
+
+
+@dlt.source(max_table_nesting=0)
+def attio_source(
+    api_key: str,
+    params: list[str],
+) -> Iterable[DltResource]:
+    attio_client = AttioClient(api_key)
+
+    @dlt.resource(
+        name="objects",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    # https://docs.attio.com/rest-api/endpoint-reference/objects/list-objects - does not support pagination
+    def fetch_objects() -> Iterator[dict]:
+        if len(params) != 0:
+            raise ValueError("Objects table must be in the format `objects`")
+
+        path = "objects"
+        yield attio_client.fetch_all(path, "get")
+
+    # https://docs.attio.com/rest-api/endpoint-reference/records/list-records
+    @dlt.resource(
+        name="records",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_records() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "Records table must be in the format `records:{object_api_slug}`"
+            )
+        object_id = params[0]
+        path = f"objects/{object_id}/records/query"
+
+        yield attio_client.fetch_paginated(path, "post")
+
+    # https://docs.attio.com/rest-api/endpoint-reference/lists/list-all-lists -- does not support pagination
+    @dlt.resource(
+        name="lists",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_lists() -> Iterator[dict]:
+        path = "lists"
+        yield attio_client.fetch_all(path, "get")
+
+    # https://docs.attio.com/rest-api/endpoint-reference/entries/list-entries
+    @dlt.resource(
+        name="list_entries",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_list_entries() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "List entries table must be in the format `list_entries:{list_id}`"
+            )
+        path = f"lists/{params[0]}/entries/query"
+
+        yield attio_client.fetch_paginated(path, "post")
+
+    @dlt.resource(
+        name="all_list_entries",
+        write_disposition="replace",
+        columns={
+            "created_at": {"data_type": "timestamp", "partition": True},
+        },
+    )
+    def fetch_all_list_entries() -> Iterator[dict]:
+        if len(params) != 1:
+            raise ValueError(
+                "All list entries table must be in the format `all_list_entries:{object_api_slug}`"
+            )
+        path = "lists"
+        for lst in attio_client.fetch_all(path, "get"):
+            if params[0] in lst["parent_object"]:
+                path = f"lists/{lst['id']['list_id']}/entries/query"
+                yield from attio_client.fetch_paginated(path, "post")
+
+    return (
+        fetch_objects,
+        fetch_records,
+        fetch_lists,
+        fetch_list_entries,
+        fetch_all_list_entries,
+    )

ingestr/src/attio/helpers.py

@@ -0,0 +1,65 @@
+from ingestr.src.http_client import create_client
+
+
+class AttioClient:
+    def __init__(self, api_key: str):
+        self.base_url = "https://api.attio.com/v2"
+        self.headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        }
+        self.client = create_client()
+
+    def fetch_paginated(self, path: str, method: str, limit: int = 1000, params=None):
+        url = f"{self.base_url}/{path}"
+        if params is None:
+            params = {}
+        offset = 0
+        while True:
+            query_params = {"limit": limit, "offset": offset, **params}
+            if method == "get":
+                response = self.client.get(
+                    url, headers=self.headers, params=query_params
+                )
+            else:
+                json_body = {**params, "limit": limit, "offset": offset}
+                response = self.client.post(url, headers=self.headers, json=json_body)
+
+            if response.status_code != 200:
+                raise Exception(f"HTTP {response.status_code} error: {response.text}")
+
+            response_data = response.json()
+            if "data" not in response_data:
+                raise Exception(
+                    "Attio API returned a response without the expected data"
+                )
+
+            data = response_data["data"]
+            for item in data:
+                flat_item = flatten_item(item)
+                yield flat_item
+            if len(data) < limit:
+                break
+
+            offset += limit
+
+    def fetch_all(self, path: str, method: str = "get", params=None):
+        url = f"{self.base_url}/{path}"
+        params = params or {}
+
+        if method == "get":
+            response = self.client.get(url, headers=self.headers, params=params)
+        else:
+            response = self.client.post(url, headers=self.headers, json=params)
+
+        response.raise_for_status()
+        data = response.json().get("data", [])
+        for item in data:
+            yield flatten_item(item)
+
+
+def flatten_item(item: dict) -> dict:
+    if "id" in item:
+        for key, value in item["id"].items():
+            item[key] = value
+    return item

ingestr/src/blob.py

@@ -1,11 +1,15 @@
 import warnings
 from typing import Tuple, TypeAlias
-from urllib.parse import ParseResult
+from urllib.parse import ParseResult, urlparse
 
 BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
 
 
+class UnsupportedEndpointError(Exception):
+    pass
+
+
 def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     """
     parse the URI of a blob storage and
@@ -14,19 +18,22 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     Supports the following Forms:
     - uri: "gs://"
       table: "bucket-name/file-glob"
+    - uri: "gs://uri-bucket-name" (uri-bucket-name is preferred)
+      table: "gs://table-bucket-name/file-glob"
+    - uri: "gs://"
+      table: "gs://bucket-name/file-glob"
     - uri: gs://bucket-name/file-glob
       table: None
     - uri: "gs://bucket-name"
       table: "file-glob"
 
-    The first form is the prefered method. Other forms are supported
-    for backward compatibility, but discouraged.
+    The first form is the prefered method. Other forms are supported but discouraged.
     """
 
     table = table.strip()
     host = uri.netloc.strip()
 
-    if table == "":
+    if table == "" or uri.path.strip() != "":
         warnings.warn(
             f"Using the form '{uri.scheme}://bucket-name/file-glob' is deprecated and will be removed in future versions.",
             DeprecationWarning,
@@ -34,16 +41,36 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         )
         return host, uri.path.lstrip("/")
 
+    table_uri = urlparse(table)
+
     if host != "":
-        warnings.warn(
-            f"Using the form '{uri.scheme}://bucket-name' is deprecated and will be removed in future versions.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return host, table.lstrip("/")
+        return host, table_uri.path.lstrip("/")
+
+    if table_uri.hostname:
+        return table_uri.hostname, table_uri.path.lstrip("/")
 
-    parts = table.lstrip("/").split("/", maxsplit=1)
+    parts = table_uri.path.lstrip("/").split("/", maxsplit=1)
     if len(parts) != 2:
         return "", parts[0]
 
     return parts[0], parts[1]
+
+
+def parse_endpoint(path: str) -> str:
+    """
+    Parse the endpoint kind from the URI.
+
+    kind is a file format. one of [csv, jsonl, parquet]
+    """
+    file_extension = path.split(".")[-1]
+    if file_extension == "gz":
+        file_extension = path.split(".")[-2]
+    if file_extension == "csv":
+        endpoint = "read_csv"
+    elif file_extension == "jsonl":
+        endpoint = "read_jsonl"
+    elif file_extension == "parquet":
+        endpoint = "read_parquet"
+    else:
+        raise UnsupportedEndpointError(f"Unsupported file format: {file_extension}")
+    return endpoint

ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.104"

ingestr/src/chess/__init__.py

@@ -75,7 +75,7 @@ def players_archives(players: List[str]) -> Iterator[List[TDataItem]]:
 
 
 @dlt.resource(
-    write_disposition="append", columns={"end_time": {"data_type": "timestamp"}}
+    write_disposition="replace", columns={"end_time": {"data_type": "timestamp"}}
 )
 def players_games(
     players: List[str], start_month: str = None, end_month: str = None

ingestr/src/clickup/__init__.py

@@ -0,0 +1,85 @@
+"""Simple ClickUp source."""
+
+from datetime import datetime
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource
+
+from .helpers import ClickupClient
+
+
+@dlt.source(max_table_nesting=0)
+def clickup_source(
+    api_token: str = dlt.secrets.value,
+    start_date: datetime = None,
+    end_date: datetime = None,
+) -> Iterable[DltResource]:
+    client = ClickupClient(api_token)
+
+    @dlt.resource(
+        name="user",
+        primary_key="id",
+        write_disposition="merge",
+    )
+    def user() -> Iterable[dict]:
+        data = client.get("/user")
+        yield data["user"]
+
+    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
+    def teams() -> Iterable[dict]:
+        for team in client.get_teams():
+            yield team
+
+    @dlt.resource(name="spaces", primary_key="id", write_disposition="merge")
+    def spaces() -> Iterable[dict]:
+        for space in client.get_spaces():
+            yield space
+
+    @dlt.resource(name="lists", write_disposition="merge", primary_key="id")
+    def lists() -> Iterable[dict]:
+        for list in client.get_lists():
+            yield list
+
+    @dlt.resource(
+        name="tasks",
+        write_disposition="merge",
+        primary_key="id",
+        columns={"date_updated": {"data_type": "timestamp"}},
+    )
+    def tasks(
+        date_updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "date_updated",
+            initial_value=ensure_pendulum_datetime(start_date).in_timezone("UTC"),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[dict]:
+        if date_updated.last_value:
+            start = ensure_pendulum_datetime(date_updated.last_value).in_timezone("UTC")
+        else:
+            start = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+
+        if date_updated.end_value is None:
+            end = pendulum.now("UTC")
+        else:
+            end = date_updated.end_value.in_timezone("UTC")
+
+        for list_obj in client.get_lists():
+            for task in client.paginated(
+                f"/list/{list_obj['id']}/task", "tasks", {"page_size": 100}
+            ):
+                task_dt = ensure_pendulum_datetime(int(task["date_updated"]) / 1000)
+                if task_dt >= start and task_dt <= end:
+                    task["date_updated"] = task_dt
+                    yield task
+
+    return (
+        user,
+        teams,
+        spaces,
+        lists,
+        tasks,
+    )

ingestr/src/clickup/helpers.py

@@ -0,0 +1,47 @@
+from typing import Iterable, Optional
+
+from ..http_client import create_client
+
+
+class ClickupClient:
+    def __init__(self, api_token: str):
+        self.session = create_client()
+        self.base_url = "https://api.clickup.com/api/v2"
+        self.headers = {"Authorization": api_token}
+
+    def get(self, endpoint: str, params: Optional[dict] = None) -> dict:
+        url = f"{self.base_url}{endpoint}"
+        resp = self.session.get(url, headers=self.headers, params=params or {})
+        resp.raise_for_status()
+        return resp.json()
+
+    def paginated(
+        self, endpoint: str, key: str, params: Optional[dict] = None
+    ) -> Iterable[dict]:
+        page = 0
+        params = params or {}
+        while True:
+            params["page"] = page
+            data = self.get(endpoint, params)
+            items = data.get(key, data)
+            if not items:
+                break
+            for item in items:
+                yield item
+            if data.get("last_page") or len(items) < params.get("page_size", 100):
+                break
+            page += 1
+
+    def get_teams(self):
+        data = self.get("/team")
+        return data.get("teams", [])
+
+    def get_spaces(self):
+        for team in self.get_teams():
+            for space in self.paginated(f"/team/{team['id']}/space", "spaces"):
+                yield space
+
+    def get_lists(self):
+        for space in self.get_spaces():
+            for lst in self.paginated(f"/space/{space['id']}/list", "lists"):
+                yield lst

ingestr/src/collector/spinner.py

@@ -0,0 +1,43 @@
+from typing import Optional
+
+from dlt.common.runtime.collector import Collector
+from rich.status import Status
+
+
+class SpinnerCollector(Collector):
+    status: Status
+    current_step: str
+    started: bool
+
+    def __init__(self) -> None:
+        self.status = Status("Ingesting data...", spinner="dots")
+        self.started = False
+
+    def update(
+        self,
+        name: str,
+        inc: int = 1,
+        total: Optional[int] = None,
+        message: Optional[str] = None,  # type: ignore
+        label: str = "",
+        **kwargs,
+    ) -> None:
+        self.status.update(self.current_step)
+
+    def _start(self, step: str) -> None:
+        self.current_step = self.__step_to_label(step)
+        self.status.start()
+
+    def __step_to_label(self, step: str) -> str:
+        verb = step.split(" ")[0].lower()
+        if verb.startswith("normalize"):
+            return "Normalizing the data"
+        elif verb.startswith("load"):
+            return "Loading the data to the destination"
+        elif verb.startswith("extract"):
+            return "Extracting the data from the source"
+
+        return f"{verb.capitalize()} the data"
+
+    def _stop(self) -> None:
+        self.status.stop()