ingestr 0.13.70__tar.gz → 0.13.72__tar.gz

{ingestr-0.13.70 → ingestr-0.13.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.70
+Version: 0.13.72
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -57,7 +57,7 @@ Requires-Dist: elastic-transport==8.17.1
 Requires-Dist: elasticsearch==8.10.1
 Requires-Dist: enum-compat==0.0.3
 Requires-Dist: et-xmlfile==2.0.0
-Requires-Dist: facebook-business==20.0.0
+Requires-Dist: facebook-business==23.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0

{ingestr-0.13.70 → ingestr-0.13.72}/docs/.vitepress/config.mjs

@@ -118,6 +118,7 @@ export default defineConfig({
               { text: "Asana", link: "/supported-sources/asana.md" },
               { text: "Attio", link: "/supported-sources/attio.md" },
               { text: "Chess.com", link: "/supported-sources/chess.md" },
+              { text: "ClickUp", link: "/supported-sources/clickup.md" },
               {
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",

ingestr-0.13.72/docs/supported-sources/clickup.md

@@ -0,0 +1,45 @@
+# ClickUp
+[ClickUp](https://clickup.com/) is a cloud-based productivity platform designed to help teams and individuals manage tasks, projects, and workflows in one place.
+
+ingestr supports ClickUp as a source.
+
+## URI format
+
+```
+clickup://?api_token=<token>
+```
+
+URI parameters:
+- `api_token` is a personal token used to authenticate with the ClickUp API.
+
+ClickUp requires a `api_token` to connect to the ClickUP API. For more information, read [here](https://developer.clickup.com/docs/authentication#generate-your-personal-api-token) to get credentail. 
+
+Once you've completed the guide, you should have `api_token`. Let's say your API Token  is `token_123`, here's a sample command that will copy the data from Clickup into a DuckDB database:
+
+To ingest tasks from ClickUp into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri "clickup://?api_token=token_123" \
+  --source-table "tasks" \
+  --dest-uri duckdb:///clickup.duckdb \
+  --dest-table "public.tasks"
+```
+<img alt="clickup_img" src="../media/clickup_ingestion.png"/>
+
+## Tables
+
+ClickUp source allows ingesting the following resources into separate tables:
+
+| **Table** | **Description** |
+|-----------|-----------------|
+| `user`   | The authorised user profile. |
+| `teams`   | Workspaces available to the authenticated user. In ClickUp, a "team" in the API is equivalent to a workspace, which is the highest-level container for all spaces, folders, lists, and tasks.
+| `spaces`  | Spaces available within a workspace. Spaces are major sections used to organize work, such as departments, projects, or business units. Each space can contain folders, lists, and tasks |
+| `lists`   | Lists contained in each space. Lists are collections of tasks, often representing projects, sprints, or workflow stages.|
+| `tasks`   | Tasks belonging to each list. |
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!WARNING]
+> ClickUp does not support incremental loading for many endpoints, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.

ingestr-0.13.72/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.72"

ingestr-0.13.72/ingestr/src/clickup/__init__.py

@@ -0,0 +1,85 @@
+"""Simple ClickUp source."""
+
+from datetime import datetime
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource
+
+from .helpers import ClickupClient
+
+
+@dlt.source(max_table_nesting=0)
+def clickup_source(
+    api_token: str = dlt.secrets.value,
+    start_date: datetime = None,
+    end_date: datetime = None,
+) -> Iterable[DltResource]:
+    client = ClickupClient(api_token)
+
+    @dlt.resource(
+        name="user",
+        primary_key="id",
+        write_disposition="merge",
+    )
+    def user() -> Iterable[dict]:
+        data = client.get("/user")
+        yield data["user"]
+
+    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
+    def teams() -> Iterable[dict]:
+        for team in client.get_teams():
+            yield team
+
+    @dlt.resource(name="spaces", primary_key="id", write_disposition="merge")
+    def spaces() -> Iterable[dict]:
+        for space in client.get_spaces():
+            yield space
+
+    @dlt.resource(name="lists", write_disposition="merge", primary_key="id")
+    def lists() -> Iterable[dict]:
+        for list in client.get_lists():
+            yield list
+
+    @dlt.resource(
+        name="tasks",
+        write_disposition="merge",
+        primary_key="id",
+        columns={"date_updated": {"data_type": "timestamp"}},
+    )
+    def tasks(
+        date_updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "date_updated",
+            initial_value=ensure_pendulum_datetime(start_date).in_timezone("UTC"),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[dict]:
+        if date_updated.last_value:
+            start = ensure_pendulum_datetime(date_updated.last_value).in_timezone("UTC")
+        else:
+            start = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+
+        if date_updated.end_value is None:
+            end = pendulum.now("UTC")
+        else:
+            end = date_updated.end_value.in_timezone("UTC")
+
+        for list_obj in client.get_lists():
+            for task in client.paginated(
+                f"/list/{list_obj['id']}/task", "tasks", {"page_size": 100}
+            ):
+                task_dt = ensure_pendulum_datetime(int(task["date_updated"]) / 1000)
+                if task_dt >= start and task_dt <= end:
+                    task["date_updated"] = task_dt
+                    yield task
+
+    return (
+        user,
+        teams,
+        spaces,
+        lists,
+        tasks,
+    )

ingestr-0.13.72/ingestr/src/clickup/helpers.py

@@ -0,0 +1,47 @@
+from typing import Iterable, Optional
+
+from ..http_client import create_client
+
+
+class ClickupClient:
+    def __init__(self, api_token: str):
+        self.session = create_client()
+        self.base_url = "https://api.clickup.com/api/v2"
+        self.headers = {"Authorization": api_token}
+
+    def get(self, endpoint: str, params: Optional[dict] = None) -> dict:
+        url = f"{self.base_url}{endpoint}"
+        resp = self.session.get(url, headers=self.headers, params=params or {})
+        resp.raise_for_status()
+        return resp.json()
+
+    def paginated(
+        self, endpoint: str, key: str, params: Optional[dict] = None
+    ) -> Iterable[dict]:
+        page = 0
+        params = params or {}
+        while True:
+            params["page"] = page
+            data = self.get(endpoint, params)
+            items = data.get(key, data)
+            if not items:
+                break
+            for item in items:
+                yield item
+            if data.get("last_page") or len(items) < params.get("page_size", 100):
+                break
+            page += 1
+
+    def get_teams(self):
+        data = self.get("/team")
+        return data.get("teams", [])
+
+    def get_spaces(self):
+        for team in self.get_teams():
+            for space in self.paginated(f"/team/{team['id']}/space", "spaces"):
+                yield space
+
+    def get_lists(self):
+        for space in self.get_spaces():
+            for lst in self.paginated(f"/space/{space['id']}/list", "lists"):
+                yield lst

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/facebook_ads/__init__.py

@@ -1,5 +1,6 @@
 """Loads campaigns, ads sets, ads, leads and insight data from Facebook Marketing API"""
 
+from datetime import datetime
 from typing import Iterator, Sequence
 
 import dlt
@@ -7,12 +8,12 @@ from dlt.common import pendulum
 from dlt.common.typing import TDataItems
 from dlt.sources import DltResource
 from facebook_business.adobjects.ad import Ad
+from dlt.common.time import ensure_pendulum_datetime
 
 from .helpers import (
     execute_job,
     get_ads_account,
     get_data_chunked,
-    get_start_date,
     process_report_item,
 )
 from .settings import (
@@ -166,15 +167,14 @@ def facebook_insights_source(
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
+            initial_value=ensure_pendulum_datetime(start_date).start_of('day').date(),
+            end_value=ensure_pendulum_datetime(end_date).end_of('day').date() if end_date else None,
             range_end="closed",
             range_start="closed",
-            lag=attribution_window_days_lag * 24 * 60 * 60,  # Convert days to seconds
         ),
     ) -> Iterator[TDataItems]:
-        start_date = get_start_date(date_start)
-        end_date = pendulum.now()
+        start_date = date_start.last_value
+        end_date = pendulum.instance(date_start.end_value) if date_start.end_value else pendulum.now()
 
         while start_date <= end_date:
             query = {
@@ -202,9 +202,10 @@ def facebook_insights_source(
             }
             job = execute_job(
                 account.get_insights(params=query, is_async=True),
-                insights_max_async_sleep_seconds=10,
+                insights_max_async_sleep_seconds=20,
             )
-            yield list(map(process_report_item, job.get_result()))
+            output = list(map(process_report_item, job.get_result()))
+            yield output
             start_date = start_date.add(days=time_increment_days)
 
     return facebook_insights

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/facebook_ads/helpers.py

@@ -3,6 +3,7 @@
 import functools
 import itertools
 import time
+from datetime import datetime
 from typing import Any, Iterator, Sequence
 
 import dlt
@@ -23,43 +24,17 @@ from facebook_business.api import FacebookResponse
 
 from .exceptions import InsightsJobTimeout
 from .settings import (
-    FACEBOOK_INSIGHTS_RETENTION_PERIOD,
     INSIGHTS_PRIMARY_KEY,
     TFbMethod,
 )
 
 
-def get_start_date(
-    incremental_start_date: dlt.sources.incremental[str],
-) -> pendulum.DateTime:
-    """
-    Get the start date for incremental loading of Facebook Insights data.
-    """
-    start_date: pendulum.DateTime = ensure_pendulum_datetime(
-        incremental_start_date.start_value
-    )
-
-    # facebook forgets insights so trim the lag and warn
-    min_start_date = pendulum.today().subtract(
-        months=FACEBOOK_INSIGHTS_RETENTION_PERIOD
-    )
-    if start_date < min_start_date:
-        logger.warning(
-            "%s: Start date is earlier than %s months ago, using %s instead. "
-            "For more information, see https://www.facebook.com/business/help/1695754927158071?id=354406972049255",
-            "facebook_insights",
-            FACEBOOK_INSIGHTS_RETENTION_PERIOD,
-            min_start_date,
-        )
-        start_date = min_start_date
-        incremental_start_date.start_value = min_start_date
-
-    # lag the incremental start date by attribution window lag
-    incremental_start_date.start_value = start_date.isoformat()
-    return start_date
-
-
 def process_report_item(item: AbstractObject) -> DictStrAny:
+    if "date_start" in item:
+        item["date_start"] = datetime.strptime(item["date_start"], "%Y-%m-%d").date()
+    if "date_stop" in item:
+        item["date_stop"] = datetime.strptime(item["date_stop"], "%Y-%m-%d").date()
+    
     d: DictStrAny = item.export_all_data()
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:
@@ -138,15 +113,20 @@ def execute_job(
     time_start = time.time()
     sleep_time = 3
     while status != "Job Completed":
+        print("-----")
+        print("waiting for job to finish")
         duration = time.time() - time_start
         job = job.api_get()
         status = job["async_status"]
         percent_complete = job["async_percent_completion"]
+        print("async_status", status)
+        print("percent_complete", percent_complete)
 
         job_id = job["id"]
         logger.info("%s, %d%% done", status, percent_complete)
 
         if status == "Job Completed":
+            print("job completed")
             return job
 
         if duration > insights_max_wait_to_start_seconds and percent_complete == 0:

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/factory.py

@@ -32,6 +32,7 @@ from ingestr.src.sources import (
     AsanaSource,
     AttioSource,
     ChessSource,
+    ClickupSource,
     DynamoDBSource,
     ElasticsearchSource,
     FacebookAdsSource,
@@ -185,6 +186,7 @@ class SourceDestinationFactory:
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
         "zoom": ZoomSource,
+        "clickup": ClickupSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/shopify/__init__.py

@@ -1690,16 +1690,6 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "nullable": True,
                 "description": "The category of the product from Shopify's Standard Product Taxonomy.",
             },
-            "combinedListing": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "A special product type that combines separate products into a single product listing.",
-            },
-            "combinedListingRole": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "The role of the product in a combined listing.",
-            },
             "compareAtPriceRange": {
                 "data_type": "json",
                 "nullable": True,
@@ -1841,12 +1831,6 @@ query products($after: String, $query: String, $first: Int)  {
       category {
         id
       }
-      combinedListing {
-        parentProduct {
-          id
-        }
-      }
-      combinedListingRole
       compareAtPriceRange {
         maxVariantCompareAtPrice {
           amount

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/solidgate/__init__.py

@@ -86,16 +86,12 @@ COLUMN_HINTS = {
         "transaction_datetime_provider": {"data_type": "timestamp"},
         "transaction_datetime_utc": {"data_type": "timestamp"},
         "accounting_date": {"data_type": "date"},
-        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount": {"data_type": "double"},
+        "amount_in_major_units": {"data_type": "double"},
         "currency": {"data_type": "text"},
         "currency_minor_units": {"data_type": "bigint"},
-        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "payout_amount_in_major_units": {
-            "data_type": "decimal",
-            "precision": 18,
-            "scale": 4,
-        },
+        "payout_amount": {"data_type": "double"},
+        "payout_amount_in_major_units": {"data_type": "double"},
         "payout_currency": {"data_type": "text"},
         "payout_currency_minor_units": {"data_type": "bigint"},
         "record_type_key": {"data_type": "text"},

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/solidgate/helpers.py

@@ -1,4 +1,5 @@
 import base64
+import csv
 import hashlib
 import hmac
 import json
@@ -123,11 +124,18 @@ class SolidgateClient:
                 except json.JSONDecodeError:
                     try:
                         csv_data = get_response.content.decode("utf-8")
-                        df = pd.read_csv(StringIO(csv_data))
-                        df["created_at"] = df["created_at"].apply(
-                            lambda x: pendulum.parse(x)
-                        )
-                        return df
+                        reader = csv.DictReader(StringIO(csv_data))
+                        rows = []
+                        for row in reader:
+                            if row["created_at"]:
+                                row["created_at"] = pendulum.parse(row["created_at"])
+                            else:
+                                row["created_at"] = None
+
+                            row2 = {k: v for k, v in row.items() if v != ''}
+                            rows.append(row2)
+                        
+                        return rows
                     except Exception as e:
                         raise Exception(f"Error reading CSV: {e}")
             else:

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/sources.py

@@ -2144,6 +2144,41 @@ class LinkedInAdsSource:
         ).with_resources("custom_reports")
 
 
+class ClickupSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        api_token = params.get("api_token")
+
+        if api_token is None:
+            raise MissingValueError("api_token", "ClickUp")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+        start_date = (
+            ensure_pendulum_datetime(interval_start).in_timezone("UTC")
+            if interval_start
+            else pendulum.datetime(2020, 1, 1, tz="UTC")
+        )
+        end_date = (
+            ensure_pendulum_datetime(interval_end).in_timezone("UTC")
+            if interval_end
+            else None
+        )
+
+        from ingestr.src.clickup import clickup_source
+
+        if table not in {"user", "teams", "lists", "tasks", "spaces"}:
+            raise UnsupportedResourceError(table, "ClickUp")
+
+        return clickup_source(
+            api_token=api_token[0], start_date=start_date, end_date=end_date
+        ).with_resources(table)
+
+
 class AppLovinSource:
     def handles_incrementality(self) -> bool:
         return True

{ingestr-0.13.70 → ingestr-0.13.72}/ingestr/src/tiktok_ads/__init__.py

@@ -112,7 +112,8 @@ def tiktok_source(
         datetime=(
             dlt.sources.incremental(
                 incremental_loading_param,
-                start_date,
+                initial_value=start_date,
+                end_value=end_date,
                 range_end="closed",
                 range_start="closed",
             )
@@ -120,15 +121,16 @@ def tiktok_source(
             else None
         ),
     ) -> Iterable[TDataItem]:
-        current_date = start_date.in_tz(timezone)
+        start_date_tz_adjusted = start_date.in_tz(timezone)
+        end_date_tz_adjusted = end_date.in_tz(timezone)
 
         if datetime is not None:
-            datetime_str = datetime.last_value
-            current_date = ensure_pendulum_datetime(datetime_str).in_tz(timezone)
+            start_date_tz_adjusted = ensure_pendulum_datetime(datetime.last_value).in_tz(timezone)
+            end_date_tz_adjusted = ensure_pendulum_datetime(datetime.end_value).in_tz(timezone)
 
         list_of_interval = find_intervals(
-            current_date=current_date,
-            end_date=end_date,
+            current_date=start_date_tz_adjusted,
+            end_date=end_date_tz_adjusted,
             interval_days=interval_days,
         )
 

{ingestr-0.13.70 → ingestr-0.13.72}/pyproject.toml

@@ -91,6 +91,7 @@ exclude = [
     'src/pipedrive/.*',
     'src/linear/.*',
     'src/zoom/.*',
+    'src/clickup/.*',
   ]
 
 [[tool.mypy.overrides]]
@@ -120,6 +121,7 @@ module = [
   "ingestr.src.pipedrive.*",
   "ingestr.src.linear.*",
   "ingestr.src.zoom.*",
+  "ingestr.src.clickup.*",
 ]
 follow_imports = "skip"
 

{ingestr-0.13.70 → ingestr-0.13.72}/requirements.in

@@ -7,7 +7,7 @@ dlt-cratedb>=0.0.1
 duckdb_engine==0.17.0
 duckdb==1.2.1
 google-ads==25.1.0
-facebook-business==20.0.0
+facebook-business==23.0.0
 google-api-python-client==2.130.0
 google-cloud-bigquery-storage==2.24.0
 mysql-connector-python==9.2.0

{ingestr-0.13.70 → ingestr-0.13.72}/requirements.txt

@@ -121,7 +121,7 @@ enum-compat==0.0.3
     # via intuit-oauth
 et-xmlfile==2.0.0
     # via openpyxl
-facebook-business==20.0.0
+facebook-business==23.0.0
     # via -r requirements.in
 filelock==3.17.0
     # via snowflake-connector-python

{ingestr-0.13.70 → ingestr-0.13.72}/requirements_arm64.txt

@@ -121,7 +121,7 @@ enum-compat==0.0.3
     # via intuit-oauth
 et-xmlfile==2.0.0
     # via openpyxl
-facebook-business==20.0.0
+facebook-business==23.0.0
     # via -r requirements.in
 filelock==3.18.0
     # via snowflake-connector-python

ingestr-0.13.70/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.70"