ingestr 0.13.71__py3-none-any.whl → 0.13.73__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.71"
+version = "v0.13.73"

ingestr/src/clickup/__init__.py

@@ -0,0 +1,85 @@
+"""Simple ClickUp source."""
+
+from datetime import datetime
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource
+
+from .helpers import ClickupClient
+
+
+@dlt.source(max_table_nesting=0)
+def clickup_source(
+    api_token: str = dlt.secrets.value,
+    start_date: datetime = None,
+    end_date: datetime = None,
+) -> Iterable[DltResource]:
+    client = ClickupClient(api_token)
+
+    @dlt.resource(
+        name="user",
+        primary_key="id",
+        write_disposition="merge",
+    )
+    def user() -> Iterable[dict]:
+        data = client.get("/user")
+        yield data["user"]
+
+    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
+    def teams() -> Iterable[dict]:
+        for team in client.get_teams():
+            yield team
+
+    @dlt.resource(name="spaces", primary_key="id", write_disposition="merge")
+    def spaces() -> Iterable[dict]:
+        for space in client.get_spaces():
+            yield space
+
+    @dlt.resource(name="lists", write_disposition="merge", primary_key="id")
+    def lists() -> Iterable[dict]:
+        for list in client.get_lists():
+            yield list
+
+    @dlt.resource(
+        name="tasks",
+        write_disposition="merge",
+        primary_key="id",
+        columns={"date_updated": {"data_type": "timestamp"}},
+    )
+    def tasks(
+        date_updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "date_updated",
+            initial_value=ensure_pendulum_datetime(start_date).in_timezone("UTC"),
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[dict]:
+        if date_updated.last_value:
+            start = ensure_pendulum_datetime(date_updated.last_value).in_timezone("UTC")
+        else:
+            start = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+
+        if date_updated.end_value is None:
+            end = pendulum.now("UTC")
+        else:
+            end = date_updated.end_value.in_timezone("UTC")
+
+        for list_obj in client.get_lists():
+            for task in client.paginated(
+                f"/list/{list_obj['id']}/task", "tasks", {"page_size": 100}
+            ):
+                task_dt = ensure_pendulum_datetime(int(task["date_updated"]) / 1000)
+                if task_dt >= start and task_dt <= end:
+                    task["date_updated"] = task_dt
+                    yield task
+
+    return (
+        user,
+        teams,
+        spaces,
+        lists,
+        tasks,
+    )

ingestr/src/clickup/helpers.py

@@ -0,0 +1,47 @@
+from typing import Iterable, Optional
+
+from ..http_client import create_client
+
+
+class ClickupClient:
+    def __init__(self, api_token: str):
+        self.session = create_client()
+        self.base_url = "https://api.clickup.com/api/v2"
+        self.headers = {"Authorization": api_token}
+
+    def get(self, endpoint: str, params: Optional[dict] = None) -> dict:
+        url = f"{self.base_url}{endpoint}"
+        resp = self.session.get(url, headers=self.headers, params=params or {})
+        resp.raise_for_status()
+        return resp.json()
+
+    def paginated(
+        self, endpoint: str, key: str, params: Optional[dict] = None
+    ) -> Iterable[dict]:
+        page = 0
+        params = params or {}
+        while True:
+            params["page"] = page
+            data = self.get(endpoint, params)
+            items = data.get(key, data)
+            if not items:
+                break
+            for item in items:
+                yield item
+            if data.get("last_page") or len(items) < params.get("page_size", 100):
+                break
+            page += 1
+
+    def get_teams(self):
+        data = self.get("/team")
+        return data.get("teams", [])
+
+    def get_spaces(self):
+        for team in self.get_teams():
+            for space in self.paginated(f"/team/{team['id']}/space", "spaces"):
+                yield space
+
+    def get_lists(self):
+        for space in self.get_spaces():
+            for lst in self.paginated(f"/space/{space['id']}/list", "lists"):
+                yield lst

ingestr/src/facebook_ads/__init__.py

@@ -1,5 +1,6 @@
 """Loads campaigns, ads sets, ads, leads and insight data from Facebook Marketing API"""
 
+from datetime import datetime
 from typing import Iterator, Sequence
 
 import dlt
@@ -7,12 +8,12 @@ from dlt.common import pendulum
 from dlt.common.typing import TDataItems
 from dlt.sources import DltResource
 from facebook_business.adobjects.ad import Ad
+from dlt.common.time import ensure_pendulum_datetime
 
 from .helpers import (
     execute_job,
     get_ads_account,
     get_data_chunked,
-    get_start_date,
     process_report_item,
 )
 from .settings import (
@@ -166,15 +167,14 @@ def facebook_insights_source(
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
             "date_start",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
+            initial_value=ensure_pendulum_datetime(start_date).start_of('day').date(),
+            end_value=ensure_pendulum_datetime(end_date).end_of('day').date() if end_date else None,
             range_end="closed",
             range_start="closed",
-            lag=attribution_window_days_lag * 24 * 60 * 60,  # Convert days to seconds
         ),
     ) -> Iterator[TDataItems]:
-        start_date = get_start_date(date_start)
-        end_date = pendulum.now()
+        start_date = date_start.last_value
+        end_date = pendulum.instance(date_start.end_value) if date_start.end_value else pendulum.now()
 
         while start_date <= end_date:
             query = {
@@ -202,9 +202,10 @@ def facebook_insights_source(
             }
             job = execute_job(
                 account.get_insights(params=query, is_async=True),
-                insights_max_async_sleep_seconds=10,
+                insights_max_async_sleep_seconds=20,
             )
-            yield list(map(process_report_item, job.get_result()))
+            output = list(map(process_report_item, job.get_result()))
+            yield output
             start_date = start_date.add(days=time_increment_days)
 
     return facebook_insights

ingestr/src/facebook_ads/helpers.py

@@ -3,6 +3,7 @@
 import functools
 import itertools
 import time
+from datetime import datetime
 from typing import Any, Iterator, Sequence
 
 import dlt
@@ -23,43 +24,17 @@ from facebook_business.api import FacebookResponse
 
 from .exceptions import InsightsJobTimeout
 from .settings import (
-    FACEBOOK_INSIGHTS_RETENTION_PERIOD,
     INSIGHTS_PRIMARY_KEY,
     TFbMethod,
 )
 
 
-def get_start_date(
-    incremental_start_date: dlt.sources.incremental[str],
-) -> pendulum.DateTime:
-    """
-    Get the start date for incremental loading of Facebook Insights data.
-    """
-    start_date: pendulum.DateTime = ensure_pendulum_datetime(
-        incremental_start_date.start_value
-    )
-
-    # facebook forgets insights so trim the lag and warn
-    min_start_date = pendulum.today().subtract(
-        months=FACEBOOK_INSIGHTS_RETENTION_PERIOD
-    )
-    if start_date < min_start_date:
-        logger.warning(
-            "%s: Start date is earlier than %s months ago, using %s instead. "
-            "For more information, see https://www.facebook.com/business/help/1695754927158071?id=354406972049255",
-            "facebook_insights",
-            FACEBOOK_INSIGHTS_RETENTION_PERIOD,
-            min_start_date,
-        )
-        start_date = min_start_date
-        incremental_start_date.start_value = min_start_date
-
-    # lag the incremental start date by attribution window lag
-    incremental_start_date.start_value = start_date.isoformat()
-    return start_date
-
-
 def process_report_item(item: AbstractObject) -> DictStrAny:
+    if "date_start" in item:
+        item["date_start"] = datetime.strptime(item["date_start"], "%Y-%m-%d").date()
+    if "date_stop" in item:
+        item["date_stop"] = datetime.strptime(item["date_stop"], "%Y-%m-%d").date()
+    
     d: DictStrAny = item.export_all_data()
     for pki in INSIGHTS_PRIMARY_KEY:
         if pki not in d:
@@ -138,15 +113,20 @@ def execute_job(
     time_start = time.time()
     sleep_time = 3
     while status != "Job Completed":
+        print("-----")
+        print("waiting for job to finish")
         duration = time.time() - time_start
         job = job.api_get()
         status = job["async_status"]
         percent_complete = job["async_percent_completion"]
+        print("async_status", status)
+        print("percent_complete", percent_complete)
 
         job_id = job["id"]
         logger.info("%s, %d%% done", status, percent_complete)
 
         if status == "Job Completed":
+            print("job completed")
             return job
 
         if duration > insights_max_wait_to_start_seconds and percent_complete == 0:

ingestr/src/factory.py

@@ -32,6 +32,7 @@ from ingestr.src.sources import (
     AsanaSource,
     AttioSource,
     ChessSource,
+    ClickupSource,
     DynamoDBSource,
     ElasticsearchSource,
     FacebookAdsSource,
@@ -185,6 +186,7 @@ class SourceDestinationFactory:
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
         "zoom": ZoomSource,
+        "clickup": ClickupSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/github/__init__.py

@@ -10,7 +10,7 @@ from dlt.sources import DltResource
 from .helpers import get_reactions_data, get_rest_pages, get_stargazers
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def github_reactions(
     owner: str,
     name: str,
@@ -114,7 +114,7 @@ def github_repo_events(
     return repo_events
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def github_stargazers(
     owner: str,
     name: str,

ingestr/src/shopify/__init__.py

@@ -1690,16 +1690,6 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "nullable": True,
                 "description": "The category of the product from Shopify's Standard Product Taxonomy.",
             },
-            "combinedListing": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "A special product type that combines separate products into a single product listing.",
-            },
-            "combinedListingRole": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "The role of the product in a combined listing.",
-            },
             "compareAtPriceRange": {
                 "data_type": "json",
                 "nullable": True,
@@ -1841,12 +1831,6 @@ query products($after: String, $query: String, $first: Int)  {
       category {
         id
       }
-      combinedListing {
-        parentProduct {
-          id
-        }
-      }
-      combinedListingRole
       compareAtPriceRange {
         maxVariantCompareAtPrice {
           amount

ingestr/src/sources.py

@@ -2144,6 +2144,41 @@ class LinkedInAdsSource:
         ).with_resources("custom_reports")
 
 
+class ClickupSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        api_token = params.get("api_token")
+
+        if api_token is None:
+            raise MissingValueError("api_token", "ClickUp")
+
+        interval_start = kwargs.get("interval_start")
+        interval_end = kwargs.get("interval_end")
+        start_date = (
+            ensure_pendulum_datetime(interval_start).in_timezone("UTC")
+            if interval_start
+            else pendulum.datetime(2020, 1, 1, tz="UTC")
+        )
+        end_date = (
+            ensure_pendulum_datetime(interval_end).in_timezone("UTC")
+            if interval_end
+            else None
+        )
+
+        from ingestr.src.clickup import clickup_source
+
+        if table not in {"user", "teams", "lists", "tasks", "spaces"}:
+            raise UnsupportedResourceError(table, "ClickUp")
+
+        return clickup_source(
+            api_token=api_token[0], start_date=start_date, end_date=end_date
+        ).with_resources(table)
+
+
 class AppLovinSource:
     def handles_incrementality(self) -> bool:
         return True

{ingestr-0.13.71.dist-info → ingestr-0.13.73.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.71
+Version: 0.13.73
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -57,7 +57,7 @@ Requires-Dist: elastic-transport==8.17.1
 Requires-Dist: elasticsearch==8.10.1
 Requires-Dist: enum-compat==0.0.3
 Requires-Dist: et-xmlfile==2.0.0
-Requires-Dist: facebook-business==20.0.0
+Requires-Dist: facebook-business==23.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0

{ingestr-0.13.71.dist-info → ingestr-0.13.73.dist-info}/RECORD

@@ -2,16 +2,16 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=wu4c_4nit3c5v2K1af68DjT2hIfEiCvzQ8EZJxOkJjo,26369
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=GlvBwB8aV5mufGGW4MafbR067L8Zr8wModuobUsy3Gs,21
+ingestr/src/buildinfo.py,sha256=pp-vE1fEyXfxgbt7NqfxCSd3aCJw-b9na5MCnvRi0-w,21
 ingestr/src/destinations.py,sha256=ivTPio0zzqLRx22i597pxZHMNClz-XvYSyCaCPuGd8g,22248
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=AJCvlK4M1sIpAAks1K-xsR_uxziIxru74mj572zixhg,6546
+ingestr/src/factory.py,sha256=1KvLgnafhD3u9Mbk3QNmKZVgQZsN0g1AwTMvTKNTyVY,6599
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=Ey3fI-SZ08fVMZf5_uCdLku87UuT3bjM8SmesgoSSY8,104292
+ingestr/src/sources.py,sha256=JGMKeoEHxpiSB64d3ScdukYfr2stfcJtJ6eU3hDoCLc,105432
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -36,12 +36,14 @@ ingestr/src/attio/helpers.py,sha256=fCySmG5E6Iyh3Nm9a-HGbHNedxPH_2_otXYMTQsCibw,
 ingestr/src/chess/__init__.py,sha256=y0Q8aKBigeKf3N7wuB_gadMQjVJzBPUT8Jhp1ObEWjk,6812
 ingestr/src/chess/helpers.py,sha256=v1HTImOMjAF7AzZUPDIuHu00e7ut0o5y1kWcVYo4QZw,549
 ingestr/src/chess/settings.py,sha256=p0RlCGgtXUacPDEvZmwzSWmzX0Apj1riwfz-nrMK89k,158
+ingestr/src/clickup/__init__.py,sha256=uvfAqNturT4bMvU4NS3E8BdL6nvDFzNuh7bMlih8HJk,2547
+ingestr/src/clickup/helpers.py,sha256=RzDKMUAHccuDhocIQ2ToBXfCERo8CBJqA3t-IPltBCE,1519
 ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
 ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
 ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
-ingestr/src/facebook_ads/__init__.py,sha256=a1A5fO1r_FotoH9UET42tamqo_-ftCm9vBrkm5lpjG0,9579
+ingestr/src/facebook_ads/__init__.py,sha256=XgMtHH7yUYxjbTQn7wfIB59Yyk2jTWW40LUtbgHaQ_g,9719
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
-ingestr/src/facebook_ads/helpers.py,sha256=EYqOAPUlhVNxwzjP_CUGjJvAXTq65nJC-v75BfyJKmg,8981
+ingestr/src/facebook_ads/helpers.py,sha256=GTXWhAmd8Hl1watEp5zEz6geOJudW76QWzSHPROAX28,8307
 ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
 ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
@@ -52,7 +54,7 @@ ingestr/src/frankfurter/helpers.py,sha256=SyrkRTDqvKdQxRHTV5kcSeVG3FEnaK5zxHyNyq
 ingestr/src/freshdesk/__init__.py,sha256=uFQW_cJyymxtHQiYb_xjzZAklc487L0n9GkgHgC7yAI,2618
 ingestr/src/freshdesk/freshdesk_client.py,sha256=3z5Yc008ADzRcJWtNc00PwjkLzG-RMI8jVIOOyYA-Rw,4088
 ingestr/src/freshdesk/settings.py,sha256=0Wr_OMnUZcTlry7BmALssLxD2yh686JW4moLNv12Jnw,409
-ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
+ingestr/src/github/__init__.py,sha256=CIhOTcn74WEebtkRwslVLaw0LU8S91GVulVykIN01D4,5931
 ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
 ingestr/src/github/settings.py,sha256=N5ahWrDIQ_4IWV9i-hTXxyYduqY9Ym2BTwqsWxcDdJ8,258
@@ -109,7 +111,7 @@ ingestr/src/pipedrive/helpers/pages.py,sha256=Klpjw2OnMuhzit3PpiHKsfzGcJ3rQPSQBl
 ingestr/src/quickbooks/__init__.py,sha256=cZUuVCOTGPHTscRj6i0DytO63_fWF-4ieMxoU4PcyTg,3727
 ingestr/src/salesforce/__init__.py,sha256=2hik5pRrxVODdDTlUEMoyccNC07zozjnxkMHcjMT1qA,4558
 ingestr/src/salesforce/helpers.py,sha256=QTdazBt-qRTBbCQMZnyclIaDQFmBixBy_RDKD00Lt-8,2492
-ingestr/src/shopify/__init__.py,sha256=PF_6VQnS065Br1UzSIekTVXBu3WtrMQL_v5CfbfaX5Y,63151
+ingestr/src/shopify/__init__.py,sha256=dp6Ybk5LIKA5suzVt923v5LzHz5rMUuDfhjTNPqSjAc,62603
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
@@ -147,8 +149,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.71.dist-info/METADATA,sha256=YIzmk8sSvhEUF2Yb7VRLRyGJpcigSaJrExacU-QtXlU,15022
-ingestr-0.13.71.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.71.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.71.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.71.dist-info/RECORD,,
+ingestr-0.13.73.dist-info/METADATA,sha256=nYpu2rxJCQx28ZDJE7aU-dVM7cCYy4IOIFt3reC8EMU,15022
+ingestr-0.13.73.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.73.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.73.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.73.dist-info/RECORD,,