ingestr 0.13.36__py3-none-any.whl → 0.13.37__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.36"
+version = "v0.13.37"

ingestr/src/factory.py

@@ -54,6 +54,7 @@ from ingestr.src.sources import (
     TikTokSource,
     ZendeskSource,
     FreshdeskSource,
+    PhantombusterSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -150,6 +151,7 @@ class SourceDestinationFactory:
         "pipedrive": PipedriveSource,
         "frankfurter": FrankfurterSource,
         "freshdesk": FreshdeskSource,
+        "phantombuster": PhantombusterSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/phantombuster/__init__.py

@@ -0,0 +1,38 @@
+from typing import Iterable, Optional
+
+import dlt
+import pendulum
+import requests
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+
+from ingestr.src.phantombuster.client import PhantombusterClient
+
+
+def retry_on_limit(
+    response: Optional[requests.Response], exception: Optional[BaseException]
+) -> bool:
+    if response is not None and response.status_code == 429:
+        return True
+    return False
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+@dlt.source(max_table_nesting=0)
+def phantombuster_source(api_key: str, agent_id: str, start_date: pendulum.DateTime, end_date: pendulum.DateTime) -> Iterable[DltResource]:
+    client = PhantombusterClient(api_key)
+
+    @dlt.resource()
+    def completed_phantoms() -> Iterable[TDataItem]:
+        yield client.fetch_containers_result(create_client(), agent_id, start_date, end_date)
+
+    return completed_phantoms

ingestr/src/phantombuster/client.py

@@ -0,0 +1,65 @@
+from typing import Union
+
+import pendulum
+import requests
+
+
+class PhantombusterClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def _get_headers(self):
+        return {
+            "X-Phantombuster-Key-1": self.api_key,
+            "accept": "application/json",
+        }
+
+    def fetch_containers_result(self, session: requests.Session, agent_id: str, start_date: pendulum.DateTime, end_date: pendulum.DateTime):
+        url = "https://api.phantombuster.com/api/v2/containers/fetch-all/"
+        before_ended_at = None
+        limit = 100
+        started_at = start_date.int_timestamp * 1000 + int(start_date.microsecond / 1000)
+        ended_at = end_date.int_timestamp * 1000 + int(end_date.microsecond / 1000)
+        while True:
+            params: dict[str, Union[str, int, float, bytes, None]] = {
+                "agentId": agent_id,
+                "limit": limit,
+                "mode": "finalized",
+            }
+
+            if before_ended_at:
+                params["beforeEndedAt"] = before_ended_at
+
+            response = session.get(url=url, headers=self._get_headers(), params=params)
+            data = response.json()
+            containers = data.get("containers", [])
+
+            for container in containers:
+                container_ended_at = container.get("endedAt")
+                if before_ended_at is None or before_ended_at > container["endedAt"]:
+                        before_ended_at = container["endedAt"]
+                        
+                if not (started_at <= container_ended_at <= ended_at):
+                    continue
+                try:
+                    result = self.fetch_result_object(session, container["id"])
+                    partition_dt = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC").to_date_string()
+                    row = {"container": container, "result": result, "partition_dt": partition_dt}
+                    yield row
+                    
+                except requests.RequestException as e:
+                    print(f"Error fetching result for container {container['id']}: {e}")
+         
+            if data["maxLimitReached"] is False:
+                break
+ 
+
+    def fetch_result_object(self, session: requests.Session, container_id: str):
+        result_url = (
+            "https://api.phantombuster.com/api/v2/containers/fetch-result-object"
+        )
+        params = {"id": container_id}
+        response = session.get(result_url, headers=self._get_headers(), params=params)
+        response.raise_for_status()
+
+        return response.json()

ingestr/src/sources.py

@@ -2231,4 +2231,44 @@ class FreshdeskSource:
         
         from ingestr.src.freshdesk import freshdesk_source
         return freshdesk_source(api_secret_key=api_key[0], domain=domain).with_resources(table)
-   
+
+class PhantombusterSource:
+    def handles_incrementality(self) -> bool:
+        return True
+    
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        #phantombuster://?api_key=<api_key>
+        #source table = phantom_results:agent_id
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Phantombuster")
+        
+        table_fields = table.replace(" ", "").split(":")
+        table_name = table_fields[0]
+        
+        agent_id = table_fields[1] if len(table_fields) > 1 else None
+        
+        if table_name not in ["completed_phantoms"]:
+            raise UnsupportedResourceError(table_name, "Phantombuster")
+        
+        if not agent_id:
+            raise MissingValueError("agent_id", "Phantombuster")
+        
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.parse("2018-01-01")
+
+        end_date = kwargs.get("interval_end")
+        
+        #doesnot support incremental loading
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date)
+        else:
+            end_date = pendulum.now()
+        
+        from ingestr.src.phantombuster import phantombuster_source
+        return phantombuster_source(api_key=api_key[0], agent_id=agent_id, start_date=start_date, end_date=end_date).with_resources(table_name)

{ingestr-0.13.36.dist-info → ingestr-0.13.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.36
+Version: 0.13.37
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.36.dist-info → ingestr-0.13.37.dist-info}/RECORD

@@ -2,15 +2,15 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=mRlGSqi2sHcZ2AKlwn5MqoMvFxXlSjcZxmPJr76rmRk,25187
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=abX1HXd_dkzG2hkJg7JdFGSvgjGi72VrEucHcTxIziA,21
+ingestr/src/buildinfo.py,sha256=zGfudKvUvWbTMFprtyFws2zsqeHGQj08eCKTrwTnVj8,21
 ingestr/src/destinations.py,sha256=Z79f01BSmEaXnQno2IQVt4Th4dmD-BiOQXlibZJ5sTw,13180
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=M0FAes6KsvqCzuTnUBcxc6DF7UVO51IlrFxy2VDpbkQ,5392
+ingestr/src/factory.py,sha256=Mm_Be60PFO4mUIeJLBMDVU_uyH0IeCiZ1dtNDFiDFSo,5463
 ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=YpaUS5Ui-YXeZYLETPAj60WhU5fWI_lP6jVA0w6J6qo,77250
+ingestr/src/sources.py,sha256=9ESEgdlaSQQszpRfp-etKvfFDvvmYfCc9sBlEPJxh3Q,78809
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -87,6 +87,8 @@ ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falP
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
 ingestr/src/personio/__init__.py,sha256=sHYpoV-rg-kA1YsflctChis0hKcTrL6mka9O0CHV4zA,11638
 ingestr/src/personio/helpers.py,sha256=EKmBN0Lf4R0lc3yqqs7D-RjoZ75E8gPcctt59xwHxrY,2901
+ingestr/src/phantombuster/__init__.py,sha256=FJJiVP0ciR48FTmXYLAasZ4JQAB1Ow4M_Hh39J6hWks,1112
+ingestr/src/phantombuster/client.py,sha256=HFJ46f_IU1NMMCA94ttoY1LBc0L7qfqeQEawczlbBvQ,2584
 ingestr/src/pipedrive/__init__.py,sha256=iRrxeMwo8_83ptgGnTFTNHV1nYvIsFfg0a3XzugPYeI,6982
 ingestr/src/pipedrive/settings.py,sha256=q119Fy4C5Ip1rMoCILX2BkHV3bwiXC_dW58KIiDUzsY,708
 ingestr/src/pipedrive/typing.py,sha256=lEMXu4hhAA3XkhVSlBUa-juqyupisd3c-qSQKxFvzoE,69
@@ -125,8 +127,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.36.dist-info/METADATA,sha256=AFJ4qtGMrtaG5luUcRCXAsp7yP8FKlL4EjP8KorvXKI,13575
-ingestr-0.13.36.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.36.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.36.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.36.dist-info/RECORD,,
+ingestr-0.13.37.dist-info/METADATA,sha256=Mmc9hAE_zCJ_b5U9hCLpJXpU0858FirZdoO-FyPuOI4,13575
+ingestr-0.13.37.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.37.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.37.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.37.dist-info/RECORD,,