ingestr 0.13.37__py3-none-any.whl → 0.13.38__py3-none-any.whl

ingestr/main.py

@@ -485,6 +485,9 @@ def ingest(
         print(
             f"[bold yellow]  Primary Key:[/bold yellow] {primary_key if primary_key else 'None'}"
         )
+        print(
+            f"[bold yellow]  Pipeline ID:[/bold yellow] {m.hexdigest()}"
+        )
         print()
 
         if not yes:

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.37"
+version = "v0.13.38"

ingestr/src/frankfurter/__init__.py

@@ -1,4 +1,4 @@
-from typing import Any, Iterator
+from typing import Any, Iterator, Optional
 
 import dlt
 from dlt.common.pendulum import pendulum
@@ -15,13 +15,13 @@ from ingestr.src.frankfurter.helpers import get_path_with_retry
 def frankfurter_source(
     start_date: TAnyDateTime,
     end_date: TAnyDateTime,
+    base_currency: str,
 ) -> Any:
     """
     A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
     various types of data: currencies, latest rates, historical rates.
     """
     date_time = dlt.sources.incremental(
-
         "date",
         initial_value=start_date,
         end_value=end_date,
@@ -31,9 +31,10 @@ def frankfurter_source(
 
     return (
         currencies(),
-        latest(),
-        exchange_rates(start_date=date_time, end_date=end_date),
-
+        latest(base_currency=base_currency),
+        exchange_rates(
+            start_date=date_time, end_date=end_date, base_currency=base_currency
+        ),
     )
 
 
@@ -61,29 +62,33 @@ def currencies() -> Iterator[dict]:
         "date": {"data_type": "text"},
         "currency_code": {"data_type": "text"},
         "rate": {"data_type": "double"},
+        "base_currency": {"data_type": "text"},
     },
-    primary_key=["date", "currency_code"],  # Composite primary key
+    primary_key=["date", "currency_code", "base_currency"],
 )
-def latest() -> Iterator[dict]:
+def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
     """
     Fetches the latest exchange rates and yields them as rows.
     """
     # Base URL
     url = "latest?"
 
+    if base_currency:
+        url += f"base={base_currency}"
+
     # Fetch data
     data = get_path_with_retry(url)
 
     # Extract rates and base currency
     rates = data["rates"]
-
     date = pendulum.parse(data["date"])
 
-    # Add the base currency (EUR) with a rate of 1.0
+    # Add the base currency with a rate of 1.0
     yield {
         "date": date,
-        "currency_code": "EUR",
+        "currency_code": base_currency,
         "rate": 1.0,
+        "base_currency": base_currency,
     }
 
     # Add all currencies and their rates
@@ -92,6 +97,7 @@ def latest() -> Iterator[dict]:
             "date": date,
             "currency_code": currency_code,
             "rate": rate,
+            "base_currency": base_currency,
         }
 
 
@@ -101,12 +107,14 @@ def latest() -> Iterator[dict]:
         "date": {"data_type": "text"},
         "currency_code": {"data_type": "text"},
         "rate": {"data_type": "double"},
+        "base_currency": {"data_type": "text"},
     },
-    primary_key=("date", "currency_code"),  # Composite primary key
+    primary_key=("date", "currency_code", "base_currency"),
 )
 def exchange_rates(
     end_date: TAnyDateTime,
     start_date: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental("date"),
+    base_currency: Optional[str] = "",
 ) -> Iterator[dict]:
     """
     Fetches exchange rates for a specified date range.
@@ -124,6 +132,9 @@ def exchange_rates(
     # Compose the URL
     url = f"{start_date_str}..{end_date_str}?"
 
+    if base_currency:
+        url += f"base={base_currency}"
+
     # Fetch data from the API
     data = get_path_with_retry(url)
 
@@ -137,8 +148,9 @@ def exchange_rates(
         # Add the base currency with a rate of 1.0
         yield {
             "date": formatted_date,
-            "currency_code": "EUR",
+            "currency_code": base_currency,
             "rate": 1.0,
+            "base_currency": base_currency,
         }
 
         # Add all other currencies and their rates
@@ -147,4 +159,5 @@ def exchange_rates(
                 "date": formatted_date,
                 "currency_code": currency_code,
                 "rate": rate,
+                "base_currency": base_currency,
             }

ingestr/src/frankfurter/helpers.py

@@ -30,3 +30,19 @@ def validate_dates(start_date: datetime, end_date: datetime) -> None:
     # Check if start_date is before end_date
     if start_date > end_date:
         raise ValueError("Interval-end cannot be before interval-start.")
+
+
+def validate_currency(currency_code: str) -> bool:
+    url = "https://api.frankfurter.dev/v1/currencies"
+
+    response = requests.get(url, timeout=5)
+    currencies = response.json()
+
+    if currency_code.upper() in currencies:
+        return True
+    else:
+        supported_currencies = list(currencies.keys())
+        print(
+            f"Invalid base currency '{currency_code}'. Supported currencies are: {supported_currencies}"
+        )
+        return False

ingestr/src/phantombuster/__init__.py

@@ -3,10 +3,11 @@ from typing import Iterable, Optional
 import dlt
 import pendulum
 import requests
-from dlt.common.typing import TDataItem
+from dlt.common.typing import TDataItem, TAnyDateTime
 from dlt.sources import DltResource
 from dlt.sources.helpers.requests import Client
 
+
 from ingestr.src.phantombuster.client import PhantombusterClient
 
 
@@ -26,13 +27,34 @@ def create_client() -> requests.Session:
         request_backoff_factor=2,
     ).session
 
-
 @dlt.source(max_table_nesting=0)
-def phantombuster_source(api_key: str, agent_id: str, start_date: pendulum.DateTime, end_date: pendulum.DateTime) -> Iterable[DltResource]:
+def phantombuster_source(api_key: str, agent_id: str, start_date: TAnyDateTime, end_date: TAnyDateTime | None) -> Iterable[DltResource]:
     client = PhantombusterClient(api_key)
-
-    @dlt.resource()
-    def completed_phantoms() -> Iterable[TDataItem]:
-        yield client.fetch_containers_result(create_client(), agent_id, start_date, end_date)
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key="container_id",
+        columns={
+            "partition_dt": {"data_type": "date", "partition": True},
+        },
+    )
+    def completed_phantoms(
+        dateTime=(
+            dlt.sources.incremental(
+                "ended_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+
+        yield client.fetch_containers_result(create_client(), agent_id, start_date=start_dt, end_date=end_dt)
 
     return completed_phantoms

ingestr/src/phantombuster/client.py

@@ -18,8 +18,10 @@ class PhantombusterClient:
         url = "https://api.phantombuster.com/api/v2/containers/fetch-all/"
         before_ended_at = None
         limit = 100
+
         started_at = start_date.int_timestamp * 1000 + int(start_date.microsecond / 1000)
         ended_at = end_date.int_timestamp * 1000 + int(end_date.microsecond / 1000)
+       
         while True:
             params: dict[str, Union[str, int, float, bytes, None]] = {
                 "agentId": agent_id,
@@ -36,15 +38,18 @@ class PhantombusterClient:
 
             for container in containers:
                 container_ended_at = container.get("endedAt")
-                if before_ended_at is None or before_ended_at > container["endedAt"]:
-                        before_ended_at = container["endedAt"]
-                        
-                if not (started_at <= container_ended_at <= ended_at):
+
+                if before_ended_at is None or before_ended_at > container_ended_at:
+                        before_ended_at = container_ended_at
+                
+                if container_ended_at < started_at or container_ended_at > ended_at:
                     continue
+              
                 try:
                     result = self.fetch_result_object(session, container["id"])
-                    partition_dt = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC").to_date_string()
-                    row = {"container": container, "result": result, "partition_dt": partition_dt}
+                    partition_dt = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC").date()
+                    container_ended_at_datetime = pendulum.from_timestamp(container_ended_at / 1000, tz="UTC")
+                    row = {"container_id": container["id"],"container": container, "result": result, "partition_dt": partition_dt, "ended_at": container_ended_at_datetime}
                     yield row
                     
                 except requests.RequestException as e:

ingestr/src/sources.py

@@ -2180,6 +2180,18 @@ class FrankfurterSource:
                 "Frankfurter takes care of incrementality on its own, you should not provide incremental_key"
             )
 
+        from ingestr.src.frankfurter import frankfurter_source
+        from ingestr.src.frankfurter.helpers import validate_currency, validate_dates
+
+        parsed_uri = urlparse(uri)
+        source_params = parse_qs(parsed_uri.query)
+        base_currency = source_params.get("base", [None])[0]
+
+        if not base_currency:
+            base_currency = "USD"
+
+        validate_currency(base_currency)
+
         if kwargs.get("interval_start"):
             start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))
             if kwargs.get("interval_end"):
@@ -2190,21 +2202,20 @@ class FrankfurterSource:
             start_date = pendulum.now()
             end_date = pendulum.now()
 
-        from ingestr.src.frankfurter import frankfurter_source
-        from ingestr.src.frankfurter.helpers import validate_dates
-
         validate_dates(start_date=start_date, end_date=end_date)
 
         src = frankfurter_source(
             start_date=start_date,
             end_date=end_date,
+            base_currency=base_currency,
         )
 
         if table not in src.resources:
             raise UnsupportedResourceError(table, "Frankfurter")
 
         return src.with_resources(table)
-    
+
+
 class FreshdeskSource:
      # freshdesk://domain?api_key=<api_key>
     def handles_incrementality(self) -> bool:
@@ -2257,18 +2268,14 @@ class PhantombusterSource:
             raise MissingValueError("agent_id", "Phantombuster")
         
         start_date = kwargs.get("interval_start")
-        if start_date is not None:
-            start_date = ensure_pendulum_datetime(start_date)
+        if start_date is None:
+            start_date = ensure_pendulum_datetime("2018-01-01").in_tz("UTC")
         else:
-            start_date = pendulum.parse("2018-01-01")
+            start_date = ensure_pendulum_datetime(start_date).in_tz("UTC")
 
         end_date = kwargs.get("interval_end")
-        
-        #doesnot support incremental loading
         if end_date is not None:
-            end_date = ensure_pendulum_datetime(end_date)
-        else:
-            end_date = pendulum.now()
+            end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
         
         from ingestr.src.phantombuster import phantombuster_source
         return phantombuster_source(api_key=api_key[0], agent_id=agent_id, start_date=start_date, end_date=end_date).with_resources(table_name)

{ingestr-0.13.37.dist-info → ingestr-0.13.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.37
+Version: 0.13.38
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -74,7 +74,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.1
+Requires-Dist: greenlet==3.2.2
 Requires-Dist: grpcio-status==1.62.3
 Requires-Dist: grpcio==1.70.0
 Requires-Dist: hdbcli==2.23.27

{ingestr-0.13.37.dist-info → ingestr-0.13.38.dist-info}/RECORD

@@ -1,8 +1,8 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
-ingestr/main.py,sha256=mRlGSqi2sHcZ2AKlwn5MqoMvFxXlSjcZxmPJr76rmRk,25187
+ingestr/main.py,sha256=QHLjpCItCgL5XHcTezBOp6Vdy_VHAPciRjqy63ZIK2s,25285
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=zGfudKvUvWbTMFprtyFws2zsqeHGQj08eCKTrwTnVj8,21
+ingestr/src/buildinfo.py,sha256=iW7GVGJjTCNtuINLzbFuur_d0ALxf_JmZSmuwD9yZ2Y,21
 ingestr/src/destinations.py,sha256=Z79f01BSmEaXnQno2IQVt4Th4dmD-BiOQXlibZJ5sTw,13180
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
 ingestr/src/factory.py,sha256=Mm_Be60PFO4mUIeJLBMDVU_uyH0IeCiZ1dtNDFiDFSo,5463
@@ -10,7 +10,7 @@ ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=9ESEgdlaSQQszpRfp-etKvfFDvvmYfCc9sBlEPJxh3Q,78809
+ingestr/src/sources.py,sha256=q9FJ3vRaeTjbL5sgau_GHPxbn5LfSI4t5TBwycVdwSc,79060
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -42,8 +42,8 @@ ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
-ingestr/src/frankfurter/__init__.py,sha256=sjxfq377-lryuFC3JswcbHBRoBjLnGLKNRTwBpDZyLw,4403
-ingestr/src/frankfurter/helpers.py,sha256=wqm087QVPcyTuMl6yj_Pl1wcuqElwcBMPz3P4773wcM,979
+ingestr/src/frankfurter/__init__.py,sha256=oVi4BiOxPRyckEVrBNunyMAHulPyMgyGRwBbhn-Xz6M,4987
+ingestr/src/frankfurter/helpers.py,sha256=SyrkRTDqvKdQxRHTV5kcSeVG3FEnaK5zxHyNyqtumZ0,1445
 ingestr/src/freshdesk/__init__.py,sha256=uFQW_cJyymxtHQiYb_xjzZAklc487L0n9GkgHgC7yAI,2618
 ingestr/src/freshdesk/freshdesk_client.py,sha256=3z5Yc008ADzRcJWtNc00PwjkLzG-RMI8jVIOOyYA-Rw,4088
 ingestr/src/freshdesk/settings.py,sha256=0Wr_OMnUZcTlry7BmALssLxD2yh686JW4moLNv12Jnw,409
@@ -87,8 +87,8 @@ ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falP
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
 ingestr/src/personio/__init__.py,sha256=sHYpoV-rg-kA1YsflctChis0hKcTrL6mka9O0CHV4zA,11638
 ingestr/src/personio/helpers.py,sha256=EKmBN0Lf4R0lc3yqqs7D-RjoZ75E8gPcctt59xwHxrY,2901
-ingestr/src/phantombuster/__init__.py,sha256=FJJiVP0ciR48FTmXYLAasZ4JQAB1Ow4M_Hh39J6hWks,1112
-ingestr/src/phantombuster/client.py,sha256=HFJ46f_IU1NMMCA94ttoY1LBc0L7qfqeQEawczlbBvQ,2584
+ingestr/src/phantombuster/__init__.py,sha256=5XGwbtNYmRxL4lmcC0TmyXgljY_21DvDIdw9zaWmgvI,1757
+ingestr/src/phantombuster/client.py,sha256=WO87AGU3Fphd9DgWTkrSAfldLH1XxcPxvNVmNxQ_5eU,2785
 ingestr/src/pipedrive/__init__.py,sha256=iRrxeMwo8_83ptgGnTFTNHV1nYvIsFfg0a3XzugPYeI,6982
 ingestr/src/pipedrive/settings.py,sha256=q119Fy4C5Ip1rMoCILX2BkHV3bwiXC_dW58KIiDUzsY,708
 ingestr/src/pipedrive/typing.py,sha256=lEMXu4hhAA3XkhVSlBUa-juqyupisd3c-qSQKxFvzoE,69
@@ -127,8 +127,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.37.dist-info/METADATA,sha256=Mmc9hAE_zCJ_b5U9hCLpJXpU0858FirZdoO-FyPuOI4,13575
-ingestr-0.13.37.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.37.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.37.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.37.dist-info/RECORD,,
+ingestr-0.13.38.dist-info/METADATA,sha256=ZknSXuae-_lb60AWDACr731hTr0Q_bX_9sUds6rHM28,13575
+ingestr-0.13.38.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.38.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.38.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.38.dist-info/RECORD,,