ingestr 0.13.24__py3-none-any.whl → 0.13.25__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.24"
+version = "v0.13.25"

ingestr/src/factory.py

@@ -28,6 +28,7 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    FrankfurterSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
@@ -146,6 +147,7 @@ class SourceDestinationFactory:
         "personio": PersonioSource,
         "kinesis": KinesisSource,
         "pipedrive": PipedriveSource,
+        "frankfurter": FrankfurterSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/frankfurter/__init__.py

@@ -0,0 +1,142 @@
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from ingestr.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    table: str,
+    start_date: Optional[TAnyDateTime] = None,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+
+    Returns the appropriate resource based on the provided parameters.
+    """
+    # Determine which resource to return based on the `table` parameter
+    if table == "currencies":
+        return currencies()
+
+    elif table == "latest":
+        return latest()
+
+    elif table == "exchange_rates":
+        return exchange_rates(start_date=start_date, end_date=end_date)
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "currency_code": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+    },
+)
+def currencies() -> Iterator[dict]:
+    """
+    Yields each currency as a separate row with two columns: currency_code and currency_name.
+    """
+    # Retrieve the list of currencies from the API
+    currencies_data = get_path_with_retry("currencies")
+
+    for currency_code, currency_name in currencies_data.items():
+        yield {"currency_code": currency_code, "currency_name": currency_name}
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "date": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+        "rate": {"data_type": "double"},
+    },
+    primary_key=["date", "currency_name"],  # Composite primary key
+)
+def latest() -> Iterator[dict]:
+    """
+    Fetches the latest exchange rates and yields them as rows.
+    """
+    # Base URL
+    url = "latest?"
+
+    # Fetch data
+    latest_data = get_path_with_retry(url)
+
+    # Extract rates and base currency
+    rates = latest_data["rates"]
+
+    # Prepare the date
+    date = pendulum.now().to_date_string()
+
+    # Add the base currency (EUR) with a rate of 1.0
+    yield {
+        "date": date,
+        "currency_name": "EUR",
+        "rate": 1.0,
+    }
+
+    # Add all currencies and their rates
+    for currency_name, rate in rates.items():
+        yield {
+            "date": date,
+            "currency_name": currency_name,
+            "rate": rate,
+        }
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "date": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+        "rate": {"data_type": "double"},
+    },
+    primary_key=["date", "currency_name"],  # Composite primary key
+)
+def exchange_rates(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime,
+) -> Iterator[dict]:
+    """
+    Fetches exchange rates for a specified date range.
+    If only start_date is provided, fetches data for that date.
+    If both start_date and end_date are provided, fetches data for each day in the range.
+    """
+    start_date_str = ensure_pendulum_datetime(start_date).format("YYYY-MM-DD")
+    end_date_str = ensure_pendulum_datetime(end_date).format("YYYY-MM-DD")
+
+    # Compose the URL
+    url = f"{start_date_str}..{end_date_str}?"
+
+    # Fetch data from the API
+    data = get_path_with_retry(url)
+
+    # Extract base currency and rates from the API response
+    base_currency = data["base"]
+    rates = data["rates"]
+
+    # Iterate over the rates dictionary (one entry per date)
+    for date, daily_rates in rates.items():
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_name": base_currency,
+            "rate": 1.0,
+        }
+
+        # Add all other currencies and their rates
+        for currency_name, rate in daily_rates.items():
+            yield {
+                "date": date,
+                "currency_name": currency_name,
+                "rate": rate,
+            }

ingestr/src/frankfurter/helpers.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import StrAny
+from dlt.sources.helpers import requests
+
+FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
+
+
+def get_url_with_retry(url: str) -> StrAny:
+    r = requests.get(url)
+    return r.json()  # type: ignore
+
+
+def get_path_with_retry(path: str) -> StrAny:
+    return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
+
+
+def validate_dates(start_date: datetime, end_date: datetime) -> None:
+    current_date = pendulum.now()
+
+    # Check if start_date is in the future
+    if start_date > current_date:
+        raise ValueError("Interval-start cannot be in the future.")
+
+    # Check if end_date is in the future
+    if end_date > current_date:
+        raise ValueError("Interval-end cannot be in the future.")
+
+    # Check if start_date is before end_date
+    if start_date > end_date:
+        raise ValueError("Interval-end cannot be before interval-start.")

ingestr/src/sources.py

@@ -67,6 +67,8 @@ from ingestr.src.errors import (
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
 from ingestr.src.filters import table_adapter_exclude_columns
+from ingestr.src.frankfurter import frankfurter_source
+from ingestr.src.frankfurter.helpers import validate_dates
 from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
 from ingestr.src.google_ads import google_ads
 from ingestr.src.google_analytics import google_analytics
@@ -2041,3 +2043,35 @@ class PipedriveSource:
         return pipedrive_source(
             pipedrive_api_key=api_key, since_timestamp=start_date
         ).with_resources(table)
+
+
+class FrankfurterSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # start and end dates only assigned and validated for exchange_rates table
+        # Note: if an end date but no start date is provided, start date and end date will be set to current date
+        if table == "exchange_rates":
+            if kwargs.get("interval_start"):
+                start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))
+                if kwargs.get("interval_end"):
+                    end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
+                else:
+                    end_date = start_date
+            else:
+                start_date = pendulum.now()
+                end_date = pendulum.now()
+            validate_dates(start_date=start_date, end_date=end_date)
+
+        # Validate table
+        if table not in ["currencies", "latest", "exchange_rates"]:
+            raise ValueError(
+                f"Table '{table}' is not supported for Frankfurter source."
+            )
+
+        return frankfurter_source(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+        )

{ingestr-0.13.24.dist-info → ingestr-0.13.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.24
+Version: 0.13.25
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.24.dist-info → ingestr-0.13.25.dist-info}/RECORD

@@ -2,15 +2,15 @@ ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=wvbRCJ2--M0Zw2cYtSH874TxTtlD0wadHREeLG3anOY,25618
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=x-bxDOuFDQ1rgDJf03eHD1bXb9Yfo3wX39XyaBE0LkU,21
+ingestr/src/buildinfo.py,sha256=lkdWOsV5VTf0j0DW9a0BdkPbOZMZEYAUvs8aGZ2V-uE,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=1jqcLv_QUUGeyg1OYN3ywrRdcDZyDRtMOongwyjDapU,5268
+ingestr/src/factory.py,sha256=659h_sVRBhtPv2dvtOK8tf3PtUhlK3KsWLrb20_iQKw,5333
 ingestr/src/filters.py,sha256=5LNpBgm8FJXdrFHGyM7dLVyphKykSpPk7yuQAZ8GML4,1133
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=kGsgFWf8Ghha0-HlC6PlDIIKX2Lriah4UmAseziGdr4,72035
+ingestr/src/sources.py,sha256=u2Kh2K9v1YhrKXss9yGGxFXr-fMp1E6pvbcwjZqYiGM,73394
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -41,6 +41,8 @@ ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_
 ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
+ingestr/src/frankfurter/__init__.py,sha256=xJUicENGYtOPsGznKP8IA_5Jt-_gJP29onrByBgUf-g,4259
+ingestr/src/frankfurter/helpers.py,sha256=RSqI-WAAJfunWnLqiBRmPuonRg7rDOqmY76beb8a6rM,967
 ingestr/src/github/__init__.py,sha256=xVijF-Wi4p88hkVJnKH-oTixismjD3aUcGqGa6Wr4e4,5889
 ingestr/src/github/helpers.py,sha256=rpv_3HzuOl4PQ-FUeA66pev-pgze9SaE8RUHIPYfZ_A,6759
 ingestr/src/github/queries.py,sha256=W34C02jUEdjFmOE7f7u9xvYyBNDMfVZAu0JIRZI2mkU,2302
@@ -119,8 +121,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.24.dist-info/METADATA,sha256=qzKyEOTPIb6cTD49Q6zC-bqSn7ax45JmCKcGh0jtJRw,13659
-ingestr-0.13.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.24.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.24.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.24.dist-info/RECORD,,
+ingestr-0.13.25.dist-info/METADATA,sha256=IiZGRzg-cRXN10snt3Y_97yODJ9KkDF-x9XjP_cx9vo,13659
+ingestr-0.13.25.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.25.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.25.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.25.dist-info/RECORD,,