ingestr 0.13.24__tar.gz → 0.13.25__tar.gz

{ingestr-0.13.24 → ingestr-0.13.25}/.gitignore

@@ -19,4 +19,5 @@ node_modules
 *.duckdb.wal
 *.db
 *.db.wal
-*.sql
+*.sql
+ingest.sh

{ingestr-0.13.24 → ingestr-0.13.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.24
+Version: 0.13.25
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.24 → ingestr-0.13.25}/docs/.vitepress/config.mjs

@@ -115,6 +115,7 @@ export default defineConfig({
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",
               },
+              { text: "Frankfurter", link: "/supported-sources/frankfurter.md" },
               { text: "Google Cloud Storage (GCS)", link: "/supported-sources/gcs.md" },
               { text: "Google Analytics", link: "/supported-sources/google_analytics.md" },
               { text: "Google Ads", link: "/supported-sources/google-ads.md" },

ingestr-0.13.25/docs/supported-sources/frankfurter.md

@@ -0,0 +1,167 @@
+# **Frankfurter Source Documentation**
+
+The `frankfurter` source in the `ingestr` pipeline is designed to fetch exchange rate data from the [Frankfurter API](https://www.frankfurter.dev/). This source supports fetching the latest exchange rates, historical exchange rates, and currency metadata. The data can be ingested into a specified destination database, such as DuckDB.
+
+---
+
+## **Command Overview**
+
+The `ingestr` command to use the `frankfurter` source is as follows:
+
+```bash
+ingestr ingest \
+--source-uri 'frankfurter://' \
+--interval-start '2025-03-27' \     # Optional. See 'exchange_rates'.
+--interval-end '2025-03-28' \       # Optional.
+--source-table '<table_name>' \     # E.g 'currencies', 'latest', 'exchange_rates'. See below.
+--dest-uri '<your-destination-uri>' \
+--dest-table '<your-schema>.<your-table_name>'
+```
+
+---
+
+## **Command Parameters**
+
+### **`--source-uri`**
+- **Description**: Specifies the source URI for the Frankfurter API.
+- **Value**: `'frankfurter://'`
+- **Purpose**: Indicates that the data will be fetched from the Frankfurter API.
+
+---
+
+### **`--interval-start` (Optional)**
+- **Description**: The start date for fetching historical exchange rates.
+- **Value**: A date in the format `YYYY-MM-DD` (e.g., `'2025-03-27'`).
+- **Purpose**: Defines the starting point for fetching historical data.
+  - For `latest` and `currencies` this parameter is ignored.
+  - For `exchange_rates`, it defaults to the current date if not provided.
+
+---
+
+### **`--interval-end` (Optional)**
+- **Description**: The end date for fetching historical exchange rates.
+- **Value**: A date in the format `YYYY-MM-DD` (e.g., `'2025-03-28'`).
+- **Purpose**: Defines the ending point for fetching historical data. If not provided, it defaults to the value of `--interval-start`.
+    - For `latest` and `currencies` this parameter is ignored.
+
+---
+
+### **`--source-table`**
+- **Description**: Specifies the table to fetch data from.
+- **Value**: One of the following:
+  - **`currencies`**: Fetches a list of the available currencies and their ISO 4217 currency code (e.g. `USD`, `EUR`, `GBP`).
+  - **`latest`**: Fetches the latest exchange rates.
+  - **`exchange_rates`**: Fetches historical exchange rates for a specified date range.
+- **Purpose**: Determines the type of data to fetch from the Frankfurter API.
+
+---
+
+### **`--dest-uri`**
+- **Description**: Specifies the destination database URI.
+- **Value**: The path to the database file (e.g., `'duckdb.db'`).
+- **Purpose**: Defines where the fetched data will be stored.
+
+---
+
+### **`--dest-table`**
+- **Description**: Specifies the destination table name.
+- **Value**: A string in the format `{schema}.{table_name}` (e.g., `'schema.my_table'`).
+- **Purpose**: Defines the schema and table name where the data will be written.
+- **Notes**:
+    - If the destination table does not yet exist in your database, ingestr will automatically create the table with name that is provided in this argument. The table will be structured according to the source table (see [Core Concepts](https://bruin-data.github.io/ingestr/getting-started/core-concepts.html)).
+
+---
+
+## **Supported Source Tables**
+
+### **`currencies`**
+- **Description**: Fetches a list of available currencies.
+- **Columns**:
+  - `currency_code`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
+  - `currency_name`: The name of the currency (e.g., `US Dollar`, `Euro`).
+- **Primary Key**: `currency_code`
+
+---
+
+### **`latest`**
+- **Description**: Fetches the latest exchange rates.
+- **Columns**:
+  - `date`: The date of the exchange rates.
+  - `currency_name`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
+  - `rate`: The exchange rate relative to the base currency.
+- **Primary Key**: Composite key of `date` and `currency_name`.
+- **Notes**:
+  - The base currency (e.g., `EUR`) is included with a rate of `1.0`.
+
+---
+
+### **`exchange_rates`**
+- **Description**: Fetches historical exchange rates for a specified date range.
+- **Columns**:
+  - `date`: The date of the exchange rates.
+  - `currency_name`: The ISO 4217 currency code (e.g., `USD`, `EUR`).
+  - `rate`: The exchange rate relative to the base currency.
+- **Primary Key**: Composite key of `date` and `currency_name`.
+- **Notes**:
+  - An optional start and end date can be added via the arguments `--interval-start` and optionally `--interval-end` to define the date range (see examples below). If no start date is specified, the date will default today's date (and thus return the latest exchange rates).
+  - If a start date but no end date is specified, then the end date will default to the start date and ingestr will retrieve data for the specified start date only.
+  - Note that the [Frankfurter API](https://www.frankfurter.dev/) only publishes updates Monday-Friday. If the given date is on the weekend, the date will default to the previous Friday. The source is however implemented in ingestr in such a way as to avoid duplicating rows of data in this case (see [Incremental Loading - Replace](https://bruin-data.github.io/ingestr/getting-started/incremental-loading.html)).
+
+#### **Example Table: Handling Weekend Dates**
+Here `--interval-start` is set to a weekend date (e.g., `2025-03-29` -- a Saturday). `--interval-end` is set to a the following Monday (`2025-03-31`). 
+
+`--interval-start` defaults to the previous Friday (`2025-03-28`) and the next data is from the following Monday (for simplicity, only a subset of currencies is shown below):
+
+| **date**     | **currency_name** | **rate** |
+|--------------|-------------------|----------|
+| 2025-03-28   | EUR               | 1.0      | 
+| 2025-03-28   | USD               | 1.0783   | 
+| 2025-03-28   | GBP               | 0.8571   | 
+| 2025-03-31   | EUR               | 1.0      | 
+| 2025-03-31   | USD               | 1.0783   |
+| 2025-03-31   | GBP               | 0.8571   |
+
+
+---
+
+## **Examples**
+
+### **1. Fetch the Latest Exchange Rates**
+```bash
+ingestr ingest \
+--source-uri 'frankfurter://' \
+--source-table 'latest' \
+--dest-uri 'duckdb.db' \
+--dest-table 'schema.latest_new_scheme'
+```
+
+---
+
+### **2. Fetch Historical Exchange Rates**
+```bash
+ingestr ingest \
+--source-uri 'frankfurter://' \
+--interval-start '2025-03-01' \
+--interval-end '2025-03-10' \
+--source-table 'exchange_rates' \
+--dest-uri 'duckdb.db' \
+--dest-table 'schema.exchange_rates'
+```
+
+---
+
+### **3. Fetch Currency Metadata**
+```bash
+ingestr ingest \
+--source-uri 'frankfurter://' \
+--source-table 'currencies' \
+--dest-uri 'duckdb.db' \
+--dest-table 'schema.currencies'
+```
+
+---
+
+## **Notes**
+- Ensure that the destination database (`--dest-uri`) is accessible and writable.
+- The `--interval-start` and `--interval-end` parameters are only applicable for the `exchange_rates` table.
+- The `latest` table always fetches the most recent exchange rates and ignores date parameters.

ingestr-0.13.25/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.25"

{ingestr-0.13.24 → ingestr-0.13.25}/ingestr/src/factory.py

@@ -28,6 +28,7 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    FrankfurterSource,
     GCSSource,
     GitHubSource,
     GoogleAdsSource,
@@ -146,6 +147,7 @@ class SourceDestinationFactory:
         "personio": PersonioSource,
         "kinesis": KinesisSource,
         "pipedrive": PipedriveSource,
+        "frankfurter": FrankfurterSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr-0.13.25/ingestr/src/frankfurter/__init__.py

@@ -0,0 +1,142 @@
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from ingestr.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    table: str,
+    start_date: Optional[TAnyDateTime] = None,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+
+    Returns the appropriate resource based on the provided parameters.
+    """
+    # Determine which resource to return based on the `table` parameter
+    if table == "currencies":
+        return currencies()
+
+    elif table == "latest":
+        return latest()
+
+    elif table == "exchange_rates":
+        return exchange_rates(start_date=start_date, end_date=end_date)
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "currency_code": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+    },
+)
+def currencies() -> Iterator[dict]:
+    """
+    Yields each currency as a separate row with two columns: currency_code and currency_name.
+    """
+    # Retrieve the list of currencies from the API
+    currencies_data = get_path_with_retry("currencies")
+
+    for currency_code, currency_name in currencies_data.items():
+        yield {"currency_code": currency_code, "currency_name": currency_name}
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "date": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+        "rate": {"data_type": "double"},
+    },
+    primary_key=["date", "currency_name"],  # Composite primary key
+)
+def latest() -> Iterator[dict]:
+    """
+    Fetches the latest exchange rates and yields them as rows.
+    """
+    # Base URL
+    url = "latest?"
+
+    # Fetch data
+    latest_data = get_path_with_retry(url)
+
+    # Extract rates and base currency
+    rates = latest_data["rates"]
+
+    # Prepare the date
+    date = pendulum.now().to_date_string()
+
+    # Add the base currency (EUR) with a rate of 1.0
+    yield {
+        "date": date,
+        "currency_name": "EUR",
+        "rate": 1.0,
+    }
+
+    # Add all currencies and their rates
+    for currency_name, rate in rates.items():
+        yield {
+            "date": date,
+            "currency_name": currency_name,
+            "rate": rate,
+        }
+
+
+@dlt.resource(
+    write_disposition="replace",
+    columns={
+        "date": {"data_type": "text"},
+        "currency_name": {"data_type": "text"},
+        "rate": {"data_type": "double"},
+    },
+    primary_key=["date", "currency_name"],  # Composite primary key
+)
+def exchange_rates(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime,
+) -> Iterator[dict]:
+    """
+    Fetches exchange rates for a specified date range.
+    If only start_date is provided, fetches data for that date.
+    If both start_date and end_date are provided, fetches data for each day in the range.
+    """
+    start_date_str = ensure_pendulum_datetime(start_date).format("YYYY-MM-DD")
+    end_date_str = ensure_pendulum_datetime(end_date).format("YYYY-MM-DD")
+
+    # Compose the URL
+    url = f"{start_date_str}..{end_date_str}?"
+
+    # Fetch data from the API
+    data = get_path_with_retry(url)
+
+    # Extract base currency and rates from the API response
+    base_currency = data["base"]
+    rates = data["rates"]
+
+    # Iterate over the rates dictionary (one entry per date)
+    for date, daily_rates in rates.items():
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_name": base_currency,
+            "rate": 1.0,
+        }
+
+        # Add all other currencies and their rates
+        for currency_name, rate in daily_rates.items():
+            yield {
+                "date": date,
+                "currency_name": currency_name,
+                "rate": rate,
+            }

ingestr-0.13.25/ingestr/src/frankfurter/helpers.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+
+from dlt.common.pendulum import pendulum
+from dlt.common.typing import StrAny
+from dlt.sources.helpers import requests
+
+FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
+
+
+def get_url_with_retry(url: str) -> StrAny:
+    r = requests.get(url)
+    return r.json()  # type: ignore
+
+
+def get_path_with_retry(path: str) -> StrAny:
+    return get_url_with_retry(f"{FRANKFURTER_API_URL}{path}")
+
+
+def validate_dates(start_date: datetime, end_date: datetime) -> None:
+    current_date = pendulum.now()
+
+    # Check if start_date is in the future
+    if start_date > current_date:
+        raise ValueError("Interval-start cannot be in the future.")
+
+    # Check if end_date is in the future
+    if end_date > current_date:
+        raise ValueError("Interval-end cannot be in the future.")
+
+    # Check if start_date is before end_date
+    if start_date > end_date:
+        raise ValueError("Interval-end cannot be before interval-start.")

{ingestr-0.13.24 → ingestr-0.13.25}/ingestr/src/sources.py

@@ -67,6 +67,8 @@ from ingestr.src.errors import (
 from ingestr.src.facebook_ads import facebook_ads_source, facebook_insights_source
 from ingestr.src.filesystem import readers
 from ingestr.src.filters import table_adapter_exclude_columns
+from ingestr.src.frankfurter import frankfurter_source
+from ingestr.src.frankfurter.helpers import validate_dates
 from ingestr.src.github import github_reactions, github_repo_events, github_stargazers
 from ingestr.src.google_ads import google_ads
 from ingestr.src.google_analytics import google_analytics
@@ -2041,3 +2043,35 @@ class PipedriveSource:
         return pipedrive_source(
             pipedrive_api_key=api_key, since_timestamp=start_date
         ).with_resources(table)
+
+
+class FrankfurterSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # start and end dates only assigned and validated for exchange_rates table
+        # Note: if an end date but no start date is provided, start date and end date will be set to current date
+        if table == "exchange_rates":
+            if kwargs.get("interval_start"):
+                start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))
+                if kwargs.get("interval_end"):
+                    end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
+                else:
+                    end_date = start_date
+            else:
+                start_date = pendulum.now()
+                end_date = pendulum.now()
+            validate_dates(start_date=start_date, end_date=end_date)
+
+        # Validate table
+        if table not in ["currencies", "latest", "exchange_rates"]:
+            raise ValueError(
+                f"Table '{table}' is not supported for Frankfurter source."
+            )
+
+        return frankfurter_source(
+            table=table,
+            start_date=start_date,
+            end_date=end_date,
+        )

ingestr-0.13.24/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.24"