ingestr 0.13.93__py3-none-any.whl → 0.14.0__py3-none-any.whl

ingestr/src/frankfurter/__init__.py

@@ -1,157 +1,157 @@
-from typing import Any, Iterator, Optional
-
-import dlt
-from dlt.common.pendulum import pendulum
-from dlt.common.time import ensure_pendulum_datetime
-from dlt.common.typing import TAnyDateTime
-
-from ingestr.src.frankfurter.helpers import get_path_with_retry
-
-
-@dlt.source(
-    name="frankfurter",
-    max_table_nesting=0,
-)
-def frankfurter_source(
-    start_date: TAnyDateTime,
-    end_date: TAnyDateTime | None,
-    base_currency: str,
-) -> Any:
-    """
-    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
-    various types of data: currencies, latest rates, historical rates.
-    """
-
-    @dlt.resource(
-        write_disposition="replace",
-    )
-    def currencies() -> Iterator[dict]:
-        """
-        Yields each currency as a separate row with two columns: currency_code and currency_name.
-        """
-        # Retrieve the list of currencies from the API
-        currencies_data = get_path_with_retry("currencies")
-
-        for currency_code, currency_name in currencies_data.items():
-            yield {"currency_code": currency_code, "currency_name": currency_name}
-
-    @dlt.resource(
-        write_disposition="merge",
-        columns={
-            "date": {"data_type": "text"},
-            "currency_code": {"data_type": "text"},
-            "rate": {"data_type": "double"},
-            "base_currency": {"data_type": "text"},
-        },
-        primary_key=["date", "currency_code", "base_currency"],
-    )
-    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
-        """
-        Fetches the latest exchange rates and yields them as rows.
-        """
-        # Base URL
-        url = "latest?"
-
-        if base_currency:
-            url += f"base={base_currency}"
-
-        # Fetch data
-        data = get_path_with_retry(url)
-
-        # Extract rates and base currency
-        rates = data["rates"]
-        date = pendulum.parse(data["date"])
-
-        # Add the base currency with a rate of 1.0
-        yield {
-            "date": date,
-            "currency_code": base_currency,
-            "rate": 1.0,
-            "base_currency": base_currency,
-        }
-
-        # Add all currencies and their rates
-        for currency_code, rate in rates.items():
-            yield {
-                "date": date,
-                "currency_code": currency_code,
-                "rate": rate,
-                "base_currency": base_currency,
-            }
-
-    @dlt.resource(
-        write_disposition="merge",
-        columns={
-            "date": {"data_type": "text"},
-            "currency_code": {"data_type": "text"},
-            "rate": {"data_type": "double"},
-            "base_currency": {"data_type": "text"},
-        },
-        primary_key=("date", "currency_code", "base_currency"),
-    )
-    def exchange_rates(
-        date_time=dlt.sources.incremental(
-            "date",
-            initial_value=start_date,
-            end_value=end_date,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[dict]:
-        """
-        Fetches exchange rates for a specified date range.
-        If only start_date is provided, fetches data until now.
-        If both start_date and end_date are provided, fetches data for each day in the range.
-        """
-        if date_time.last_value is not None:
-            start_date = date_time.last_value
-        else:
-            start_date = start_date
-
-        if date_time.end_value is not None:
-            end_date = date_time.end_value
-        else:
-            end_date = pendulum.now()
-
-        # Ensure start_date.last_value is a pendulum.DateTime object
-        start_date_obj = ensure_pendulum_datetime(start_date)  # type: ignore
-        start_date_str = start_date_obj.format("YYYY-MM-DD")
-
-        # Ensure end_date is a pendulum.DateTime object
-        end_date_obj = ensure_pendulum_datetime(end_date)
-        end_date_str = end_date_obj.format("YYYY-MM-DD")
-
-        # Compose the URL
-        url = f"{start_date_str}..{end_date_str}?"
-
-        if base_currency:
-            url += f"base={base_currency}"
-
-        # Fetch data from the API
-        data = get_path_with_retry(url)
-
-        # Extract base currency and rates from the API response
-        rates = data["rates"]
-
-        # Iterate over the rates dictionary (one entry per date)
-        for date, daily_rates in rates.items():
-            formatted_date = pendulum.parse(date)
-
-            # Add the base currency with a rate of 1.0
-            yield {
-                "date": formatted_date,
-                "currency_code": base_currency,
-                "rate": 1.0,
-                "base_currency": base_currency,
-            }
-
-            # Add all other currencies and their rates
-            for currency_code, rate in daily_rates.items():
-                yield {
-                    "date": formatted_date,
-                    "currency_code": currency_code,
-                    "rate": rate,
-                    "base_currency": base_currency,
-                }
-
-    return currencies, latest, exchange_rates
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from ingestr.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime | None,
+    base_currency: str,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+    """
+
+    @dlt.resource(
+        write_disposition="replace",
+    )
+    def currencies() -> Iterator[dict]:
+        """
+        Yields each currency as a separate row with two columns: currency_code and currency_name.
+        """
+        # Retrieve the list of currencies from the API
+        currencies_data = get_path_with_retry("currencies")
+
+        for currency_code, currency_name in currencies_data.items():
+            yield {"currency_code": currency_code, "currency_name": currency_name}
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=["date", "currency_code", "base_currency"],
+    )
+    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
+        """
+        Fetches the latest exchange rates and yields them as rows.
+        """
+        # Base URL
+        url = "latest?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data
+        data = get_path_with_retry(url)
+
+        # Extract rates and base currency
+        rates = data["rates"]
+        date = pendulum.parse(data["date"])
+
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_code": base_currency,
+            "rate": 1.0,
+            "base_currency": base_currency,
+        }
+
+        # Add all currencies and their rates
+        for currency_code, rate in rates.items():
+            yield {
+                "date": date,
+                "currency_code": currency_code,
+                "rate": rate,
+                "base_currency": base_currency,
+            }
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=("date", "currency_code", "base_currency"),
+    )
+    def exchange_rates(
+        date_time=dlt.sources.incremental(
+            "date",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        """
+        Fetches exchange rates for a specified date range.
+        If only start_date is provided, fetches data until now.
+        If both start_date and end_date are provided, fetches data for each day in the range.
+        """
+        if date_time.last_value is not None:
+            start_date = date_time.last_value
+        else:
+            start_date = start_date
+
+        if date_time.end_value is not None:
+            end_date = date_time.end_value
+        else:
+            end_date = pendulum.now()
+
+        # Ensure start_date.last_value is a pendulum.DateTime object
+        start_date_obj = ensure_pendulum_datetime(start_date)  # type: ignore
+        start_date_str = start_date_obj.format("YYYY-MM-DD")
+
+        # Ensure end_date is a pendulum.DateTime object
+        end_date_obj = ensure_pendulum_datetime(end_date)
+        end_date_str = end_date_obj.format("YYYY-MM-DD")
+
+        # Compose the URL
+        url = f"{start_date_str}..{end_date_str}?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data from the API
+        data = get_path_with_retry(url)
+
+        # Extract base currency and rates from the API response
+        rates = data["rates"]
+
+        # Iterate over the rates dictionary (one entry per date)
+        for date, daily_rates in rates.items():
+            formatted_date = pendulum.parse(date)
+
+            # Add the base currency with a rate of 1.0
+            yield {
+                "date": formatted_date,
+                "currency_code": base_currency,
+                "rate": 1.0,
+                "base_currency": base_currency,
+            }
+
+            # Add all other currencies and their rates
+            for currency_code, rate in daily_rates.items():
+                yield {
+                    "date": formatted_date,
+                    "currency_code": currency_code,
+                    "rate": rate,
+                    "base_currency": base_currency,
+                }
+
+    return currencies, latest, exchange_rates

ingestr/src/fundraiseup/__init__.py

@@ -0,0 +1,49 @@
+"""Fundraiseup source for ingesting donations, events, fundraisers, recurring plans, and supporters."""
+
+from typing import Any, Dict, Generator, Iterable
+
+import dlt
+from dlt.sources import DltResource
+
+from .client import FundraiseupClient
+
+
+@dlt.source(name="fundraiseup", max_table_nesting=0)
+def fundraiseup_source(api_key: str) -> Iterable[DltResource]:
+    """
+    Return resources for Fundraiseup API.
+
+    Args:
+        api_key: API key for authentication
+
+    Returns:
+        Iterable of DLT resources
+    """
+    client = FundraiseupClient(api_key=api_key)
+
+    # Define available resources and their configurations
+    resources = {
+        "donations": {"write_disposition": "replace", "primary_key": "id"},
+        "events": {"write_disposition": "replace", "primary_key": "id"},
+        "fundraisers": {"write_disposition": "replace", "primary_key": "id"},
+        "recurring_plans": {"write_disposition": "replace", "primary_key": "id"},
+        "supporters": {"write_disposition": "replace", "primary_key": "id"},
+    }
+
+    def create_resource(resource_name: str, config: Dict[str, Any]) -> DltResource:
+        """Create a DLT resource dynamically."""
+
+        @dlt.resource(
+            name=resource_name,
+            write_disposition=config["write_disposition"],
+            primary_key=config["primary_key"],
+        )
+        def generic_resource() -> Generator[Dict[str, Any], None, None]:
+            """Generic resource that yields batches directly."""
+            for batch in client.get_paginated_data(resource_name):
+                yield batch  # type: ignore[misc]
+
+        return generic_resource()
+
+    # Return all resources
+    return [create_resource(name, config) for name, config in resources.items()]

ingestr/src/fundraiseup/client.py

@@ -0,0 +1,81 @@
+"""Fundraiseup API Client for handling authentication and paginated requests."""
+
+from typing import Any, Dict, Iterator, Optional
+
+from ingestr.src.http_client import create_client
+
+
+class FundraiseupClient:
+    """Client for interacting with Fundraiseup API v1."""
+
+    def __init__(self, api_key: str):
+        """
+        Initialize Fundraiseup API client.
+
+        Args:
+            api_key: API key for authentication
+        """
+        self.api_key = api_key
+        self.base_url = "https://api.fundraiseup.com/v1"
+        # Use shared HTTP client with retry logic for rate limiting
+        self.client = create_client(retry_status_codes=[429, 500, 502, 503, 504])
+
+    def get_paginated_data(
+        self,
+        endpoint: str,
+        params: Optional[Dict[str, Any]] = None,
+        page_size: int = 100,
+    ) -> Iterator[list[Dict[str, Any]]]:
+        """
+        Fetch paginated data from a Fundraiseup API endpoint using cursor-based pagination.
+
+        Args:
+            endpoint: API endpoint path (e.g., "donations")
+            params: Additional query parameters
+            page_size: Number of items per page (default 100)
+
+        Yields:
+            Batches of items from the API
+        """
+        url = f"{self.base_url}/{endpoint}"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        if params is None:
+            params = {}
+
+        params["limit"] = page_size
+        starting_after = None
+
+        while True:
+            # Add cursor for pagination if not first page
+            if starting_after:
+                params["starting_after"] = starting_after
+
+            response = self.client.get(url=url, headers=headers, params=params)
+            response.raise_for_status()
+
+            data = response.json()
+
+            # Handle both list response and object with data array
+            if isinstance(data, list):
+                items = data
+                has_more = len(items) == page_size
+            else:
+                items = data.get("data", [])
+                has_more = data.get("has_more", False)
+
+            if not items:
+                break
+
+            yield items
+
+            # Set cursor for next page
+            if has_more and items:
+                starting_after = items[-1].get("id")
+                if not starting_after:
+                    break
+            else:
+                break

ingestr/src/google_analytics/__init__.py

@@ -7,7 +7,7 @@ from typing import Iterator, List, Optional, Union
 import dlt
 from dlt.common import pendulum
 from dlt.common.typing import DictStrAny, TDataItem
-from dlt.extract import DltResource
+from dlt.sources import DltResource
 from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
 from google.analytics.data_v1beta import BetaAnalyticsDataClient
 from google.analytics.data_v1beta.types import (

ingestr/src/mongodb/__init__.py

@@ -107,7 +107,7 @@ def mongodb_collection(
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
     pymongoarrow_schema: Optional[Any] = None,
     custom_query: Optional[List[Dict[str, Any]]] = None,
-) -> Any:
+) -> DltResource:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.