ingestr 0.13.94__py3-none-any.whl → 0.14.0__py3-none-any.whl

ingestr/src/frankfurter/__init__.py

@@ -1,157 +1,157 @@
-from typing import Any, Iterator, Optional
-
-import dlt
-from dlt.common.pendulum import pendulum
-from dlt.common.time import ensure_pendulum_datetime
-from dlt.common.typing import TAnyDateTime
-
-from ingestr.src.frankfurter.helpers import get_path_with_retry
-
-
-@dlt.source(
-    name="frankfurter",
-    max_table_nesting=0,
-)
-def frankfurter_source(
-    start_date: TAnyDateTime,
-    end_date: TAnyDateTime | None,
-    base_currency: str,
-) -> Any:
-    """
-    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
-    various types of data: currencies, latest rates, historical rates.
-    """
-
-    @dlt.resource(
-        write_disposition="replace",
-    )
-    def currencies() -> Iterator[dict]:
-        """
-        Yields each currency as a separate row with two columns: currency_code and currency_name.
-        """
-        # Retrieve the list of currencies from the API
-        currencies_data = get_path_with_retry("currencies")
-
-        for currency_code, currency_name in currencies_data.items():
-            yield {"currency_code": currency_code, "currency_name": currency_name}
-
-    @dlt.resource(
-        write_disposition="merge",
-        columns={
-            "date": {"data_type": "text"},
-            "currency_code": {"data_type": "text"},
-            "rate": {"data_type": "double"},
-            "base_currency": {"data_type": "text"},
-        },
-        primary_key=["date", "currency_code", "base_currency"],
-    )
-    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
-        """
-        Fetches the latest exchange rates and yields them as rows.
-        """
-        # Base URL
-        url = "latest?"
-
-        if base_currency:
-            url += f"base={base_currency}"
-
-        # Fetch data
-        data = get_path_with_retry(url)
-
-        # Extract rates and base currency
-        rates = data["rates"]
-        date = pendulum.parse(data["date"])
-
-        # Add the base currency with a rate of 1.0
-        yield {
-            "date": date,
-            "currency_code": base_currency,
-            "rate": 1.0,
-            "base_currency": base_currency,
-        }
-
-        # Add all currencies and their rates
-        for currency_code, rate in rates.items():
-            yield {
-                "date": date,
-                "currency_code": currency_code,
-                "rate": rate,
-                "base_currency": base_currency,
-            }
-
-    @dlt.resource(
-        write_disposition="merge",
-        columns={
-            "date": {"data_type": "text"},
-            "currency_code": {"data_type": "text"},
-            "rate": {"data_type": "double"},
-            "base_currency": {"data_type": "text"},
-        },
-        primary_key=("date", "currency_code", "base_currency"),
-    )
-    def exchange_rates(
-        date_time=dlt.sources.incremental(
-            "date",
-            initial_value=start_date,
-            end_value=end_date,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[dict]:
-        """
-        Fetches exchange rates for a specified date range.
-        If only start_date is provided, fetches data until now.
-        If both start_date and end_date are provided, fetches data for each day in the range.
-        """
-        if date_time.last_value is not None:
-            start_date = date_time.last_value
-        else:
-            start_date = start_date
-
-        if date_time.end_value is not None:
-            end_date = date_time.end_value
-        else:
-            end_date = pendulum.now()
-
-        # Ensure start_date.last_value is a pendulum.DateTime object
-        start_date_obj = ensure_pendulum_datetime(start_date)  # type: ignore
-        start_date_str = start_date_obj.format("YYYY-MM-DD")
-
-        # Ensure end_date is a pendulum.DateTime object
-        end_date_obj = ensure_pendulum_datetime(end_date)
-        end_date_str = end_date_obj.format("YYYY-MM-DD")
-
-        # Compose the URL
-        url = f"{start_date_str}..{end_date_str}?"
-
-        if base_currency:
-            url += f"base={base_currency}"
-
-        # Fetch data from the API
-        data = get_path_with_retry(url)
-
-        # Extract base currency and rates from the API response
-        rates = data["rates"]
-
-        # Iterate over the rates dictionary (one entry per date)
-        for date, daily_rates in rates.items():
-            formatted_date = pendulum.parse(date)
-
-            # Add the base currency with a rate of 1.0
-            yield {
-                "date": formatted_date,
-                "currency_code": base_currency,
-                "rate": 1.0,
-                "base_currency": base_currency,
-            }
-
-            # Add all other currencies and their rates
-            for currency_code, rate in daily_rates.items():
-                yield {
-                    "date": formatted_date,
-                    "currency_code": currency_code,
-                    "rate": rate,
-                    "base_currency": base_currency,
-                }
-
-    return currencies, latest, exchange_rates
+from typing import Any, Iterator, Optional
+
+import dlt
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+
+from ingestr.src.frankfurter.helpers import get_path_with_retry
+
+
+@dlt.source(
+    name="frankfurter",
+    max_table_nesting=0,
+)
+def frankfurter_source(
+    start_date: TAnyDateTime,
+    end_date: TAnyDateTime | None,
+    base_currency: str,
+) -> Any:
+    """
+    A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
+    various types of data: currencies, latest rates, historical rates.
+    """
+
+    @dlt.resource(
+        write_disposition="replace",
+    )
+    def currencies() -> Iterator[dict]:
+        """
+        Yields each currency as a separate row with two columns: currency_code and currency_name.
+        """
+        # Retrieve the list of currencies from the API
+        currencies_data = get_path_with_retry("currencies")
+
+        for currency_code, currency_name in currencies_data.items():
+            yield {"currency_code": currency_code, "currency_name": currency_name}
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=["date", "currency_code", "base_currency"],
+    )
+    def latest(base_currency: Optional[str] = "") -> Iterator[dict]:
+        """
+        Fetches the latest exchange rates and yields them as rows.
+        """
+        # Base URL
+        url = "latest?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data
+        data = get_path_with_retry(url)
+
+        # Extract rates and base currency
+        rates = data["rates"]
+        date = pendulum.parse(data["date"])
+
+        # Add the base currency with a rate of 1.0
+        yield {
+            "date": date,
+            "currency_code": base_currency,
+            "rate": 1.0,
+            "base_currency": base_currency,
+        }
+
+        # Add all currencies and their rates
+        for currency_code, rate in rates.items():
+            yield {
+                "date": date,
+                "currency_code": currency_code,
+                "rate": rate,
+                "base_currency": base_currency,
+            }
+
+    @dlt.resource(
+        write_disposition="merge",
+        columns={
+            "date": {"data_type": "text"},
+            "currency_code": {"data_type": "text"},
+            "rate": {"data_type": "double"},
+            "base_currency": {"data_type": "text"},
+        },
+        primary_key=("date", "currency_code", "base_currency"),
+    )
+    def exchange_rates(
+        date_time=dlt.sources.incremental(
+            "date",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        """
+        Fetches exchange rates for a specified date range.
+        If only start_date is provided, fetches data until now.
+        If both start_date and end_date are provided, fetches data for each day in the range.
+        """
+        if date_time.last_value is not None:
+            start_date = date_time.last_value
+        else:
+            start_date = start_date
+
+        if date_time.end_value is not None:
+            end_date = date_time.end_value
+        else:
+            end_date = pendulum.now()
+
+        # Ensure start_date.last_value is a pendulum.DateTime object
+        start_date_obj = ensure_pendulum_datetime(start_date)  # type: ignore
+        start_date_str = start_date_obj.format("YYYY-MM-DD")
+
+        # Ensure end_date is a pendulum.DateTime object
+        end_date_obj = ensure_pendulum_datetime(end_date)
+        end_date_str = end_date_obj.format("YYYY-MM-DD")
+
+        # Compose the URL
+        url = f"{start_date_str}..{end_date_str}?"
+
+        if base_currency:
+            url += f"base={base_currency}"
+
+        # Fetch data from the API
+        data = get_path_with_retry(url)
+
+        # Extract base currency and rates from the API response
+        rates = data["rates"]
+
+        # Iterate over the rates dictionary (one entry per date)
+        for date, daily_rates in rates.items():
+            formatted_date = pendulum.parse(date)
+
+            # Add the base currency with a rate of 1.0
+            yield {
+                "date": formatted_date,
+                "currency_code": base_currency,
+                "rate": 1.0,
+                "base_currency": base_currency,
+            }
+
+            # Add all other currencies and their rates
+            for currency_code, rate in daily_rates.items():
+                yield {
+                    "date": formatted_date,
+                    "currency_code": currency_code,
+                    "rate": rate,
+                    "base_currency": base_currency,
+                }
+
+    return currencies, latest, exchange_rates

ingestr/src/fundraiseup/__init__.py

@@ -0,0 +1,49 @@
+"""Fundraiseup source for ingesting donations, events, fundraisers, recurring plans, and supporters."""
+
+from typing import Any, Dict, Generator, Iterable
+
+import dlt
+from dlt.sources import DltResource
+
+from .client import FundraiseupClient
+
+
+@dlt.source(name="fundraiseup", max_table_nesting=0)
+def fundraiseup_source(api_key: str) -> Iterable[DltResource]:
+    """
+    Return resources for Fundraiseup API.
+
+    Args:
+        api_key: API key for authentication
+
+    Returns:
+        Iterable of DLT resources
+    """
+    client = FundraiseupClient(api_key=api_key)
+
+    # Define available resources and their configurations
+    resources = {
+        "donations": {"write_disposition": "replace", "primary_key": "id"},
+        "events": {"write_disposition": "replace", "primary_key": "id"},
+        "fundraisers": {"write_disposition": "replace", "primary_key": "id"},
+        "recurring_plans": {"write_disposition": "replace", "primary_key": "id"},
+        "supporters": {"write_disposition": "replace", "primary_key": "id"},
+    }
+
+    def create_resource(resource_name: str, config: Dict[str, Any]) -> DltResource:
+        """Create a DLT resource dynamically."""
+
+        @dlt.resource(
+            name=resource_name,
+            write_disposition=config["write_disposition"],
+            primary_key=config["primary_key"],
+        )
+        def generic_resource() -> Generator[Dict[str, Any], None, None]:
+            """Generic resource that yields batches directly."""
+            for batch in client.get_paginated_data(resource_name):
+                yield batch  # type: ignore[misc]
+
+        return generic_resource()
+
+    # Return all resources
+    return [create_resource(name, config) for name, config in resources.items()]

ingestr/src/fundraiseup/client.py

@@ -0,0 +1,81 @@
+"""Fundraiseup API Client for handling authentication and paginated requests."""
+
+from typing import Any, Dict, Iterator, Optional
+
+from ingestr.src.http_client import create_client
+
+
+class FundraiseupClient:
+    """Client for interacting with Fundraiseup API v1."""
+
+    def __init__(self, api_key: str):
+        """
+        Initialize Fundraiseup API client.
+
+        Args:
+            api_key: API key for authentication
+        """
+        self.api_key = api_key
+        self.base_url = "https://api.fundraiseup.com/v1"
+        # Use shared HTTP client with retry logic for rate limiting
+        self.client = create_client(retry_status_codes=[429, 500, 502, 503, 504])
+
+    def get_paginated_data(
+        self,
+        endpoint: str,
+        params: Optional[Dict[str, Any]] = None,
+        page_size: int = 100,
+    ) -> Iterator[list[Dict[str, Any]]]:
+        """
+        Fetch paginated data from a Fundraiseup API endpoint using cursor-based pagination.
+
+        Args:
+            endpoint: API endpoint path (e.g., "donations")
+            params: Additional query parameters
+            page_size: Number of items per page (default 100)
+
+        Yields:
+            Batches of items from the API
+        """
+        url = f"{self.base_url}/{endpoint}"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        if params is None:
+            params = {}
+
+        params["limit"] = page_size
+        starting_after = None
+
+        while True:
+            # Add cursor for pagination if not first page
+            if starting_after:
+                params["starting_after"] = starting_after
+
+            response = self.client.get(url=url, headers=headers, params=params)
+            response.raise_for_status()
+
+            data = response.json()
+
+            # Handle both list response and object with data array
+            if isinstance(data, list):
+                items = data
+                has_more = len(items) == page_size
+            else:
+                items = data.get("data", [])
+                has_more = data.get("has_more", False)
+
+            if not items:
+                break
+
+            yield items
+
+            # Set cursor for next page
+            if has_more and items:
+                starting_after = items[-1].get("id")
+                if not starting_after:
+                    break
+            else:
+                break

ingestr/src/google_analytics/__init__.py

@@ -7,7 +7,7 @@ from typing import Iterator, List, Optional, Union
 import dlt
 from dlt.common import pendulum
 from dlt.common.typing import DictStrAny, TDataItem
-from dlt.extract import DltResource
+from dlt.sources import DltResource
 from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
 from google.analytics.data_v1beta import BetaAnalyticsDataClient
 from google.analytics.data_v1beta.types import (

ingestr/src/mongodb/__init__.py

@@ -107,7 +107,7 @@ def mongodb_collection(
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,
     pymongoarrow_schema: Optional[Any] = None,
     custom_query: Optional[List[Dict[str, Any]]] = None,
-) -> Any:
+) -> DltResource:
     """
     A DLT source which loads a collection from a mongo database using PyMongo.
 

ingestr/src/mongodb/helpers.py

@@ -1,4 +1,4 @@
-"""Mongo database source helpers"""
+"""Mongo database source helpers and destination utilities"""
 
 import re
 from itertools import islice
@@ -23,6 +23,7 @@ from bson.timestamp import Timestamp
 from dlt.common import logger
 from dlt.common.configuration.specs import BaseConfiguration, configspec
 from dlt.common.data_writers import TDataItemFormat
+from dlt.common.schema import TTableSchema
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import TDataItem
 from dlt.common.utils import map_nested_in_place
@@ -945,3 +946,70 @@ def convert_mongo_shell_to_extended_json(query_string: str) -> str:
 
 
 __source_name__ = "mongodb"
+
+
+# MongoDB destination helper functions
+def process_file_items(file_path: str) -> list[dict]:
+    """Process items from a file path (JSONL format)."""
+    import json
+
+    documents = []
+    with open(file_path, "r") as f:
+        for line in f:
+            if line.strip():
+                doc = json.loads(line.strip())
+                documents.append(doc)  # Include all fields including DLT metadata
+    return documents
+
+
+def mongodb_insert(uri: str, database: str):
+    """Creates a dlt.destination for inserting data into a MongoDB collection.
+
+    Args:
+        uri (str): MongoDB connection URI.
+        database (str): Name of the MongoDB database.
+
+    Returns:
+        dlt.destination: A DLT destination object configured for MongoDB.
+    """
+
+    state = {"first_batch": True}
+
+    def destination(items: TDataItem, table: TTableSchema) -> None:
+        import pyarrow
+        from pymongo import MongoClient
+
+        # Extract database name from connection string
+        # Get collection name from table metadata
+        collection_name = table["name"]
+
+        # Connect to MongoDB
+        client: MongoClient
+
+        with MongoClient(uri) as client:
+            db = client[database]
+            collection = db[collection_name]
+
+            # Process and insert documents
+            if isinstance(items, str):
+                documents = process_file_items(items)
+            elif isinstance(items, pyarrow.RecordBatch):
+                documents = [item for item in items.to_pylist()]
+            else:
+                documents = [item for item in items if isinstance(item, dict)]
+
+            if state["first_batch"] and documents:
+                collection.delete_many({})
+                state["first_batch"] = False
+
+            if documents:
+                collection.insert_many(documents)  # Insert all new data
+
+    return dlt.destination(
+        destination,
+        name="mongodb",
+        loader_file_format="typed-jsonl",
+        batch_size=1000,
+        naming_convention="snake_case",
+        loader_parallelism_strategy="sequential",
+    )

ingestr/src/sources.py

@@ -237,6 +237,9 @@ class SqlSource:
                 backend_kwargs: Dict[str, Any] = None,  # type: ignore
                 type_adapter_callback: Optional[TTypeAdapter] = None,
                 included_columns: Optional[List[str]] = None,
+                excluded_columns: Optional[
+                    List[str]
+                ] = None,  # Added for dlt 1.16.0 compatibility
                 query_adapter_callback: Optional[TQueryAdapter] = None,
                 resolve_foreign_keys: bool = False,
             ) -> Iterator[TDataItem]:
@@ -3623,3 +3626,80 @@ class WiseSource:
             start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class FundraiseupSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Fundraiseup")
+
+        if table not in [
+            "donations",
+            "events",
+            "fundraisers",
+            "recurring_plans",
+            "supporters",
+        ]:
+            raise UnsupportedResourceError(table, "Fundraiseup")
+
+        from ingestr.src.fundraiseup import fundraiseup_source
+
+        return fundraiseup_source(
+            api_key=api_key[0],
+        ).with_resources(table)
+
+
+class AnthropicSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # anthropic://?api_key=<admin_api_key>
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Anthropic")
+
+        if table not in [
+            "claude_code_usage",
+            "usage_report",
+            "cost_report",
+            "organization",
+            "workspaces",
+            "api_keys",
+            "invites",
+            "users",
+            "workspace_members",
+        ]:
+            raise UnsupportedResourceError(table, "Anthropic")
+
+        # Get start and end dates from kwargs
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            # Default to 2023-01-01
+            start_date = pendulum.datetime(2023, 1, 1)
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date)
+        else:
+            end_date = None
+
+        from ingestr.src.anthropic import anthropic_source
+
+        return anthropic_source(
+            api_key=api_key[0],
+            initial_start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

ingestr/tests/unit/test_smartsheets.py

@@ -1,6 +1,6 @@
 import sys
 import unittest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import smartsheet  # type: ignore
 from smartsheet.models import Cell, Column, Row, Sheet  # type: ignore