ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/freshdesk/freshdesk_client.py

@@ -4,9 +4,14 @@ import logging
 import time
 from typing import Any, Dict, Iterable, Optional
 
+import pendulum
 from dlt.common.typing import TDataItem
 from dlt.sources.helpers import requests
 
+from ingestr.src.errors import HTTPError
+
+TICKETS_QUERY_MAX_PAGE = 10
+
 
 class FreshdeskClient:
     """
@@ -61,13 +66,15 @@ class FreshdeskClient:
                 else:
                     # If the error is not a rate limit (429), raise the exception to be
                     # handled elsewhere or stop execution
-                    raise
+                    raise HTTPError(e) from e
 
     def paginated_response(
         self,
         endpoint: str,
         per_page: int,
-        updated_at: Optional[str] = None,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+        query: Optional[str] = None,
     ) -> Iterable[TDataItem]:
         """
         Fetches a paginated response from a specified endpoint.
@@ -77,6 +84,11 @@ class FreshdeskClient:
         updated at the specified timestamp.
         """
         page = 1
+        if query is not None:
+            query = query.replace('"', "").strip()
+
+        is_tickets_query = query and endpoint == "tickets"
+
         while True:
             # Construct the URL for the specific endpoint
             url = f"{self.base_url}/{endpoint}"
@@ -88,15 +100,38 @@ class FreshdeskClient:
                 param_key = (
                     "updated_since" if endpoint == "tickets" else "_updated_since"
                 )
-                if updated_at:
-                    params[param_key] = updated_at
+
+                params[param_key] = start_date.to_iso8601_string()
+
+            if is_tickets_query:
+                url = f"{self.base_url}/search/tickets"
+                params = {
+                    "query": f'"{query}"',
+                    "page": page,
+                }
 
             # Handle requests with rate-limiting
             # A maximum of 300 pages (30000 tickets) will be returned.
             response = self._request_with_rate_limit(url, params=params)
             data = response.json()
 
+            if query and endpoint == "tickets":
+                data = data["results"]
+
             if not data:
                 break  # Stop if no data or max page limit reached
-            yield data
+
+            filtered_data = [
+                item
+                for item in data
+                if "updated_at" in item
+                and pendulum.parse(item["updated_at"]) <= end_date
+            ]
+            if not filtered_data:
+                break
+            yield filtered_data
             page += 1
+
+            # https://developers.freshdesk.com/api/#filter_tickets
+            if is_tickets_query and page > TICKETS_QUERY_MAX_PAGE:
+                break

ingestr/src/fundraiseup/__init__.py

@@ -0,0 +1,49 @@
+"""Fundraiseup source for ingesting donations, events, fundraisers, recurring plans, and supporters."""
+
+from typing import Any, Dict, Generator, Iterable
+
+import dlt
+from dlt.sources import DltResource
+
+from .client import FundraiseupClient
+
+
+@dlt.source(name="fundraiseup", max_table_nesting=0)
+def fundraiseup_source(api_key: str) -> Iterable[DltResource]:
+    """
+    Return resources for Fundraiseup API.
+
+    Args:
+        api_key: API key for authentication
+
+    Returns:
+        Iterable of DLT resources
+    """
+    client = FundraiseupClient(api_key=api_key)
+
+    # Define available resources and their configurations
+    resources = {
+        "donations": {"write_disposition": "replace", "primary_key": "id"},
+        "events": {"write_disposition": "replace", "primary_key": "id"},
+        "fundraisers": {"write_disposition": "replace", "primary_key": "id"},
+        "recurring_plans": {"write_disposition": "replace", "primary_key": "id"},
+        "supporters": {"write_disposition": "replace", "primary_key": "id"},
+    }
+
+    def create_resource(resource_name: str, config: Dict[str, Any]) -> DltResource:
+        """Create a DLT resource dynamically."""
+
+        @dlt.resource(
+            name=resource_name,
+            write_disposition=config["write_disposition"],
+            primary_key=config["primary_key"],
+        )
+        def generic_resource() -> Generator[Dict[str, Any], None, None]:
+            """Generic resource that yields batches directly."""
+            for batch in client.get_paginated_data(resource_name):
+                yield batch  # type: ignore[misc]
+
+        return generic_resource()
+
+    # Return all resources
+    return [create_resource(name, config) for name, config in resources.items()]

ingestr/src/fundraiseup/client.py

@@ -0,0 +1,81 @@
+"""Fundraiseup API Client for handling authentication and paginated requests."""
+
+from typing import Any, Dict, Iterator, Optional
+
+from ingestr.src.http_client import create_client
+
+
+class FundraiseupClient:
+    """Client for interacting with Fundraiseup API v1."""
+
+    def __init__(self, api_key: str):
+        """
+        Initialize Fundraiseup API client.
+
+        Args:
+            api_key: API key for authentication
+        """
+        self.api_key = api_key
+        self.base_url = "https://api.fundraiseup.com/v1"
+        # Use shared HTTP client with retry logic for rate limiting
+        self.client = create_client(retry_status_codes=[429, 500, 502, 503, 504])
+
+    def get_paginated_data(
+        self,
+        endpoint: str,
+        params: Optional[Dict[str, Any]] = None,
+        page_size: int = 100,
+    ) -> Iterator[list[Dict[str, Any]]]:
+        """
+        Fetch paginated data from a Fundraiseup API endpoint using cursor-based pagination.
+
+        Args:
+            endpoint: API endpoint path (e.g., "donations")
+            params: Additional query parameters
+            page_size: Number of items per page (default 100)
+
+        Yields:
+            Batches of items from the API
+        """
+        url = f"{self.base_url}/{endpoint}"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        if params is None:
+            params = {}
+
+        params["limit"] = page_size
+        starting_after = None
+
+        while True:
+            # Add cursor for pagination if not first page
+            if starting_after:
+                params["starting_after"] = starting_after
+
+            response = self.client.get(url=url, headers=headers, params=params)
+            response.raise_for_status()
+
+            data = response.json()
+
+            # Handle both list response and object with data array
+            if isinstance(data, list):
+                items = data
+                has_more = len(items) == page_size
+            else:
+                items = data.get("data", [])
+                has_more = data.get("has_more", False)
+
+            if not items:
+                break
+
+            yield items
+
+            # Set cursor for next page
+            if has_more and items:
+                starting_after = items[-1].get("id")
+                if not starting_after:
+                    break
+            else:
+                break

ingestr/src/github/__init__.py

@@ -72,7 +72,7 @@ def github_repo_events(
     name: str,
     access_token: str,
     start_date: pendulum.DateTime,
-    end_date: pendulum.DateTime,
+    end_date: Optional[pendulum.DateTime] = None,
 ) -> DltResource:
     """Gets events for repository `name` with owner `owner` incrementally.
 
@@ -91,12 +91,14 @@ def github_repo_events(
     """
 
     # use naming function in table name to generate separate tables for each event
-    @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
+    @dlt.resource(
+        primary_key="id", table_name=lambda i: i["type"], write_disposition="merge"
+    )
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "created_at",
             initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
             last_value_func=max,
             range_end="closed",
             range_start="closed",
@@ -113,7 +115,7 @@ def github_repo_events(
         end_filter = (
             pendulum.parse(last_created_at.end_value)
             if last_created_at.end_value
-            else None
+            else pendulum.now()
         )
 
         for page in get_rest_pages(access_token, repos_path + "?per_page=100"):

ingestr/src/google_analytics/__init__.py

@@ -7,7 +7,7 @@ from typing import Iterator, List, Optional, Union
 import dlt
 from dlt.common import pendulum
 from dlt.common.typing import DictStrAny, TDataItem
-from dlt.extract import DltResource
+from dlt.sources import DltResource
 from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
 from google.analytics.data_v1beta import BetaAnalyticsDataClient
 from google.analytics.data_v1beta.types import (

ingestr/src/hostaway/__init__.py

@@ -0,0 +1,302 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import HostawayClient
+
+
+@dlt.source(max_table_nesting=0)
+def hostaway_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    """
+    Hostaway API source for fetching listings and fee settings data.
+
+    Args:
+        api_key: Hostaway API key for Bearer token authentication
+        start_date: Start date for incremental loading
+        end_date: End date for incremental loading (defaults to current time)
+
+    Returns:
+        Iterable[DltResource]: DLT resources for listings and/or fee settings
+    """
+
+    client = HostawayClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listings",
+        primary_key="id",
+    )
+    def listings(
+        datetime=dlt.sources.incremental(
+            "latestActivityOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listings from Hostaway API with incremental loading.
+        Uses latestActivityOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_listings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listing_fee_settings",
+        primary_key="id",
+    )
+    def listing_fee_settings(
+        datetime=dlt.sources.incremental(
+            "updatedOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listing fee settings from Hostaway API with incremental loading.
+        Uses updatedOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_all_listing_fee_settings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_agreements",
+    )
+    def listing_agreements() -> Iterable[TDataItem]:
+        """
+        Fetch listing agreements from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_agreements(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_pricing_settings",
+    )
+    def listing_pricing_settings() -> Iterable[TDataItem]:
+        """
+        Fetch listing pricing settings from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_pricing_settings(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies",
+    )
+    def cancellation_policies() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_airbnb",
+    )
+    def cancellation_policies_airbnb() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_airbnb()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_marriott",
+    )
+    def cancellation_policies_marriott() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_marriott()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_vrbo",
+    )
+    def cancellation_policies_vrbo() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_vrbo()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservations",
+        selected=False,
+    )
+    def reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_reservations()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="finance_fields",
+    )
+    def finance_fields(reservation_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_finance_field(res_id):
+            return list(client.fetch_finance_field(res_id))
+
+        reservation_id_val = reservation_item.get("id")
+        if reservation_id_val:
+            yield _get_finance_field(reservation_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservation_payment_methods",
+    )
+    def reservation_payment_methods() -> Iterable[TDataItem]:
+        yield from client.fetch_reservation_payment_methods()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="reservation_rental_agreements",
+    )
+    def reservation_rental_agreements(
+        reservation_item: TDataItem,
+    ) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_rental_agreement(res_id):
+            return list(client.fetch_reservation_rental_agreement(res_id))
+
+        reservation_id = reservation_item.get("id")
+        if reservation_id:
+            yield _get_rental_agreement(reservation_id)
+
+    @dlt.transformer(
+        data_from=listings,
+        write_disposition="replace",
+        name="listing_calendars",
+    )
+    def listing_calendars(listing_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_calendar(lst_id):
+            return list(client.fetch_listing_calendar(lst_id))
+
+        listing_id_val = listing_item.get("id")
+        if listing_id_val:
+            yield _get_calendar(listing_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="conversations",
+    )
+    def conversations() -> Iterable[TDataItem]:
+        yield from client.fetch_conversations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="message_templates",
+    )
+    def message_templates() -> Iterable[TDataItem]:
+        yield from client.fetch_message_templates()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="bed_types",
+    )
+    def bed_types() -> Iterable[TDataItem]:
+        yield from client.fetch_bed_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="property_types",
+    )
+    def property_types() -> Iterable[TDataItem]:
+        yield from client.fetch_property_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="countries",
+    )
+    def countries() -> Iterable[TDataItem]:
+        yield from client.fetch_countries()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="account_tax_settings",
+    )
+    def account_tax_settings() -> Iterable[TDataItem]:
+        yield from client.fetch_account_tax_settings()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="user_groups",
+    )
+    def user_groups() -> Iterable[TDataItem]:
+        yield from client.fetch_user_groups()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="guest_payment_charges",
+    )
+    def guest_payment_charges() -> Iterable[TDataItem]:
+        yield from client.fetch_guest_payment_charges()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="coupons",
+    )
+    def coupons() -> Iterable[TDataItem]:
+        yield from client.fetch_coupons()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="webhook_reservations",
+    )
+    def webhook_reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_webhook_reservations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="tasks",
+    )
+    def tasks() -> Iterable[TDataItem]:
+        yield from client.fetch_tasks()
+
+    return (
+        listings,
+        listing_fee_settings,
+        listing_agreements,
+        listing_pricing_settings,
+        cancellation_policies,
+        cancellation_policies_airbnb,
+        cancellation_policies_marriott,
+        cancellation_policies_vrbo,
+        reservations,
+        finance_fields,
+        reservation_payment_methods,
+        reservation_rental_agreements,
+        listing_calendars,
+        conversations,
+        message_templates,
+        bed_types,
+        property_types,
+        countries,
+        account_tax_settings,
+        user_groups,
+        guest_payment_charges,
+        coupons,
+        webhook_reservations,
+        tasks,
+    )