ingestr 0.14.93__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/hostaway/__init__.py

@@ -0,0 +1,302 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import HostawayClient
+
+
+@dlt.source(max_table_nesting=0)
+def hostaway_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    """
+    Hostaway API source for fetching listings and fee settings data.
+
+    Args:
+        api_key: Hostaway API key for Bearer token authentication
+        start_date: Start date for incremental loading
+        end_date: End date for incremental loading (defaults to current time)
+
+    Returns:
+        Iterable[DltResource]: DLT resources for listings and/or fee settings
+    """
+
+    client = HostawayClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listings",
+        primary_key="id",
+    )
+    def listings(
+        datetime=dlt.sources.incremental(
+            "latestActivityOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listings from Hostaway API with incremental loading.
+        Uses latestActivityOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_listings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listing_fee_settings",
+        primary_key="id",
+    )
+    def listing_fee_settings(
+        datetime=dlt.sources.incremental(
+            "updatedOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listing fee settings from Hostaway API with incremental loading.
+        Uses updatedOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_all_listing_fee_settings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_agreements",
+    )
+    def listing_agreements() -> Iterable[TDataItem]:
+        """
+        Fetch listing agreements from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_agreements(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_pricing_settings",
+    )
+    def listing_pricing_settings() -> Iterable[TDataItem]:
+        """
+        Fetch listing pricing settings from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_pricing_settings(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies",
+    )
+    def cancellation_policies() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_airbnb",
+    )
+    def cancellation_policies_airbnb() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_airbnb()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_marriott",
+    )
+    def cancellation_policies_marriott() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_marriott()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_vrbo",
+    )
+    def cancellation_policies_vrbo() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_vrbo()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservations",
+        selected=False,
+    )
+    def reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_reservations()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="finance_fields",
+    )
+    def finance_fields(reservation_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_finance_field(res_id):
+            return list(client.fetch_finance_field(res_id))
+
+        reservation_id_val = reservation_item.get("id")
+        if reservation_id_val:
+            yield _get_finance_field(reservation_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservation_payment_methods",
+    )
+    def reservation_payment_methods() -> Iterable[TDataItem]:
+        yield from client.fetch_reservation_payment_methods()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="reservation_rental_agreements",
+    )
+    def reservation_rental_agreements(
+        reservation_item: TDataItem,
+    ) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_rental_agreement(res_id):
+            return list(client.fetch_reservation_rental_agreement(res_id))
+
+        reservation_id = reservation_item.get("id")
+        if reservation_id:
+            yield _get_rental_agreement(reservation_id)
+
+    @dlt.transformer(
+        data_from=listings,
+        write_disposition="replace",
+        name="listing_calendars",
+    )
+    def listing_calendars(listing_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_calendar(lst_id):
+            return list(client.fetch_listing_calendar(lst_id))
+
+        listing_id_val = listing_item.get("id")
+        if listing_id_val:
+            yield _get_calendar(listing_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="conversations",
+    )
+    def conversations() -> Iterable[TDataItem]:
+        yield from client.fetch_conversations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="message_templates",
+    )
+    def message_templates() -> Iterable[TDataItem]:
+        yield from client.fetch_message_templates()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="bed_types",
+    )
+    def bed_types() -> Iterable[TDataItem]:
+        yield from client.fetch_bed_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="property_types",
+    )
+    def property_types() -> Iterable[TDataItem]:
+        yield from client.fetch_property_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="countries",
+    )
+    def countries() -> Iterable[TDataItem]:
+        yield from client.fetch_countries()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="account_tax_settings",
+    )
+    def account_tax_settings() -> Iterable[TDataItem]:
+        yield from client.fetch_account_tax_settings()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="user_groups",
+    )
+    def user_groups() -> Iterable[TDataItem]:
+        yield from client.fetch_user_groups()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="guest_payment_charges",
+    )
+    def guest_payment_charges() -> Iterable[TDataItem]:
+        yield from client.fetch_guest_payment_charges()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="coupons",
+    )
+    def coupons() -> Iterable[TDataItem]:
+        yield from client.fetch_coupons()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="webhook_reservations",
+    )
+    def webhook_reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_webhook_reservations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="tasks",
+    )
+    def tasks() -> Iterable[TDataItem]:
+        yield from client.fetch_tasks()
+
+    return (
+        listings,
+        listing_fee_settings,
+        listing_agreements,
+        listing_pricing_settings,
+        cancellation_policies,
+        cancellation_policies_airbnb,
+        cancellation_policies_marriott,
+        cancellation_policies_vrbo,
+        reservations,
+        finance_fields,
+        reservation_payment_methods,
+        reservation_rental_agreements,
+        listing_calendars,
+        conversations,
+        message_templates,
+        bed_types,
+        property_types,
+        countries,
+        account_tax_settings,
+        user_groups,
+        guest_payment_charges,
+        coupons,
+        webhook_reservations,
+        tasks,
+    )

ingestr/src/hostaway/client.py

@@ -0,0 +1,288 @@
+from typing import Callable, Iterable, Optional
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class HostawayClient:
+    BASE_URL = "https://api.hostaway.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    def _fetch_single(self, url: str, params: Optional[dict] = None) -> Iterable[dict]:
+        response = self.session.get(url, params=params, timeout=30)
+        response.raise_for_status()
+        response_data = response.json()
+
+        if isinstance(response_data, dict) and "result" in response_data:
+            items = response_data["result"]
+        elif isinstance(response_data, list):
+            items = response_data
+        else:
+            items = []
+
+        if isinstance(items, list):
+            for item in items:
+                yield item
+        elif isinstance(items, dict):
+            yield items
+
+    def _paginate(
+        self,
+        url: str,
+        params: Optional[dict] = None,
+        limit: int = 100,
+        process_item: Optional[Callable[[dict], dict]] = None,
+    ) -> Iterable[dict]:
+        offset = 0
+        if params is None:
+            params = {}
+
+        while True:
+            page_params = {**params, "limit": limit, "offset": offset}
+            response = self.session.get(url, params=page_params, timeout=30)
+            response.raise_for_status()
+            response_data = response.json()
+
+            if isinstance(response_data, dict) and "result" in response_data:
+                items = response_data["result"]
+            elif isinstance(response_data, list):
+                items = response_data
+            else:
+                items = []
+
+            if not items or (isinstance(items, list) and len(items) == 0):
+                break
+
+            if isinstance(items, list):
+                for item in items:
+                    if process_item:
+                        item = process_item(item)
+                    yield item
+            elif isinstance(items, dict):
+                if process_item:
+                    items = process_item(items)
+                yield items
+
+            if isinstance(items, list) and len(items) < limit:
+                break
+            elif isinstance(items, dict):
+                break
+
+            offset += limit
+
+    def fetch_listings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        def process_listing(listing: dict) -> dict:
+            if "latestActivityOn" in listing and listing["latestActivityOn"]:
+                try:
+                    listing["latestActivityOn"] = pendulum.parse(
+                        listing["latestActivityOn"]
+                    )
+                except Exception:
+                    listing["latestActivityOn"] = pendulum.datetime(
+                        1970, 1, 1, tz="UTC"
+                    )
+            else:
+                listing["latestActivityOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            return listing
+
+        url = f"{self.BASE_URL}/v1/listings"
+        for listing in self._paginate(url, process_item=process_listing):
+            if start_time <= listing["latestActivityOn"] <= end_time:
+                yield listing
+
+    def fetch_listing_fee_settings(
+        self,
+        listing_id,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        def process_fee(fee: dict) -> dict:
+            if "updatedOn" in fee and fee["updatedOn"]:
+                try:
+                    fee["updatedOn"] = pendulum.parse(fee["updatedOn"])
+                except Exception:
+                    fee["updatedOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            else:
+                fee["updatedOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            return fee
+
+        url = f"{self.BASE_URL}/v1/listingFeeSettings/{str(listing_id)}"
+        for fee in self._paginate(url, process_item=process_fee):
+            if start_time <= fee["updatedOn"] <= end_time:
+                yield fee
+
+    def fetch_all_listing_fee_settings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_fee_settings(
+                        listing_id, start_time, end_time
+                    )
+                except Exception:
+                    continue
+
+    def fetch_listing_agreement(
+        self,
+        listing_id,
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listingAgreement/{str(listing_id)}"
+        yield from self._paginate(url)
+
+    def fetch_listing_pricing_settings(
+        self,
+        listing_id,
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listing/pricingSettings/{str(listing_id)}"
+        yield from self._paginate(url)
+
+    def fetch_all_listing_pricing_settings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_pricing_settings(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_all_listing_agreements(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_agreement(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_cancellation_policies(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_airbnb(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/airbnb"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_marriott(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/marriott"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_vrbo(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/vrbo"
+        yield from self._fetch_single(url)
+
+    def fetch_reservations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations"
+        yield from self._paginate(url)
+
+    def fetch_finance_field(self, reservation_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/financeField/{str(reservation_id)}"
+        yield from self._fetch_single(url)
+
+    def fetch_all_finance_fields(self) -> Iterable[dict]:
+        for reservation in self.fetch_reservations():
+            reservation_id = reservation.get("id")
+            if reservation_id:
+                try:
+                    yield from self.fetch_finance_field(reservation_id)
+                except Exception:
+                    continue
+
+    def fetch_reservation_payment_methods(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations/paymentMethods"
+        yield from self._fetch_single(url)
+
+    def fetch_reservation_rental_agreement(self, reservation_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations/{str(reservation_id)}/rentalAgreement"
+        try:
+            yield from self._fetch_single(url)
+        except Exception:
+            return
+
+    def fetch_all_reservation_rental_agreements(self) -> Iterable[dict]:
+        for reservation in self.fetch_reservations():
+            reservation_id = reservation.get("id")
+            if reservation_id:
+                try:
+                    yield from self.fetch_reservation_rental_agreement(reservation_id)
+                except Exception:
+                    continue
+
+    def fetch_listing_calendar(self, listing_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listings/{str(listing_id)}/calendar"
+        yield from self._fetch_single(url)
+
+    def fetch_all_listing_calendars(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_calendar(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_conversations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/conversations"
+        yield from self._paginate(url)
+
+    def fetch_message_templates(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/messageTemplates"
+        yield from self._fetch_single(url)
+
+    def fetch_bed_types(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/bedTypes"
+        yield from self._fetch_single(url)
+
+    def fetch_property_types(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/propertyTypes"
+        yield from self._fetch_single(url)
+
+    def fetch_countries(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/countries"
+        yield from self._fetch_single(url)
+
+    def fetch_account_tax_settings(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/accountTaxSettings"
+        yield from self._fetch_single(url)
+
+    def fetch_user_groups(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/userGroups"
+        yield from self._fetch_single(url)
+
+    def fetch_guest_payment_charges(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/guestPayments/charges"
+        yield from self._paginate(url)
+
+    def fetch_coupons(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/coupons"
+        yield from self._fetch_single(url)
+
+    def fetch_webhook_reservations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/webhooks/reservations"
+        yield from self._fetch_single(url)
+
+    def fetch_tasks(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/tasks"
+        yield from self._fetch_single(url)

ingestr/src/revenuecat/__init__.py

@@ -1,4 +1,3 @@
-import asyncio
 from typing import Any, Dict, Iterable, Iterator
 
 import aiohttp
@@ -40,51 +39,26 @@ def revenuecat_source(
                 yield project
 
     @dlt.resource(
-        name="customers", primary_key="id", write_disposition="merge", parallelized=True
+        name="customer_ids",
+        write_disposition="replace",
+        selected=False,
+        parallelized=True,
     )
-    def customers() -> Iterator[Dict[str, Any]]:
-        """Get list of customers with nested purchases and subscriptions."""
+    def customer_ids():
         if project_id is None:
             raise ValueError("project_id is required for customers resource")
-        endpoint = f"/projects/{project_id}/customers"
 
-        async def process_customer_batch(customer_batch):
-            """Process a batch of customers with async operations."""
-            async with aiohttp.ClientSession() as session:
-                tasks = []
-                for customer in customer_batch:
-                    task = process_customer_with_nested_resources_async(
-                        session, api_key, project_id, customer
-                    )
-                    tasks.append(task)
+        yield _paginate(api_key, f"/projects/{project_id}/customers")
 
-                return await asyncio.gather(*tasks)
-
-        def process_customers_sync():
-            """Process customers in batches using asyncio."""
-            batch_size = 50  # Conservative batch size due to 60 req/min rate limit
-            current_batch = []
-
-            for customer in _paginate(api_key, endpoint):
-                current_batch.append(customer)
-
-                if len(current_batch) >= batch_size:
-                    # Process the batch asynchronously
-                    processed_customers = asyncio.run(
-                        process_customer_batch(current_batch)
-                    )
-                    for processed_customer in processed_customers:
-                        yield processed_customer
-                    current_batch = []
-
-            # Process any remaining customers in the final batch
-            if current_batch:
-                processed_customers = asyncio.run(process_customer_batch(current_batch))
-                for processed_customer in processed_customers:
-                    yield processed_customer
-
-        # Yield each processed customer
-        yield from process_customers_sync()
+    @dlt.transformer(
+        data_from=customer_ids, write_disposition="replace", parallelized=True
+    )
+    async def customers(customers) -> Iterator[Dict[str, Any]]:
+        async with aiohttp.ClientSession() as session:
+            for customer in customers:
+                yield await process_customer_with_nested_resources_async(
+                    session, api_key, project_id, customer
+                )
 
     # Create project-dependent resources dynamically
     project_resources = []
@@ -103,6 +77,7 @@ def revenuecat_source(
 
     return [
         projects,
+        customer_ids,
         customers,
         *project_resources,
     ]