ingestr 0.14.94__py3-none-any.whl → 0.14.97__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.14.94"
+version = "v0.14.97"

ingestr/src/factory.py

@@ -54,6 +54,7 @@ from ingestr.src.sources import (
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HostawaySource,
     HttpSource,
     HubspotSource,
     InfluxDBSource,
@@ -84,6 +85,7 @@ from ingestr.src.sources import (
     ShopifySource,
     SlackSource,
     SmartsheetSource,
+    SocrataSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
@@ -177,6 +179,7 @@ class SourceDestinationFactory:
         "facebookads": FacebookAdsSource,
         "fluxx": FluxxSource,
         "slack": SlackSource,
+        "hostaway": HostawaySource,
         "hubspot": HubspotSource,
         "intercom": IntercomSource,
         "jira": JiraSource,
@@ -218,6 +221,7 @@ class SourceDestinationFactory:
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
         "revenuecat": RevenueCatSource,
+        "socrata": SocrataSource,
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,

ingestr/src/hostaway/__init__.py

@@ -0,0 +1,302 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import HostawayClient
+
+
+@dlt.source(max_table_nesting=0)
+def hostaway_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    """
+    Hostaway API source for fetching listings and fee settings data.
+
+    Args:
+        api_key: Hostaway API key for Bearer token authentication
+        start_date: Start date for incremental loading
+        end_date: End date for incremental loading (defaults to current time)
+
+    Returns:
+        Iterable[DltResource]: DLT resources for listings and/or fee settings
+    """
+
+    client = HostawayClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listings",
+        primary_key="id",
+    )
+    def listings(
+        datetime=dlt.sources.incremental(
+            "latestActivityOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listings from Hostaway API with incremental loading.
+        Uses latestActivityOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_listings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listing_fee_settings",
+        primary_key="id",
+    )
+    def listing_fee_settings(
+        datetime=dlt.sources.incremental(
+            "updatedOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listing fee settings from Hostaway API with incremental loading.
+        Uses updatedOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_all_listing_fee_settings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_agreements",
+    )
+    def listing_agreements() -> Iterable[TDataItem]:
+        """
+        Fetch listing agreements from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_agreements(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_pricing_settings",
+    )
+    def listing_pricing_settings() -> Iterable[TDataItem]:
+        """
+        Fetch listing pricing settings from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_pricing_settings(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies",
+    )
+    def cancellation_policies() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_airbnb",
+    )
+    def cancellation_policies_airbnb() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_airbnb()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_marriott",
+    )
+    def cancellation_policies_marriott() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_marriott()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_vrbo",
+    )
+    def cancellation_policies_vrbo() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_vrbo()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservations",
+        selected=False,
+    )
+    def reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_reservations()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="finance_fields",
+    )
+    def finance_fields(reservation_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_finance_field(res_id):
+            return list(client.fetch_finance_field(res_id))
+
+        reservation_id_val = reservation_item.get("id")
+        if reservation_id_val:
+            yield _get_finance_field(reservation_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservation_payment_methods",
+    )
+    def reservation_payment_methods() -> Iterable[TDataItem]:
+        yield from client.fetch_reservation_payment_methods()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="reservation_rental_agreements",
+    )
+    def reservation_rental_agreements(
+        reservation_item: TDataItem,
+    ) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_rental_agreement(res_id):
+            return list(client.fetch_reservation_rental_agreement(res_id))
+
+        reservation_id = reservation_item.get("id")
+        if reservation_id:
+            yield _get_rental_agreement(reservation_id)
+
+    @dlt.transformer(
+        data_from=listings,
+        write_disposition="replace",
+        name="listing_calendars",
+    )
+    def listing_calendars(listing_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_calendar(lst_id):
+            return list(client.fetch_listing_calendar(lst_id))
+
+        listing_id_val = listing_item.get("id")
+        if listing_id_val:
+            yield _get_calendar(listing_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="conversations",
+    )
+    def conversations() -> Iterable[TDataItem]:
+        yield from client.fetch_conversations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="message_templates",
+    )
+    def message_templates() -> Iterable[TDataItem]:
+        yield from client.fetch_message_templates()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="bed_types",
+    )
+    def bed_types() -> Iterable[TDataItem]:
+        yield from client.fetch_bed_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="property_types",
+    )
+    def property_types() -> Iterable[TDataItem]:
+        yield from client.fetch_property_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="countries",
+    )
+    def countries() -> Iterable[TDataItem]:
+        yield from client.fetch_countries()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="account_tax_settings",
+    )
+    def account_tax_settings() -> Iterable[TDataItem]:
+        yield from client.fetch_account_tax_settings()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="user_groups",
+    )
+    def user_groups() -> Iterable[TDataItem]:
+        yield from client.fetch_user_groups()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="guest_payment_charges",
+    )
+    def guest_payment_charges() -> Iterable[TDataItem]:
+        yield from client.fetch_guest_payment_charges()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="coupons",
+    )
+    def coupons() -> Iterable[TDataItem]:
+        yield from client.fetch_coupons()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="webhook_reservations",
+    )
+    def webhook_reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_webhook_reservations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="tasks",
+    )
+    def tasks() -> Iterable[TDataItem]:
+        yield from client.fetch_tasks()
+
+    return (
+        listings,
+        listing_fee_settings,
+        listing_agreements,
+        listing_pricing_settings,
+        cancellation_policies,
+        cancellation_policies_airbnb,
+        cancellation_policies_marriott,
+        cancellation_policies_vrbo,
+        reservations,
+        finance_fields,
+        reservation_payment_methods,
+        reservation_rental_agreements,
+        listing_calendars,
+        conversations,
+        message_templates,
+        bed_types,
+        property_types,
+        countries,
+        account_tax_settings,
+        user_groups,
+        guest_payment_charges,
+        coupons,
+        webhook_reservations,
+        tasks,
+    )

ingestr/src/hostaway/client.py

@@ -0,0 +1,288 @@
+from typing import Callable, Iterable, Optional
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class HostawayClient:
+    BASE_URL = "https://api.hostaway.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    def _fetch_single(self, url: str, params: Optional[dict] = None) -> Iterable[dict]:
+        response = self.session.get(url, params=params, timeout=30)
+        response.raise_for_status()
+        response_data = response.json()
+
+        if isinstance(response_data, dict) and "result" in response_data:
+            items = response_data["result"]
+        elif isinstance(response_data, list):
+            items = response_data
+        else:
+            items = []
+
+        if isinstance(items, list):
+            for item in items:
+                yield item
+        elif isinstance(items, dict):
+            yield items
+
+    def _paginate(
+        self,
+        url: str,
+        params: Optional[dict] = None,
+        limit: int = 100,
+        process_item: Optional[Callable[[dict], dict]] = None,
+    ) -> Iterable[dict]:
+        offset = 0
+        if params is None:
+            params = {}
+
+        while True:
+            page_params = {**params, "limit": limit, "offset": offset}
+            response = self.session.get(url, params=page_params, timeout=30)
+            response.raise_for_status()
+            response_data = response.json()
+
+            if isinstance(response_data, dict) and "result" in response_data:
+                items = response_data["result"]
+            elif isinstance(response_data, list):
+                items = response_data
+            else:
+                items = []
+
+            if not items or (isinstance(items, list) and len(items) == 0):
+                break
+
+            if isinstance(items, list):
+                for item in items:
+                    if process_item:
+                        item = process_item(item)
+                    yield item
+            elif isinstance(items, dict):
+                if process_item:
+                    items = process_item(items)
+                yield items
+
+            if isinstance(items, list) and len(items) < limit:
+                break
+            elif isinstance(items, dict):
+                break
+
+            offset += limit
+
+    def fetch_listings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        def process_listing(listing: dict) -> dict:
+            if "latestActivityOn" in listing and listing["latestActivityOn"]:
+                try:
+                    listing["latestActivityOn"] = pendulum.parse(
+                        listing["latestActivityOn"]
+                    )
+                except Exception:
+                    listing["latestActivityOn"] = pendulum.datetime(
+                        1970, 1, 1, tz="UTC"
+                    )
+            else:
+                listing["latestActivityOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            return listing
+
+        url = f"{self.BASE_URL}/v1/listings"
+        for listing in self._paginate(url, process_item=process_listing):
+            if start_time <= listing["latestActivityOn"] <= end_time:
+                yield listing
+
+    def fetch_listing_fee_settings(
+        self,
+        listing_id,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        def process_fee(fee: dict) -> dict:
+            if "updatedOn" in fee and fee["updatedOn"]:
+                try:
+                    fee["updatedOn"] = pendulum.parse(fee["updatedOn"])
+                except Exception:
+                    fee["updatedOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            else:
+                fee["updatedOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            return fee
+
+        url = f"{self.BASE_URL}/v1/listingFeeSettings/{str(listing_id)}"
+        for fee in self._paginate(url, process_item=process_fee):
+            if start_time <= fee["updatedOn"] <= end_time:
+                yield fee
+
+    def fetch_all_listing_fee_settings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_fee_settings(
+                        listing_id, start_time, end_time
+                    )
+                except Exception:
+                    continue
+
+    def fetch_listing_agreement(
+        self,
+        listing_id,
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listingAgreement/{str(listing_id)}"
+        yield from self._paginate(url)
+
+    def fetch_listing_pricing_settings(
+        self,
+        listing_id,
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listing/pricingSettings/{str(listing_id)}"
+        yield from self._paginate(url)
+
+    def fetch_all_listing_pricing_settings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_pricing_settings(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_all_listing_agreements(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_agreement(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_cancellation_policies(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_airbnb(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/airbnb"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_marriott(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/marriott"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_vrbo(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/vrbo"
+        yield from self._fetch_single(url)
+
+    def fetch_reservations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations"
+        yield from self._paginate(url)
+
+    def fetch_finance_field(self, reservation_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/financeField/{str(reservation_id)}"
+        yield from self._fetch_single(url)
+
+    def fetch_all_finance_fields(self) -> Iterable[dict]:
+        for reservation in self.fetch_reservations():
+            reservation_id = reservation.get("id")
+            if reservation_id:
+                try:
+                    yield from self.fetch_finance_field(reservation_id)
+                except Exception:
+                    continue
+
+    def fetch_reservation_payment_methods(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations/paymentMethods"
+        yield from self._fetch_single(url)
+
+    def fetch_reservation_rental_agreement(self, reservation_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations/{str(reservation_id)}/rentalAgreement"
+        try:
+            yield from self._fetch_single(url)
+        except Exception:
+            return
+
+    def fetch_all_reservation_rental_agreements(self) -> Iterable[dict]:
+        for reservation in self.fetch_reservations():
+            reservation_id = reservation.get("id")
+            if reservation_id:
+                try:
+                    yield from self.fetch_reservation_rental_agreement(reservation_id)
+                except Exception:
+                    continue
+
+    def fetch_listing_calendar(self, listing_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listings/{str(listing_id)}/calendar"
+        yield from self._fetch_single(url)
+
+    def fetch_all_listing_calendars(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_calendar(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_conversations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/conversations"
+        yield from self._paginate(url)
+
+    def fetch_message_templates(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/messageTemplates"
+        yield from self._fetch_single(url)
+
+    def fetch_bed_types(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/bedTypes"
+        yield from self._fetch_single(url)
+
+    def fetch_property_types(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/propertyTypes"
+        yield from self._fetch_single(url)
+
+    def fetch_countries(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/countries"
+        yield from self._fetch_single(url)
+
+    def fetch_account_tax_settings(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/accountTaxSettings"
+        yield from self._fetch_single(url)
+
+    def fetch_user_groups(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/userGroups"
+        yield from self._fetch_single(url)
+
+    def fetch_guest_payment_charges(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/guestPayments/charges"
+        yield from self._paginate(url)
+
+    def fetch_coupons(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/coupons"
+        yield from self._fetch_single(url)
+
+    def fetch_webhook_reservations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/webhooks/reservations"
+        yield from self._fetch_single(url)
+
+    def fetch_tasks(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/tasks"
+        yield from self._fetch_single(url)

ingestr/src/socrata_source/__init__.py

@@ -0,0 +1,83 @@
+"""A source loading data from Socrata open data platform"""
+
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+
+from .helpers import fetch_data
+
+
+@dlt.source(name="socrata", max_table_nesting=0)
+def source(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental: Optional[Any] = None,
+    primary_key: Optional[str] = None,
+    write_disposition: Optional[str] = dlt.config.value,
+):
+    """
+    A dlt source for the Socrata open data platform.
+
+    Supports both full refresh (replace) and incremental loading (merge).
+
+    Args:
+        domain: The Socrata domain (e.g., "evergreen.data.socrata.com")
+        dataset_id: The dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits (recommended)
+        username: Username for authentication (if dataset is private)
+        password: Password for authentication (if dataset is private)
+        incremental: DLT incremental object for incremental loading
+        primary_key: Primary key field for merge operations (default: ":id")
+        write_disposition: Write disposition ("replace", "append", "merge").
+            If not provided, automatically determined based on incremental setting.
+
+    Returns:
+        A dlt source with a single "dataset" resource
+    """
+
+    @dlt.resource(
+        write_disposition=write_disposition or "replace",
+        primary_key=primary_key,  # type: ignore[call-overload]
+    )
+    def dataset(
+        incremental: Optional[dlt.sources.incremental] = incremental,  # type: ignore[type-arg]
+    ) -> Iterator[Dict[str, Any]]:
+        """
+        Yields records from a Socrata dataset.
+
+        Supports both full refresh (replace) and incremental loading (merge).
+        When incremental is provided, filters data using SoQL WHERE clause on the server side.
+
+        Yields:
+            Dict[str, Any]: Individual records from the dataset
+        """
+        fetch_kwargs: Dict[str, Any] = {
+            "domain": domain,
+            "dataset_id": dataset_id,
+            "app_token": app_token,
+            "username": username,
+            "password": password,
+        }
+
+        if incremental and incremental.cursor_path:
+            fetch_kwargs["incremental_key"] = incremental.cursor_path
+            fetch_kwargs["start_value"] = (
+                str(incremental.last_value)
+                if incremental.last_value is not None
+                else None
+            )
+            if getattr(incremental, "end_value", None) is not None:
+                ev = incremental.end_value  # type: ignore[attr-defined]
+                fetch_kwargs["end_value"] = (
+                    ev.isoformat()  # type: ignore[union-attr]
+                    if hasattr(ev, "isoformat")
+                    else str(ev)
+                )
+
+        # Fetch and yield records
+        yield from fetch_data(**fetch_kwargs)
+
+    return (dataset,)

ingestr/src/socrata_source/helpers.py

@@ -0,0 +1,85 @@
+"""Socrata API helpers"""
+
+from typing import Any, Dict, Iterator, Optional
+
+from dlt.sources.helpers import requests
+
+from .settings import DEFAULT_PAGE_SIZE, REQUEST_TIMEOUT
+
+
+def fetch_data(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental_key: Optional[str] = None,
+    start_value: Optional[str] = None,
+    end_value: Optional[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch records from Socrata dataset with pagination and optional filtering.
+
+    Uses offset-based pagination to get all records, not just first 50000.
+    Supports incremental loading via SoQL WHERE clause for server-side filtering.
+
+    Args:
+        domain: Socrata domain (e.g., "data.seattle.gov")
+        dataset_id: Dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits
+        username: Username for authentication
+        password: Password for authentication
+        start_value: Minimum value for incremental_key (inclusive)
+        end_value: Maximum value for incremental_key (exclusive)
+
+    Yields:
+        Lists of records (one list per page)
+
+    Raises:
+        requests.HTTPError: If API request fails
+    """
+    url = f"https://{domain}/resource/{dataset_id}.json"
+
+    headers = {"Accept": "application/json"}
+    if app_token:
+        headers["X-App-Token"] = app_token
+
+    auth = (username, password) if username and password else None
+
+    limit = DEFAULT_PAGE_SIZE
+    offset = 0
+
+    while True:
+        params: Dict[str, Any] = {"$limit": limit, "$offset": offset}
+
+        if incremental_key and start_value:
+            start_value_iso = str(start_value).replace(" ", "T")
+            where_conditions = [f"{incremental_key} >= '{start_value_iso}'"]
+
+            if end_value:
+                end_value_iso = str(end_value).replace(" ", "T")
+                where_conditions.append(f"{incremental_key} < '{end_value_iso}'")
+
+            params["$where"] = " AND ".join(where_conditions)
+            params["$order"] = f"{incremental_key} ASC"
+
+        response = requests.get(
+            url,
+            headers=headers,
+            auth=auth,
+            params=params,
+            timeout=REQUEST_TIMEOUT,
+        )
+        response.raise_for_status()
+
+        data = response.json()
+
+        if not data:
+            break
+
+        yield data
+
+        if len(data) < limit:
+            break
+
+        offset += limit

ingestr/src/socrata_source/settings.py

@@ -0,0 +1,8 @@
+"""Socrata API settings and constants"""
+
+# Request timeout in seconds
+REQUEST_TIMEOUT = 30
+
+# Maximum number of records to fetch per page
+# Socrata API supports up to 50000 records per request
+DEFAULT_PAGE_SIZE = 50000

ingestr/src/sources.py

@@ -4233,3 +4233,183 @@ class CouchbaseSource:
         table_instance.max_table_nesting = 1
 
         return table_instance
+
+
+class SocrataSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        """
+        Creates a DLT source for Socrata open data platform.
+
+        URI format: socrata://domain?app_token=TOKEN
+        Table: dataset_id (e.g., "6udu-fhnu")
+
+        Args:
+            uri: Socrata connection URI with domain and optional auth params
+            table: Dataset ID (e.g., "6udu-fhnu")
+            **kwargs: Additional arguments:
+                - incremental_key: Field to use for incremental loading (e.g., ":updated_at")
+                - interval_start: Start date for initial load
+                - interval_end: End date for load
+                - primary_key: Primary key field for merge operations
+
+        Returns:
+            DltResource for the Socrata dataset
+        """
+        from urllib.parse import parse_qs, urlparse
+
+        parsed = urlparse(uri)
+
+        domain = parsed.netloc
+        if not domain:
+            raise ValueError(
+                "Domain must be provided in the URI.\n"
+                "Format: socrata://domain?app_token=TOKEN\n"
+                "Example: socrata://evergreen.data.socrata.com?app_token=mytoken"
+            )
+
+        query_params = parse_qs(parsed.query)
+
+        dataset_id = table
+        if not dataset_id:
+            raise ValueError(
+                "Dataset ID must be provided as the table parameter.\n"
+                "Example: --source-table 6udu-fhnu"
+            )
+
+        app_token = query_params.get("app_token", [None])[0]
+        username = query_params.get("username", [None])[0]
+        password = query_params.get("password", [None])[0]
+
+        incremental = None
+        if kwargs.get("incremental_key"):
+            start_value = kwargs.get("interval_start")
+            end_value = kwargs.get("interval_end")
+
+            if start_value:
+                start_value = (
+                    start_value.isoformat()
+                    if hasattr(start_value, "isoformat")
+                    else str(start_value)
+                )
+
+            if end_value:
+                end_value = (
+                    end_value.isoformat()
+                    if hasattr(end_value, "isoformat")
+                    else str(end_value)
+                )
+
+            incremental = dlt_incremental(
+                kwargs.get("incremental_key", ""),
+                initial_value=start_value,
+                end_value=end_value,
+                range_end="open",
+                range_start="closed",
+            )
+
+        primary_key = kwargs.get("primary_key")
+
+        from ingestr.src.socrata_source import source
+
+        return source(
+            domain=domain,
+            dataset_id=dataset_id,
+            app_token=app_token,
+            username=username,
+            password=password,
+            incremental=incremental,
+            primary_key=primary_key,
+        ).with_resources("dataset")
+
+
+class HostawaySource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Hostaway takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        source_parts = urlparse(uri)
+        source_params = parse_qs(source_parts.query)
+        api_key = source_params.get("api_key")
+
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Hostaway")
+
+        match table:
+            case "listings":
+                resource_name = "listings"
+            case "listing_fee_settings":
+                resource_name = "listing_fee_settings"
+            case "listing_agreements":
+                resource_name = "listing_agreements"
+            case "listing_pricing_settings":
+                resource_name = "listing_pricing_settings"
+            case "cancellation_policies":
+                resource_name = "cancellation_policies"
+            case "cancellation_policies_airbnb":
+                resource_name = "cancellation_policies_airbnb"
+            case "cancellation_policies_marriott":
+                resource_name = "cancellation_policies_marriott"
+            case "cancellation_policies_vrbo":
+                resource_name = "cancellation_policies_vrbo"
+            case "reservations":
+                resource_name = "reservations"
+            case "finance_fields":
+                resource_name = "finance_fields"
+            case "reservation_payment_methods":
+                resource_name = "reservation_payment_methods"
+            case "reservation_rental_agreements":
+                resource_name = "reservation_rental_agreements"
+            case "listing_calendars":
+                resource_name = "listing_calendars"
+            case "conversations":
+                resource_name = "conversations"
+            case "message_templates":
+                resource_name = "message_templates"
+            case "bed_types":
+                resource_name = "bed_types"
+            case "property_types":
+                resource_name = "property_types"
+            case "countries":
+                resource_name = "countries"
+            case "account_tax_settings":
+                resource_name = "account_tax_settings"
+            case "user_groups":
+                resource_name = "user_groups"
+            case "guest_payment_charges":
+                resource_name = "guest_payment_charges"
+            case "coupons":
+                resource_name = "coupons"
+            case "webhook_reservations":
+                resource_name = "webhook_reservations"
+            case "tasks":
+                resource_name = "tasks"
+            case _:
+                raise ValueError(
+                    f"Resource '{table}' is not supported for Hostaway source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+                )
+
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+        else:
+            start_date = pendulum.datetime(1970, 1, 1).in_timezone("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date).in_timezone("UTC")
+
+        from ingestr.src.hostaway import hostaway_source
+
+        return hostaway_source(
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(resource_name)

{ingestr-0.14.94.dist-info → ingestr-0.14.97.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.94
+Version: 0.14.97
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.94.dist-info → ingestr-0.14.97.dist-info}/RECORD

@@ -2,17 +2,17 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=CcaXu7ayEOiHaFuTFCgGdFP5zqt7HZWvn7bvO_OyRhk,21
+ingestr/src/buildinfo.py,sha256=9YBWD_D5PtKLUdQnBqW9Onk5dyG9HVr8nLFni4TM9S8,21
 ingestr/src/destinations.py,sha256=QtjE0AGs0WkPHaI2snWPHJ8HHi4lwXUBYLJPklz8Mvk,27772
 ingestr/src/errors.py,sha256=fhJ2BxOqOsBfOxuTDKfZblvawBrPG3x_1VikIxMZBRI,874
-ingestr/src/factory.py,sha256=WBc2y5N-9HH3WOTUSe7sDEftarScDAipka7CSW-A4L4,7829
+ingestr/src/factory.py,sha256=j8vVlaE737p2MBn-b7JuR8OZP1On6uB3_kNbJkgS344,7938
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=XEimy9ph8QDW8vjNocjovxM6cdrUkIPmFGZ9eLXZWVk,148842
+ingestr/src/sources.py,sha256=KM1Y4eZtdjTGLFfVmIHXAGXWYQrYGjcBEslVZd7IX80,155470
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -86,6 +86,8 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
+ingestr/src/hostaway/__init__.py,sha256=sq7qG5J4XcyoYoHBSgAszYPByN9bMLzWjhSmvzJuTeI,8887
+ingestr/src/hostaway/client.py,sha256=omzoT4gPQ_nvMWDcm7-bm2AyFwwRDgV8D1sI0gkkydw,10452
 ingestr/src/http/__init__.py,sha256=Y9mQIE0RolHOh6dPjW41qzYXSG8BC0GPKxEtz2CJGpU,902
 ingestr/src/http/readers.py,sha256=rgBwYG5SOQ7P2uzBAFMOQIevKxV51ZW41VSiRTZ0Xvo,3863
 ingestr/src/hubspot/__init__.py,sha256=FCqjLeOjijdc9JC_NoDwtRqy3FDyY-szDi6UV7CdDN0,11548
@@ -154,6 +156,9 @@ ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM
 ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
 ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
 ingestr/src/smartsheets/__init__.py,sha256=RIEfN1T2TMFg8T0RvN4o6sqC58YusJRDrmE9Isos5P4,2375
+ingestr/src/socrata_source/__init__.py,sha256=K5DVpsVXTMfunZd5YoEsn1nipfo1zavFS59g3m2tsc8,2984
+ingestr/src/socrata_source/helpers.py,sha256=KbVojFSmMLXb0ajh8bhqfZfxDHH7rQ3nyI8p2jxVifA,2500
+ingestr/src/socrata_source/settings.py,sha256=DLfu-4HOa5nR7h9tbOySEa2ye3w_Z6TYZ9_zPqWaNQk,220
 ingestr/src/solidgate/__init__.py,sha256=Ts83j-JSnFsFuF4tDhVOfZKg7H0-bIpfn3kg1ZOR58A,8003
 ingestr/src/solidgate/helpers.py,sha256=mAsW_1hpD7ab3Y2vw8fxHi4yD3aT1geLdIYZ7ycyxBc,5690
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -186,8 +191,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
-ingestr-0.14.94.dist-info/METADATA,sha256=B9NePP8xKNVs7YZ6hZV6gDfTGBxdRwxs7aMd5xPL6As,15359
-ingestr-0.14.94.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.14.94.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.14.94.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.14.94.dist-info/RECORD,,
+ingestr-0.14.97.dist-info/METADATA,sha256=XtW1-uOtM8YkyoK59fjqCsF3ZAQ8hBPXY-hd0pljoag,15359
+ingestr-0.14.97.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.14.97.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.14.97.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.14.97.dist-info/RECORD,,