ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/hostaway/client.py

@@ -0,0 +1,288 @@
+from typing import Callable, Iterable, Optional
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class HostawayClient:
+    BASE_URL = "https://api.hostaway.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    def _fetch_single(self, url: str, params: Optional[dict] = None) -> Iterable[dict]:
+        response = self.session.get(url, params=params, timeout=30)
+        response.raise_for_status()
+        response_data = response.json()
+
+        if isinstance(response_data, dict) and "result" in response_data:
+            items = response_data["result"]
+        elif isinstance(response_data, list):
+            items = response_data
+        else:
+            items = []
+
+        if isinstance(items, list):
+            for item in items:
+                yield item
+        elif isinstance(items, dict):
+            yield items
+
+    def _paginate(
+        self,
+        url: str,
+        params: Optional[dict] = None,
+        limit: int = 100,
+        process_item: Optional[Callable[[dict], dict]] = None,
+    ) -> Iterable[dict]:
+        offset = 0
+        if params is None:
+            params = {}
+
+        while True:
+            page_params = {**params, "limit": limit, "offset": offset}
+            response = self.session.get(url, params=page_params, timeout=30)
+            response.raise_for_status()
+            response_data = response.json()
+
+            if isinstance(response_data, dict) and "result" in response_data:
+                items = response_data["result"]
+            elif isinstance(response_data, list):
+                items = response_data
+            else:
+                items = []
+
+            if not items or (isinstance(items, list) and len(items) == 0):
+                break
+
+            if isinstance(items, list):
+                for item in items:
+                    if process_item:
+                        item = process_item(item)
+                    yield item
+            elif isinstance(items, dict):
+                if process_item:
+                    items = process_item(items)
+                yield items
+
+            if isinstance(items, list) and len(items) < limit:
+                break
+            elif isinstance(items, dict):
+                break
+
+            offset += limit
+
+    def fetch_listings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        def process_listing(listing: dict) -> dict:
+            if "latestActivityOn" in listing and listing["latestActivityOn"]:
+                try:
+                    listing["latestActivityOn"] = pendulum.parse(
+                        listing["latestActivityOn"]
+                    )
+                except Exception:
+                    listing["latestActivityOn"] = pendulum.datetime(
+                        1970, 1, 1, tz="UTC"
+                    )
+            else:
+                listing["latestActivityOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            return listing
+
+        url = f"{self.BASE_URL}/v1/listings"
+        for listing in self._paginate(url, process_item=process_listing):
+            if start_time <= listing["latestActivityOn"] <= end_time:
+                yield listing
+
+    def fetch_listing_fee_settings(
+        self,
+        listing_id,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        def process_fee(fee: dict) -> dict:
+            if "updatedOn" in fee and fee["updatedOn"]:
+                try:
+                    fee["updatedOn"] = pendulum.parse(fee["updatedOn"])
+                except Exception:
+                    fee["updatedOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            else:
+                fee["updatedOn"] = pendulum.datetime(1970, 1, 1, tz="UTC")
+            return fee
+
+        url = f"{self.BASE_URL}/v1/listingFeeSettings/{str(listing_id)}"
+        for fee in self._paginate(url, process_item=process_fee):
+            if start_time <= fee["updatedOn"] <= end_time:
+                yield fee
+
+    def fetch_all_listing_fee_settings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_fee_settings(
+                        listing_id, start_time, end_time
+                    )
+                except Exception:
+                    continue
+
+    def fetch_listing_agreement(
+        self,
+        listing_id,
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listingAgreement/{str(listing_id)}"
+        yield from self._paginate(url)
+
+    def fetch_listing_pricing_settings(
+        self,
+        listing_id,
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listing/pricingSettings/{str(listing_id)}"
+        yield from self._paginate(url)
+
+    def fetch_all_listing_pricing_settings(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_pricing_settings(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_all_listing_agreements(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_agreement(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_cancellation_policies(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_airbnb(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/airbnb"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_marriott(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/marriott"
+        yield from self._fetch_single(url)
+
+    def fetch_cancellation_policies_vrbo(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/cancellationPolicies/vrbo"
+        yield from self._fetch_single(url)
+
+    def fetch_reservations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations"
+        yield from self._paginate(url)
+
+    def fetch_finance_field(self, reservation_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/financeField/{str(reservation_id)}"
+        yield from self._fetch_single(url)
+
+    def fetch_all_finance_fields(self) -> Iterable[dict]:
+        for reservation in self.fetch_reservations():
+            reservation_id = reservation.get("id")
+            if reservation_id:
+                try:
+                    yield from self.fetch_finance_field(reservation_id)
+                except Exception:
+                    continue
+
+    def fetch_reservation_payment_methods(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations/paymentMethods"
+        yield from self._fetch_single(url)
+
+    def fetch_reservation_rental_agreement(self, reservation_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/reservations/{str(reservation_id)}/rentalAgreement"
+        try:
+            yield from self._fetch_single(url)
+        except Exception:
+            return
+
+    def fetch_all_reservation_rental_agreements(self) -> Iterable[dict]:
+        for reservation in self.fetch_reservations():
+            reservation_id = reservation.get("id")
+            if reservation_id:
+                try:
+                    yield from self.fetch_reservation_rental_agreement(reservation_id)
+                except Exception:
+                    continue
+
+    def fetch_listing_calendar(self, listing_id) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/listings/{str(listing_id)}/calendar"
+        yield from self._fetch_single(url)
+
+    def fetch_all_listing_calendars(
+        self,
+        start_time: pendulum.DateTime,
+        end_time: pendulum.DateTime,
+    ) -> Iterable[dict]:
+        for listing in self.fetch_listings(start_time, end_time):
+            listing_id = listing.get("id")
+            if listing_id:
+                try:
+                    yield from self.fetch_listing_calendar(listing_id)
+                except Exception:
+                    continue
+
+    def fetch_conversations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/conversations"
+        yield from self._paginate(url)
+
+    def fetch_message_templates(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/messageTemplates"
+        yield from self._fetch_single(url)
+
+    def fetch_bed_types(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/bedTypes"
+        yield from self._fetch_single(url)
+
+    def fetch_property_types(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/propertyTypes"
+        yield from self._fetch_single(url)
+
+    def fetch_countries(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/countries"
+        yield from self._fetch_single(url)
+
+    def fetch_account_tax_settings(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/accountTaxSettings"
+        yield from self._fetch_single(url)
+
+    def fetch_user_groups(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/userGroups"
+        yield from self._fetch_single(url)
+
+    def fetch_guest_payment_charges(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/guestPayments/charges"
+        yield from self._paginate(url)
+
+    def fetch_coupons(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/coupons"
+        yield from self._fetch_single(url)
+
+    def fetch_webhook_reservations(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/webhooks/reservations"
+        yield from self._fetch_single(url)
+
+    def fetch_tasks(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v1/tasks"
+        yield from self._fetch_single(url)

ingestr/src/http/__init__.py

@@ -0,0 +1,35 @@
+"""HTTP source for reading CSV, JSON, and Parquet files from public URLs"""
+
+from typing import Any, Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .readers import HttpReader
+
+
+@dlt.source
+def http_source(
+    url: str,
+    file_format: Optional[str] = None,
+    **kwargs: Any,
+) -> DltResource:
+    """Source for reading files from HTTP URLs.
+
+    Supports CSV, JSON, and Parquet file formats.
+
+    Args:
+        url (str): The HTTP(S) URL to the file
+        file_format (str, optional): File format ('csv', 'json', 'parquet').
+            If not provided, will be inferred from URL extension.
+        **kwargs: Additional arguments passed to the reader functions
+
+    Returns:
+        DltResource: A dlt resource that yields the file data
+    """
+    reader = HttpReader(url, file_format)
+
+    return dlt.resource(
+        reader.read_file(**kwargs),
+        name="http_data",
+    )

ingestr/src/http/readers.py

@@ -0,0 +1,114 @@
+"""Readers for HTTP file sources"""
+
+import io
+from typing import Any, Iterator, Optional
+from urllib.parse import urlparse
+
+import requests
+from dlt.sources import TDataItems
+
+
+class HttpReader:
+    """Reader for HTTP-based file sources"""
+
+    def __init__(self, url: str, file_format: Optional[str] = None):
+        self.url = url
+        self.file_format = file_format or self._infer_format(url)
+
+        if self.file_format not in ["csv", "json", "parquet"]:
+            raise ValueError(
+                f"Unsupported file format: {self.file_format}. "
+                "Supported formats: csv, json, parquet"
+            )
+
+    def _infer_format(self, url: str) -> str:
+        """Infer file format from URL extension"""
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+
+        if path.endswith(".csv"):
+            return "csv"
+        elif path.endswith(".json") or path.endswith(".jsonl"):
+            return "json"
+        elif path.endswith(".parquet"):
+            return "parquet"
+        else:
+            raise ValueError(
+                f"Cannot infer file format from URL: {url}. "
+                "Please specify file_format parameter."
+            )
+
+    def _download_file(self) -> bytes:
+        """Download file from URL"""
+        response = requests.get(self.url, stream=True, timeout=30)
+        response.raise_for_status()
+        return response.content
+
+    def read_file(self, **kwargs: Any) -> Iterator[TDataItems]:
+        """Read file and yield data in chunks"""
+        content = self._download_file()
+
+        if self.file_format == "csv":
+            yield from self._read_csv(content, **kwargs)
+        elif self.file_format == "json":
+            yield from self._read_json(content, **kwargs)
+        elif self.file_format == "parquet":
+            yield from self._read_parquet(content, **kwargs)
+
+    def _read_csv(
+        self, content: bytes, chunksize: int = 10000, **pandas_kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read CSV file with Pandas chunk by chunk"""
+        import pandas as pd  # type: ignore
+
+        kwargs = {**{"header": "infer", "chunksize": chunksize}, **pandas_kwargs}
+
+        file_obj = io.BytesIO(content)
+        for df in pd.read_csv(file_obj, **kwargs):
+            yield df.to_dict(orient="records")
+
+    def _read_json(
+        self, content: bytes, chunksize: int = 1000, **kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read JSON or JSONL file"""
+        from dlt.common import json
+
+        file_obj = io.BytesIO(content)
+        text = file_obj.read().decode("utf-8")
+
+        # Try to detect if it's JSONL format (one JSON object per line)
+        lines = text.strip().split("\n")
+
+        if len(lines) > 1:
+            # Likely JSONL format
+            lines_chunk = []
+            for line in lines:
+                if line.strip():
+                    lines_chunk.append(json.loads(line))
+                    if len(lines_chunk) >= chunksize:
+                        yield lines_chunk
+                        lines_chunk = []
+            if lines_chunk:
+                yield lines_chunk
+        else:
+            # Single JSON object or array
+            data = json.loads(text)
+            if isinstance(data, list):
+                # Chunk the list
+                for i in range(0, len(data), chunksize):
+                    yield data[i : i + chunksize]
+            else:
+                # Single object
+                yield [data]
+
+    def _read_parquet(
+        self, content: bytes, chunksize: int = 10000, **kwargs: Any
+    ) -> Iterator[TDataItems]:
+        """Read Parquet file"""
+        from pyarrow import parquet as pq  # type: ignore
+
+        file_obj = io.BytesIO(content)
+        parquet_file = pq.ParquetFile(file_obj)
+
+        for batch in parquet_file.iter_batches(batch_size=chunksize):
+            yield batch.to_pylist()

ingestr/src/hubspot/__init__.py

@@ -93,7 +93,6 @@ def hubspot(
     def companies(
         api_key: str = api_key,
         include_history: bool = include_history,
-        props: Sequence[str] = DEFAULT_COMPANY_PROPS,
         include_custom_props: bool = include_custom_props,
     ) -> Iterator[TDataItems]:
         """Hubspot companies resource"""
@@ -101,7 +100,7 @@ def hubspot(
             "company",
             api_key,
             include_history=include_history,
-            props=props,
+            props=DEFAULT_COMPANY_PROPS,
             include_custom_props=include_custom_props,
         )
 
@@ -109,7 +108,6 @@ def hubspot(
     def contacts(
         api_key: str = api_key,
         include_history: bool = include_history,
-        props: Sequence[str] = DEFAULT_CONTACT_PROPS,
         include_custom_props: bool = include_custom_props,
     ) -> Iterator[TDataItems]:
         """Hubspot contacts resource"""
@@ -117,7 +115,7 @@ def hubspot(
             "contact",
             api_key,
             include_history,
-            props,
+            DEFAULT_CONTACT_PROPS,
             include_custom_props,
         )
 
@@ -125,7 +123,6 @@ def hubspot(
     def deals(
         api_key: str = api_key,
         include_history: bool = include_history,
-        props: Sequence[str] = DEFAULT_DEAL_PROPS,
         include_custom_props: bool = include_custom_props,
     ) -> Iterator[TDataItems]:
         """Hubspot deals resource"""
@@ -133,7 +130,7 @@ def hubspot(
             "deal",
             api_key,
             include_history,
-            props,
+            DEFAULT_DEAL_PROPS,
             include_custom_props,
         )
 
@@ -141,7 +138,6 @@ def hubspot(
     def tickets(
         api_key: str = api_key,
         include_history: bool = include_history,
-        props: Sequence[str] = DEFAULT_TICKET_PROPS,
         include_custom_props: bool = include_custom_props,
     ) -> Iterator[TDataItems]:
         """Hubspot tickets resource"""
@@ -149,7 +145,7 @@ def hubspot(
             "ticket",
             api_key,
             include_history,
-            props,
+            DEFAULT_TICKET_PROPS,
             include_custom_props,
         )
 
@@ -157,7 +153,6 @@ def hubspot(
     def products(
         api_key: str = api_key,
         include_history: bool = include_history,
-        props: Sequence[str] = DEFAULT_PRODUCT_PROPS,
         include_custom_props: bool = include_custom_props,
     ) -> Iterator[TDataItems]:
         """Hubspot products resource"""
@@ -165,7 +160,7 @@ def hubspot(
             "product",
             api_key,
             include_history,
-            props,
+            DEFAULT_PRODUCT_PROPS,
             include_custom_props,
         )
 
@@ -180,7 +175,6 @@ def hubspot(
     def quotes(
         api_key: str = api_key,
         include_history: bool = include_history,
-        props: Sequence[str] = DEFAULT_QUOTE_PROPS,
         include_custom_props: bool = include_custom_props,
     ) -> Iterator[TDataItems]:
         """Hubspot quotes resource"""
@@ -188,7 +182,7 @@ def hubspot(
             "quote",
             api_key,
             include_history,
-            props,
+            DEFAULT_QUOTE_PROPS,
             include_custom_props,
         )
 

ingestr/src/influxdb/__init__.py

@@ -7,6 +7,7 @@ from dlt.sources import DltResource
 
 from .client import InfluxClient
 
+
 @dlt.source(max_table_nesting=0)
 def influxdb_source(
     measurement: str,

ingestr/src/intercom/__init__.py

@@ -0,0 +1,142 @@
+"""
+Intercom source implementation for data ingestion.
+
+This module provides DLT sources for retrieving data from Intercom API endpoints
+including contacts, companies, conversations, tickets, and more.
+"""
+
+from typing import Optional, Sequence
+
+import dlt
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+from dlt.sources import DltResource, DltSource
+
+from .helpers import (
+    IntercomAPIClient,
+    IntercomCredentialsAccessToken,
+    TIntercomCredentials,
+    convert_datetime_to_timestamp,
+    create_resource_from_config,
+    transform_company,
+    transform_contact,
+    transform_conversation,
+)
+from .helpers import (
+    IntercomCredentialsOAuth as IntercomCredentialsOAuth,
+)
+from .settings import (
+    DEFAULT_START_DATE,
+    RESOURCE_CONFIGS,
+)
+
+
+@dlt.source(name="intercom", max_table_nesting=0)
+def intercom_source(
+    credentials: TIntercomCredentials = dlt.secrets.value,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Sequence[DltResource]:
+    """
+    A DLT source that retrieves data from Intercom API.
+
+    This source provides access to various Intercom resources including contacts,
+    companies, conversations, tickets, and more. It supports incremental loading
+    for resources that track updated timestamps.
+
+    Args:
+        credentials: Intercom API credentials (AccessToken or OAuth).
+            Defaults to dlt.secrets.value.
+        start_date: The start date for incremental loading.
+            Defaults to January 1, 2020.
+        end_date: Optional end date for incremental loading.
+            If not provided, loads all data from start_date to present.
+
+    Returns:
+        Sequence of DLT resources for different Intercom endpoints.
+
+    Example:
+        >>> source = intercom_source(
+        ...     credentials=IntercomCredentialsAccessToken(
+        ...         access_token="your_token",
+        ...         region="us"
+        ...     ),
+        ...     start_date=datetime(2024, 1, 1)
+        ... )
+    """
+    # Initialize API client
+    api_client = IntercomAPIClient(credentials)
+
+    # Convert dates to pendulum and then to unix timestamps for Intercom API
+    start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+    # Convert to unix timestamps for API compatibility
+    # Use default start date if none provided
+    if not start_date_obj:
+        from .settings import DEFAULT_START_DATE
+
+        start_date_obj = ensure_pendulum_datetime(DEFAULT_START_DATE)
+
+    start_timestamp = convert_datetime_to_timestamp(start_date_obj)
+    end_timestamp = (
+        convert_datetime_to_timestamp(end_date_obj) if end_date_obj else None
+    )
+
+    # Transform function mapping
+    transform_functions = {
+        "transform_contact": transform_contact,
+        "transform_company": transform_company,
+        "transform_conversation": transform_conversation,
+    }
+
+    # Generate all resources from configuration
+    resources = []
+    for resource_name, config in RESOURCE_CONFIGS.items():
+        resource_func = create_resource_from_config(
+            resource_name,
+            config,
+            api_client,
+            start_timestamp,
+            end_timestamp,
+            transform_functions,
+        )
+
+        # Call the resource function to get the actual resource
+        resources.append(resource_func())
+
+    return resources
+
+
+def intercom(
+    api_key: str,
+    region: str = "us",
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> DltSource:
+    """
+    Convenience function to create Intercom source with access token.
+
+    Args:
+        api_key: Intercom API access token.
+        region: Data region (us, eu, or au). Defaults to "us".
+        start_date: Start date for incremental loading.
+        end_date: Optional end date for incremental loading.
+
+    Returns:
+        Sequence of DLT resources.
+
+    Example:
+        >>> source = intercom(
+        ...     api_key="your_access_token",
+        ...     region="us",
+        ...     start_date=datetime(2024, 1, 1)
+        ... )
+    """
+    credentials = IntercomCredentialsAccessToken(access_token=api_key, region=region)
+
+    return intercom_source(
+        credentials=credentials,
+        start_date=start_date,
+        end_date=end_date,
+    )