ingestr 0.14.96__tar.gz → 0.14.97__tar.gz

{ingestr-0.14.96 → ingestr-0.14.97}/.github/workflows/tests.yml

@@ -16,21 +16,21 @@ jobs:
   check-changes:
     runs-on: ubuntu-latest
     outputs:
-      docs-only: ${{ steps.filter.outputs.docs-only }}
+      code-changes: ${{ steps.filter.outputs.code-changes }}
     steps:
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
-            docs-only:
-              - '!**'
-              - '**/*.md'
-              - 'docs/**'
+            code-changes:
+              - '**'
+              - '!**/*.md'
+              - '!docs/**'
 
   tests:
     needs: check-changes
-    if: needs.check-changes.outputs.docs-only != 'true'
+    if: needs.check-changes.outputs.code-changes == 'true'
     strategy:
       matrix:
         # I tried running stuff on macOS but it was too slow and unreliable.

{ingestr-0.14.96 → ingestr-0.14.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.96
+Version: 0.14.97
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.96 → ingestr-0.14.97}/docs/.vitepress/config.mjs

@@ -150,6 +150,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "Google Ads", link: "/supported-sources/google-ads.md" },
               { text: "GitHub", link: "/supported-sources/github.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
+              { text: "Hostaway", link: "/supported-sources/hostaway.md" },
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Internet Society Pulse", link: "/supported-sources/isoc-pulse.md" },

ingestr-0.14.97/docs/supported-sources/hostaway.md

@@ -0,0 +1,104 @@
+# Hostaway
+
+[Hostaway](https://www.hostaway.com/) is a property management system (PMS) designed for vacation rental managers and hosts. It provides tools for managing listings, reservations, channels, and guest communications across multiple booking platforms.
+
+ingestr supports Hostaway as a source.
+
+## URI format
+
+The URI format for Hostaway is as follows:
+
+```plaintext
+hostaway://?api_key=<api-key-here>
+```
+
+URI parameters:
+
+- `api_key`: the API access token used for authentication with the Hostaway API
+
+The URI is used to connect to the Hostaway API for extracting data. More details on the Hostaway API can be found in the [official API documentation](https://api-docs.hostaway.com/).
+
+## Setting up a Hostaway Integration
+
+Hostaway uses OAuth 2.0 client credentials for authentication. Follow these steps to obtain an API access token:
+
+### 1. Get Your Credentials
+
+First, you need your Hostaway account credentials:
+- `client_id`: Your Hostaway account ID
+- `client_secret`: Your API client secret (available in Hostaway settings)
+
+### 2. Generate an Access Token
+
+Use the following curl command to generate an access token:
+
+```bash
+curl -X POST https://api.hostaway.com/v1/accessTokens \
+  -H 'Cache-control: no-cache' \
+  -H 'Content-type: application/x-www-form-urlencoded' \
+  -d 'grant_type=client_credentials&client_id=YOUR_ACCOUNT_ID&client_secret=YOUR_CLIENT_SECRET&scope=general'
+```
+
+The response will contain an access token (JWT) that you'll use for authentication.
+
+### 3. Use the Access Token
+
+Once you have your access token, you can use it with ingestr. Here's a sample command that will copy listings data from Hostaway into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri 'hostaway://?api_key=YOUR_ACCESS_TOKEN' \
+  --source-table 'listings' \
+  --dest-uri 'duckdb:///hostaway.duckdb' \
+  --dest-table 'main.listings' \
+  --interval-start '2020-01-01' \
+  --interval-end '2025-12-31'
+```
+
+### 4. Revoking Access Tokens
+
+To revoke an access token when it's no longer needed:
+
+```bash
+curl -X DELETE 'https://api.hostaway.com/v1/accessTokens?token=YOUR_ACCESS_TOKEN' \
+  -H 'Content-type: application/x-www-form-urlencoded'
+```
+
+## Tables
+
+Hostaway source allows ingesting the following resources into separate tables:
+
+| Table | PK | Inc Key | Inc Strategy | Description |
+| ----- | -- | ------- | ------------ | ----------- |
+| `listings` | id | latestActivityOn | merge | Property listings managed in Hostaway |
+| `listing_fee_settings` | id | updatedOn | merge | Fee settings configured for each listing |
+| `listing_pricing_settings` | - | - | replace | Pricing rules and settings for listings |
+| `listing_agreements` | - | - | replace | Rental agreements associated with listings |
+| `listing_calendars` | - | - | replace | Calendar availability data for each listing. Uses parallelization for performance |
+| `cancellation_policies` | - | - | replace | General cancellation policies |
+| `cancellation_policies_airbnb` | - | - | replace | Airbnb-specific cancellation policies |
+| `cancellation_policies_marriott` | - | - | replace | Marriott-specific cancellation policies |
+| `cancellation_policies_vrbo` | - | - | replace | VRBO-specific cancellation policies |
+| `reservations` | - | - | replace | Booking reservations across all channels |
+| `finance_fields` | - | - | replace | Financial data for each reservation. Uses parallelization for performance |
+| `reservation_payment_methods` | - | - | replace | Available payment methods for reservations |
+| `reservation_rental_agreements` | - | - | replace | Rental agreements for specific reservations. Uses parallelization for performance |
+| `conversations` | - | - | replace | Guest communication threads |
+| `message_templates` | - | - | replace | Pre-configured message templates |
+| `bed_types` | - | - | replace | Available bed type configurations |
+| `property_types` | - | - | replace | Property type classifications |
+| `countries` | - | - | replace | Supported countries and their codes |
+| `account_tax_settings` | - | - | replace | Tax configuration for the account |
+| `user_groups` | - | - | replace | User groups and permissions |
+| `guest_payment_charges` | - | - | replace | Guest payment transaction records |
+| `coupons` | - | - | replace | Discount coupons and promotional codes |
+| `webhook_reservations` | - | - | replace | Webhook configurations for reservation events |
+| `tasks` | - | - | replace | Tasks and to-dos within the system |
+
+Use these table names as the `--source-table` parameter in the `ingestr ingest` command.
+
+## Notes
+
+- The `finance_fields`, `reservation_rental_agreements`, and `listing_calendars` tables use parallelization for improved performance when fetching nested resource data
+- Some endpoints like `listings` and `reservations` support incremental loading using `--interval-start` and `--interval-end` parameters
+- Access tokens are JWTs with configurable expiration times - manage them securely and rotate them as needed

ingestr-0.14.97/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.97"

{ingestr-0.14.96 → ingestr-0.14.97}/ingestr/src/factory.py

@@ -54,6 +54,7 @@ from ingestr.src.sources import (
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HostawaySource,
     HttpSource,
     HubspotSource,
     InfluxDBSource,
@@ -178,6 +179,7 @@ class SourceDestinationFactory:
         "facebookads": FacebookAdsSource,
         "fluxx": FluxxSource,
         "slack": SlackSource,
+        "hostaway": HostawaySource,
         "hubspot": HubspotSource,
         "intercom": IntercomSource,
         "jira": JiraSource,

ingestr-0.14.97/ingestr/src/hostaway/__init__.py

@@ -0,0 +1,302 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import HostawayClient
+
+
+@dlt.source(max_table_nesting=0)
+def hostaway_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    """
+    Hostaway API source for fetching listings and fee settings data.
+
+    Args:
+        api_key: Hostaway API key for Bearer token authentication
+        start_date: Start date for incremental loading
+        end_date: End date for incremental loading (defaults to current time)
+
+    Returns:
+        Iterable[DltResource]: DLT resources for listings and/or fee settings
+    """
+
+    client = HostawayClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listings",
+        primary_key="id",
+    )
+    def listings(
+        datetime=dlt.sources.incremental(
+            "latestActivityOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listings from Hostaway API with incremental loading.
+        Uses latestActivityOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_listings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listing_fee_settings",
+        primary_key="id",
+    )
+    def listing_fee_settings(
+        datetime=dlt.sources.incremental(
+            "updatedOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listing fee settings from Hostaway API with incremental loading.
+        Uses updatedOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_all_listing_fee_settings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_agreements",
+    )
+    def listing_agreements() -> Iterable[TDataItem]:
+        """
+        Fetch listing agreements from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_agreements(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_pricing_settings",
+    )
+    def listing_pricing_settings() -> Iterable[TDataItem]:
+        """
+        Fetch listing pricing settings from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_pricing_settings(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies",
+    )
+    def cancellation_policies() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_airbnb",
+    )
+    def cancellation_policies_airbnb() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_airbnb()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_marriott",
+    )
+    def cancellation_policies_marriott() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_marriott()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_vrbo",
+    )
+    def cancellation_policies_vrbo() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_vrbo()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservations",
+        selected=False,
+    )
+    def reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_reservations()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="finance_fields",
+    )
+    def finance_fields(reservation_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_finance_field(res_id):
+            return list(client.fetch_finance_field(res_id))
+
+        reservation_id_val = reservation_item.get("id")
+        if reservation_id_val:
+            yield _get_finance_field(reservation_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservation_payment_methods",
+    )
+    def reservation_payment_methods() -> Iterable[TDataItem]:
+        yield from client.fetch_reservation_payment_methods()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="reservation_rental_agreements",
+    )
+    def reservation_rental_agreements(
+        reservation_item: TDataItem,
+    ) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_rental_agreement(res_id):
+            return list(client.fetch_reservation_rental_agreement(res_id))
+
+        reservation_id = reservation_item.get("id")
+        if reservation_id:
+            yield _get_rental_agreement(reservation_id)
+
+    @dlt.transformer(
+        data_from=listings,
+        write_disposition="replace",
+        name="listing_calendars",
+    )
+    def listing_calendars(listing_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_calendar(lst_id):
+            return list(client.fetch_listing_calendar(lst_id))
+
+        listing_id_val = listing_item.get("id")
+        if listing_id_val:
+            yield _get_calendar(listing_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="conversations",
+    )
+    def conversations() -> Iterable[TDataItem]:
+        yield from client.fetch_conversations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="message_templates",
+    )
+    def message_templates() -> Iterable[TDataItem]:
+        yield from client.fetch_message_templates()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="bed_types",
+    )
+    def bed_types() -> Iterable[TDataItem]:
+        yield from client.fetch_bed_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="property_types",
+    )
+    def property_types() -> Iterable[TDataItem]:
+        yield from client.fetch_property_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="countries",
+    )
+    def countries() -> Iterable[TDataItem]:
+        yield from client.fetch_countries()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="account_tax_settings",
+    )
+    def account_tax_settings() -> Iterable[TDataItem]:
+        yield from client.fetch_account_tax_settings()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="user_groups",
+    )
+    def user_groups() -> Iterable[TDataItem]:
+        yield from client.fetch_user_groups()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="guest_payment_charges",
+    )
+    def guest_payment_charges() -> Iterable[TDataItem]:
+        yield from client.fetch_guest_payment_charges()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="coupons",
+    )
+    def coupons() -> Iterable[TDataItem]:
+        yield from client.fetch_coupons()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="webhook_reservations",
+    )
+    def webhook_reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_webhook_reservations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="tasks",
+    )
+    def tasks() -> Iterable[TDataItem]:
+        yield from client.fetch_tasks()
+
+    return (
+        listings,
+        listing_fee_settings,
+        listing_agreements,
+        listing_pricing_settings,
+        cancellation_policies,
+        cancellation_policies_airbnb,
+        cancellation_policies_marriott,
+        cancellation_policies_vrbo,
+        reservations,
+        finance_fields,
+        reservation_payment_methods,
+        reservation_rental_agreements,
+        listing_calendars,
+        conversations,
+        message_templates,
+        bed_types,
+        property_types,
+        countries,
+        account_tax_settings,
+        user_groups,
+        guest_payment_charges,
+        coupons,
+        webhook_reservations,
+        tasks,
+    )