ingestr 0.14.94__tar.gz → 0.14.97__tar.gz

{ingestr-0.14.94 → ingestr-0.14.97}/.github/workflows/tests.yml

@@ -16,21 +16,21 @@ jobs:
   check-changes:
     runs-on: ubuntu-latest
     outputs:
-      docs-only: ${{ steps.filter.outputs.docs-only }}
+      code-changes: ${{ steps.filter.outputs.code-changes }}
     steps:
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
-            docs-only:
-              - '!**'
-              - '**/*.md'
-              - 'docs/**'
+            code-changes:
+              - '**'
+              - '!**/*.md'
+              - '!docs/**'
 
   tests:
     needs: check-changes
-    if: needs.check-changes.outputs.docs-only != 'true'
+    if: needs.check-changes.outputs.code-changes == 'true'
     strategy:
       matrix:
         # I tried running stuff on macOS but it was too slow and unreliable.

{ingestr-0.14.94 → ingestr-0.14.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.94
+Version: 0.14.97
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.94 → ingestr-0.14.97}/docs/.vitepress/config.mjs

@@ -108,6 +108,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "Postgres", link: "/supported-sources/postgres.md" },
               { text: "SAP Hana", link: "/supported-sources/sap-hana.md" },
               { text: "Snowflake", link: "/supported-sources/snowflake.md" },
+              { text: "Socrata", link: "/supported-sources/socrata.md" },
               { text: "SQLite", link: "/supported-sources/sqlite.md" },
               {
                 text: "Experimental",
@@ -149,6 +150,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "Google Ads", link: "/supported-sources/google-ads.md" },
               { text: "GitHub", link: "/supported-sources/github.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
+              { text: "Hostaway", link: "/supported-sources/hostaway.md" },
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Internet Society Pulse", link: "/supported-sources/isoc-pulse.md" },

ingestr-0.14.97/docs/supported-sources/hostaway.md

@@ -0,0 +1,104 @@
+# Hostaway
+
+[Hostaway](https://www.hostaway.com/) is a property management system (PMS) designed for vacation rental managers and hosts. It provides tools for managing listings, reservations, channels, and guest communications across multiple booking platforms.
+
+ingestr supports Hostaway as a source.
+
+## URI format
+
+The URI format for Hostaway is as follows:
+
+```plaintext
+hostaway://?api_key=<api-key-here>
+```
+
+URI parameters:
+
+- `api_key`: the API access token used for authentication with the Hostaway API
+
+The URI is used to connect to the Hostaway API for extracting data. More details on the Hostaway API can be found in the [official API documentation](https://api-docs.hostaway.com/).
+
+## Setting up a Hostaway Integration
+
+Hostaway uses OAuth 2.0 client credentials for authentication. Follow these steps to obtain an API access token:
+
+### 1. Get Your Credentials
+
+First, you need your Hostaway account credentials:
+- `client_id`: Your Hostaway account ID
+- `client_secret`: Your API client secret (available in Hostaway settings)
+
+### 2. Generate an Access Token
+
+Use the following curl command to generate an access token:
+
+```bash
+curl -X POST https://api.hostaway.com/v1/accessTokens \
+  -H 'Cache-control: no-cache' \
+  -H 'Content-type: application/x-www-form-urlencoded' \
+  -d 'grant_type=client_credentials&client_id=YOUR_ACCOUNT_ID&client_secret=YOUR_CLIENT_SECRET&scope=general'
+```
+
+The response will contain an access token (JWT) that you'll use for authentication.
+
+### 3. Use the Access Token
+
+Once you have your access token, you can use it with ingestr. Here's a sample command that will copy listings data from Hostaway into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri 'hostaway://?api_key=YOUR_ACCESS_TOKEN' \
+  --source-table 'listings' \
+  --dest-uri 'duckdb:///hostaway.duckdb' \
+  --dest-table 'main.listings' \
+  --interval-start '2020-01-01' \
+  --interval-end '2025-12-31'
+```
+
+### 4. Revoking Access Tokens
+
+To revoke an access token when it's no longer needed:
+
+```bash
+curl -X DELETE 'https://api.hostaway.com/v1/accessTokens?token=YOUR_ACCESS_TOKEN' \
+  -H 'Content-type: application/x-www-form-urlencoded'
+```
+
+## Tables
+
+Hostaway source allows ingesting the following resources into separate tables:
+
+| Table | PK | Inc Key | Inc Strategy | Description |
+| ----- | -- | ------- | ------------ | ----------- |
+| `listings` | id | latestActivityOn | merge | Property listings managed in Hostaway |
+| `listing_fee_settings` | id | updatedOn | merge | Fee settings configured for each listing |
+| `listing_pricing_settings` | - | - | replace | Pricing rules and settings for listings |
+| `listing_agreements` | - | - | replace | Rental agreements associated with listings |
+| `listing_calendars` | - | - | replace | Calendar availability data for each listing. Uses parallelization for performance |
+| `cancellation_policies` | - | - | replace | General cancellation policies |
+| `cancellation_policies_airbnb` | - | - | replace | Airbnb-specific cancellation policies |
+| `cancellation_policies_marriott` | - | - | replace | Marriott-specific cancellation policies |
+| `cancellation_policies_vrbo` | - | - | replace | VRBO-specific cancellation policies |
+| `reservations` | - | - | replace | Booking reservations across all channels |
+| `finance_fields` | - | - | replace | Financial data for each reservation. Uses parallelization for performance |
+| `reservation_payment_methods` | - | - | replace | Available payment methods for reservations |
+| `reservation_rental_agreements` | - | - | replace | Rental agreements for specific reservations. Uses parallelization for performance |
+| `conversations` | - | - | replace | Guest communication threads |
+| `message_templates` | - | - | replace | Pre-configured message templates |
+| `bed_types` | - | - | replace | Available bed type configurations |
+| `property_types` | - | - | replace | Property type classifications |
+| `countries` | - | - | replace | Supported countries and their codes |
+| `account_tax_settings` | - | - | replace | Tax configuration for the account |
+| `user_groups` | - | - | replace | User groups and permissions |
+| `guest_payment_charges` | - | - | replace | Guest payment transaction records |
+| `coupons` | - | - | replace | Discount coupons and promotional codes |
+| `webhook_reservations` | - | - | replace | Webhook configurations for reservation events |
+| `tasks` | - | - | replace | Tasks and to-dos within the system |
+
+Use these table names as the `--source-table` parameter in the `ingestr ingest` command.
+
+## Notes
+
+- The `finance_fields`, `reservation_rental_agreements`, and `listing_calendars` tables use parallelization for improved performance when fetching nested resource data
+- Some endpoints like `listings` and `reservations` support incremental loading using `--interval-start` and `--interval-end` parameters
+- Access tokens are JWTs with configurable expiration times - manage them securely and rotate them as needed

ingestr-0.14.97/docs/supported-sources/socrata.md

@@ -0,0 +1,127 @@
+# Socrata
+
+[Socrata](https://dev.socrata.com/) is an open data platform used by governments and organizations to publish and share public datasets. The platform powers thousands of open data portals worldwide, including data.gov and many city, state, and federal government sites.
+
+`ingestr` allows ingesting data from any Socrata-powered open data portal using the [Socrata Open Data API (SODA)](https://dev.socrata.com/docs/endpoints.html).
+
+## URI Format
+
+The URI format for Socrata is as follows:
+```
+socrata://?domain=<domain>&dataset_id=<dataset_id>&app_token=<app_token>
+```
+
+URI Parameters:
+* `domain`: The Socrata domain (e.g., `data.seattle.gov`, `data.cityofnewyork.us`)
+* `dataset_id`: The dataset identifier (4x4 format like `6udu-fhnu`)
+* `app_token`: Socrata app token for API access (required)
+* `username` (optional): Username for authentication (required for private datasets)
+* `password` (optional): Password for authentication (required for private datasets)
+
+## Setting up Socrata Integration
+
+### Finding Domain and Dataset ID
+
+1. Navigate to any Socrata-powered open data portal
+2. Find the dataset you want to ingest
+3. The domain is the base URL (e.g., `data.seattle.gov`)
+4. The dataset ID is in the URL or API endpoint (e.g., `6udu-fhnu`)
+
+Example: For `https://data.seattle.gov/City-Business/City-of-Seattle-Wage-Data/2khk-5ukd`, the domain is `data.seattle.gov` and the dataset ID is `2khk-5ukd`.
+
+### Generate an App Token
+
+You need to obtain an app token to access the Socrata API.
+
+1. Sign up for a free account at the Socrata portal you're using
+2. Navigate to the developer settings or API documentation
+3. Generate an app token
+4. Use this token in the `app_token` parameter
+
+### Example: Loading a Public Dataset
+
+For this example, we'll load the Seattle City wage data:
+* Domain: `data.seattle.gov`
+* Dataset ID: `2khk-5ukd`
+* App token: `your_app_token_here`
+
+We will run `ingestr` to save this data to a [duckdb](https://duckdb.org/) database called `socrata.db` under the table name `public.wage_data`.
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=data.seattle.gov&dataset_id=2khk-5ukd&app_token=your_app_token_here" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.wage_data"
+```
+
+### Example: Incremental Loading
+
+Socrata supports incremental loading using the `:updated_at` system field or any other date/timestamp field in your dataset that gets updated. You must specify the incremental key using the `--incremental-key` flag.
+
+First, run an initial load for a specific time range using `:updated_at` as the incremental key:
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=data.seattle.gov&dataset_id=2khk-5ukd&app_token=your_app_token_here" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.wage_data" \
+    --incremental-key ":updated_at" \
+    --interval-start "2024-01-01" \
+    --interval-end "2024-06-30" \
+    --incremental-strategy "merge"
+```
+
+
+
+Now, we will run `ingestr` again without specifying dates to load only new or updated records:
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=data.seattle.gov&dataset_id=2khk-5ukd&app_token=your_app_token_here" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.wage_data" \
+    --incremental-key ":updated_at"
+```
+
+This will automatically fetch only records that were created or updated after the last ingestion.
+
+### Example: Loading Private Datasets
+
+For private datasets that require authentication:
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=your.domain.com&dataset_id=xxxx-xxxx&app_token=your_token&username=your_username&password=your_password" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.private_data"
+```
+
+## Tables
+
+Socrata source provides a single table called `dataset` that represents the Socrata dataset.
+
+| Name | Merge Key | Inc Key | Inc Strategy | Details |
+| --- | --- | --- | --- | --- |
+| `dataset` | `:id` | user-defined | replace/merge | Loads all records from the specified Socrata dataset. Uses `replace` by default, or `merge` when `--incremental-key` is specified |
+
+
+
+
+
+## Troubleshooting
+
+### Rate Limit Errors
+If you hit rate limits, register for an app token or reduce the frequency of requests.
+
+### Authentication Errors
+For private datasets, ensure username and password are correct and that you have access to the dataset.
+
+### Invalid Dataset ID
+Verify the dataset ID is in the correct 4x4 format (e.g., `xxxx-xxxx`) and exists on the specified domain.
+
+### Timeout Errors
+For very large datasets, the initial load may take time. Consider using date ranges to break up large loads.

ingestr-0.14.97/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.97"

{ingestr-0.14.94 → ingestr-0.14.97}/ingestr/src/factory.py

@@ -54,6 +54,7 @@ from ingestr.src.sources import (
     GoogleAnalyticsSource,
     GoogleSheetsSource,
     GorgiasSource,
+    HostawaySource,
     HttpSource,
     HubspotSource,
     InfluxDBSource,
@@ -84,6 +85,7 @@ from ingestr.src.sources import (
     ShopifySource,
     SlackSource,
     SmartsheetSource,
+    SocrataSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
@@ -177,6 +179,7 @@ class SourceDestinationFactory:
         "facebookads": FacebookAdsSource,
         "fluxx": FluxxSource,
         "slack": SlackSource,
+        "hostaway": HostawaySource,
         "hubspot": HubspotSource,
         "intercom": IntercomSource,
         "jira": JiraSource,
@@ -218,6 +221,7 @@ class SourceDestinationFactory:
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
         "revenuecat": RevenueCatSource,
+        "socrata": SocrataSource,
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,

ingestr-0.14.97/ingestr/src/hostaway/__init__.py

@@ -0,0 +1,302 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import HostawayClient
+
+
+@dlt.source(max_table_nesting=0)
+def hostaway_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    """
+    Hostaway API source for fetching listings and fee settings data.
+
+    Args:
+        api_key: Hostaway API key for Bearer token authentication
+        start_date: Start date for incremental loading
+        end_date: End date for incremental loading (defaults to current time)
+
+    Returns:
+        Iterable[DltResource]: DLT resources for listings and/or fee settings
+    """
+
+    client = HostawayClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listings",
+        primary_key="id",
+    )
+    def listings(
+        datetime=dlt.sources.incremental(
+            "latestActivityOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listings from Hostaway API with incremental loading.
+        Uses latestActivityOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_listings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="merge",
+        name="listing_fee_settings",
+        primary_key="id",
+    )
+    def listing_fee_settings(
+        datetime=dlt.sources.incremental(
+            "updatedOn",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        """
+        Fetch listing fee settings from Hostaway API with incremental loading.
+        Uses updatedOn field as the incremental cursor.
+        """
+        start_dt = datetime.last_value
+        end_dt = (
+            datetime.end_value
+            if datetime.end_value is not None
+            else pendulum.now(tz="UTC")
+        )
+
+        yield from client.fetch_all_listing_fee_settings(start_dt, end_dt)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_agreements",
+    )
+    def listing_agreements() -> Iterable[TDataItem]:
+        """
+        Fetch listing agreements from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_agreements(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="listing_pricing_settings",
+    )
+    def listing_pricing_settings() -> Iterable[TDataItem]:
+        """
+        Fetch listing pricing settings from Hostaway API.
+
+        Note: Uses replace mode, so no incremental loading.
+        """
+        very_old_date = pendulum.datetime(1970, 1, 1, tz="UTC")
+        now = pendulum.now(tz="UTC")
+        yield from client.fetch_all_listing_pricing_settings(very_old_date, now)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies",
+    )
+    def cancellation_policies() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_airbnb",
+    )
+    def cancellation_policies_airbnb() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_airbnb()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_marriott",
+    )
+    def cancellation_policies_marriott() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_marriott()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="cancellation_policies_vrbo",
+    )
+    def cancellation_policies_vrbo() -> Iterable[TDataItem]:
+        yield from client.fetch_cancellation_policies_vrbo()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservations",
+        selected=False,
+    )
+    def reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_reservations()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="finance_fields",
+    )
+    def finance_fields(reservation_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_finance_field(res_id):
+            return list(client.fetch_finance_field(res_id))
+
+        reservation_id_val = reservation_item.get("id")
+        if reservation_id_val:
+            yield _get_finance_field(reservation_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="reservation_payment_methods",
+    )
+    def reservation_payment_methods() -> Iterable[TDataItem]:
+        yield from client.fetch_reservation_payment_methods()
+
+    @dlt.transformer(
+        data_from=reservations,
+        write_disposition="replace",
+        name="reservation_rental_agreements",
+    )
+    def reservation_rental_agreements(
+        reservation_item: TDataItem,
+    ) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_rental_agreement(res_id):
+            return list(client.fetch_reservation_rental_agreement(res_id))
+
+        reservation_id = reservation_item.get("id")
+        if reservation_id:
+            yield _get_rental_agreement(reservation_id)
+
+    @dlt.transformer(
+        data_from=listings,
+        write_disposition="replace",
+        name="listing_calendars",
+    )
+    def listing_calendars(listing_item: TDataItem) -> Iterable[TDataItem]:
+        @dlt.defer
+        def _get_calendar(lst_id):
+            return list(client.fetch_listing_calendar(lst_id))
+
+        listing_id_val = listing_item.get("id")
+        if listing_id_val:
+            yield _get_calendar(listing_id_val)
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="conversations",
+    )
+    def conversations() -> Iterable[TDataItem]:
+        yield from client.fetch_conversations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="message_templates",
+    )
+    def message_templates() -> Iterable[TDataItem]:
+        yield from client.fetch_message_templates()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="bed_types",
+    )
+    def bed_types() -> Iterable[TDataItem]:
+        yield from client.fetch_bed_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="property_types",
+    )
+    def property_types() -> Iterable[TDataItem]:
+        yield from client.fetch_property_types()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="countries",
+    )
+    def countries() -> Iterable[TDataItem]:
+        yield from client.fetch_countries()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="account_tax_settings",
+    )
+    def account_tax_settings() -> Iterable[TDataItem]:
+        yield from client.fetch_account_tax_settings()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="user_groups",
+    )
+    def user_groups() -> Iterable[TDataItem]:
+        yield from client.fetch_user_groups()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="guest_payment_charges",
+    )
+    def guest_payment_charges() -> Iterable[TDataItem]:
+        yield from client.fetch_guest_payment_charges()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="coupons",
+    )
+    def coupons() -> Iterable[TDataItem]:
+        yield from client.fetch_coupons()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="webhook_reservations",
+    )
+    def webhook_reservations() -> Iterable[TDataItem]:
+        yield from client.fetch_webhook_reservations()
+
+    @dlt.resource(
+        write_disposition="replace",
+        name="tasks",
+    )
+    def tasks() -> Iterable[TDataItem]:
+        yield from client.fetch_tasks()
+
+    return (
+        listings,
+        listing_fee_settings,
+        listing_agreements,
+        listing_pricing_settings,
+        cancellation_policies,
+        cancellation_policies_airbnb,
+        cancellation_policies_marriott,
+        cancellation_policies_vrbo,
+        reservations,
+        finance_fields,
+        reservation_payment_methods,
+        reservation_rental_agreements,
+        listing_calendars,
+        conversations,
+        message_templates,
+        bed_types,
+        property_types,
+        countries,
+        account_tax_settings,
+        user_groups,
+        guest_payment_charges,
+        coupons,
+        webhook_reservations,
+        tasks,
+    )