ingestr 0.14.93__tar.gz → 0.14.96__tar.gz

{ingestr-0.14.93 → ingestr-0.14.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.93
+Version: 0.14.96
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -39,6 +39,7 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: couchbase==4.3.6
 Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1

{ingestr-0.14.93 → ingestr-0.14.96}/docs/.vitepress/config.mjs

@@ -82,6 +82,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
+              { text: "Couchbase", link: "/supported-sources/couchbase.md" },
               { text: "CrateDB", link: "/supported-sources/cratedb.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },
@@ -107,6 +108,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "Postgres", link: "/supported-sources/postgres.md" },
               { text: "SAP Hana", link: "/supported-sources/sap-hana.md" },
               { text: "Snowflake", link: "/supported-sources/snowflake.md" },
+              { text: "Socrata", link: "/supported-sources/socrata.md" },
               { text: "SQLite", link: "/supported-sources/sqlite.md" },
               {
                 text: "Experimental",

ingestr-0.14.96/docs/supported-sources/couchbase.md

@@ -0,0 +1,136 @@
+# Couchbase
+
+[Couchbase](https://www.couchbase.com/) is a distributed NoSQL cloud database that delivers unmatched performance, scalability, and flexibility for building modern applications.
+
+ingestr supports Couchbase as a source.
+
+## URI format
+
+### Standard format (without SSL)
+```plaintext
+couchbase://username:password@host
+```
+
+### With SSL/TLS enabled
+```plaintext
+couchbase://username:password@host?ssl=true
+```
+
+### Including bucket in URI
+```plaintext
+couchbase://username:password@host/bucket
+couchbase://username:password@host/bucket?ssl=true
+```
+
+URI parameters:
+- `username`: the username to connect to the Couchbase cluster
+- `password`: the password for the user
+- `host`: the host address of the Couchbase server
+- `bucket`: optional bucket name in the URI path
+- `ssl`: SSL/TLS connection parameter
+  - `ssl=true`: Required for Couchbase Capella (cloud) deployments
+  - `ssl=false` or omitted: Use for Couchbase Server (self-hosted/on-premises) deployments
+
+> [!NOTE]
+> **SSL Parameter Usage:**
+> - Use `ssl=true` when connecting to **Couchbase Capella (cloud)**
+> - Use `ssl=false` or omit the parameter when connecting to **Couchbase Server (self-hosted/on-premises)**
+
+The URI structure can be used for connecting to both local/self-hosted Couchbase instances and Couchbase Capella (cloud).
+
+## Source table format
+
+The `--source-table` option for Couchbase supports two formats depending on whether the bucket is specified in the URI:
+
+### When bucket is NOT in URI
+```plaintext
+bucket.scope.collection
+```
+
+### When bucket IS in URI path
+```plaintext
+scope.collection
+```
+
+For default scope and collection, you can use:
+```plaintext
+bucket._default._default
+```
+
+## Using Couchbase as a source
+
+### Local/self-hosted Couchbase
+
+#### Basic connection without SSL
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost" \
+  --source-table "mybucket.myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+#### For Couchbase Capella (Cloud)
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost?ssl=true" \
+  --source-table "mybucket._default._default" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+#### With bucket in URI
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost/mybucket" \
+  --source-table "myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+### Couchbase Capella (Cloud)
+
+> [!IMPORTANT]
+> Couchbase Capella (cloud) **requires SSL connections**. You must use `?ssl=true` in your connection URI and prefix the host with `cb.`
+
+> [!TIP]
+> You can obtain the connection string for Capella from the SDK connection details in your Couchbase Capella dashboard.
+
+Use the `couchbase://` scheme with `ssl=true` parameter. Note the `cb.` prefix in the hostname:
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://username:password@cb.xxx.cloud.couchbase.com?ssl=true" \
+  --source-table "travel-sample.inventory.airport" \
+  --dest-uri "duckdb:///airports.db" \
+  --dest-table "main.airports"
+```
+
+With bucket in URI for Couchbase Capella
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://username:password@cb.xxx.cloud.couchbase.com/travel-sample?ssl=true" \
+  --source-table "inventory.airport" \
+  --dest-uri "duckdb:///airports.db" \
+  --dest-table "main.airports"
+```
+
+
+### With URL-encoded password
+
+> [!IMPORTANT]
+> When using ingestr CLI, passwords containing special characters (`@`, `:`, `/`, `#`, `?`, etc.) **must be URL-encoded** in the connection URI.
+
+If your password contains special characters, you need to URL-encode them:
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:MyPass%40123%21@localhost" \
+  --source-table "mybucket.myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+This example encodes the password `MyPass@123!` as `MyPass%40123%21`.
+

ingestr-0.14.96/docs/supported-sources/socrata.md

@@ -0,0 +1,127 @@
+# Socrata
+
+[Socrata](https://dev.socrata.com/) is an open data platform used by governments and organizations to publish and share public datasets. The platform powers thousands of open data portals worldwide, including data.gov and many city, state, and federal government sites.
+
+`ingestr` allows ingesting data from any Socrata-powered open data portal using the [Socrata Open Data API (SODA)](https://dev.socrata.com/docs/endpoints.html).
+
+## URI Format
+
+The URI format for Socrata is as follows:
+```
+socrata://?domain=<domain>&dataset_id=<dataset_id>&app_token=<app_token>
+```
+
+URI Parameters:
+* `domain`: The Socrata domain (e.g., `data.seattle.gov`, `data.cityofnewyork.us`)
+* `dataset_id`: The dataset identifier (4x4 format like `6udu-fhnu`)
+* `app_token`: Socrata app token for API access (required)
+* `username` (optional): Username for authentication (required for private datasets)
+* `password` (optional): Password for authentication (required for private datasets)
+
+## Setting up Socrata Integration
+
+### Finding Domain and Dataset ID
+
+1. Navigate to any Socrata-powered open data portal
+2. Find the dataset you want to ingest
+3. The domain is the base URL (e.g., `data.seattle.gov`)
+4. The dataset ID is in the URL or API endpoint (e.g., `6udu-fhnu`)
+
+Example: For `https://data.seattle.gov/City-Business/City-of-Seattle-Wage-Data/2khk-5ukd`, the domain is `data.seattle.gov` and the dataset ID is `2khk-5ukd`.
+
+### Generate an App Token
+
+You need to obtain an app token to access the Socrata API.
+
+1. Sign up for a free account at the Socrata portal you're using
+2. Navigate to the developer settings or API documentation
+3. Generate an app token
+4. Use this token in the `app_token` parameter
+
+### Example: Loading a Public Dataset
+
+For this example, we'll load the Seattle City wage data:
+* Domain: `data.seattle.gov`
+* Dataset ID: `2khk-5ukd`
+* App token: `your_app_token_here`
+
+We will run `ingestr` to save this data to a [duckdb](https://duckdb.org/) database called `socrata.db` under the table name `public.wage_data`.
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=data.seattle.gov&dataset_id=2khk-5ukd&app_token=your_app_token_here" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.wage_data"
+```
+
+### Example: Incremental Loading
+
+Socrata supports incremental loading using the `:updated_at` system field or any other date/timestamp field in your dataset that gets updated. You must specify the incremental key using the `--incremental-key` flag.
+
+First, run an initial load for a specific time range using `:updated_at` as the incremental key:
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=data.seattle.gov&dataset_id=2khk-5ukd&app_token=your_app_token_here" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.wage_data" \
+    --incremental-key ":updated_at" \
+    --interval-start "2024-01-01" \
+    --interval-end "2024-06-30" \
+    --incremental-strategy "merge"
+```
+
+
+
+Now, we will run `ingestr` again without specifying dates to load only new or updated records:
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=data.seattle.gov&dataset_id=2khk-5ukd&app_token=your_app_token_here" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.wage_data" \
+    --incremental-key ":updated_at"
+```
+
+This will automatically fetch only records that were created or updated after the last ingestion.
+
+### Example: Loading Private Datasets
+
+For private datasets that require authentication:
+
+```sh
+ingestr ingest \
+    --source-uri "socrata://?domain=your.domain.com&dataset_id=xxxx-xxxx&app_token=your_token&username=your_username&password=your_password" \
+    --source-table "dataset" \
+    --dest-uri "duckdb:///socrata.db" \
+    --dest-table "public.private_data"
+```
+
+## Tables
+
+Socrata source provides a single table called `dataset` that represents the Socrata dataset.
+
+| Name | Merge Key | Inc Key | Inc Strategy | Details |
+| --- | --- | --- | --- | --- |
+| `dataset` | `:id` | user-defined | replace/merge | Loads all records from the specified Socrata dataset. Uses `replace` by default, or `merge` when `--incremental-key` is specified |
+
+
+
+
+
+## Troubleshooting
+
+### Rate Limit Errors
+If you hit rate limits, register for an app token or reduce the frequency of requests.
+
+### Authentication Errors
+For private datasets, ensure username and password are correct and that you have access to the dataset.
+
+### Invalid Dataset ID
+Verify the dataset ID is in the correct 4x4 format (e.g., `xxxx-xxxx`) and exists on the specified domain.
+
+### Timeout Errors
+For very large datasets, the initial load may take time. Consider using date ranges to break up large loads.

ingestr-0.14.96/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.96"

ingestr-0.14.96/ingestr/src/couchbase_source/__init__.py

@@ -0,0 +1,118 @@
+"""Source that loads data from Couchbase buckets, supports incremental loads."""
+
+from typing import Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import (
+    CouchbaseConfiguration,
+    client_from_credentials,
+    fetch_documents,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def couchbase_source(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+) -> DltResource:
+    """
+    A DLT source which loads data from a Couchbase bucket using Couchbase Python SDK.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    resource_name = f"{bucket}_{scope}_{collection}"
+
+    return dlt.resource(  # type: ignore[call-overload, arg-type]
+        fetch_documents,
+        name=resource_name,
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+        spec=CouchbaseConfiguration,
+        max_table_nesting=0,
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+    )
+
+
+@dlt.resource(
+    name=lambda args: f"{args['bucket']}_{args['scope']}_{args['collection']}",
+    standalone=True,
+    spec=CouchbaseConfiguration,  # type: ignore[arg-type]
+)
+def couchbase_collection(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> DltResource:
+    """
+    A DLT resource which loads a collection from Couchbase.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+        chunk_size (Optional[int]): The number of documents to load in each batch.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    return dlt.resource(  # type: ignore[call-overload]
+        fetch_documents,
+        name=f"{bucket}_{scope}_{collection}",
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+        chunk_size=chunk_size,
+    )

ingestr-0.14.96/ingestr/src/couchbase_source/helpers.py

@@ -0,0 +1,135 @@
+"""Helper functions for Couchbase source."""
+
+from datetime import datetime, timedelta
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+from couchbase.auth import PasswordAuthenticator  # type: ignore[import-untyped]
+from couchbase.cluster import Cluster  # type: ignore[import-untyped]
+from couchbase.options import (  # type: ignore[import-untyped]
+    ClusterOptions,
+    QueryOptions,
+)
+from dlt.common.configuration import configspec
+from dlt.common.time import ensure_pendulum_datetime
+
+
+@configspec
+class CouchbaseConfiguration:
+    """Configuration for Couchbase source."""
+
+    connection_string: str = dlt.secrets.value
+    username: str = dlt.secrets.value
+    password: str = dlt.secrets.value
+    bucket: str = dlt.config.value
+    scope: Optional[str] = dlt.config.value
+    collection: Optional[str] = dlt.config.value
+
+
+def client_from_credentials(
+    connection_string: str, username: str, password: str
+) -> Cluster:
+    """
+    Create a Couchbase cluster client from credentials.
+
+    Args:
+        connection_string: Couchbase connection string
+            - Local/self-hosted: 'couchbase://localhost'
+            - Capella (cloud): 'couchbases://your-instance.cloud.couchbase.com'
+        username: Couchbase username
+        password: Couchbase password
+
+    Returns:
+        Cluster: Connected Couchbase cluster instance
+    """
+    auth = PasswordAuthenticator(username, password)
+    options = ClusterOptions(auth)
+
+    # Apply wan_development profile for Capella (couchbases://) connections
+    # This helps avoid latency issues when accessing from different networks
+    if connection_string.startswith("couchbases://"):
+        options.apply_profile("wan_development")
+
+    cluster = Cluster(connection_string, options)
+    cluster.wait_until_ready(timedelta(seconds=30))
+
+    return cluster
+
+
+def fetch_documents(
+    cluster: Cluster,
+    bucket_name: str,
+    scope_name: str,
+    collection_name: str,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch documents from a Couchbase collection using N1QL queries.
+
+    Args:
+        cluster: Couchbase cluster instance
+        bucket_name: Name of the bucket
+        scope_name: Name of the scope
+        collection_name: Name of the collection
+        incremental: Incremental loading configuration
+        limit: Maximum number of documents to fetch
+        chunk_size: Number of documents to fetch per batch
+
+    Yields:
+        Dict[str, Any]: Document data
+    """
+    # Build N1QL query with full path
+    full_collection_path = f"`{bucket_name}`.`{scope_name}`.`{collection_name}`"
+    n1ql_query = f"SELECT META().id as id, c.* FROM {full_collection_path} c"
+
+    # Add incremental filter if provided
+    if incremental and incremental.cursor_path:
+        where_clause = f" WHERE {incremental.cursor_path} >= $start_value"
+        if incremental.end_value is not None:
+            where_clause += f" AND {incremental.cursor_path} < $end_value"
+        n1ql_query += where_clause
+
+    # Add limit if provided
+    if limit:
+        n1ql_query += f" LIMIT {limit}"
+
+    # Execute query
+    try:
+        query_options = QueryOptions()
+
+        # Add parameters if incremental
+        if incremental and incremental.cursor_path:
+            named_parameters = {"start_value": incremental.last_value}
+            if incremental.end_value is not None:
+                named_parameters["end_value"] = incremental.end_value
+            query_options = QueryOptions(named_parameters=named_parameters)
+
+        result = cluster.query(n1ql_query, query_options)
+
+        # Yield documents
+        count = 0
+        for row in result:
+            doc = dict(row)
+
+            # Convert datetime fields to proper format
+            if (
+                incremental
+                and incremental.cursor_path
+                and incremental.cursor_path in doc
+            ):
+                cursor_value = doc[incremental.cursor_path]
+                if isinstance(cursor_value, (str, datetime)):
+                    doc[incremental.cursor_path] = ensure_pendulum_datetime(
+                        cursor_value
+                    )
+
+            yield doc
+
+            count += 1
+            if limit and count >= limit:
+                break
+
+    except Exception as e:
+        raise Exception(f"Error executing Couchbase N1QL query: {str(e)}")

{ingestr-0.14.93 → ingestr-0.14.96}/ingestr/src/factory.py

@@ -39,6 +39,7 @@ from ingestr.src.sources import (
     AttioSource,
     ChessSource,
     ClickupSource,
+    CouchbaseSource,
     DoceboSource,
     DynamoDBSource,
     ElasticsearchSource,
@@ -83,6 +84,7 @@ from ingestr.src.sources import (
     ShopifySource,
     SlackSource,
     SmartsheetSource,
+    SocrataSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
@@ -160,6 +162,7 @@ class SourceDestinationFactory:
         "allium": AlliumSource,
         "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
+        "couchbase": CouchbaseSource,
         "docebo": DoceboSource,
         "http": HttpSource,
         "https": HttpSource,
@@ -216,6 +219,7 @@ class SourceDestinationFactory:
         "sftp": SFTPSource,
         "pinterest": PinterestSource,
         "revenuecat": RevenueCatSource,
+        "socrata": SocrataSource,
         "zoom": ZoomSource,
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,

{ingestr-0.14.93 → ingestr-0.14.96}/ingestr/src/revenuecat/__init__.py

@@ -1,4 +1,3 @@
-import asyncio
 from typing import Any, Dict, Iterable, Iterator
 
 import aiohttp
@@ -40,51 +39,26 @@ def revenuecat_source(
                 yield project
 
     @dlt.resource(
-        name="customers", primary_key="id", write_disposition="merge", parallelized=True
+        name="customer_ids",
+        write_disposition="replace",
+        selected=False,
+        parallelized=True,
     )
-    def customers() -> Iterator[Dict[str, Any]]:
-        """Get list of customers with nested purchases and subscriptions."""
+    def customer_ids():
         if project_id is None:
             raise ValueError("project_id is required for customers resource")
-        endpoint = f"/projects/{project_id}/customers"
 
-        async def process_customer_batch(customer_batch):
-            """Process a batch of customers with async operations."""
-            async with aiohttp.ClientSession() as session:
-                tasks = []
-                for customer in customer_batch:
-                    task = process_customer_with_nested_resources_async(
-                        session, api_key, project_id, customer
-                    )
-                    tasks.append(task)
+        yield _paginate(api_key, f"/projects/{project_id}/customers")
 
-                return await asyncio.gather(*tasks)
-
-        def process_customers_sync():
-            """Process customers in batches using asyncio."""
-            batch_size = 50  # Conservative batch size due to 60 req/min rate limit
-            current_batch = []
-
-            for customer in _paginate(api_key, endpoint):
-                current_batch.append(customer)
-
-                if len(current_batch) >= batch_size:
-                    # Process the batch asynchronously
-                    processed_customers = asyncio.run(
-                        process_customer_batch(current_batch)
-                    )
-                    for processed_customer in processed_customers:
-                        yield processed_customer
-                    current_batch = []
-
-            # Process any remaining customers in the final batch
-            if current_batch:
-                processed_customers = asyncio.run(process_customer_batch(current_batch))
-                for processed_customer in processed_customers:
-                    yield processed_customer
-
-        # Yield each processed customer
-        yield from process_customers_sync()
+    @dlt.transformer(
+        data_from=customer_ids, write_disposition="replace", parallelized=True
+    )
+    async def customers(customers) -> Iterator[Dict[str, Any]]:
+        async with aiohttp.ClientSession() as session:
+            for customer in customers:
+                yield await process_customer_with_nested_resources_async(
+                    session, api_key, project_id, customer
+                )
 
     # Create project-dependent resources dynamically
     project_resources = []
@@ -103,6 +77,7 @@ def revenuecat_source(
 
     return [
         projects,
+        customer_ids,
         customers,
         *project_resources,
     ]