ingestr 0.14.93__tar.gz → 0.14.94__tar.gz

{ingestr-0.14.93 → ingestr-0.14.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.93
+Version: 0.14.94
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -39,6 +39,7 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: couchbase==4.3.6
 Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1

{ingestr-0.14.93 → ingestr-0.14.94}/docs/.vitepress/config.mjs

@@ -82,6 +82,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
+              { text: "Couchbase", link: "/supported-sources/couchbase.md" },
               { text: "CrateDB", link: "/supported-sources/cratedb.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },

ingestr-0.14.94/docs/supported-sources/couchbase.md

@@ -0,0 +1,136 @@
+# Couchbase
+
+[Couchbase](https://www.couchbase.com/) is a distributed NoSQL cloud database that delivers unmatched performance, scalability, and flexibility for building modern applications.
+
+ingestr supports Couchbase as a source.
+
+## URI format
+
+### Standard format (without SSL)
+```plaintext
+couchbase://username:password@host
+```
+
+### With SSL/TLS enabled
+```plaintext
+couchbase://username:password@host?ssl=true
+```
+
+### Including bucket in URI
+```plaintext
+couchbase://username:password@host/bucket
+couchbase://username:password@host/bucket?ssl=true
+```
+
+URI parameters:
+- `username`: the username to connect to the Couchbase cluster
+- `password`: the password for the user
+- `host`: the host address of the Couchbase server
+- `bucket`: optional bucket name in the URI path
+- `ssl`: SSL/TLS connection parameter
+  - `ssl=true`: Required for Couchbase Capella (cloud) deployments
+  - `ssl=false` or omitted: Use for Couchbase Server (self-hosted/on-premises) deployments
+
+> [!NOTE]
+> **SSL Parameter Usage:**
+> - Use `ssl=true` when connecting to **Couchbase Capella (cloud)**
+> - Use `ssl=false` or omit the parameter when connecting to **Couchbase Server (self-hosted/on-premises)**
+
+The URI structure can be used for connecting to both local/self-hosted Couchbase instances and Couchbase Capella (cloud).
+
+## Source table format
+
+The `--source-table` option for Couchbase supports two formats depending on whether the bucket is specified in the URI:
+
+### When bucket is NOT in URI
+```plaintext
+bucket.scope.collection
+```
+
+### When bucket IS in URI path
+```plaintext
+scope.collection
+```
+
+For default scope and collection, you can use:
+```plaintext
+bucket._default._default
+```
+
+## Using Couchbase as a source
+
+### Local/self-hosted Couchbase
+
+#### Basic connection without SSL
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost" \
+  --source-table "mybucket.myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+#### For Couchbase Capella (Cloud)
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost?ssl=true" \
+  --source-table "mybucket._default._default" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+#### With bucket in URI
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost/mybucket" \
+  --source-table "myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+### Couchbase Capella (Cloud)
+
+> [!IMPORTANT]
+> Couchbase Capella (cloud) **requires SSL connections**. You must use `?ssl=true` in your connection URI and prefix the host with `cb.`
+
+> [!TIP]
+> You can obtain the connection string for Capella from the SDK connection details in your Couchbase Capella dashboard.
+
+Use the `couchbase://` scheme with `ssl=true` parameter. Note the `cb.` prefix in the hostname:
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://username:password@cb.xxx.cloud.couchbase.com?ssl=true" \
+  --source-table "travel-sample.inventory.airport" \
+  --dest-uri "duckdb:///airports.db" \
+  --dest-table "main.airports"
+```
+
+With bucket in URI for Couchbase Capella
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://username:password@cb.xxx.cloud.couchbase.com/travel-sample?ssl=true" \
+  --source-table "inventory.airport" \
+  --dest-uri "duckdb:///airports.db" \
+  --dest-table "main.airports"
+```
+
+
+### With URL-encoded password
+
+> [!IMPORTANT]
+> When using ingestr CLI, passwords containing special characters (`@`, `:`, `/`, `#`, `?`, etc.) **must be URL-encoded** in the connection URI.
+
+If your password contains special characters, you need to URL-encode them:
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:MyPass%40123%21@localhost" \
+  --source-table "mybucket.myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+This example encodes the password `MyPass@123!` as `MyPass%40123%21`.
+

ingestr-0.14.94/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.94"

ingestr-0.14.94/ingestr/src/couchbase_source/__init__.py

@@ -0,0 +1,118 @@
+"""Source that loads data from Couchbase buckets, supports incremental loads."""
+
+from typing import Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import (
+    CouchbaseConfiguration,
+    client_from_credentials,
+    fetch_documents,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def couchbase_source(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+) -> DltResource:
+    """
+    A DLT source which loads data from a Couchbase bucket using Couchbase Python SDK.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    resource_name = f"{bucket}_{scope}_{collection}"
+
+    return dlt.resource(  # type: ignore[call-overload, arg-type]
+        fetch_documents,
+        name=resource_name,
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+        spec=CouchbaseConfiguration,
+        max_table_nesting=0,
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+    )
+
+
+@dlt.resource(
+    name=lambda args: f"{args['bucket']}_{args['scope']}_{args['collection']}",
+    standalone=True,
+    spec=CouchbaseConfiguration,  # type: ignore[arg-type]
+)
+def couchbase_collection(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> DltResource:
+    """
+    A DLT resource which loads a collection from Couchbase.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+        chunk_size (Optional[int]): The number of documents to load in each batch.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    return dlt.resource(  # type: ignore[call-overload]
+        fetch_documents,
+        name=f"{bucket}_{scope}_{collection}",
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+        chunk_size=chunk_size,
+    )

ingestr-0.14.94/ingestr/src/couchbase_source/helpers.py

@@ -0,0 +1,135 @@
+"""Helper functions for Couchbase source."""
+
+from datetime import datetime, timedelta
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+from couchbase.auth import PasswordAuthenticator  # type: ignore[import-untyped]
+from couchbase.cluster import Cluster  # type: ignore[import-untyped]
+from couchbase.options import (  # type: ignore[import-untyped]
+    ClusterOptions,
+    QueryOptions,
+)
+from dlt.common.configuration import configspec
+from dlt.common.time import ensure_pendulum_datetime
+
+
+@configspec
+class CouchbaseConfiguration:
+    """Configuration for Couchbase source."""
+
+    connection_string: str = dlt.secrets.value
+    username: str = dlt.secrets.value
+    password: str = dlt.secrets.value
+    bucket: str = dlt.config.value
+    scope: Optional[str] = dlt.config.value
+    collection: Optional[str] = dlt.config.value
+
+
+def client_from_credentials(
+    connection_string: str, username: str, password: str
+) -> Cluster:
+    """
+    Create a Couchbase cluster client from credentials.
+
+    Args:
+        connection_string: Couchbase connection string
+            - Local/self-hosted: 'couchbase://localhost'
+            - Capella (cloud): 'couchbases://your-instance.cloud.couchbase.com'
+        username: Couchbase username
+        password: Couchbase password
+
+    Returns:
+        Cluster: Connected Couchbase cluster instance
+    """
+    auth = PasswordAuthenticator(username, password)
+    options = ClusterOptions(auth)
+
+    # Apply wan_development profile for Capella (couchbases://) connections
+    # This helps avoid latency issues when accessing from different networks
+    if connection_string.startswith("couchbases://"):
+        options.apply_profile("wan_development")
+
+    cluster = Cluster(connection_string, options)
+    cluster.wait_until_ready(timedelta(seconds=30))
+
+    return cluster
+
+
+def fetch_documents(
+    cluster: Cluster,
+    bucket_name: str,
+    scope_name: str,
+    collection_name: str,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch documents from a Couchbase collection using N1QL queries.
+
+    Args:
+        cluster: Couchbase cluster instance
+        bucket_name: Name of the bucket
+        scope_name: Name of the scope
+        collection_name: Name of the collection
+        incremental: Incremental loading configuration
+        limit: Maximum number of documents to fetch
+        chunk_size: Number of documents to fetch per batch
+
+    Yields:
+        Dict[str, Any]: Document data
+    """
+    # Build N1QL query with full path
+    full_collection_path = f"`{bucket_name}`.`{scope_name}`.`{collection_name}`"
+    n1ql_query = f"SELECT META().id as id, c.* FROM {full_collection_path} c"
+
+    # Add incremental filter if provided
+    if incremental and incremental.cursor_path:
+        where_clause = f" WHERE {incremental.cursor_path} >= $start_value"
+        if incremental.end_value is not None:
+            where_clause += f" AND {incremental.cursor_path} < $end_value"
+        n1ql_query += where_clause
+
+    # Add limit if provided
+    if limit:
+        n1ql_query += f" LIMIT {limit}"
+
+    # Execute query
+    try:
+        query_options = QueryOptions()
+
+        # Add parameters if incremental
+        if incremental and incremental.cursor_path:
+            named_parameters = {"start_value": incremental.last_value}
+            if incremental.end_value is not None:
+                named_parameters["end_value"] = incremental.end_value
+            query_options = QueryOptions(named_parameters=named_parameters)
+
+        result = cluster.query(n1ql_query, query_options)
+
+        # Yield documents
+        count = 0
+        for row in result:
+            doc = dict(row)
+
+            # Convert datetime fields to proper format
+            if (
+                incremental
+                and incremental.cursor_path
+                and incremental.cursor_path in doc
+            ):
+                cursor_value = doc[incremental.cursor_path]
+                if isinstance(cursor_value, (str, datetime)):
+                    doc[incremental.cursor_path] = ensure_pendulum_datetime(
+                        cursor_value
+                    )
+
+            yield doc
+
+            count += 1
+            if limit and count >= limit:
+                break
+
+    except Exception as e:
+        raise Exception(f"Error executing Couchbase N1QL query: {str(e)}")

{ingestr-0.14.93 → ingestr-0.14.94}/ingestr/src/factory.py

@@ -39,6 +39,7 @@ from ingestr.src.sources import (
     AttioSource,
     ChessSource,
     ClickupSource,
+    CouchbaseSource,
     DoceboSource,
     DynamoDBSource,
     ElasticsearchSource,
@@ -160,6 +161,7 @@ class SourceDestinationFactory:
         "allium": AlliumSource,
         "anthropic": AnthropicSource,
         "csv": LocalCsvSource,
+        "couchbase": CouchbaseSource,
         "docebo": DoceboSource,
         "http": HttpSource,
         "https": HttpSource,

{ingestr-0.14.93 → ingestr-0.14.94}/ingestr/src/revenuecat/__init__.py

@@ -1,4 +1,3 @@
-import asyncio
 from typing import Any, Dict, Iterable, Iterator
 
 import aiohttp
@@ -40,51 +39,26 @@ def revenuecat_source(
                 yield project
 
     @dlt.resource(
-        name="customers", primary_key="id", write_disposition="merge", parallelized=True
+        name="customer_ids",
+        write_disposition="replace",
+        selected=False,
+        parallelized=True,
     )
-    def customers() -> Iterator[Dict[str, Any]]:
-        """Get list of customers with nested purchases and subscriptions."""
+    def customer_ids():
         if project_id is None:
             raise ValueError("project_id is required for customers resource")
-        endpoint = f"/projects/{project_id}/customers"
 
-        async def process_customer_batch(customer_batch):
-            """Process a batch of customers with async operations."""
-            async with aiohttp.ClientSession() as session:
-                tasks = []
-                for customer in customer_batch:
-                    task = process_customer_with_nested_resources_async(
-                        session, api_key, project_id, customer
-                    )
-                    tasks.append(task)
+        yield _paginate(api_key, f"/projects/{project_id}/customers")
 
-                return await asyncio.gather(*tasks)
-
-        def process_customers_sync():
-            """Process customers in batches using asyncio."""
-            batch_size = 50  # Conservative batch size due to 60 req/min rate limit
-            current_batch = []
-
-            for customer in _paginate(api_key, endpoint):
-                current_batch.append(customer)
-
-                if len(current_batch) >= batch_size:
-                    # Process the batch asynchronously
-                    processed_customers = asyncio.run(
-                        process_customer_batch(current_batch)
-                    )
-                    for processed_customer in processed_customers:
-                        yield processed_customer
-                    current_batch = []
-
-            # Process any remaining customers in the final batch
-            if current_batch:
-                processed_customers = asyncio.run(process_customer_batch(current_batch))
-                for processed_customer in processed_customers:
-                    yield processed_customer
-
-        # Yield each processed customer
-        yield from process_customers_sync()
+    @dlt.transformer(
+        data_from=customer_ids, write_disposition="replace", parallelized=True
+    )
+    async def customers(customers) -> Iterator[Dict[str, Any]]:
+        async with aiohttp.ClientSession() as session:
+            for customer in customers:
+                yield await process_customer_with_nested_resources_async(
+                    session, api_key, project_id, customer
+                )
 
     # Create project-dependent resources dynamically
     project_resources = []
@@ -103,6 +77,7 @@ def revenuecat_source(
 
     return [
         projects,
+        customer_ids,
         customers,
         *project_resources,
     ]

{ingestr-0.14.93 → ingestr-0.14.94}/ingestr/src/revenuecat/helpers.py

@@ -64,12 +64,9 @@ def _paginate(
     while True:
         data = _make_request(api_key, endpoint, current_params)
 
-        # Yield items from the current page
         if "items" in data and data["items"] is not None:
-            for item in data["items"]:
-                yield item
+            yield data["items"]
 
-        # Check if there's a next page
         if "next_page" not in data:
             break
 
@@ -88,7 +85,6 @@ def convert_timestamps_to_iso(
     """Convert timestamp fields from milliseconds to ISO format."""
     for field in timestamp_fields:
         if field in record and record[field] is not None:
-            # Convert from milliseconds timestamp to ISO datetime string
             timestamp_ms = record[field]
             dt = pendulum.from_timestamp(timestamp_ms / 1000)
             record[field] = dt.to_iso8601_string()
@@ -177,87 +173,37 @@ async def _paginate_async(
     return items
 
 
-async def fetch_and_process_nested_resource_async(
-    session: aiohttp.ClientSession,
-    api_key: str,
-    project_id: str,
-    customer_id: str,
-    customer: Dict[str, Any],
-    resource_name: str,
-    timestamp_fields: Optional[List[str]] = None,
-) -> None:
-    """
-    Fetch and process any nested resource for a customer asynchronously.
-
-    Args:
-        session: aiohttp ClientSession
-        api_key: RevenueCat API key
-        project_id: Project ID
-        customer_id: Customer ID
-        customer: Customer data dictionary to modify
-        resource_name: Name of the nested resource (e.g., 'purchases', 'subscriptions', 'events')
-        timestamp_fields: List of timestamp fields to convert to ISO format
-    """
-    # If resource not included in customer data, fetch separately
-    if resource_name not in customer or customer[resource_name] is None:
-        endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
-        customer[resource_name] = await _paginate_async(session, api_key, endpoint)
-
-    # Convert timestamps if fields specified
-    if (
-        timestamp_fields
-        and resource_name in customer
-        and customer[resource_name] is not None
-    ):
-        for item in customer[resource_name]:
-            convert_timestamps_to_iso(item, timestamp_fields)
-
-
 async def process_customer_with_nested_resources_async(
     session: aiohttp.ClientSession,
     api_key: str,
     project_id: str,
     customer: Dict[str, Any],
 ) -> Dict[str, Any]:
-    """
-    Process a customer and fetch nested resources concurrently.
-
-    Args:
-        session: aiohttp ClientSession
-        api_key: RevenueCat API key
-        project_id: Project ID
-        customer: Customer data to process
-
-    Returns:
-        Customer data with nested resources populated
-    """
     customer_id = customer["id"]
-
-    # Convert customer timestamps
     customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
-
-    # Define nested resources to fetch concurrently
     nested_resources = [
         ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
         ("purchases", ["purchased_at", "expires_at"]),
     ]
 
-    # Create concurrent tasks for fetching nested resources
-    tasks = []
-    for resource_name, timestamp_fields in nested_resources:
-        task = fetch_and_process_nested_resource_async(
-            session,
-            api_key,
-            project_id,
-            customer_id,
-            customer,
-            resource_name,
-            timestamp_fields,
-        )
-        tasks.append(task)
-
-    # Wait for all nested resources to be fetched
-    await asyncio.gather(*tasks)
+    async def fetch_and_convert(resource_name, timestamp_fields):
+        if resource_name not in customer or customer[resource_name] is None:
+            endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+            customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+        if (
+            timestamp_fields
+            and resource_name in customer
+            and customer[resource_name] is not None
+        ):
+            for item in customer[resource_name]:
+                convert_timestamps_to_iso(item, timestamp_fields)
+
+    await asyncio.gather(
+        *[
+            fetch_and_convert(resource_name, timestamp_fields)
+            for resource_name, timestamp_fields in nested_resources
+        ]
+    )
 
     return customer