ingestr 0.14.92__tar.gz → 0.14.94__tar.gz

{ingestr-0.14.92 → ingestr-0.14.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.92
+Version: 0.14.94
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -39,6 +39,7 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: couchbase==4.3.6
 Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1

{ingestr-0.14.92 → ingestr-0.14.94}/docs/.vitepress/config.mjs

@@ -82,6 +82,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
               { text: "AWS Athena", link: "/supported-sources/athena.md" },
               { text: "AWS Redshift", link: "/supported-sources/redshift.md" },
               { text: "ClickHouse", link: "/supported-sources/clickhouse.md" },
+              { text: "Couchbase", link: "/supported-sources/couchbase.md" },
               { text: "CrateDB", link: "/supported-sources/cratedb.md" },
               { text: "Databricks", link: "/supported-sources/databricks.md" },
               { text: "DuckDB", link: "/supported-sources/duckdb.md" },

ingestr-0.14.94/docs/supported-sources/allium.md

@@ -0,0 +1,164 @@
+# Allium
+
+[Allium](https://allium.so/) is a blockchain data platform that provides access to indexed blockchain data through a powerful query interface.
+
+ingestr supports Allium as a source.
+
+## URI format
+
+The URI format for Allium is as follows:
+
+```plaintext
+allium://?api_key=<api-key>
+```
+
+URI parameters:
+
+- `api_key`: The API key used for authentication with the Allium API (required)
+
+The URI is used to connect to the Allium API for extracting blockchain data.
+
+Query parameters should be passed using ingestr's `--interval-start` and `--interval-end` flags.
+
+## Setting up an Allium Integration
+
+To get your Allium API credentials:
+
+1. Sign up for an Allium account at [allium.so](https://allium.so/)
+2. Navigate to your account settings
+3. Generate an API key
+4. Find your query ID from the Allium explorer interface
+
+Once you have your credentials, here's a sample command that will copy the data from Allium into a DuckDB database:
+
+```sh
+ingestr ingest \
+  --source-uri 'allium://?api_key=your_api_key' \
+  --source-table 'query:your_query_id' \
+  --interval-start '2025-02-01' \
+  --interval-end '2025-02-02' \
+  --dest-uri duckdb:///allium.duckdb \
+  --dest-table 'allium.query_results'
+```
+
+The result of this command will be a table in the `allium.duckdb` database.
+
+## Query ID Format
+
+Allium source uses query IDs to identify which query to execute. The query ID should be passed as the `--source-table` parameter with the `query:` prefix.
+
+| Format | Example | Description |
+|--------|---------|-------------|
+| `query:<query_id>` | `query:abc123def456` | The query ID from Allium explorer |
+| `query:<query_id>:<params>` | `query:abc123def456:network=ethereum&min_value=1000` | Query ID with custom parameters |
+| `query:<query_id>:<params>` | `query:abc123def456:limit=5000&compute_profile=standard` | Query ID with run_config parameters |
+
+Each query ID represents a specific blockchain data query that you've created in the Allium explorer.
+
+### Run Config Parameters
+
+Special parameters that control query execution (part of `run_config`):
+
+- `limit`: Limit the number of rows in the result (max 250,000)
+- `compute_profile`: Compute profile identifier
+
+These parameters are passed in the same format as custom parameters but are used for query execution control.
+
+### Custom Query Parameters
+
+You can pass additional custom parameters to your Allium query using the format:
+```
+query:<query_id>:param1=value1&param2=value2
+```
+
+These custom parameters will be merged with the default date parameters. If a custom parameter has the same name as a default parameter, the custom value will take precedence.
+
+## How it works
+
+The Allium source connector:
+
+1. **Starts an async query execution** using your query ID and parameters
+2. **Polls for completion status** every 5 seconds (max 12 hours)
+3. **Fetches and returns the results** once the query completes successfully
+
+## Examples
+
+### Basic Query Ingestion (without date filters)
+
+```sh
+ingestr ingest \
+  --source-uri 'allium://?api_key=your_api_key' \
+  --source-table 'query:abc123def456' \
+  --dest-uri duckdb:///allium.duckdb \
+  --dest-table 'allium.transactions'
+```
+
+### Query with Date Parameters
+
+```sh
+ingestr ingest \
+  --source-uri 'allium://?api_key=your_api_key' \
+  --source-table 'query:abc123def456' \
+  --interval-start '2025-02-01' \
+  --interval-end '2025-02-02' \
+  --dest-uri duckdb:///allium.duckdb \
+  --dest-table 'allium.daily_transactions'
+```
+
+### Query with Custom Parameters
+
+```sh
+ingestr ingest \
+  --source-uri 'allium://?api_key=your_api_key' \
+  --source-table 'query:abc123def456:network=ethereum&min_value=1000' \
+  --interval-start '2025-02-01' \
+  --interval-end '2025-02-02' \
+  --dest-uri duckdb:///allium.duckdb \
+  --dest-table 'allium.filtered_transactions'
+```
+
+In this example, the query will receive both the default date parameters and the custom parameters `network` and `min_value`.
+
+### Query with Run Config Parameters
+
+```sh
+ingestr ingest \
+  --source-uri 'allium://?api_key=your_api_key' \
+  --source-table 'query:abc123def456:limit=5000&compute_profile=standard' \
+  --interval-start '2025-02-01' \
+  --interval-end '2025-02-02' \
+  --dest-uri duckdb:///allium.duckdb \
+  --dest-table 'allium.query_results'
+```
+
+This example limits the result to 5000 rows and uses the 'standard' compute profile.
+
+### Query with Both Custom and Run Config Parameters
+
+```sh
+ingestr ingest \
+  --source-uri 'allium://?api_key=your_api_key' \
+  --source-table 'query:abc123def456:network=ethereum&limit=10000&compute_profile=large' \
+  --interval-start '2025-02-01' \
+  --interval-end '2025-02-02' \
+  --dest-uri duckdb:///allium.duckdb \
+  --dest-table 'allium.filtered_transactions'
+```
+
+This example combines custom query parameters (`network`) with run config parameters (`limit` and `compute_profile`).
+
+## Notes
+
+> [!NOTE]
+> - Query execution is asynchronous and may take time depending on the complexity of your query
+> - The connector will wait up to 12 hours for query completion
+> - Use `--interval-start` and `--interval-end` flags to pass date parameters to your Allium query
+> - The dates will be automatically converted to:
+>   - `start_date` and `end_date` parameters in the format `YYYY-MM-DD`
+>   - `start_timestamp` and `end_timestamp` parameters as Unix timestamps (seconds since epoch)
+> - **Default dates**: If not specified, defaults to 2 days ago (00:00) to yesterday (00:00)
+> - Custom parameters can be added to the source table format: `query:your_query_id:param1=value1&param2=value2`
+> - Run config parameters (`limit`, `compute_profile`) are also passed in the source table format
+> - Custom parameters will override default parameters if they have the same name
+> - The `limit` parameter has a maximum value of 250,000 rows
+> - Make sure your query ID is valid and accessible with your API key

ingestr-0.14.94/docs/supported-sources/couchbase.md

@@ -0,0 +1,136 @@
+# Couchbase
+
+[Couchbase](https://www.couchbase.com/) is a distributed NoSQL cloud database that delivers unmatched performance, scalability, and flexibility for building modern applications.
+
+ingestr supports Couchbase as a source.
+
+## URI format
+
+### Standard format (without SSL)
+```plaintext
+couchbase://username:password@host
+```
+
+### With SSL/TLS enabled
+```plaintext
+couchbase://username:password@host?ssl=true
+```
+
+### Including bucket in URI
+```plaintext
+couchbase://username:password@host/bucket
+couchbase://username:password@host/bucket?ssl=true
+```
+
+URI parameters:
+- `username`: the username to connect to the Couchbase cluster
+- `password`: the password for the user
+- `host`: the host address of the Couchbase server
+- `bucket`: optional bucket name in the URI path
+- `ssl`: SSL/TLS connection parameter
+  - `ssl=true`: Required for Couchbase Capella (cloud) deployments
+  - `ssl=false` or omitted: Use for Couchbase Server (self-hosted/on-premises) deployments
+
+> [!NOTE]
+> **SSL Parameter Usage:**
+> - Use `ssl=true` when connecting to **Couchbase Capella (cloud)**
+> - Use `ssl=false` or omit the parameter when connecting to **Couchbase Server (self-hosted/on-premises)**
+
+The URI structure can be used for connecting to both local/self-hosted Couchbase instances and Couchbase Capella (cloud).
+
+## Source table format
+
+The `--source-table` option for Couchbase supports two formats depending on whether the bucket is specified in the URI:
+
+### When bucket is NOT in URI
+```plaintext
+bucket.scope.collection
+```
+
+### When bucket IS in URI path
+```plaintext
+scope.collection
+```
+
+For default scope and collection, you can use:
+```plaintext
+bucket._default._default
+```
+
+## Using Couchbase as a source
+
+### Local/self-hosted Couchbase
+
+#### Basic connection without SSL
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost" \
+  --source-table "mybucket.myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+#### For Couchbase Capella (Cloud)
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost?ssl=true" \
+  --source-table "mybucket._default._default" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+#### With bucket in URI
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:password123@localhost/mybucket" \
+  --source-table "myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+### Couchbase Capella (Cloud)
+
+> [!IMPORTANT]
+> Couchbase Capella (cloud) **requires SSL connections**. You must use `?ssl=true` in your connection URI and prefix the host with `cb.`
+
+> [!TIP]
+> You can obtain the connection string for Capella from the SDK connection details in your Couchbase Capella dashboard.
+
+Use the `couchbase://` scheme with `ssl=true` parameter. Note the `cb.` prefix in the hostname:
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://username:password@cb.xxx.cloud.couchbase.com?ssl=true" \
+  --source-table "travel-sample.inventory.airport" \
+  --dest-uri "duckdb:///airports.db" \
+  --dest-table "main.airports"
+```
+
+With bucket in URI for Couchbase Capella
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://username:password@cb.xxx.cloud.couchbase.com/travel-sample?ssl=true" \
+  --source-table "inventory.airport" \
+  --dest-uri "duckdb:///airports.db" \
+  --dest-table "main.airports"
+```
+
+
+### With URL-encoded password
+
+> [!IMPORTANT]
+> When using ingestr CLI, passwords containing special characters (`@`, `:`, `/`, `#`, `?`, etc.) **must be URL-encoded** in the connection URI.
+
+If your password contains special characters, you need to URL-encode them:
+
+```bash
+ingestr ingest \
+  --source-uri "couchbase://admin:MyPass%40123%21@localhost" \
+  --source-table "mybucket.myscope.mycollection" \
+  --dest-uri "duckdb:///output.db" \
+  --dest-table "main.couchbase_data"
+```
+
+This example encodes the password `MyPass@123!` as `MyPass%40123%21`.
+

ingestr-0.14.94/ingestr/src/allium/__init__.py

@@ -0,0 +1,128 @@
+"""
+Allium source for data extraction via REST API.
+
+This source provides access to Allium blockchain data via asynchronous query execution.
+"""
+
+import time
+from typing import Any, Iterator
+
+import dlt
+
+from ingestr.src.http_client import create_client
+
+
+@dlt.source(max_table_nesting=0, name="allium_source")
+def allium_source(
+    api_key: str,
+    query_id: str,
+    parameters: dict[str, Any] | None = None,
+    limit: int | None = None,
+    compute_profile: str | None = None,
+) -> Any:
+    """
+    Allium data source for blockchain data extraction.
+
+    This source connects to Allium API, runs async queries, and fetches results.
+
+    Args:
+        api_key: Allium API key for authentication
+        query_id: The query ID to execute (e.g., 'abc123')
+        parameters: Optional parameters for the query (e.g., {'start_date': '2025-02-01', 'end_date': '2025-02-02'})
+        limit: Limit the number of rows in the result (max 250,000)
+        compute_profile: Compute profile identifier
+
+    Yields:
+        DltResource: Data resources for Allium query results
+    """
+    base_url = "https://api.allium.so/api/v1/explorer"
+    session = create_client()
+    headers = {"X-API-Key": api_key}
+
+    @dlt.resource(
+        name="query_results",
+        write_disposition="replace",
+    )
+    def fetch_query_results() -> Iterator[dict[str, Any]]:
+        """
+        Fetch query results from Allium.
+
+        This function:
+        1. Starts an async query execution
+        2. Polls for completion status
+        3. Fetches and yields the results
+        """
+        # Step 1: Start async query execution
+        run_config: dict[str, Any] = {}
+        if limit is not None:
+            run_config["limit"] = limit
+        if compute_profile is not None:
+            run_config["compute_profile"] = compute_profile
+
+        run_payload = {"parameters": parameters or {}, "run_config": run_config}
+
+        run_response = session.post(
+            f"{base_url}/queries/{query_id}/run-async",
+            json=run_payload,
+            headers=headers,
+        )
+
+        run_data = run_response.json()
+
+        if "run_id" not in run_data:
+            raise ValueError(f"Failed to start query execution: {run_data}")
+
+        run_id = run_data["run_id"]
+
+        # Step 2: Poll for completion
+        max_retries = 8640  # Max 12 hours with 5-second intervals
+        retry_count = 0
+        poll_interval = 5  # seconds
+
+        while retry_count < max_retries:
+            status_response = session.get(
+                f"{base_url}/query-runs/{run_id}/status",
+                headers=headers,
+            )
+            status_response.raise_for_status()
+            status_data = status_response.json()
+
+            # Handle both string and dict responses
+            if isinstance(status_data, str):
+                status = status_data
+            else:
+                status = status_data.get("status")
+
+            if status == "success":
+                break
+            elif status == "failed":
+                error_msg = (
+                    status_data.get("error", "Unknown error")
+                    if isinstance(status_data, dict)
+                    else "Unknown error"
+                )
+                raise ValueError(f"Query execution failed: {error_msg}")
+            elif status in ["pending", "running", "queued"]:
+                time.sleep(poll_interval)
+                retry_count += 1
+            else:
+                raise ValueError(f"Unknown status: {status}")
+
+        if retry_count >= max_retries:
+            raise TimeoutError(
+                f"Query execution timed out after {max_retries * poll_interval} seconds"
+            )
+
+        # Step 3: Fetch results
+        results_response = session.get(
+            f"{base_url}/query-runs/{run_id}/results",
+            headers=headers,
+            params={"f": "json"},
+        )
+        results_response.raise_for_status()
+        query_output = results_response.json()
+
+        # Extract and yield all data
+        yield query_output.get("data", [])
+
+    return (fetch_query_results,)

ingestr-0.14.94/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.94"

ingestr-0.14.94/ingestr/src/couchbase_source/__init__.py

@@ -0,0 +1,118 @@
+"""Source that loads data from Couchbase buckets, supports incremental loads."""
+
+from typing import Optional
+
+import dlt
+from dlt.sources import DltResource
+
+from .helpers import (
+    CouchbaseConfiguration,
+    client_from_credentials,
+    fetch_documents,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def couchbase_source(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+) -> DltResource:
+    """
+    A DLT source which loads data from a Couchbase bucket using Couchbase Python SDK.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+            E.g., `incremental=dlt.sources.incremental('updated_at', pendulum.parse('2022-01-01T00:00:00Z'))`
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    resource_name = f"{bucket}_{scope}_{collection}"
+
+    return dlt.resource(  # type: ignore[call-overload, arg-type]
+        fetch_documents,
+        name=resource_name,
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+        spec=CouchbaseConfiguration,
+        max_table_nesting=0,
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+    )
+
+
+@dlt.resource(
+    name=lambda args: f"{args['bucket']}_{args['scope']}_{args['collection']}",
+    standalone=True,
+    spec=CouchbaseConfiguration,  # type: ignore[arg-type]
+)
+def couchbase_collection(
+    connection_string: str = dlt.secrets.value,
+    username: str = dlt.secrets.value,
+    password: str = dlt.secrets.value,
+    bucket: str = dlt.config.value,
+    scope: Optional[str] = dlt.config.value,
+    collection: Optional[str] = dlt.config.value,
+    incremental: Optional[dlt.sources.incremental] = None,  # type: ignore[type-arg]
+    write_disposition: Optional[str] = dlt.config.value,
+    limit: Optional[int] = None,
+    chunk_size: Optional[int] = 1000,
+) -> DltResource:
+    """
+    A DLT resource which loads a collection from Couchbase.
+
+    Args:
+        connection_string (str): Couchbase connection string (e.g., 'couchbase://localhost')
+        username (str): Couchbase username
+        password (str): Couchbase password
+        bucket (str): Bucket name to load data from
+        scope (Optional[str]): Scope name (defaults to '_default')
+        collection (Optional[str]): Collection name (defaults to '_default')
+        incremental (Optional[dlt.sources.incremental]): Option to enable incremental loading.
+        write_disposition (str): Write disposition of the resource.
+        limit (Optional[int]): The maximum number of documents to load.
+        chunk_size (Optional[int]): The number of documents to load in each batch.
+
+    Returns:
+        DltResource: A DLT resource for the Couchbase collection.
+    """
+    # Set up Couchbase client
+    cluster = client_from_credentials(connection_string, username, password)
+
+    return dlt.resource(  # type: ignore[call-overload]
+        fetch_documents,
+        name=f"{bucket}_{scope}_{collection}",
+        primary_key="id",
+        write_disposition=write_disposition or "replace",
+    )(
+        cluster=cluster,
+        bucket_name=bucket,
+        scope_name=scope,
+        collection_name=collection,
+        incremental=incremental,
+        limit=limit,
+        chunk_size=chunk_size,
+    )