quackpipe 0.6.1__tar.gz

quackpipe-0.6.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ekiourk consulting ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

quackpipe-0.6.1/PKG-INFO

@@ -0,0 +1,193 @@
+Metadata-Version: 2.4
+Name: quackpipe
+Version: 0.6.1
+Summary: A configuration-driven and programmatic ETL helper for DuckDB.
+License: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml
+Requires-Dist: duckdb>=0.9.0
+Requires-Dist: pandas
+Requires-Dist: python-dotenv
+Requires-Dist: azure-storage-blob
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: quackpipe[fixtures]; extra == "dev"
+Requires-Dist: ipdb; extra == "dev"
+Provides-Extra: fixtures
+Requires-Dist: testcontainers==4.10.0; extra == "fixtures"
+Requires-Dist: sqlalchemy; extra == "fixtures"
+Requires-Dist: testcontainers-postgres; extra == "fixtures"
+Requires-Dist: testcontainers-minio; extra == "fixtures"
+Requires-Dist: testcontainers-azurite; extra == "fixtures"
+Requires-Dist: httpx; extra == "fixtures"
+Provides-Extra: lint
+Requires-Dist: ruff; extra == "lint"
+Provides-Extra: logging
+Requires-Dist: structlog>=23.0.0; extra == "logging"
+Requires-Dist: colorlog>=6.0.0; extra == "logging"
+Provides-Extra: postgres
+Requires-Dist: psycopg; extra == "postgres"
+Provides-Extra: s3
+Requires-Dist: pyarrow; extra == "s3"
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka; extra == "kafka"
+Dynamic: license-file
+
+# Quackpipe
+
+**The missing link between your Python scripts and your data infrastructure.**
+
+Quackpipe is a powerful ETL helper library that uses **DuckDB** to create a unified, high-performance data plane for Python applications. It bridges the gap between writing raw, complex connection code and adopting a full-scale data transformation framework.
+
+With a simple YAML configuration, you can instantly connect to multiple data sources like **PostgreSQL**, **S3**, **Azure Blob Storage**, and **SQLite**, and even orchestrate complex **DuckLake** setups, all from a single, clean Python interface.
+
+[![codecov](https://codecov.io/github/ekiourk/quackpipe/graph/badge.svg?token=5LF2QD9MEW)](https://codecov.io/github/ekiourk/quackpipe)
+
+## What Gap Does Quackpipe Fill?
+
+In the modern data stack, you often face a choice:
+
+* **Low-Level:** Write boilerplate code with multiple database drivers (`psycopg2`, `boto3`, etc.) to connect and move data manually. This is flexible but repetitive and error-prone.
+* **High-Level:** Adopt a full DataOps framework like **SQLMesh** or **dbt**. These are powerful for building production-grade data warehouses but can be overkill for ad-hoc analysis, rapid prototyping, or simple scripting.
+
+**Quackpipe provides the perfect middle ground.** It gives you the power of a unified query engine and the simplicity of a Python library, allowing you to:
+
+* **Prototype Rapidly:** Spin up a multi-source data environment in seconds.
+* **Simplify ETL Scripts:** Replace complex driver code with a single, clean `session` or a one-line `move_data` command.
+* **Explore Data Interactively:** Use the built-in CLI to launch a web UI with all your sources pre-connected for instant ad-hoc querying.
+* **Bridge to Production:** Automatically generate configuration for frameworks like **SQLMesh** when you're ready to graduate from a script to a versioned data model.
+
+## Core Capabilities
+
+* **Unified Data Access:** Query across PostgreSQL, S3, Azure, and SQLite as if they were all schemas in a single database.
+* **Declarative Configuration:** Define all your data sources in one human-readable `config.yml` file.
+* **Powerful ETL Utilities:** Move data between any two configured sources with the `move_data()` function.
+* **Programmatic API:** Use the `QuackpipeBuilder` for dynamic, on-the-fly connection setups in your code.
+* **Secure Secret Management:** Load credentials safely from `.env` files, keeping them out of your code and configuration.
+* **Interactive UI:** Launch an interactive DuckDB web UI with all your sources pre-connected using a single CLI command.
+* **Framework Integration:** Automatically generate a `sqlmesh_config.yml` file to seamlessly transition your project to a full DataOps framework.
+
+## Installation
+
+```bash
+pip install quackpipe
+```
+
+Install support for the sources you need:
+
+```bash
+# Example: Install support for Postgres, S3, Azure, and the UI
+pip install "quackpipe[postgres,s3,azure,ui]"
+```
+
+## Configuration
+
+`quackpipe` uses a simple `config.yml` file to define your sources and an `.env` file to manage your secrets.
+
+### `config.yml` Example
+
+```yaml
+# config.yml
+sources:
+  # A writeable PostgreSQL database.
+  pg_warehouse:
+    type: postgres
+    secret_name: "pg_prod" # See Secret Management section below
+    read_only: false       # Allows writing data back to this source
+
+  # An S3 data lake for Parquet files.
+  s3_datalake:
+    type: s3
+    secret_name: "aws_prod"
+    region: "us-east-1"
+
+  # An Azure Blob Storage container.
+  azure_datalake:
+    type: azure
+    provider: connection_string
+    secret_name: "azure_prod"
+
+  # A composite DuckLake source.
+  my_lake:
+    type: ducklake
+    catalog:
+      type: sqlite
+      path: "/path/to/lake_catalog.db"
+    storage:
+      type: local
+      path: "/path/to/lake_storage/"
+```
+
+### Secret Management with `.env`
+
+Quackpipe uses a `secret_name` in the config to refer to a bundle of credentials. These are loaded from an `.env` file using a simple prefix convention: `SECRET_NAME_KEY`.
+
+Create an `.env` file in your project root:
+
+```dotenv
+# .env
+
+# Secrets for secret_name: "pg_prod"
+PG_PROD_HOST=db.example.com
+PG_PROD_USER=myuser
+PG_PROD_PASSWORD=mypassword
+PG_PROD_DATABASE=production
+
+# Secrets for secret_name: "aws_prod"
+AWS_PROD_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+AWS_PROD_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_KEY
+
+# Secrets for secret_name: "azure_prod"
+AZURE_PROD_CONNECTION_STRING="DefaultEndpointsProtocol=https..."
+```
+
+## Usage Highlights
+
+### 1. Interactive Querying with `session`
+
+Need to join a CSV in S3 with a table in Postgres? `quackpipe` makes it trivial.
+
+```python
+import quackpipe
+
+# quackpipe automatically loads your .env file
+with quackpipe.session(config_path="config.yml", env_file=".env") as con:
+    df = con.execute("""
+        SELECT u.name, o.order_total
+        FROM pg_warehouse.users u
+        JOIN read_parquet('s3://my-bucket/orders/*.parquet') o ON u.id = o.user_id
+        WHERE u.signup_date > '2024-01-01';
+    """).fetchdf()
+
+    print(df.head())
+```
+
+### 2. One-Line Data Movement with `move_data`
+
+Archive old records from your production database to your data lake with a single command.
+
+```python
+from quackpipe.etl_utils import move_data
+
+move_data(
+    config_path="config.yml",
+    env_file=".env",
+    source_query="SELECT * FROM pg_warehouse.logs WHERE timestamp < '2024-01-01'",
+    destination_name="s3_datalake",
+    table_name="logs_archive_2023"
+)
+```
+
+### 3. Instant Data Exploration with the CLI
+
+Launch a web browser UI with all your sources attached and ready for ad-hoc queries.
+
+```bash
+# This command reads your config.yml and .env file
+quackpipe ui
+
+# Or connect to specific sources
+quackpipe ui pg_warehouse s3_datalake

quackpipe-0.6.1/README.md

@@ -0,0 +1,155 @@
+# Quackpipe
+
+**The missing link between your Python scripts and your data infrastructure.**
+
+Quackpipe is a powerful ETL helper library that uses **DuckDB** to create a unified, high-performance data plane for Python applications. It bridges the gap between writing raw, complex connection code and adopting a full-scale data transformation framework.
+
+With a simple YAML configuration, you can instantly connect to multiple data sources like **PostgreSQL**, **S3**, **Azure Blob Storage**, and **SQLite**, and even orchestrate complex **DuckLake** setups, all from a single, clean Python interface.
+
+[![codecov](https://codecov.io/github/ekiourk/quackpipe/graph/badge.svg?token=5LF2QD9MEW)](https://codecov.io/github/ekiourk/quackpipe)
+
+## What Gap Does Quackpipe Fill?
+
+In the modern data stack, you often face a choice:
+
+* **Low-Level:** Write boilerplate code with multiple database drivers (`psycopg2`, `boto3`, etc.) to connect and move data manually. This is flexible but repetitive and error-prone.
+* **High-Level:** Adopt a full DataOps framework like **SQLMesh** or **dbt**. These are powerful for building production-grade data warehouses but can be overkill for ad-hoc analysis, rapid prototyping, or simple scripting.
+
+**Quackpipe provides the perfect middle ground.** It gives you the power of a unified query engine and the simplicity of a Python library, allowing you to:
+
+* **Prototype Rapidly:** Spin up a multi-source data environment in seconds.
+* **Simplify ETL Scripts:** Replace complex driver code with a single, clean `session` or a one-line `move_data` command.
+* **Explore Data Interactively:** Use the built-in CLI to launch a web UI with all your sources pre-connected for instant ad-hoc querying.
+* **Bridge to Production:** Automatically generate configuration for frameworks like **SQLMesh** when you're ready to graduate from a script to a versioned data model.
+
+## Core Capabilities
+
+* **Unified Data Access:** Query across PostgreSQL, S3, Azure, and SQLite as if they were all schemas in a single database.
+* **Declarative Configuration:** Define all your data sources in one human-readable `config.yml` file.
+* **Powerful ETL Utilities:** Move data between any two configured sources with the `move_data()` function.
+* **Programmatic API:** Use the `QuackpipeBuilder` for dynamic, on-the-fly connection setups in your code.
+* **Secure Secret Management:** Load credentials safely from `.env` files, keeping them out of your code and configuration.
+* **Interactive UI:** Launch an interactive DuckDB web UI with all your sources pre-connected using a single CLI command.
+* **Framework Integration:** Automatically generate a `sqlmesh_config.yml` file to seamlessly transition your project to a full DataOps framework.
+
+## Installation
+
+```bash
+pip install quackpipe
+```
+
+Install support for the sources you need:
+
+```bash
+# Example: Install support for Postgres, S3, Azure, and the UI
+pip install "quackpipe[postgres,s3,azure,ui]"
+```
+
+## Configuration
+
+`quackpipe` uses a simple `config.yml` file to define your sources and an `.env` file to manage your secrets.
+
+### `config.yml` Example
+
+```yaml
+# config.yml
+sources:
+  # A writeable PostgreSQL database.
+  pg_warehouse:
+    type: postgres
+    secret_name: "pg_prod" # See Secret Management section below
+    read_only: false       # Allows writing data back to this source
+
+  # An S3 data lake for Parquet files.
+  s3_datalake:
+    type: s3
+    secret_name: "aws_prod"
+    region: "us-east-1"
+
+  # An Azure Blob Storage container.
+  azure_datalake:
+    type: azure
+    provider: connection_string
+    secret_name: "azure_prod"
+
+  # A composite DuckLake source.
+  my_lake:
+    type: ducklake
+    catalog:
+      type: sqlite
+      path: "/path/to/lake_catalog.db"
+    storage:
+      type: local
+      path: "/path/to/lake_storage/"
+```
+
+### Secret Management with `.env`
+
+Quackpipe uses a `secret_name` in the config to refer to a bundle of credentials. These are loaded from an `.env` file using a simple prefix convention: `SECRET_NAME_KEY`.
+
+Create an `.env` file in your project root:
+
+```dotenv
+# .env
+
+# Secrets for secret_name: "pg_prod"
+PG_PROD_HOST=db.example.com
+PG_PROD_USER=myuser
+PG_PROD_PASSWORD=mypassword
+PG_PROD_DATABASE=production
+
+# Secrets for secret_name: "aws_prod"
+AWS_PROD_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
+AWS_PROD_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_KEY
+
+# Secrets for secret_name: "azure_prod"
+AZURE_PROD_CONNECTION_STRING="DefaultEndpointsProtocol=https..."
+```
+
+## Usage Highlights
+
+### 1. Interactive Querying with `session`
+
+Need to join a CSV in S3 with a table in Postgres? `quackpipe` makes it trivial.
+
+```python
+import quackpipe
+
+# quackpipe automatically loads your .env file
+with quackpipe.session(config_path="config.yml", env_file=".env") as con:
+    df = con.execute("""
+        SELECT u.name, o.order_total
+        FROM pg_warehouse.users u
+        JOIN read_parquet('s3://my-bucket/orders/*.parquet') o ON u.id = o.user_id
+        WHERE u.signup_date > '2024-01-01';
+    """).fetchdf()
+
+    print(df.head())
+```
+
+### 2. One-Line Data Movement with `move_data`
+
+Archive old records from your production database to your data lake with a single command.
+
+```python
+from quackpipe.etl_utils import move_data
+
+move_data(
+    config_path="config.yml",
+    env_file=".env",
+    source_query="SELECT * FROM pg_warehouse.logs WHERE timestamp < '2024-01-01'",
+    destination_name="s3_datalake",
+    table_name="logs_archive_2023"
+)
+```
+
+### 3. Instant Data Exploration with the CLI
+
+Launch a web browser UI with all your sources attached and ready for ad-hoc queries.
+
+```bash
+# This command reads your config.yml and .env file
+quackpipe ui
+
+# Or connect to specific sources
+quackpipe ui pg_warehouse s3_datalake

quackpipe-0.6.1/pyproject.toml

@@ -0,0 +1,83 @@
+[project]
+name = "quackpipe"
+version = "0.6.1"
+requires-python = ">=3.12"
+description = "A configuration-driven and programmatic ETL helper for DuckDB."
+license = {text = "MIT"}
+readme = "README.md"
+dependencies = [
+    "pyyaml",
+    "duckdb>=0.9.0",
+    "pandas",
+    "python-dotenv",
+    "azure-storage-blob"
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-cov",
+    "quackpipe[fixtures]",
+    "ipdb"
+]
+fixtures = [
+    "testcontainers==4.10.0",
+    "sqlalchemy",
+    "testcontainers-postgres",
+    "testcontainers-minio",
+    "testcontainers-azurite",
+    "httpx"
+]
+lint = [
+    "ruff"
+]
+logging = [
+    "structlog>=23.0.0",
+    "colorlog>=6.0.0",
+]
+postgres = ["psycopg"]
+s3 = ["pyarrow"]
+kafka = ["confluent-kafka"]
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+src = ["src"]
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",    # pycodestyle errors
+    "W",    # pycodestyle warnings
+    "F",    # pyflakes
+    "I",    # isort
+    "B",    # flake8-bugbear
+    "C4",   # flake8-comprehensions
+    "UP",   # pyupgrade
+]
+# Allow logging format strings
+ignore = [
+    "G201",
+    "G202",
+    "E501",  # line too long (handled by formatter)
+    "B008",  # do not perform function calls in argument defaults
+]
+
+[tool.ruff.format]
+# Use double quotes
+quote-style = "double"
+
+# Indent with spaces
+indent-style = "space"
+
+[project.scripts]
+quackpipe = "quackpipe.cli:main"
+

quackpipe-0.6.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

quackpipe-0.6.1/src/quackpipe/__init__.py

@@ -0,0 +1,45 @@
+"""
+quackpipe - A configuration-driven ETL helper for DuckDB.
+
+This library provides simple, high-level functions to connect DuckDB
+to various data sources based on a YAML configuration file or a
+programmatic builder.
+"""
+
+import logging
+import os
+
+# Expose the primary user-facing functions and classes.
+from .builder import QuackpipeBuilder
+from .config import SourceConfig, SourceType
+from .core import session, with_session
+from .exceptions import ConfigError, QuackpipeError, SecretError
+from .secrets import configure_secret_provider
+
+# Set up the library's top-level logger
+_default_level = os.getenv('QUACKPIPE_LOG_LEVEL', 'WARNING').upper()
+_root_logger = logging.getLogger(__name__)
+_root_logger.setLevel(getattr(logging, _default_level, logging.WARNING))
+_root_logger.addHandler(logging.NullHandler())
+
+
+__all__ = [
+    # Core API
+    "session",
+    "with_session",
+
+    # Builder API
+    "QuackpipeBuilder",
+
+    # Configuration Types
+    "SourceConfig",
+    "SourceType",
+
+    # Secret Management
+    "configure_secret_provider",
+
+    # Exceptions
+    "QuackpipeError",
+    "ConfigError",
+    "SecretError",
+]

quackpipe-0.6.1/src/quackpipe/builder.py

@@ -0,0 +1,58 @@
+"""
+The Builder API for programmatically constructing a quackpipe session.
+"""
+from typing import Any, Self
+
+from .config import SourceConfig, SourceType
+from .core import session as core_session  # Avoid circular import
+
+
+class QuackpipeBuilder:
+    """A fluent builder for creating a quackpipe session without a YAML file."""
+
+    def __init__(self):
+        self._sources: list[SourceConfig] = []
+
+    def add_source(self, name: str, type: SourceType, config: dict[str, Any] = None, secret_name: str = None) -> Self:
+        """
+        Adds a data source to the configuration.
+
+        Args:
+            name: The name for the data source (e.g., 'pg_main').
+            type: The type of the source, using the SourceType enum.
+            config: A dictionary of non-secret parameters.
+            secret_name: The logical name of the secret bundle.
+
+        Returns:
+            The builder instance for chaining.
+        """
+        source = SourceConfig(
+            name=name,
+            type=type,
+            config=config or {},
+            secret_name=secret_name
+        )
+        self._sources.append(source)
+        return self
+
+    def get_configs(self) -> list[SourceConfig]:
+        """
+        Returns the list of SourceConfig objects that have been added to the builder.
+        This is useful for passing to high-level utilities like `move_data`.
+        """
+        return self._sources
+
+    def session(self, **kwargs):
+        """
+        Builds and enters the session context manager. Can accept the same arguments
+        as the core session function, like `sources=['source_a']`.
+
+        Returns:
+            A context manager yielding a configured DuckDB connection.
+        """
+        if not self._sources:
+            raise ValueError("Cannot build a session with no sources defined.")
+
+        # Pass the built configs and any extra arguments (like `sources`)
+        # to the core session manager.
+        return core_session(configs=self._sources, **kwargs)

quackpipe-0.6.1/src/quackpipe/cli.py

@@ -0,0 +1,28 @@
+"""
+cli.py
+
+This module provides the main entry point for the quackpipe command-line interface.
+It discovers and registers commands from the 'commands' submodule.
+"""
+import argparse
+
+# Import the registration functions from each command module
+from .commands import generate_sqlmesh_config, ui
+
+
+def main():
+    """Main function to parse arguments and dispatch commands."""
+    parser = argparse.ArgumentParser(description="quackpipe: A DuckDB ETL Helper CLI.")
+    subparsers = parser.add_subparsers(dest="command", required=True, help="Available commands")
+
+    # Register all available commands
+    generate_sqlmesh_config.register_command(subparsers)
+    ui.register_command(subparsers)
+
+    # Parse the arguments and call the handler function assigned by the subparser
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

quackpipe-0.6.1/src/quackpipe/commands/common.py

@@ -0,0 +1,43 @@
+"""
+src/quackpipe/commands/common.py
+
+This module contains common utilities shared across CLI command modules.
+"""
+import logging
+import sys
+
+
+def setup_cli_logging(verbose_level: int = 0):
+    """
+    Configures the root logger for quackpipe to ensure CLI output is visible.
+
+    Args:
+        verbose_level (int): The verbosity level. 0 for WARNING, 1 for INFO, 2+ for DEBUG.
+    """
+    # Map the integer verbosity level to a logging level
+    if verbose_level >= 2:
+        level = logging.DEBUG
+    elif verbose_level == 1:
+        level = logging.INFO
+    else:
+        # Default to WARNING to avoid being too noisy
+        level = logging.WARNING
+
+    # Get the top-level logger for the library
+    log = logging.getLogger("quackpipe")
+    log.setLevel(level)
+
+    # Create a handler to write messages to the console (stdout)
+    handler = logging.StreamHandler(sys.stdout)
+
+    # Create a formatter and add it to the handler
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    handler.setFormatter(formatter)
+
+    # Add the handler to the logger. This ensures messages will be output.
+    # We clear existing handlers to avoid duplicate messages if run in a notebook.
+    if log.hasHandlers():
+        log.handlers.clear()
+    log.addHandler(handler)
+
+    return log