quackpipe 0.6.1__py3-none-any.whl

quackpipe/__init__.py

@@ -0,0 +1,45 @@
+"""
+quackpipe - A configuration-driven ETL helper for DuckDB.
+
+This library provides simple, high-level functions to connect DuckDB
+to various data sources based on a YAML configuration file or a
+programmatic builder.
+"""
+
+import logging
+import os
+
+# Expose the primary user-facing functions and classes.
+from .builder import QuackpipeBuilder
+from .config import SourceConfig, SourceType
+from .core import session, with_session
+from .exceptions import ConfigError, QuackpipeError, SecretError
+from .secrets import configure_secret_provider
+
+# Set up the library's top-level logger
+_default_level = os.getenv('QUACKPIPE_LOG_LEVEL', 'WARNING').upper()
+_root_logger = logging.getLogger(__name__)
+_root_logger.setLevel(getattr(logging, _default_level, logging.WARNING))
+_root_logger.addHandler(logging.NullHandler())
+
+
+__all__ = [
+    # Core API
+    "session",
+    "with_session",
+
+    # Builder API
+    "QuackpipeBuilder",
+
+    # Configuration Types
+    "SourceConfig",
+    "SourceType",
+
+    # Secret Management
+    "configure_secret_provider",
+
+    # Exceptions
+    "QuackpipeError",
+    "ConfigError",
+    "SecretError",
+]

quackpipe/builder.py

@@ -0,0 +1,58 @@
+"""
+The Builder API for programmatically constructing a quackpipe session.
+"""
+from typing import Any, Self
+
+from .config import SourceConfig, SourceType
+from .core import session as core_session  # Avoid circular import
+
+
+class QuackpipeBuilder:
+    """A fluent builder for creating a quackpipe session without a YAML file."""
+
+    def __init__(self):
+        self._sources: list[SourceConfig] = []
+
+    def add_source(self, name: str, type: SourceType, config: dict[str, Any] = None, secret_name: str = None) -> Self:
+        """
+        Adds a data source to the configuration.
+
+        Args:
+            name: The name for the data source (e.g., 'pg_main').
+            type: The type of the source, using the SourceType enum.
+            config: A dictionary of non-secret parameters.
+            secret_name: The logical name of the secret bundle.
+
+        Returns:
+            The builder instance for chaining.
+        """
+        source = SourceConfig(
+            name=name,
+            type=type,
+            config=config or {},
+            secret_name=secret_name
+        )
+        self._sources.append(source)
+        return self
+
+    def get_configs(self) -> list[SourceConfig]:
+        """
+        Returns the list of SourceConfig objects that have been added to the builder.
+        This is useful for passing to high-level utilities like `move_data`.
+        """
+        return self._sources
+
+    def session(self, **kwargs):
+        """
+        Builds and enters the session context manager. Can accept the same arguments
+        as the core session function, like `sources=['source_a']`.
+
+        Returns:
+            A context manager yielding a configured DuckDB connection.
+        """
+        if not self._sources:
+            raise ValueError("Cannot build a session with no sources defined.")
+
+        # Pass the built configs and any extra arguments (like `sources`)
+        # to the core session manager.
+        return core_session(configs=self._sources, **kwargs)

quackpipe/cli.py

@@ -0,0 +1,28 @@
+"""
+cli.py
+
+This module provides the main entry point for the quackpipe command-line interface.
+It discovers and registers commands from the 'commands' submodule.
+"""
+import argparse
+
+# Import the registration functions from each command module
+from .commands import generate_sqlmesh_config, ui
+
+
+def main():
+    """Main function to parse arguments and dispatch commands."""
+    parser = argparse.ArgumentParser(description="quackpipe: A DuckDB ETL Helper CLI.")
+    subparsers = parser.add_subparsers(dest="command", required=True, help="Available commands")
+
+    # Register all available commands
+    generate_sqlmesh_config.register_command(subparsers)
+    ui.register_command(subparsers)
+
+    # Parse the arguments and call the handler function assigned by the subparser
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

quackpipe/commands/common.py

@@ -0,0 +1,43 @@
+"""
+src/quackpipe/commands/common.py
+
+This module contains common utilities shared across CLI command modules.
+"""
+import logging
+import sys
+
+
+def setup_cli_logging(verbose_level: int = 0):
+    """
+    Configures the root logger for quackpipe to ensure CLI output is visible.
+
+    Args:
+        verbose_level (int): The verbosity level. 0 for WARNING, 1 for INFO, 2+ for DEBUG.
+    """
+    # Map the integer verbosity level to a logging level
+    if verbose_level >= 2:
+        level = logging.DEBUG
+    elif verbose_level == 1:
+        level = logging.INFO
+    else:
+        # Default to WARNING to avoid being too noisy
+        level = logging.WARNING
+
+    # Get the top-level logger for the library
+    log = logging.getLogger("quackpipe")
+    log.setLevel(level)
+
+    # Create a handler to write messages to the console (stdout)
+    handler = logging.StreamHandler(sys.stdout)
+
+    # Create a formatter and add it to the handler
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    handler.setFormatter(formatter)
+
+    # Add the handler to the logger. This ensures messages will be output.
+    # We clear existing handlers to avoid duplicate messages if run in a notebook.
+    if log.hasHandlers():
+        log.handlers.clear()
+    log.addHandler(handler)
+
+    return log

quackpipe/commands/generate_sqlmesh_config.py

@@ -0,0 +1,85 @@
+"""
+src/quackpipe/commands/generate_sqlmesh_config.py
+
+This module contains the implementation for the 'generate-sqlmesh-config' CLI command.
+"""
+from argparse import _SubParsersAction
+
+import yaml
+
+from ..config import SourceConfig
+from ..core import SOURCE_HANDLER_REGISTRY
+from ..secrets import configure_secret_provider, fetch_raw_secret_bundle
+from ..utils import get_configs
+
+
+def _generate_raw_sql(configs: list[SourceConfig]) -> str:
+    """Instantiates handlers and generates the full setup SQL with resolved secrets."""
+    all_sql_statements = []
+    for cfg in configs:
+        HandlerClass = SOURCE_HANDLER_REGISTRY.get(cfg.type)
+        if not HandlerClass:
+            continue
+        full_context = {**cfg.config, "connection_name": cfg.name, "secret_name": cfg.secret_name}
+        handler_instance = HandlerClass(full_context)
+        sql = handler_instance.render_sql()
+        all_sql_statements.append(sql)
+    return "\n\n".join(filter(None, all_sql_statements))
+
+
+def _replace_secrets_with_placeholders(sql_string: str, configs: list[SourceConfig]) -> str:
+    """Takes a raw SQL string and replaces secret values with environment variable placeholders."""
+    final_sql = sql_string
+    for cfg in configs:
+        if cfg.secret_name:
+            resolved_secrets = fetch_raw_secret_bundle(cfg.secret_name)
+            value_to_placeholder = {}
+            for env_var_name, value in resolved_secrets.items():
+                placeholder = f"${{{env_var_name}}}"
+                value_to_placeholder[f"'{value}'"] = f"'{placeholder}'"
+                value_to_placeholder[str(value)] = placeholder
+            for val, placeholder in sorted(value_to_placeholder.items(), key=lambda item: len(item[0]), reverse=True):
+                final_sql = final_sql.replace(val, placeholder)
+    return final_sql
+
+
+def _build_sqlmesh_dict(init_sql_block: str, gateway_name: str, state_db: str) -> dict:
+    """Constructs the Python dictionary for the SQLMesh config YAML."""
+    return {'gateways': {gateway_name: {'connection': {'type': 'duckdb', 'init': init_sql_block},
+                                        'state_connection': {'type': 'duckdb', 'database': state_db}}},
+            'default_gateway': gateway_name}
+
+
+def handler(args):
+    """The main handler function for the generate-sqlmesh-config command."""
+    configure_secret_provider(env_file=args.env_file)
+    print(f"Reading quackpipe configuration from: {args.config}")
+    quackpipe_configs = get_configs(config_path=args.config)
+    raw_sql = _generate_raw_sql(quackpipe_configs)
+    final_sql_with_placeholders = _replace_secrets_with_placeholders(raw_sql, quackpipe_configs)
+    sqlmesh_config_dict = _build_sqlmesh_dict(final_sql_with_placeholders, args.gateway_name, args.state_db)
+    try:
+        with open(args.output, 'w') as f:
+            yaml.dump(sqlmesh_config_dict, f, sort_keys=False, default_flow_style=False, indent=2)
+        print(f"✅ Successfully generated SQLMesh config at: {args.output}")
+    except Exception as e:
+        print(f"❌ Failed to write output file: {e}")
+
+
+def register_command(subparsers: _SubParsersAction):
+    """Registers the command and its arguments to the main CLI parser."""
+    parser_gen = subparsers.add_parser(
+        "generate-sqlmesh-config",
+        help="Generate a SQLMesh config file from a quackpipe config."
+    )
+    parser_gen.add_argument("-c", "--config", default="config.yml",
+                            help="Path to the input quackpipe config.yml file. (Default: config.yml)")
+    parser_gen.add_argument("-o", "--output", default="sqlmesh_config.yml",
+                            help="Path for the output SQLMesh config file. (Default: sqlmesh_config.yml)")
+    parser_gen.add_argument("--gateway-name", default="quackpipe_gateway",
+                            help="The name for the gateway in the SQLMesh config. (Default: quackpipe_gateway)")
+    parser_gen.add_argument("--state-db", default=".sqlmesh/state.db",
+                            help="The path for the SQLMesh state database. (Default: .sqlmesh/state.db)")
+    parser_gen.add_argument("--env-file", default=".env",
+                            help="Path to the environment file to load secrets from. (Default: .env)")
+    parser_gen.set_defaults(func=handler)

quackpipe/commands/ui.py

@@ -0,0 +1,74 @@
+"""
+src/quackpipe/commands/ui.py
+
+This module contains the implementation for the 'ui' CLI command.
+"""
+from argparse import _SubParsersAction
+
+from .. import ConfigError
+from ..core import session
+from .common import setup_cli_logging
+
+
+def handler(args):
+    """The main handler function for the ui command."""
+    log = setup_cli_logging(args.verbose)
+
+    sources_to_load = args.sources if args.sources else "all configured sources"
+    log.info(f"Attempting to start UI session for: {sources_to_load}")
+    log.debug(f"Using config file: {args.config} and env file: {args.env_file}")
+
+    try:
+        with session(config_path=args.config, env_file=args.env_file, sources=args.sources) as con:
+            log.info("Session created.")
+
+            log.info(f"Setting UI port to {args.port}...")
+            con.execute(f"SET ui_local_port = {args.port};")
+
+            log.info("Starting DuckDB UI server...")
+            con.execute("CALL start_ui_server();")
+
+            log.warning(f"✅ DuckDB UI is running at: http://localhost:{args.port}")
+            log.info("All sources from your config are attached and ready to query.")
+
+            try:
+                # Wait for user input to keep the server alive.
+                input("Press Enter or Ctrl+C to exit and shut down the UI server...")
+            except KeyboardInterrupt:
+                # Handle Ctrl+C gracefully by just printing a newline and proceeding.
+                print()  # Move to the next line after the ^C character
+                pass
+
+            log.info("Stopping DuckDB UI server...")
+            con.execute("CALL stop_ui_server();")
+
+    except Exception as e:
+        log_msg = f"❌ Failed to start UI session: {e}"
+        if isinstance(e, ConfigError):
+            log.warning(log_msg)
+        else:
+            log.error(log_msg, exc_info=True)
+    finally:
+        log.info("Shutting down.")
+
+
+def register_command(subparsers: _SubParsersAction):
+    """Registers the command and its arguments to the main CLI parser."""
+    parser_ui = subparsers.add_parser(
+        "ui",
+        help="Launch an interactive DuckDB UI with pre-configured sources."
+    )
+    parser_ui.add_argument("-c", "--config", default="config.yml",
+                           help="Path to the input quackpipe config.yml file. (Default: config.yml)")
+    parser_ui.add_argument("--env-file", default=".env",
+                           help="Path to the environment file to load secrets from. (Default: .env)")
+    parser_ui.add_argument("-p", "--port", type=int, default=4213, help="Port to run the DuckDB UI on. (Default: 4213)")
+    parser_ui.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        help="Increase output verbosity. Use -v for INFO and -vv for DEBUG."
+    )
+    parser_ui.add_argument("sources", nargs='*',
+                           help="Optional: A space-separated list of specific sources to load. If omitted, all sources are loaded.")
+    parser_ui.set_defaults(func=handler)

quackpipe/config.py

@@ -0,0 +1,35 @@
+"""
+Defines the typed configuration objects for quackpipe.
+"""
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Plugin:
+    """A structured definition for a DuckDB plugin that may require special installation."""
+    name: str
+    repository: str | None = None
+
+
+class SourceType(Enum):
+    """Enumeration of supported source types."""
+    POSTGRES = "postgres"
+    S3 = "s3"
+    AZURE = "azure"
+    DUCKLAKE = "ducklake"
+    SQLITE = "sqlite"
+    PARQUET = "parquet"
+    CSV = "csv"
+
+
+@dataclass
+class SourceConfig:
+    """
+    A structured configuration object for a single data source.
+    """
+    name: str
+    type: SourceType
+    config: dict[str, Any] = field(default_factory=dict)
+    secret_name: str | None = None

quackpipe/core.py

@@ -0,0 +1,123 @@
+"""
+The core logic of quackpipe.
+"""
+import logging
+from collections.abc import Generator
+from contextlib import contextmanager
+from functools import wraps
+
+import duckdb
+
+from quackpipe.config import Plugin, SourceConfig, SourceType
+from quackpipe.secrets import configure_secret_provider
+
+# Import all handlers
+from quackpipe.sources import azure_blob, ducklake, postgres, s3, sqlite
+from quackpipe.utils import get_configs
+
+logger = logging.getLogger(__name__)
+
+# The registry stores the handler CLASSES, not instances.
+SOURCE_HANDLER_REGISTRY = {
+    SourceType.POSTGRES: postgres.PostgresHandler,
+    SourceType.S3: s3.S3Handler,
+    SourceType.AZURE: azure_blob.AzureBlobHandler,
+    SourceType.DUCKLAKE: ducklake.DuckLakeHandler,
+    SourceType.SQLITE: sqlite.SQLiteHandler,
+}
+
+
+def _prepare_connection(con: duckdb.DuckDBPyConnection, configs: list[SourceConfig]):
+    """Configures a DuckDB connection from a list of SourceConfig objects."""
+    if not configs:
+        return
+
+    # 1. Instantiate all handlers first
+    instantiated_handlers = []
+    for cfg in configs:
+        HandlerClass = SOURCE_HANDLER_REGISTRY.get(cfg.type)
+        if not HandlerClass:
+            logger.warning("Warning: No handler class found for source type '%s'. Skipping.", cfg.type.value)
+            continue
+
+        full_context = {
+            **cfg.config,
+            "connection_name": cfg.name,
+            "secret_name": cfg.secret_name,
+        }
+        handler_instance = HandlerClass(full_context)
+        instantiated_handlers.append(handler_instance)
+
+    # 2. Gather all required plugins from the instantiated handlers
+    required_plugins = set()
+    for handler in instantiated_handlers:
+        required_plugins.update(handler.required_plugins)
+
+    # 3. Install and load all extensions
+    for plugin_def in required_plugins:
+        if isinstance(plugin_def, Plugin):
+            # It's a structured Plugin object with extra parameters
+            plugin_name = plugin_def.name
+            install_params = {'repository': plugin_def.repository}
+            # Filter out None values to avoid passing `repository=None`
+            clean_params = {k: v for k, v in install_params.items() if v is not None}
+            con.install_extension(plugin_name, **clean_params)
+        else:
+            # It's a simple string (the name of the plugin)
+            plugin_name = plugin_def
+            con.install_extension(plugin_name)
+
+        # Loading the extension only requires the name
+        con.load_extension(plugin_name)
+
+    # 4. Render and execute the setup SQL for each handler
+    for handler in instantiated_handlers:
+        setup_sql = handler.render_sql()
+        logger.debug(setup_sql)
+        try:
+            con.execute(setup_sql)
+        except (duckdb.ParserException, duckdb.IOException):
+            logger.exception(setup_sql)
+            raise
+
+
+@contextmanager
+def session(
+        config_path: str | None = None,
+        configs: list[SourceConfig] | None = None,
+        sources: list[str] | None = None,
+        env_file: str | None = None
+) -> Generator[duckdb.DuckDBPyConnection, None, None]:
+    """
+    A context manager providing a pre-configured DuckDB connection.
+    """
+    configure_secret_provider(env_file=env_file)
+
+    all_configs = get_configs(config_path, configs)
+
+    active_configs = all_configs
+    if sources:
+        active_configs = [c for c in all_configs if c.name in sources]
+
+    con = duckdb.connect(database=':memory:')
+    try:
+        _prepare_connection(con, active_configs)
+        yield con
+    finally:
+        con.close()
+
+
+def with_session(**session_kwargs):
+    """
+    A decorator to inject a pre-configured DuckDB connection into a function.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with session(**session_kwargs) as con:
+                return func(con, *args, **kwargs)
+
+        return wrapper
+
+    return decorator

quackpipe/etl_utils.py

@@ -0,0 +1,110 @@
+"""
+High-level utility functions for common ETL operations.
+"""
+import logging
+
+import duckdb
+import pandas as pd
+
+from .config import SourceConfig, SourceType
+
+# Import the session context manager from core and config loader from utils
+from .core import session
+from .utils import get_configs
+
+logger = logging.getLogger(__name__)
+
+
+def to_df(con: duckdb.DuckDBPyConnection, query: str) -> pd.DataFrame:
+    """Executes a query and returns the result as a pandas DataFrame."""
+    return con.execute(query).fetchdf()
+
+
+def create_table_from_df(con: duckdb.DuckDBPyConnection, df: pd.DataFrame, table_name: str):
+    """Creates a new table in DuckDB from a pandas DataFrame, replacing if it exists."""
+    con.execute(f"CREATE OR REPLACE TABLE {table_name} AS SELECT * FROM df")
+
+
+def move_data(
+        source_query: str,
+        destination_name: str,
+        table_name: str,
+        config_path: str | None = None,
+        configs: list[SourceConfig] | None = None,
+        env_file: str | None = None,
+        mode: str = 'replace',
+        format: str = 'parquet'
+):
+    """
+    A self-contained utility to move data from a source query to a destination.
+    This function creates and manages its own quackpipe session.
+
+    Args:
+        source_query: The SELECT query to execute for the source data.
+        destination_name: The logical name of the destination source from the config.
+        table_name: The name of the table or file to create at the destination.
+        config_path: Path to the YAML configuration file.
+        configs: A direct list of SourceConfig objects.
+        env_file: Path to an env file to use.
+        mode: Write mode. 'replace' or 'append'.
+        format: The file format for file-based destinations (e.g., 'parquet', 'csv').
+    """
+    # Load all configurations using the shared helper function.
+    all_configs = get_configs(config_path, configs)
+
+    try:
+        # Find the destination config to determine its type.
+        dest_config = next(c for c in all_configs if c.name == destination_name)
+    except StopIteration as e:
+        raise ValueError(f"Destination '{destination_name}' not found in the provided configuration.") from e
+
+    # This utility creates its own session to perform the work.
+    with session(configs=all_configs, env_file=env_file) as con:
+        if dest_config.type == SourceType.S3:
+            base_path = dest_config.config.get('path', f"s3://{destination_name}/")
+            if not base_path.endswith('/'):
+                base_path += '/'
+            full_path = f"{base_path}{table_name}.{format}"
+            sql = f"COPY ({source_query}) TO '{full_path}' (FORMAT {format.upper()});"
+            con.execute(sql)
+            logger.info("Data successfully copied to %s", full_path)
+
+        elif dest_config.type == SourceType.DUCKLAKE:
+            full_table_name = f"{destination_name}.{table_name}"
+            if mode == 'replace':
+                sql = f"CREATE OR REPLACE TABLE {full_table_name} AS ({source_query});"
+            elif mode == 'append':
+                sql = f"INSERT INTO {full_table_name} ({source_query});"
+            else:
+                raise ValueError(f"Invalid mode '{mode}'. Use 'replace' or 'append'.")
+            con.execute(sql)
+            logger.info("Data successfully moved to table %s", full_table_name)
+
+        elif dest_config.type in [SourceType.POSTGRES, SourceType.SQLITE]:
+            is_read_only = dest_config.config.get('read_only', True)
+            if is_read_only:
+                raise PermissionError(
+                    f"Cannot write to destination '{destination_name}' because it is configured as read-only. "
+                    "To enable writing, set 'read_only: false' in your configuration for this source."
+                )
+
+            full_table_name = f"{destination_name}.{table_name}"
+            if mode == 'replace':
+                con.execute(f"DROP TABLE IF EXISTS {full_table_name};")
+                sql = f"CREATE TABLE {full_table_name} AS ({source_query});"
+            elif mode == 'append':
+                sql = f"INSERT INTO {full_table_name} ({source_query});"
+            else:
+                raise ValueError(f"Invalid mode '{mode}'. Use 'replace' or 'append'.")
+            con.execute(sql)
+            logger.info("Data successfully moved to table %s", full_table_name)
+
+        else:
+            if mode == 'replace':
+                sql = f"CREATE OR REPLACE TABLE {table_name} AS ({source_query});"
+            elif mode == 'append':
+                sql = f"INSERT INTO {table_name} ({source_query});"
+            else:
+                raise ValueError(f"Invalid mode '{mode}'. Use 'replace' or 'append'.")
+            con.execute(sql)
+            logger.info("Data successfully moved to in-memory table '%s'", table_name)

quackpipe/exceptions.py

@@ -0,0 +1,15 @@
+"""
+Exception classes for quackpipe.
+"""
+
+class QuackpipeError(Exception):
+    """Base exception for quackpipe."""
+    pass
+
+class ConfigError(QuackpipeError):
+    """Raised when there's an error with configuration."""
+    pass
+
+class SecretError(QuackpipeError):
+    """Raised when there's an error with secret management."""
+    pass