anysite-cli 0.1.2__py3-none-any.whl

anysite/db/config.py

@@ -0,0 +1,92 @@
+"""Database connection configuration."""
+
+from __future__ import annotations
+
+import os
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, model_validator
+
+
+class DatabaseType(str, Enum):
+    """Supported database types."""
+
+    SQLITE = "sqlite"
+    POSTGRES = "postgres"
+    MYSQL = "mysql"
+    DUCKDB = "duckdb"
+
+
+class ConnectionConfig(BaseModel):
+    """Configuration for a database connection."""
+
+    name: str
+    type: DatabaseType
+    host: str | None = None
+    port: int | None = None
+    database: str | None = None
+    user: str | None = None
+    password_env: str | None = None
+    url_env: str | None = None
+    path: str | None = None
+    ssl: bool = False
+    options: dict[str, Any] = {}
+
+    @model_validator(mode="after")
+    def validate_config(self) -> ConnectionConfig:
+        """Validate that required fields are present for the database type."""
+        if self.type in (DatabaseType.SQLITE, DatabaseType.DUCKDB) and not self.path:
+            raise ValueError(f"{self.type.value} requires 'path'")
+        if self.type in (DatabaseType.POSTGRES, DatabaseType.MYSQL) and not self.url_env and not self.host:
+            raise ValueError(f"{self.type.value} requires 'host' or 'url_env'")
+        return self
+
+    def get_password(self) -> str | None:
+        """Resolve password from environment variable."""
+        if self.password_env:
+            value = os.environ.get(self.password_env)
+            if value is None:
+                raise ValueError(
+                    f"Environment variable '{self.password_env}' is not set"
+                )
+            return value
+        return None
+
+    def get_url(self) -> str | None:
+        """Resolve connection URL from environment variable."""
+        if self.url_env:
+            value = os.environ.get(self.url_env)
+            if value is None:
+                raise ValueError(
+                    f"Environment variable '{self.url_env}' is not set"
+                )
+            return value
+        return None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for YAML serialization, omitting None values."""
+        data: dict[str, Any] = {"type": self.type.value}
+        for field in ("host", "port", "database", "user", "password_env", "url_env", "path"):
+            value = getattr(self, field)
+            if value is not None:
+                data[field] = value
+        if self.ssl:
+            data["ssl"] = True
+        if self.options:
+            data["options"] = self.options
+        return data
+
+    @classmethod
+    def from_dict(cls, name: str, data: dict[str, Any]) -> ConnectionConfig:
+        """Create a ConnectionConfig from a dictionary."""
+        return cls(name=name, **data)
+
+
+class OnConflict(str, Enum):
+    """Conflict resolution strategy for inserts."""
+
+    ERROR = "error"
+    IGNORE = "ignore"
+    REPLACE = "replace"
+    UPDATE = "update"

anysite/db/manager.py

@@ -0,0 +1,166 @@
+"""Connection manager for database connections."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from anysite.config.paths import ensure_config_dir, get_config_dir
+from anysite.db.adapters.base import DatabaseAdapter
+from anysite.db.config import ConnectionConfig, DatabaseType
+
+
+def get_connections_path() -> Path:
+    """Get the path to the connections YAML file."""
+    return get_config_dir() / "connections.yaml"
+
+
+class ConnectionManager:
+    """Manages named database connections stored in YAML."""
+
+    def __init__(self, path: Path | None = None) -> None:
+        self.path = path or get_connections_path()
+        self._connections: dict[str, ConnectionConfig] | None = None
+
+    def _load(self) -> dict[str, ConnectionConfig]:
+        """Load connections from YAML file."""
+        if self._connections is not None:
+            return self._connections
+
+        connections: dict[str, ConnectionConfig] = {}
+        if self.path.exists():
+            with open(self.path) as f:
+                data = yaml.safe_load(f) or {}
+            raw = data.get("connections", {})
+            for name, config_data in raw.items():
+                connections[name] = ConnectionConfig.from_dict(name, config_data)
+
+        self._connections = connections
+        return connections
+
+    def _save(self) -> None:
+        """Save connections to YAML file."""
+        connections = self._load()
+        data: dict[str, Any] = {
+            "connections": {
+                name: config.to_dict()
+                for name, config in connections.items()
+            }
+        }
+        ensure_config_dir()
+        with open(self.path, "w") as f:
+            yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+
+    def add(self, config: ConnectionConfig) -> None:
+        """Add or update a connection.
+
+        Args:
+            config: Connection configuration.
+        """
+        connections = self._load()
+        connections[config.name] = config
+        self._save()
+
+    def remove(self, name: str) -> bool:
+        """Remove a connection by name.
+
+        Args:
+            name: Connection name.
+
+        Returns:
+            True if the connection was removed, False if not found.
+        """
+        connections = self._load()
+        if name not in connections:
+            return False
+        del connections[name]
+        self._save()
+        return True
+
+    def get(self, name: str) -> ConnectionConfig | None:
+        """Get a connection config by name.
+
+        Args:
+            name: Connection name.
+
+        Returns:
+            ConnectionConfig or None if not found.
+        """
+        connections = self._load()
+        return connections.get(name)
+
+    def list(self) -> list[ConnectionConfig]:
+        """List all connections.
+
+        Returns:
+            List of connection configs.
+        """
+        connections = self._load()
+        return list(connections.values())
+
+    def test(self, name: str) -> dict[str, str]:
+        """Test a connection by connecting and getting server info.
+
+        Args:
+            name: Connection name.
+
+        Returns:
+            Server info dictionary.
+
+        Raises:
+            ValueError: If connection not found.
+            Exception: If connection fails.
+        """
+        config = self.get(name)
+        if config is None:
+            raise ValueError(f"Connection '{name}' not found")
+
+        adapter = self.get_adapter(config)
+        with adapter:
+            return adapter.get_server_info()
+
+    def get_adapter(self, config: ConnectionConfig) -> DatabaseAdapter:
+        """Get a database adapter for the given config.
+
+        Args:
+            config: Connection configuration.
+
+        Returns:
+            Appropriate DatabaseAdapter instance.
+
+        Raises:
+            ValueError: If database type is not supported.
+        """
+        if config.type == DatabaseType.SQLITE:
+            from anysite.db.adapters.sqlite import SQLiteAdapter
+
+            return SQLiteAdapter(config)
+
+        elif config.type == DatabaseType.POSTGRES:
+            from anysite.db import check_db_deps
+            from anysite.db.adapters.postgres import PostgresAdapter
+
+            check_db_deps("postgres")
+            return PostgresAdapter(config)
+
+        else:
+            raise ValueError(f"Unsupported database type: {config.type.value}")
+
+    def get_adapter_by_name(self, name: str) -> DatabaseAdapter:
+        """Get a database adapter by connection name.
+
+        Args:
+            name: Connection name.
+
+        Returns:
+            Appropriate DatabaseAdapter instance.
+
+        Raises:
+            ValueError: If connection not found or type unsupported.
+        """
+        config = self.get(name)
+        if config is None:
+            raise ValueError(f"Connection '{name}' not found")
+        return self.get_adapter(config)

anysite/db/operations/__init__.py

@@ -0,0 +1 @@
+"""Database operations."""

anysite/db/operations/insert.py

@@ -0,0 +1,199 @@
+"""Insert operations for streaming JSON data into database tables."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import IO, Any, TextIO
+
+from anysite.db.adapters.base import DatabaseAdapter
+from anysite.db.config import OnConflict
+from anysite.db.schema.inference import infer_table_schema
+from anysite.db.utils.sanitize import sanitize_table_name
+
+
+def insert_from_stream(
+    adapter: DatabaseAdapter,
+    table: str,
+    stream: TextIO | IO[str],
+    on_conflict: OnConflict = OnConflict.ERROR,
+    conflict_columns: list[str] | None = None,
+    auto_create: bool = False,
+    primary_key: str | None = None,
+    batch_size: int = 100,
+    quiet: bool = False,
+) -> int:
+    """Read JSONL from a stream and insert rows into a database table.
+
+    Each line is parsed as a JSON object. If the input is a JSON array,
+    the array elements are used as rows.
+
+    Args:
+        adapter: Connected database adapter.
+        table: Target table name.
+        stream: Input stream (stdin or file).
+        on_conflict: Conflict resolution strategy.
+        conflict_columns: Columns for upsert conflict detection.
+        auto_create: Create the table automatically if it doesn't exist.
+        primary_key: Primary key column for auto-created tables.
+        batch_size: Number of rows per batch insert.
+        quiet: Suppress progress output.
+
+    Returns:
+        Total number of rows inserted.
+    """
+    rows = _read_json_stream(stream)
+    if not rows:
+        return 0
+
+    # Auto-create table if requested
+    if auto_create and not adapter.table_exists(table):
+        schema = infer_table_schema(table, rows)
+        dialect = _get_dialect(adapter)
+        sql_types = schema.to_sql_types(dialect)
+        adapter.create_table(table, sql_types, primary_key=primary_key)
+        if not quiet:
+            import typer
+
+            safe = sanitize_table_name(table)
+            typer.echo(f"Created table {safe} with {len(sql_types)} columns", err=True)
+
+    total = 0
+    for i in range(0, len(rows), batch_size):
+        batch = rows[i : i + batch_size]
+        count = adapter.insert_batch(
+            table, batch, on_conflict=on_conflict, conflict_columns=conflict_columns
+        )
+        total += count
+
+    return total
+
+
+def _read_json_stream(stream: TextIO | IO[str]) -> list[dict[str, Any]]:
+    """Read JSON or JSONL data from a stream.
+
+    Handles three formats:
+    1. JSON array: [{"a": 1}, {"b": 2}]
+    2. JSONL: one JSON object per line
+    3. Single JSON object: {"a": 1}
+
+    Args:
+        stream: Input stream.
+
+    Returns:
+        List of row dictionaries.
+    """
+    content = stream.read().strip()
+    if not content:
+        return []
+
+    # Try parsing as a JSON array or single object first
+    try:
+        data = json.loads(content)
+        if isinstance(data, list):
+            return [row for row in data if isinstance(row, dict)]
+        elif isinstance(data, dict):
+            return [data]
+    except json.JSONDecodeError:
+        pass
+
+    # Parse as JSONL (one JSON object per line)
+    rows: list[dict[str, Any]] = []
+    for line in content.split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+            if isinstance(obj, dict):
+                rows.append(obj)
+        except json.JSONDecodeError:
+            continue
+
+    return rows
+
+
+def insert_from_file(
+    adapter: DatabaseAdapter,
+    table: str,
+    file_path: Path,
+    on_conflict: OnConflict = OnConflict.ERROR,
+    conflict_columns: list[str] | None = None,
+    auto_create: bool = False,
+    primary_key: str | None = None,
+    batch_size: int = 100,
+    quiet: bool = False,
+) -> int:
+    """Read JSONL from a file and insert rows into a database table.
+
+    Args:
+        adapter: Connected database adapter.
+        table: Target table name.
+        file_path: Path to JSONL/JSON file.
+        on_conflict: Conflict resolution strategy.
+        conflict_columns: Columns for upsert conflict detection.
+        auto_create: Create the table automatically if it doesn't exist.
+        primary_key: Primary key column for auto-created tables.
+        batch_size: Number of rows per batch insert.
+        quiet: Suppress progress output.
+
+    Returns:
+        Total number of rows inserted.
+    """
+    with open(file_path) as f:
+        return insert_from_stream(
+            adapter,
+            table,
+            f,
+            on_conflict=on_conflict,
+            conflict_columns=conflict_columns,
+            auto_create=auto_create,
+            primary_key=primary_key,
+            batch_size=batch_size,
+            quiet=quiet,
+        )
+
+
+def insert_from_stdin(
+    adapter: DatabaseAdapter,
+    table: str,
+    on_conflict: OnConflict = OnConflict.ERROR,
+    conflict_columns: list[str] | None = None,
+    auto_create: bool = False,
+    primary_key: str | None = None,
+    batch_size: int = 100,
+    quiet: bool = False,
+) -> int:
+    """Read JSONL from stdin and insert rows.
+
+    Args:
+        adapter: Connected database adapter.
+        table: Target table name.
+        on_conflict: Conflict resolution strategy.
+        conflict_columns: Columns for upsert conflict detection.
+        auto_create: Create the table automatically if it doesn't exist.
+        primary_key: Primary key column for auto-created tables.
+        batch_size: Number of rows per batch insert.
+        quiet: Suppress progress output.
+
+    Returns:
+        Total number of rows inserted.
+    """
+    return insert_from_stream(
+        adapter,
+        table,
+        sys.stdin,
+        on_conflict=on_conflict,
+        conflict_columns=conflict_columns,
+        auto_create=auto_create,
+        primary_key=primary_key,
+        batch_size=batch_size,
+        quiet=quiet,
+    )
+
+
+def _get_dialect(adapter: DatabaseAdapter) -> str:
+    """Get the SQL dialect name from an adapter."""
+    info = adapter.get_server_info()
+    return info.get("type", "sqlite")

anysite/db/operations/query.py

@@ -0,0 +1,43 @@
+"""Query operations for database tables."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from anysite.db.adapters.base import DatabaseAdapter
+
+
+def execute_query(
+    adapter: DatabaseAdapter,
+    sql: str,
+    params: tuple[Any, ...] | None = None,
+) -> list[dict[str, Any]]:
+    """Execute a SQL query and return results.
+
+    Args:
+        adapter: Connected database adapter.
+        sql: SQL query string.
+        params: Optional query parameters.
+
+    Returns:
+        List of row dictionaries.
+    """
+    return adapter.fetch_all(sql, params)
+
+
+def execute_query_from_file(
+    adapter: DatabaseAdapter,
+    file_path: Path,
+) -> list[dict[str, Any]]:
+    """Execute a SQL query from a file.
+
+    Args:
+        adapter: Connected database adapter.
+        file_path: Path to SQL file.
+
+    Returns:
+        List of row dictionaries.
+    """
+    sql = file_path.read_text().strip()
+    return execute_query(adapter, sql)

anysite/db/schema/__init__.py

@@ -0,0 +1 @@
+"""Schema inference for database tables."""