ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/persistence/db/dialect.py

@@ -0,0 +1,380 @@
+"""
+Database dialect adapter for SQLite and PostgreSQL compatibility.
+
+Provides lightweight SQL translation to support both SQLite and PostgreSQL
+backends with a single connection string configuration.
+"""
+
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Optional
+from urllib.parse import unquote, urlparse
+
+if TYPE_CHECKING:
+    from ciris_engine.logic.persistence.db.query_builder import QueryBuilder
+
+
+class Dialect(str, Enum):
+    """Supported database dialects."""
+
+    SQLITE = "sqlite"
+    POSTGRESQL = "postgresql"
+
+
+def parse_postgres_url(url: str) -> tuple[str, str, str, int, str, str, str]:
+    """Parse PostgreSQL URL handling special characters in password.
+
+    Handles passwords with special characters (@, }, ], {, etc.) that break urlparse.
+
+    Args:
+        url: PostgreSQL connection string
+
+    Returns:
+        Tuple of (scheme, username, password, port, host, database, params)
+
+    Raises:
+        ValueError: If URL format is invalid
+    """
+    import re
+
+    # Pattern: postgresql://username:password@host:port/database?params
+    # Password may contain special chars INCLUDING @, so we need to parse carefully
+    # The password is everything after the first : and before the LAST @
+    # Use non-greedy match for scheme/user, greedy for password (can contain @ or be empty)
+    pattern = r"^(postgres(?:ql)?):\/\/([^:]+):(.*)@([^:\/\?]+):(\d+)\/([^?]+)(\?.*)?$"
+    match = re.match(pattern, url)
+
+    if not match:
+        raise ValueError(f"Invalid PostgreSQL URL format: {url}")
+
+    scheme, username, password_and_host, host, port, database, params = match.groups()
+
+    # Now we need to split password from host at the LAST @
+    # Find the last @ that's followed by host:port pattern
+    # The host part should not contain @ (it's the part after the last @)
+    last_at_idx = password_and_host.rfind("@")
+    if last_at_idx == -1:
+        # No @ found - this means password_and_host is just the password (empty host captured)
+        password = password_and_host
+    else:
+        # Split at the last @
+        password = password_and_host[:last_at_idx]
+        # The part after @ should match our captured host
+        rest = password_and_host[last_at_idx + 1 :]
+        if rest != host:
+            # Host mismatch - means our regex didn't capture correctly
+            # Fall back to treating everything as password
+            password = password_and_host
+
+    # URL-decode the password component (handles %XX encoding)
+    password = unquote(password) if password else ""
+    params = params or ""
+
+    return scheme, username, password, int(port), host, database, params
+
+
+class DialectAdapter:
+    """Translates SQL between SQLite and PostgreSQL dialects.
+
+    This lightweight adapter enables the CIRIS persistence layer to work
+    with both SQLite (development/small deployments) and PostgreSQL
+    (production/scale) without code changes.
+
+    Design Philosophy:
+    - Minimal abstraction (no ORM overhead)
+    - Strategic translation of 5 key patterns
+    - Backward compatible (SQLite default)
+    - Connection string determines dialect
+    """
+
+    def __init__(self, connection_string: str):
+        """Initialize adapter from connection string.
+
+        Args:
+            connection_string: Database URL (sqlite://path or postgresql://...)
+        """
+        # Quick check for PostgreSQL scheme
+        if connection_string.startswith(("postgresql://", "postgres://")):
+            # Use robust parser that handles special characters in passwords
+            try:
+                _scheme, _user, _password, _port, _host, _database, _params = parse_postgres_url(connection_string)
+                self.dialect = Dialect.POSTGRESQL
+                self.db_url = connection_string
+                self.db_path = ""  # Empty string for PostgreSQL (not a file path)
+            except ValueError as e:
+                # Fall back to urlparse if custom parser fails
+                import logging
+
+                logger = logging.getLogger(__name__)
+                logger.warning(f"Failed to parse PostgreSQL URL with custom parser: {e}")
+                logger.warning("Falling back to standard urlparse - may fail with special chars in password")
+
+                parsed = urlparse(connection_string)
+                self.dialect = Dialect.POSTGRESQL
+                self.db_url = connection_string
+                self.db_path = ""
+        else:
+            # Use standard urlparse for SQLite and other schemes
+            parsed = urlparse(connection_string)
+
+            if parsed.scheme in ("sqlite", "sqlite3", ""):
+                self.dialect = Dialect.SQLITE
+                # For SQLite, store the path (with or without leading //)
+                self.db_path = parsed.path or connection_string
+                self.db_url = connection_string
+            else:
+                # Default to SQLite for backward compatibility
+                self.dialect = Dialect.SQLITE
+                self.db_path = connection_string
+                self.db_url = connection_string
+
+    def upsert(
+        self,
+        table: str,
+        columns: list[str],
+        conflict_columns: list[str],
+        update_columns: Optional[list[str]] = None,
+    ) -> str:
+        """Generate UPSERT statement for the target dialect.
+
+        Translates INSERT OR REPLACE (SQLite) to INSERT ... ON CONFLICT (Postgres).
+
+        Args:
+            table: Table name
+            columns: All column names to insert
+            conflict_columns: Columns that define uniqueness constraint
+            update_columns: Columns to update on conflict (defaults to all non-conflict columns)
+
+        Returns:
+            Dialect-specific UPSERT SQL statement
+        """
+        if update_columns is None:
+            # Update all columns except conflict columns
+            update_columns = [col for col in columns if col not in conflict_columns]
+
+        placeholders = ", ".join([self.placeholder()] * len(columns))
+        columns_str = ", ".join(columns)
+
+        if self.dialect == Dialect.SQLITE:
+            # SQLite: INSERT OR REPLACE
+            return f"""
+INSERT OR REPLACE INTO {table}
+({columns_str})
+VALUES ({placeholders})
+"""
+
+        # PostgreSQL: INSERT ... ON CONFLICT ... DO UPDATE
+        conflict_str = ", ".join(conflict_columns)
+        updates = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_columns])
+
+        return f"""
+INSERT INTO {table}
+({columns_str})
+VALUES ({placeholders})
+ON CONFLICT ({conflict_str})
+DO UPDATE SET {updates}
+"""
+
+    def json_extract(self, column: str, json_path: str) -> str:
+        """Generate JSON field extraction for the target dialect.
+
+        Translates json_extract() (SQLite) to JSONB operators (Postgres).
+
+        Args:
+            column: Column name containing JSON
+            json_path: JSON path ($.field.subfield)
+
+        Returns:
+            Dialect-specific JSON extraction expression
+        """
+        if self.dialect == Dialect.SQLITE:
+            # SQLite: json_extract(column, '$.path')
+            return f"json_extract({column}, '{json_path}')"
+
+        # PostgreSQL: column->'field'->>'subfield' or column->>'field'
+        # Convert $.field.subfield to JSONB path
+        path_parts = json_path.lstrip("$").strip(".").split(".")
+
+        if not path_parts or not path_parts[0]:
+            return f"{column}"
+
+        # Build JSONB accessor chain
+        # All intermediate paths use -> (returns JSONB)
+        # Final path uses ->> (returns text)
+        expr = column
+        for i, part in enumerate(path_parts):
+            if i == len(path_parts) - 1:
+                # Last element: extract as text
+                expr = f"{expr}->>'{part}'"
+            else:
+                # Intermediate: keep as JSONB
+                expr = f"{expr}->'{part}'"
+
+        return expr
+
+    def insert_or_ignore(self, table: str, columns: list[str], conflict_columns: Optional[list[str]] = None) -> str:
+        """Generate INSERT OR IGNORE statement for the target dialect.
+
+        Args:
+            table: Table name
+            columns: Column names to insert
+            conflict_columns: Columns that define uniqueness (required for PostgreSQL)
+
+        Returns:
+            Dialect-specific INSERT OR IGNORE SQL statement
+        """
+        placeholders = ", ".join(["?"] * len(columns))
+        columns_str = ", ".join(columns)
+
+        if self.dialect == Dialect.SQLITE:
+            # SQLite: INSERT OR IGNORE
+            return f"INSERT OR IGNORE INTO {table} ({columns_str}) VALUES ({placeholders})"
+
+        # PostgreSQL: INSERT ... ON CONFLICT DO NOTHING
+        # Requires conflict columns to be specified
+        if not conflict_columns:
+            # If no conflict columns specified, use all columns
+            conflict_columns = columns
+
+        conflict_str = ", ".join(conflict_columns)
+        return f"INSERT INTO {table} ({columns_str}) VALUES ({placeholders}) ON CONFLICT ({conflict_str}) DO NOTHING"
+
+    def pragma(self, statement: str) -> Optional[str]:
+        """Handle PRAGMA statements (SQLite-specific).
+
+        Args:
+            statement: PRAGMA statement
+
+        Returns:
+            Statement for SQLite, None for PostgreSQL
+        """
+        if self.dialect == Dialect.SQLITE:
+            return statement
+        # PostgreSQL doesn't use PRAGMA - return None to skip
+        return None
+
+    def placeholder(self) -> str:
+        """Return parameter placeholder for the target dialect.
+
+        Returns:
+            '?' for SQLite, '%s' for PostgreSQL
+        """
+        if self.dialect == Dialect.SQLITE:
+            return "?"
+        return "%s"
+
+    def is_sqlite(self) -> bool:
+        """Check if using SQLite dialect."""
+        return self.dialect == Dialect.SQLITE
+
+    def is_postgresql(self) -> bool:
+        """Check if using PostgreSQL dialect."""
+        return self.dialect == Dialect.POSTGRESQL
+
+    def translate_placeholders(self, sql: str) -> str:
+        """Translate SQLite '?' placeholders to PostgreSQL '%s' placeholders."""
+        if not self.is_postgresql():
+            return sql
+        # Replace ? with %s for PostgreSQL
+        translated = sql.replace("?", "%s")
+        if "?" in sql:
+            import logging
+
+            logger = logging.getLogger(__name__)
+            logger.debug(f"DEBUG translate_placeholders: {sql[:100]}... -> {translated[:100]}...")
+        return translated
+
+    def extract_scalar(self, row: Optional[Any]) -> Optional[Any]:
+        """Extract scalar value from cursor fetchone() result.
+
+        Handles differences between SQLite and PostgreSQL row objects.
+
+        Args:
+            row: Result from cursor.fetchone()
+
+        Returns:
+            First column value from the row, or None if row is None
+
+        Note:
+            - SQLite: Row objects support both integer and dict-like indexing
+            - PostgreSQL: Row objects may only support dict-like indexing
+        """
+        if row is None:
+            return None
+
+        # Try integer indexing first (works for SQLite and some PostgreSQL drivers)
+        try:
+            return row[0]
+        except (KeyError, TypeError):
+            # Fall back to dict-like access for PostgreSQL
+            # Get first column name and access by key
+            if hasattr(row, "keys"):
+                keys = row.keys()
+                if keys:
+                    # Convert keys to list since dict_keys/odict_keys don't support indexing
+                    return row[list(keys)[0]]
+            return None
+
+    def get_query_builder(self) -> "QueryBuilder":
+        """Get a query builder for this dialect.
+
+        Returns:
+            QueryBuilder instance configured for this dialect
+        """
+        from ciris_engine.logic.persistence.db.query_builder import QueryBuilder
+
+        return QueryBuilder(self)
+
+    def insert_ignore_node_sql(self) -> str:
+        """Get INSERT OR IGNORE SQL for graph_nodes.
+
+        Handles dialect-specific conflict resolution for the graph_nodes table.
+        Uses PRIMARY KEY constraint (node_id, scope).
+
+        Returns:
+            SQL string with ? placeholders (will be translated by cursor wrapper)
+        """
+        return self.get_query_builder().insert_ignore_node()
+
+    def insert_ignore_edge_sql(self) -> str:
+        """Get INSERT OR IGNORE SQL for graph_edges.
+
+        Handles dialect-specific conflict resolution for the graph_edges table.
+        Uses PRIMARY KEY constraint (edge_id).
+
+        Returns:
+            SQL string with ? placeholders (will be translated by cursor wrapper)
+        """
+        return self.get_query_builder().insert_ignore_edge()
+
+
+# Global adapter instance
+_adapter: Optional[DialectAdapter] = None
+
+
+def init_dialect(connection_string: str = "data/ciris.db") -> DialectAdapter:
+    """Initialize global dialect adapter.
+
+    Args:
+        connection_string: Database URL (defaults to SQLite for backward compatibility)
+
+    Returns:
+        Initialized DialectAdapter instance
+    """
+    global _adapter
+    _adapter = DialectAdapter(connection_string)
+    return _adapter
+
+
+def get_adapter() -> DialectAdapter:
+    """Get global dialect adapter instance.
+
+    Returns:
+        Global DialectAdapter instance
+
+    Raises:
+        RuntimeError: If adapter not initialized
+    """
+    if _adapter is None:
+        # Auto-initialize with SQLite default for backward compatibility
+        return init_dialect()
+    return _adapter

ciris_engine/logic/persistence/db/execution_helpers.py

@@ -0,0 +1,216 @@
+"""Database execution helpers for PostgreSQL/SQLite abstraction.
+
+This module provides database-agnostic helper functions to reduce cognitive
+complexity in database initialization and migration code.
+"""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, List, Optional, Set
+
+from ciris_engine.logic.persistence.db.dialect import get_adapter
+
+logger = logging.getLogger(__name__)
+
+
+def execute_sql_statements(conn: Any, sql_statements: List[str], adapter: Any) -> None:
+    """Execute SQL statements using appropriate method for database backend.
+
+    Args:
+        conn: Database connection object
+        sql_statements: List of SQL statements to execute
+        adapter: Database adapter for dialect detection
+
+    PostgreSQL requires individual cursor.execute() calls,
+    SQLite can use executescript() for batch execution.
+    """
+    if adapter.is_postgresql():
+        cursor = conn.cursor()
+        try:
+            for statement in sql_statements:
+                if statement.strip():
+                    cursor.execute(statement)
+        finally:
+            cursor.close()
+    else:
+        # SQLite uses executescript for batch execution
+        combined_sql = ";\n".join(sql_statements)
+        conn.executescript(combined_sql)
+
+
+def _update_dollar_quote_state(
+    tag: str, in_dollar_quote: bool, current_tag: Optional[str]
+) -> tuple[bool, Optional[str]]:
+    """Update dollar quote tracking state based on matched tag.
+
+    Args:
+        tag: Matched dollar quote tag (e.g., "$$", "$func$")
+        in_dollar_quote: Current state - inside dollar-quoted block?
+        current_tag: Current opening tag we're tracking
+
+    Returns:
+        Tuple of (new_in_dollar_quote, new_current_tag)
+    """
+    if not in_dollar_quote:
+        # Starting a new dollar-quoted block
+        return True, tag
+    elif tag == current_tag:
+        # Found matching closing tag
+        return False, None
+    return in_dollar_quote, current_tag
+
+
+def _should_finalize_statement(line: str, in_dollar_quote: bool) -> bool:
+    """Check if line terminates a statement (semicolon at end, not in dollar quote).
+
+    Args:
+        line: SQL line to check
+        in_dollar_quote: Whether we're currently inside a dollar-quoted block
+
+    Returns:
+        True if this line terminates a statement
+    """
+    return ";" in line and not in_dollar_quote and line.strip().endswith(";")
+
+
+def _finalize_statement(current_statement: List[str]) -> Optional[str]:
+    """Finalize current statement by joining and stripping.
+
+    Args:
+        current_statement: List of lines making up the statement
+
+    Returns:
+        Finalized statement string, or None if empty
+    """
+    statement = "\n".join(current_statement).strip()
+    return statement if statement else None
+
+
+def split_sql_statements(table_sql: str) -> List[str]:
+    """Split multi-statement SQL into individual statements.
+
+    Handles PostgreSQL dollar-quoted blocks correctly, including both
+    simple ($$) and tagged ($identifier$) dollar quotes.
+
+    Args:
+        table_sql: SQL string potentially containing multiple statements
+
+    Returns:
+        List of individual SQL statements (stripped, non-empty)
+    """
+    import re
+
+    # Quick check: if no dollar quotes, use simple split
+    if "$" not in table_sql:
+        return [s.strip() for s in table_sql.split(";") if s.strip()]
+
+    statements: List[str] = []
+    current_statement: List[str] = []
+    in_dollar_quote = False
+    current_tag: Optional[str] = None
+
+    # Regex to match dollar quote tags: $$ or $identifier$
+    # Matches: $$, $func$, $BODY$, $tag123$, etc.
+    dollar_quote_pattern = re.compile(r"\$([a-zA-Z_]\w*)?\$")
+
+    lines = table_sql.split("\n")
+    for line in lines:
+        # Find all dollar quotes in this line and update state
+        matches = list(dollar_quote_pattern.finditer(line))
+        for match in matches:
+            tag = match.group(0)  # Full match like $$ or $func$
+            in_dollar_quote, current_tag = _update_dollar_quote_state(tag, in_dollar_quote, current_tag)
+
+        current_statement.append(line)
+
+        # Split on semicolon only if not in dollar-quoted block
+        if _should_finalize_statement(line, in_dollar_quote):
+            statement = _finalize_statement(current_statement)
+            if statement:
+                statements.append(statement)
+            current_statement = []
+
+    # Add any remaining statement
+    statement = _finalize_statement(current_statement)
+    if statement:
+        statements.append(statement)
+
+    return statements
+
+
+def mask_password_in_url(db_url: str) -> str:
+    """Mask password in database URL for safe logging.
+
+    Args:
+        db_url: Database URL (e.g., postgresql://user:pass@host/db)
+
+    Returns:
+        URL with password replaced by ****
+
+    Examples:
+        postgresql://user:PASSWORD@host/db -> postgresql://user:****@host/db
+        sqlite:///path/to/db.sqlite -> sqlite:///path/to/db.sqlite
+    """
+    if "://" in db_url and "@" in db_url:
+        protocol, rest = db_url.split("://", 1)
+        # Find the last @ symbol (separates credentials from host)
+        if "@" in rest:
+            last_at_index = rest.rfind("@")
+            credentials = rest[:last_at_index]
+            host = rest[last_at_index + 1 :]
+            if ":" in credentials:
+                username, _ = credentials.split(":", 1)
+                return f"{protocol}://{username}:****@{host}"
+    return db_url
+
+
+def get_applied_migrations(conn: Any) -> Set[str]:
+    """Retrieve set of already-applied migration names.
+
+    Args:
+        conn: Database connection with migrations table
+
+    Returns:
+        Set of migration names that have been applied
+    """
+    cursor = conn.cursor()
+    cursor.execute("SELECT migration_name FROM migrations")
+    return {row["migration_name"] for row in cursor.fetchall()}
+
+
+def get_pending_migrations(migrations_dir: Path, applied_migrations: Set[str]) -> List[Path]:
+    """Get list of pending migration files in order.
+
+    Args:
+        migrations_dir: Directory containing migration SQL files
+        applied_migrations: Set of migration names already applied
+
+    Returns:
+        List of migration file paths sorted by name, excluding applied ones
+    """
+    all_migrations = sorted(migrations_dir.glob("*.sql"))
+    return [m for m in all_migrations if m.name not in applied_migrations]
+
+
+def record_migration(conn: Any, migration_name: str, adapter: Any) -> None:
+    """Record a migration as applied.
+
+    Args:
+        conn: Database connection
+        migration_name: Name of migration file (e.g., "001_initial.sql")
+        adapter: Database adapter for parameter placeholder detection
+    """
+    cursor = conn.cursor()
+    applied_at = datetime.now().isoformat()
+
+    if adapter.is_postgresql():
+        cursor.execute(
+            "INSERT INTO migrations (migration_name, applied_at) VALUES (%s, %s)",
+            (migration_name, applied_at),
+        )
+    else:
+        cursor.execute(
+            "INSERT INTO migrations (migration_name, applied_at) VALUES (?, ?)",
+            (migration_name, applied_at),
+        )