icdev 0.0.3__py3-none-any.whl

tools/db/storage.py

@@ -0,0 +1,1080 @@
+# CUI // SP-CTI
+"""Storage Abstraction Layer — database-agnostic connection factory (D-DB-20).
+
+Provides a unified interface over SQLite and PostgreSQL so that all 255+ tool
+files can use the same API regardless of which backend is configured.
+
+Design principles:
+  - Drop-in compatible: returned connections expose the same API as sqlite3
+    (.execute, .fetchall, .fetchone, .commit, .close, .row_factory-like behavior)
+  - Placeholder translation: SQL using ``?`` (sqlite3 style) is auto-translated
+    to ``%s`` (psycopg2 style) when PostgreSQL is the backend
+  - Context manager: ``with get_connection() as conn:`` for auto-cleanup
+  - Config-driven: env vars + args/storage_config.yaml, defaults to SQLite
+  - Zero new deps for SQLite path; psycopg2 only required for PostgreSQL
+
+Usage (drop-in replacement for sqlite3.connect):
+    from tools.db.storage import get_connection
+
+    with get_connection() as conn:
+        rows = conn.execute("SELECT * FROM projects WHERE id = ?", (pid,)).fetchall()
+        for row in rows:
+            print(row["name"])  # dict-like access
+
+    # Or without context manager (legacy compatibility):
+    conn = get_connection()
+    try:
+        conn.execute("INSERT INTO t (a) VALUES (?)", (1,))
+        conn.commit()
+    finally:
+        conn.close()
+
+Configuration (args/storage_config.yaml or env vars):
+    ICDEV_STORAGE_BACKEND=postgresql   # or sqlite (default)
+    ICDEV_PG_HOST=localhost
+    ICDEV_PG_PORT=5432
+    ICDEV_PG_DATABASE=icdev
+    ICDEV_PG_USERNAME=icdev
+    ICDEV_PG_PASSWORD=secret           # or use ICDEV_PG_SECRET_REF=env:PG_PASS
+"""
+
+import logging
+import os
+import re
+import sqlite3
+import threading
+from contextlib import contextmanager
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union
+
+logger = logging.getLogger("icdev.db.storage")
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+
+
+# ---------------------------------------------------------------------------
+# Backend enum
+# ---------------------------------------------------------------------------
+
+class StorageBackend(Enum):
+    """Supported database backends."""
+    SQLITE = "sqlite"
+    POSTGRESQL = "postgresql"
+
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+class StorageConfig:
+    """Resolved storage configuration from env vars and/or YAML.
+
+    Resolution order (highest priority first):
+      1. Explicit keyword arguments
+      2. Environment variables (ICDEV_STORAGE_BACKEND, ICDEV_PG_*, ICDEV_DB_PATH)
+      3. args/storage_config.yaml
+      4. Defaults (SQLite at data/icdev.db)
+    """
+
+    def __init__(
+        self,
+        backend: Optional[str] = None,
+        db_path: Optional[Union[str, Path]] = None,
+        pg_host: Optional[str] = None,
+        pg_port: Optional[int] = None,
+        pg_database: Optional[str] = None,
+        pg_username: Optional[str] = None,
+        pg_password: Optional[str] = None,
+        pg_connection_string: Optional[str] = None,
+        pg_pool_size: int = 20,
+    ):
+        # Load YAML config as base (lowest priority)
+        yaml_cfg = self._load_yaml_config()
+
+        # Backend resolution
+        raw_backend = (
+            backend
+            or os.environ.get("ICDEV_STORAGE_BACKEND")
+            or yaml_cfg.get("backend", "sqlite")
+        )
+        self.backend = StorageBackend(raw_backend.lower().strip())
+
+        # SQLite config
+        self.db_path = Path(
+            db_path
+            or os.environ.get("ICDEV_DB_PATH")
+            or yaml_cfg.get("sqlite", {}).get("path")
+            or str(_PROJECT_ROOT / "data" / "icdev.db")
+        )
+
+        # PostgreSQL config
+        pg_yaml = yaml_cfg.get("postgresql", {})
+        self.pg_host = (
+            pg_host
+            or os.environ.get("ICDEV_PG_HOST")
+            or pg_yaml.get("host", "localhost")
+        )
+        self.pg_port = int(
+            pg_port
+            or os.environ.get("ICDEV_PG_PORT")
+            or pg_yaml.get("port", 5432)
+        )
+        self.pg_database = (
+            pg_database
+            or os.environ.get("ICDEV_PG_DATABASE")
+            or pg_yaml.get("database", "icdev")
+        )
+        self.pg_username = (
+            pg_username
+            or os.environ.get("ICDEV_PG_USERNAME")
+            or pg_yaml.get("username", "icdev")
+        )
+        self.pg_password = self._resolve_password(
+            pg_password, pg_yaml
+        )
+        self.pg_connection_string = (
+            pg_connection_string
+            or os.environ.get("ICDEV_PG_CONNECTION_STRING")
+            or pg_yaml.get("connection_string")
+        )
+        self.pg_pool_size = int(
+            os.environ.get("ICDEV_PG_POOL_SIZE")
+            or pg_yaml.get("pool_size", pg_pool_size)
+        )
+
+    @staticmethod
+    def _resolve_password(explicit: Optional[str], yaml_cfg: Dict) -> str:
+        """Resolve PostgreSQL password from explicit, env var, or YAML."""
+        if explicit:
+            return explicit
+
+        env_pw = os.environ.get("ICDEV_PG_PASSWORD")
+        if env_pw:
+            return env_pw
+
+        # Support secret reference pattern (D-DB-13)
+        secret_ref = (
+            os.environ.get("ICDEV_PG_SECRET_REF")
+            or yaml_cfg.get("secret_ref", "")
+        )
+        if secret_ref.startswith("env:"):
+            var_name = secret_ref[4:]
+            env_val = os.environ.get(var_name)
+            if env_val:
+                return env_val
+            # Fall through to YAML password if env var not set
+
+        return yaml_cfg.get("password", "")
+
+    @staticmethod
+    def _load_yaml_config() -> Dict[str, Any]:
+        """Load args/storage_config.yaml if it exists."""
+        config_path = _PROJECT_ROOT / "args" / "storage_config.yaml"
+        if not config_path.exists():
+            return {}
+        try:
+            import yaml
+            with open(config_path, "r") as f:
+                return yaml.safe_load(f) or {}
+        except ImportError:
+            # Fall back to basic parsing if PyYAML not available
+            return {}
+        except Exception as exc:
+            logger.warning("Failed to load storage_config.yaml: %s", exc)
+            return {}
+
+    def pg_dsn(self) -> str:
+        """Build PostgreSQL DSN string."""
+        if self.pg_connection_string:
+            return self.pg_connection_string
+        pw_part = f":{self.pg_password}" if self.pg_password else ""
+        return (
+            f"host={self.pg_host} port={self.pg_port} "
+            f"dbname={self.pg_database} user={self.pg_username} "
+            f"password={self.pg_password}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Placeholder translation
+# ---------------------------------------------------------------------------
+
+# Regex to find ? placeholders that are NOT inside single-quoted strings.
+# This handles the common case; edge cases (escaped quotes, $$ strings)
+# are rare in ICDEV's tool SQL.
+_QMARK_RE = re.compile(
+    r"""
+    '(?:[^'\\]|\\.)*'   # single-quoted string — skip
+    |
+    (\?)                 # ? placeholder — capture group 1
+    """,
+    re.VERBOSE | re.DOTALL,
+)
+
+
+def _translate_placeholders(sql: str) -> str:
+    """Convert sqlite3-style ``?`` placeholders to psycopg2-style ``%s``.
+
+    Leaves ``?`` inside string literals untouched.
+    Also escapes literal ``%`` inside single-quoted strings so psycopg2
+    doesn't misinterpret them as format specifiers (e.g. LIKE '%foo%').
+    """
+    def _replace(m: re.Match) -> str:
+        if m.group(1):
+            return "%s"
+        # It's a quoted string — escape any % inside it for psycopg2
+        text = m.group(0)
+        if "%" in text:
+            # Double the % to escape for psycopg2
+            return text.replace("%", "%%")
+        return text
+
+    return _QMARK_RE.sub(_replace, sql)
+
+
+# Regex for INSERT OR REPLACE/IGNORE translation
+_INSERT_OR_REPLACE_RE = re.compile(
+    r"\bINSERT\s+OR\s+REPLACE\b", re.IGNORECASE
+)
+_INSERT_OR_IGNORE_RE = re.compile(
+    r"\bINSERT\s+OR\s+IGNORE\b", re.IGNORECASE
+)
+
+# Cache for unique constraint lookups
+_UNIQUE_CONSTRAINT_CACHE: Dict[str, List[str]] = {}
+
+
+def _get_unique_constraint_cols(table_name: str, insert_cols: List[str]) -> Optional[List[str]]:
+    """Find unique constraint columns for a table that overlap with insert columns.
+
+    Uses a module-level cache to avoid repeated PG metadata queries.
+    Returns the best matching unique constraint columns, or None.
+    """
+    cache_key = table_name.lower()
+    if cache_key in _UNIQUE_CONSTRAINT_CACHE:
+        cached = _UNIQUE_CONSTRAINT_CACHE[cache_key]
+        return cached if cached else None
+
+    try:
+        import psycopg2
+        cfg = StorageConfig()
+        if cfg.backend != StorageBackend.POSTGRESQL:
+            _UNIQUE_CONSTRAINT_CACHE[cache_key] = []
+            return None
+        conn = psycopg2.connect(cfg.pg_dsn())
+        cur = conn.cursor()
+        # Find UNIQUE constraints (not PK) on this table
+        cur.execute("""
+            SELECT kcu.column_name
+            FROM information_schema.table_constraints tc
+            JOIN information_schema.key_column_usage kcu
+              ON tc.constraint_name = kcu.constraint_name
+             AND tc.table_schema = kcu.table_schema
+            WHERE tc.table_name = %s
+              AND tc.constraint_type = 'UNIQUE'
+            ORDER BY kcu.ordinal_position
+        """, (table_name,))
+        unique_cols = [row[0] for row in cur.fetchall()]
+        cur.close()
+        conn.close()
+
+        insert_set = {c.lower() for c in insert_cols}
+        matching = [c for c in unique_cols if c.lower() in insert_set]
+        _UNIQUE_CONSTRAINT_CACHE[cache_key] = matching
+        return matching if matching else None
+    except Exception:
+        _UNIQUE_CONSTRAINT_CACHE[cache_key] = []
+        return None
+
+
+def _translate_sqlite_sql(sql: str) -> str:
+    """Translate SQLite-specific SQL syntax to PostgreSQL.
+
+    Handles:
+      - ? -> %s placeholders
+      - INSERT OR REPLACE -> INSERT ... ON CONFLICT DO UPDATE
+      - INSERT OR IGNORE -> INSERT ... ON CONFLICT DO NOTHING
+      - datetime('now') -> NOW()
+    """
+    sql = _translate_placeholders(sql)
+
+    # INSERT OR IGNORE -> INSERT ... ON CONFLICT DO NOTHING
+    if _INSERT_OR_IGNORE_RE.search(sql):
+        sql = _INSERT_OR_IGNORE_RE.sub("INSERT", sql)
+        # Add ON CONFLICT DO NOTHING before any RETURNING clause or at end
+        if "RETURNING" in sql.upper():
+            sql = re.sub(
+                r"(\s+RETURNING)", r" ON CONFLICT DO NOTHING\1",
+                sql, count=1, flags=re.IGNORECASE,
+            )
+        else:
+            sql = sql.rstrip().rstrip(";") + " ON CONFLICT DO NOTHING"
+
+    # INSERT OR REPLACE -> INSERT ... ON CONFLICT DO UPDATE
+    elif _INSERT_OR_REPLACE_RE.search(sql):
+        sql = _INSERT_OR_REPLACE_RE.sub("INSERT", sql)
+        # Extract table name and column names from INSERT INTO table (col1, col2, ...)
+        table_match = re.search(
+            r"INSERT\s+INTO\s+(\w+)\s*\(([^)]+)\)", sql, re.IGNORECASE
+        )
+        if table_match:
+            table_name = table_match.group(1)
+            cols = [c.strip() for c in table_match.group(2).split(",")]
+            # Determine conflict target:
+            # If first column is 'id', use it as PK.
+            # Otherwise, look for unique constraint columns among the inserted cols.
+            conflict_cols = None
+            if cols[0].lower() == "id":
+                conflict_cols = [cols[0]]
+            else:
+                # Try to find unique constraint columns from the connection cache
+                conflict_cols = _get_unique_constraint_cols(table_name, cols)
+                if not conflict_cols:
+                    # Fallback: use first column
+                    conflict_cols = [cols[0]]
+
+            conflict_str = ", ".join(conflict_cols)
+            update_cols = [c for c in cols if c.lower() not in
+                          {cc.lower() for cc in conflict_cols}]
+            if update_cols:
+                set_clause = ", ".join(
+                    f"{c} = EXCLUDED.{c}" for c in update_cols
+                )
+                conflict = f" ON CONFLICT ({conflict_str}) DO UPDATE SET {set_clause}"
+            else:
+                conflict = f" ON CONFLICT ({conflict_str}) DO NOTHING"
+            sql = sql.rstrip().rstrip(";") + conflict
+
+    # datetime('now', '-N days/hours/...') -> NOW() - INTERVAL 'N days/hours/...'
+    def _datetime_replace(m):
+        modifiers = re.findall(r"'([^']*)'", m.group(0))
+        # First arg is 'now'; remaining are modifiers like '-30 days'
+        result = "NOW()"
+        for mod in modifiers[1:]:
+            mod = mod.strip()
+            if mod.startswith("-"):
+                result += f" - INTERVAL '{mod[1:]}'"
+            elif mod.startswith("+"):
+                result += f" + INTERVAL '{mod[1:]}'"
+            else:
+                result += f" + INTERVAL '{mod}'"
+        return result
+
+    sql = re.sub(
+        r"datetime\s*\(\s*'now'\s*(?:,\s*'[^']*'\s*)*\)",
+        _datetime_replace,
+        sql,
+        flags=re.IGNORECASE,
+    )
+
+    # AUTOINCREMENT -> removed (PostgreSQL SERIAL handles this natively)
+    sql = re.sub(r"\s+AUTOINCREMENT\b", "", sql, flags=re.IGNORECASE)
+
+    # INTEGER PRIMARY KEY -> SERIAL PRIMARY KEY (for CREATE TABLE)
+    if re.search(r"\bCREATE\s+TABLE\b", sql, re.IGNORECASE):
+        sql = re.sub(
+            r"\bINTEGER\s+PRIMARY\s+KEY\b",
+            "SERIAL PRIMARY KEY",
+            sql,
+            count=1,
+            flags=re.IGNORECASE,
+        )
+
+    # ROUND(double_precision, int) needs cast in PostgreSQL
+    sql = re.sub(
+        r"\bROUND\(([^,)]+),\s*(\d+)\)",
+        r"ROUND((\1)::numeric, \2)",
+        sql,
+        flags=re.IGNORECASE,
+    )
+
+    # GROUP_CONCAT(x) -> STRING_AGG(x, ',')
+    # GROUP_CONCAT(DISTINCT x) -> STRING_AGG(DISTINCT x, ',')
+    sql = re.sub(
+        r"\bGROUP_CONCAT\(\s*DISTINCT\s+(\w+)\s*\)",
+        r"STRING_AGG(DISTINCT \1::text, ',')",
+        sql,
+        flags=re.IGNORECASE,
+    )
+    sql = re.sub(
+        r"\bGROUP_CONCAT\(([^)]+)\)",
+        r"STRING_AGG(\1::text, ',')",
+        sql,
+        flags=re.IGNORECASE,
+    )
+
+    return sql
+
+
+# ---------------------------------------------------------------------------
+# Row wrapper — supports both integer index and key access
+# ---------------------------------------------------------------------------
+
+class _DualAccessRow:
+    """Wrap a PostgreSQL RealDictRow to support both dict-like and index access.
+
+    SQLite's Row supports row[0] and row["col"]. PostgreSQL's RealDictRow
+    only supports row["col"]. This wrapper adds row[0] support for backward
+    compatibility with code that uses integer indexing.
+    """
+    __slots__ = ("_data", "_keys")
+
+    def __init__(self, row_dict):
+        self._data = row_dict
+        self._keys = list(row_dict.keys())
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self._data[self._keys[key]]
+        return self._data[key]
+
+    def __contains__(self, key):
+        return key in self._data
+
+    def __len__(self):
+        return len(self._data)
+
+    def __iter__(self):
+        return iter(self._data.values())
+
+    def __repr__(self):
+        return repr(self._data)
+
+    def keys(self):
+        return self._data.keys()
+
+    def values(self):
+        return self._data.values()
+
+    def items(self):
+        return self._data.items()
+
+    def get(self, key, default=None):
+        return self._data.get(key, default)
+
+
+def _wrap_row(row, backend):
+    """Wrap a row for dual access if PostgreSQL backend."""
+    if row is None or backend == StorageBackend.SQLITE:
+        return row
+    return _DualAccessRow(row)
+
+
+def _wrap_rows(rows, backend):
+    """Wrap multiple rows for dual access if PostgreSQL backend."""
+    if backend == StorageBackend.SQLITE:
+        return rows
+    return [_DualAccessRow(r) for r in rows]
+
+
+# ---------------------------------------------------------------------------
+# Cursor wrapper
+# ---------------------------------------------------------------------------
+
+class _CursorWrapper:
+    """Wraps a database cursor to provide a uniform API.
+
+    For SQLite, this wraps sqlite3.Cursor directly.
+    For PostgreSQL, this wraps psycopg2 cursor with dict-like Row access
+    and sqlite3-compatible execute (with ? placeholder translation).
+    """
+
+    def __init__(self, cursor: Any, backend: StorageBackend):
+        self._cursor = cursor
+        self._backend = backend
+        self._description = None
+
+    def execute(self, sql: str, params: Any = None) -> "_CursorWrapper":
+        """Execute SQL. Translates SQLite syntax to PostgreSQL."""
+        if self._backend == StorageBackend.POSTGRESQL:
+            sql = _translate_sqlite_sql(sql)
+        if params is None:
+            self._cursor.execute(sql)
+        else:
+            # Convert list to tuple for psycopg2 compatibility
+            if isinstance(params, list):
+                params = tuple(params)
+            self._cursor.execute(sql, params)
+        self._description = self._cursor.description
+        return self
+
+    def executemany(self, sql: str, param_seq: Any) -> "_CursorWrapper":
+        """Execute SQL with multiple parameter sets."""
+        if self._backend == StorageBackend.POSTGRESQL:
+            sql = _translate_sqlite_sql(sql)
+        self._cursor.executemany(sql, param_seq)
+        self._description = self._cursor.description
+        return self
+
+    def executescript(self, sql: str) -> "_CursorWrapper":
+        """Execute a SQL script (multiple statements).
+
+        SQLite has native executescript; PostgreSQL splits on semicolons
+        and executes each statement individually, skipping statements
+        that contain SQLite-specific syntax (AUTOINCREMENT, etc.) since
+        the tables already exist via pg_migrate.py.
+        """
+        if self._backend == StorageBackend.SQLITE:
+            self._cursor.executescript(sql)
+        else:
+            # Split into individual statements and execute each
+            for stmt in sql.split(";"):
+                stmt = stmt.strip()
+                if not stmt:
+                    continue
+                # Translate SQLite-specific syntax to PostgreSQL
+                stmt = _translate_sqlite_sql(stmt)
+                try:
+                    self._cursor.execute(stmt)
+                except Exception as exc:
+                    # Skip "already exists" errors for IF NOT EXISTS
+                    err_msg = str(exc).lower()
+                    if "already exists" in err_msg:
+                        continue
+                    logger.warning("executescript statement failed: %s", exc)
+        return self
+
+    def fetchall(self) -> List[Any]:
+        """Fetch all rows. Supports both integer and key access."""
+        return _wrap_rows(self._cursor.fetchall(), self._backend)
+
+    def fetchone(self) -> Any:
+        """Fetch one row. Supports both integer and key access."""
+        return _wrap_row(self._cursor.fetchone(), self._backend)
+
+    def fetchmany(self, size: int = 100) -> List[Any]:
+        """Fetch many rows. Supports both integer and key access."""
+        return _wrap_rows(self._cursor.fetchmany(size), self._backend)
+
+    @property
+    def description(self):
+        return self._cursor.description
+
+    @property
+    def rowcount(self) -> int:
+        return self._cursor.rowcount
+
+    @property
+    def lastrowid(self) -> Optional[int]:
+        """Last inserted row ID. Native on SQLite; emulated on PostgreSQL."""
+        if self._backend == StorageBackend.SQLITE:
+            return self._cursor.lastrowid
+        # PostgreSQL doesn't have lastrowid on cursor; use RETURNING in SQL
+        return getattr(self._cursor, "lastrowid", None)
+
+    def close(self) -> None:
+        self._cursor.close()
+
+    def __iter__(self):
+        if self._backend == StorageBackend.SQLITE:
+            return iter(self._cursor)
+        return (_DualAccessRow(row) for row in self._cursor)
+
+
+# ---------------------------------------------------------------------------
+# Connection wrapper
+# ---------------------------------------------------------------------------
+
+class StorageConnection:
+    """Database-agnostic connection wrapper.
+
+    Provides the same API as sqlite3.Connection:
+      - execute(sql, params) -> CursorWrapper
+      - commit()
+      - rollback()
+      - close()
+      - cursor()
+      - Context manager support (__enter__ / __exit__)
+
+    Row access is dict-like (column names as keys) for both backends.
+    """
+
+    def __init__(self, raw_conn: Any, backend: StorageBackend,
+                 pool_return=None):
+        self._conn = raw_conn
+        self._backend = backend
+        self._closed = False
+        self._pool_return = pool_return  # callback to return conn to pool
+
+    @property
+    def backend(self) -> StorageBackend:
+        return self._backend
+
+    @property
+    def raw(self) -> Any:
+        """Access the underlying raw connection (sqlite3.Connection or psycopg2 conn).
+
+        Use sparingly — only for backend-specific operations like
+        sqlite3.backup() or PostgreSQL COPY.
+        """
+        return self._conn
+
+    def execute(self, sql: str, params: Any = None) -> _CursorWrapper:
+        """Execute SQL and return a cursor wrapper."""
+        cursor = self.cursor()
+        return cursor.execute(sql, params)
+
+    def executemany(self, sql: str, param_seq: Any) -> _CursorWrapper:
+        """Execute SQL with multiple parameter sets."""
+        cursor = self.cursor()
+        return cursor.executemany(sql, param_seq)
+
+    def executescript(self, sql: str) -> _CursorWrapper:
+        """Execute a SQL script."""
+        cursor = self.cursor()
+        return cursor.executescript(sql)
+
+    def cursor(self) -> _CursorWrapper:
+        """Create a new cursor wrapper."""
+        if self._backend == StorageBackend.POSTGRESQL:
+            import psycopg2.extras
+            raw_cursor = self._conn.cursor(
+                cursor_factory=psycopg2.extras.RealDictCursor
+            )
+        else:
+            raw_cursor = self._conn.cursor()
+        return _CursorWrapper(raw_cursor, self._backend)
+
+    def commit(self) -> None:
+        self._conn.commit()
+
+    def rollback(self) -> None:
+        self._conn.rollback()
+
+    def close(self) -> None:
+        if not self._closed:
+            if self._pool_return:
+                self._pool_return(self._conn)
+            else:
+                self._conn.close()
+            self._closed = True
+
+    def __enter__(self) -> "StorageConnection":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        if exc_type:
+            self.rollback()
+        self.close()
+
+    # Compatibility: allow conn.row_factory reads (no-op setter for PG)
+    @property
+    def row_factory(self):
+        if self._backend == StorageBackend.SQLITE:
+            return self._conn.row_factory
+        return None
+
+    @row_factory.setter
+    def row_factory(self, factory):
+        if self._backend == StorageBackend.SQLITE:
+            self._conn.row_factory = factory
+
+
+# ---------------------------------------------------------------------------
+# PostgreSQL type adapters — return strings for datetime (SQLite compat)
+# ---------------------------------------------------------------------------
+
+_pg_dates_registered = False
+
+
+def _register_pg_string_dates():
+    """Register psycopg2 type casters that return datetime as ISO strings.
+
+    SQLite returns datetime columns as strings. Without this adapter,
+    PostgreSQL returns datetime objects which break json.dumps() across
+    100+ tool files. Registering globally ensures consistent behavior.
+    """
+    global _pg_dates_registered
+    if _pg_dates_registered:
+        return
+    try:
+        import psycopg2.extensions as ext
+
+        # TIMESTAMP / TIMESTAMPTZ → ISO string
+        def _cast_timestamp(value, cur):
+            if value is None:
+                return None
+            return str(value)
+
+        # DATE → ISO string
+        def _cast_date(value, cur):
+            if value is None:
+                return None
+            return str(value)
+
+        # OIDs: 1114=TIMESTAMP, 1184=TIMESTAMPTZ, 1082=DATE
+        TIMESTAMP = ext.new_type((1114,), "TIMESTAMP_STR", _cast_timestamp)
+        TIMESTAMPTZ = ext.new_type((1184,), "TIMESTAMPTZ_STR", _cast_timestamp)
+        DATE = ext.new_type((1082,), "DATE_STR", _cast_date)
+
+        ext.register_type(TIMESTAMP)
+        ext.register_type(TIMESTAMPTZ)
+        ext.register_type(DATE)
+
+        _pg_dates_registered = True
+        logger.debug("Registered psycopg2 string date adapters")
+    except ImportError:
+        pass
+
+
+# ---------------------------------------------------------------------------
+# Connection pool (PostgreSQL only)
+# ---------------------------------------------------------------------------
+
+class _PGPool:
+    """Simple PostgreSQL connection pool using psycopg2.pool.
+
+    Thread-safe. Falls back to direct connections if pooling unavailable.
+    """
+
+    def __init__(self, config: StorageConfig):
+        self._config = config
+        self._pool = None
+        self._lock = threading.Lock()
+
+    def _ensure_pool(self):
+        if self._pool is not None:
+            return
+        with self._lock:
+            if self._pool is not None:
+                return
+            try:
+                from psycopg2 import pool as pg_pool
+                import psycopg2.extensions
+                # Register type adapters so PostgreSQL returns strings for
+                # datetime/date columns, matching SQLite behavior. Without
+                # this, json.dumps() fails across 100+ tools.
+                _register_pg_string_dates()
+                self._pool = pg_pool.ThreadedConnectionPool(
+                    minconn=1,
+                    maxconn=self._config.pg_pool_size,
+                    dsn=self._config.pg_dsn(),
+                )
+                logger.info(
+                    "PostgreSQL pool created: %s:%s/%s (pool_size=%d)",
+                    self._config.pg_host,
+                    self._config.pg_port,
+                    self._config.pg_database,
+                    self._config.pg_pool_size,
+                )
+            except Exception as exc:
+                logger.error("Failed to create PG pool: %s", exc)
+                raise
+
+    def get_connection(self) -> Any:
+        """Get a connection from the pool.
+
+        Sets autocommit=True by default to match SQLite behavior where
+        each statement auto-commits unless wrapped in BEGIN/COMMIT.
+        Without this, psycopg2 wraps everything in a transaction and
+        a single error aborts all subsequent statements.
+        """
+        self._ensure_pool()
+        conn = self._pool.getconn()
+        conn.autocommit = True
+        return conn
+
+    def put_connection(self, conn: Any) -> None:
+        """Return a connection to the pool."""
+        if self._pool:
+            self._pool.putconn(conn)
+
+    def close_all(self) -> None:
+        """Close all pooled connections."""
+        if self._pool:
+            self._pool.closeall()
+            self._pool = None
+
+
+# ---------------------------------------------------------------------------
+# Engine (singleton per database)
+# ---------------------------------------------------------------------------
+
+class StorageEngine:
+    """Database engine managing configuration and connection lifecycle.
+
+    Typically used as a singleton via ``get_engine()``.
+    """
+
+    def __init__(self, config: Optional[StorageConfig] = None):
+        self._config = config or StorageConfig()
+        self._pg_pool: Optional[_PGPool] = None
+
+        if self._config.backend == StorageBackend.POSTGRESQL:
+            self._pg_pool = _PGPool(self._config)
+            logger.info("StorageEngine initialized: PostgreSQL")
+        else:
+            logger.info(
+                "StorageEngine initialized: SQLite (%s)", self._config.db_path
+            )
+
+    @property
+    def backend(self) -> StorageBackend:
+        return self._config.backend
+
+    @property
+    def config(self) -> StorageConfig:
+        return self._config
+
+    def connect(
+        self,
+        db_path: Optional[Union[str, Path]] = None,
+        row_factory: bool = True,
+        validate: bool = False,
+    ) -> StorageConnection:
+        """Create a new database connection.
+
+        Args:
+            db_path: Override SQLite path (ignored for PostgreSQL).
+            row_factory: Enable dict-like row access (default True).
+            validate: Raise FileNotFoundError if SQLite DB missing.
+
+        Returns:
+            StorageConnection wrapping the appropriate backend.
+        """
+        if self._config.backend == StorageBackend.POSTGRESQL:
+            return self._connect_pg()
+        return self._connect_sqlite(db_path, row_factory, validate)
+
+    def _connect_sqlite(
+        self,
+        db_path: Optional[Union[str, Path]] = None,
+        row_factory: bool = True,
+        validate: bool = False,
+    ) -> StorageConnection:
+        """Create a SQLite connection."""
+        path = Path(db_path) if db_path else self._config.db_path
+        if validate and not path.exists():
+            raise FileNotFoundError(
+                f"Database not found: {path}\n"
+                "Run: python tools/db/init_icdev_db.py"
+            )
+        conn = sqlite3.connect(str(path))
+        if row_factory:
+            conn.row_factory = sqlite3.Row
+        return StorageConnection(conn, StorageBackend.SQLITE)
+
+    def _connect_pg(self) -> StorageConnection:
+        """Create a PostgreSQL connection from the pool."""
+        if not self._pg_pool:
+            raise RuntimeError("PostgreSQL pool not initialized")
+        raw_conn = self._pg_pool.get_connection()
+        return StorageConnection(
+            raw_conn, StorageBackend.POSTGRESQL,
+            pool_return=self._pg_pool.put_connection,
+        )
+
+    def close(self) -> None:
+        """Shut down the engine and release all resources."""
+        if self._pg_pool:
+            self._pg_pool.close_all()
+            self._pg_pool = None
+
+    def health_check(self) -> Dict[str, Any]:
+        """Check backend health."""
+        import time
+        start = time.time()
+        try:
+            with self.connect() as conn:
+                conn.execute("SELECT 1")
+            latency_ms = int((time.time() - start) * 1000)
+            return {
+                "status": "healthy",
+                "backend": self._config.backend.value,
+                "latency_ms": latency_ms,
+            }
+        except Exception as exc:
+            return {
+                "status": "unhealthy",
+                "backend": self._config.backend.value,
+                "error": str(exc),
+            }
+
+
+# ---------------------------------------------------------------------------
+# Singleton engine + public API
+# ---------------------------------------------------------------------------
+
+_engine: Optional[StorageEngine] = None
+_engine_lock = threading.Lock()
+
+
+def get_engine(config: Optional[StorageConfig] = None) -> StorageEngine:
+    """Get or create the singleton StorageEngine.
+
+    Args:
+        config: Optional explicit config. If provided on first call, it
+                becomes the engine config. Subsequent calls ignore it.
+
+    Returns:
+        The global StorageEngine instance.
+    """
+    global _engine
+    if _engine is not None:
+        return _engine
+    with _engine_lock:
+        if _engine is not None:
+            return _engine
+        _engine = StorageEngine(config)
+        return _engine
+
+
+def reset_engine() -> None:
+    """Shut down and reset the global engine. Primarily for testing."""
+    global _engine
+    with _engine_lock:
+        if _engine:
+            _engine.close()
+        _engine = None
+
+
+def get_connection(
+    db_path: Optional[Union[str, Path]] = None,
+    row_factory: bool = True,
+    validate: bool = False,
+) -> StorageConnection:
+    """Get a database connection from the global engine.
+
+    This is the primary public API — a drop-in replacement for
+    ``sqlite3.connect()`` and ``tools.compat.db_utils.get_db_connection()``.
+
+    Args:
+        db_path: Override SQLite path (ignored for PostgreSQL).
+        row_factory: Enable dict-like row access (default True).
+        validate: Raise FileNotFoundError if SQLite DB missing.
+
+    Returns:
+        StorageConnection (context manager) wrapping the configured backend.
+
+    Example:
+        with get_connection() as conn:
+            rows = conn.execute("SELECT * FROM projects").fetchall()
+    """
+    return get_engine().connect(db_path=db_path, row_factory=row_factory, validate=validate)
+
+
+# ---------------------------------------------------------------------------
+# Convenience: per-database connections (matching db_utils.py API)
+# ---------------------------------------------------------------------------
+
+def get_icdev_connection(
+    db_path: Optional[Union[str, Path]] = None,
+    validate: bool = False,
+    row_factory: bool = True,
+) -> StorageConnection:
+    """Get connection to the ICDEV database.
+
+    For PostgreSQL backend, connects to the configured PG database.
+    For SQLite backend, connects to data/icdev.db (or override).
+    """
+    engine = get_engine()
+    if engine.backend == StorageBackend.POSTGRESQL:
+        return engine.connect()
+
+    # SQLite: resolve path with env var fallback
+    path = Path(
+        db_path
+        or os.environ.get("ICDEV_DB_PATH")
+        or str(_PROJECT_ROOT / "data" / "icdev.db")
+    )
+    return engine.connect(db_path=path, row_factory=row_factory, validate=validate)
+
+
+def get_sparkpilot_connection(
+    db_path: Optional[Union[str, Path]] = None,
+    validate: bool = False,
+    row_factory: bool = True,
+) -> StorageConnection:
+    """Get connection to the SparkPilot database.
+
+    For PostgreSQL backend, connects to the configured PG database
+    (all tables live in one PG database with schema separation).
+    For SQLite backend, connects to data/icdev.db.
+    """
+    engine = get_engine()
+    if engine.backend == StorageBackend.POSTGRESQL:
+        return engine.connect()
+
+    path = Path(
+        db_path
+        or os.environ.get("SPARKPILOT_DB_PATH")
+        or str(_PROJECT_ROOT / "data" / "icdev.db")
+    )
+    return engine.connect(db_path=path, row_factory=row_factory, validate=validate)
+
+
+def get_memory_connection(
+    db_path: Optional[Union[str, Path]] = None,
+    validate: bool = False,
+    row_factory: bool = True,
+) -> StorageConnection:
+    """Get connection to the memory database."""
+    engine = get_engine()
+    if engine.backend == StorageBackend.POSTGRESQL:
+        return engine.connect()
+
+    path = Path(
+        db_path
+        or os.environ.get("ICDEV_MEMORY_DB_PATH")
+        or str(_PROJECT_ROOT / "data" / "memory.db")
+    )
+    return engine.connect(db_path=path, row_factory=row_factory, validate=validate)
+
+
+# ---------------------------------------------------------------------------
+# CLI: health check and info
+# ---------------------------------------------------------------------------
+
+def _main():
+    """CLI entry point for health check and configuration display."""
+    import argparse
+    import json
+
+    parser = argparse.ArgumentParser(description="ICDEV Storage Layer (D-DB-20)")
+    parser.add_argument("--health", action="store_true", help="Run health check")
+    parser.add_argument("--info", action="store_true", help="Show configuration")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    args = parser.parse_args()
+
+    engine = get_engine()
+
+    if args.health:
+        result = engine.health_check()
+        if args.json:
+            print(json.dumps(result, indent=2))
+        else:
+            status = result["status"]
+            backend = result["backend"]
+            latency = result.get("latency_ms", "N/A")
+            print(f"Backend: {backend}")
+            print(f"Status:  {status}")
+            print(f"Latency: {latency}ms")
+        return
+
+    if args.info:
+        cfg = engine.config
+        info = {
+            "backend": cfg.backend.value,
+            "sqlite_path": str(cfg.db_path),
+            "pg_host": cfg.pg_host,
+            "pg_port": cfg.pg_port,
+            "pg_database": cfg.pg_database,
+            "pg_username": cfg.pg_username,
+            "pg_pool_size": cfg.pg_pool_size,
+            "pg_has_connection_string": bool(cfg.pg_connection_string),
+        }
+        if args.json:
+            print(json.dumps(info, indent=2))
+        else:
+            for k, v in info.items():
+                print(f"{k}: {v}")
+        return
+
+    # Default: show backend
+    print(f"Storage backend: {engine.config.backend.value}")
+    print(f"Run --health for connectivity test, --info for configuration")
+
+
+if __name__ == "__main__":
+    _main()