ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/persistence/db/query_builder.py

@@ -0,0 +1,232 @@
+"""
+Database-agnostic query builder with dialect support.
+
+Provides a fluent interface for building SQL queries that automatically
+adapt to the target dialect (SQLite, PostgreSQL, MySQL, etc.).
+
+This module centralizes all SQL generation logic, making it easy to:
+1. Add support for new database backends
+2. Test SQL generation without a database
+3. Keep business logic free of SQL dialect knowledge
+"""
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+if TYPE_CHECKING:
+    from ciris_engine.logic.persistence.db.dialect import DialectAdapter
+
+
+class ConflictResolution(str, Enum):
+    """How to handle conflicts during INSERT operations."""
+
+    IGNORE = "ignore"  # INSERT OR IGNORE / ON CONFLICT DO NOTHING
+    REPLACE = "replace"  # INSERT OR REPLACE / ON CONFLICT DO UPDATE
+    ERROR = "error"  # Let database raise error (default behavior)
+
+
+@dataclass
+class InsertQuery:
+    """Builder for INSERT statements with dialect awareness.
+
+    Examples:
+        >>> query = InsertQuery(
+        ...     table="users",
+        ...     columns=["id", "name", "email"],
+        ...     conflict_resolution=ConflictResolution.IGNORE,
+        ...     conflict_columns=["id"]
+        ... )
+        >>> sql = query.to_sql(adapter)
+    """
+
+    table: str
+    columns: List[str]
+    values_placeholder: bool = True  # Use ? placeholders
+    conflict_resolution: ConflictResolution = ConflictResolution.ERROR
+    conflict_columns: Optional[List[str]] = None  # For ON CONFLICT
+    update_columns: Optional[List[str]] = None  # For ON CONFLICT DO UPDATE
+
+    def to_sql(self, adapter: "DialectAdapter") -> str:
+        """Generate SQL for the target dialect.
+
+        Args:
+            adapter: Dialect adapter to use for SQL generation
+
+        Returns:
+            SQL string with ? placeholders (will be translated by cursor wrapper)
+        """
+        from ciris_engine.logic.persistence.db.dialect import Dialect
+
+        placeholders = ", ".join(["?"] * len(self.columns))
+        columns_str = ", ".join(self.columns)
+
+        if self.conflict_resolution == ConflictResolution.IGNORE:
+            if adapter.dialect == Dialect.SQLITE:
+                return f"INSERT OR IGNORE INTO {self.table} ({columns_str}) VALUES ({placeholders})"
+            else:  # PostgreSQL
+                conflict_cols = ", ".join(self.conflict_columns or self.columns)
+                return f"INSERT INTO {self.table} ({columns_str}) VALUES ({placeholders}) ON CONFLICT ({conflict_cols}) DO NOTHING"
+
+        elif self.conflict_resolution == ConflictResolution.REPLACE:
+            if adapter.dialect == Dialect.SQLITE:
+                return f"INSERT OR REPLACE INTO {self.table} ({columns_str}) VALUES ({placeholders})"
+            else:  # PostgreSQL
+                conflict_cols = ", ".join(self.conflict_columns or self.columns)
+                update_cols = self.update_columns or [c for c in self.columns if c not in (self.conflict_columns or [])]
+                updates = ", ".join([f"{col} = EXCLUDED.{col}" for col in update_cols])
+                return f"INSERT INTO {self.table} ({columns_str}) VALUES ({placeholders}) ON CONFLICT ({conflict_cols}) DO UPDATE SET {updates}"
+
+        # ERROR (default)
+        return f"INSERT INTO {self.table} ({columns_str}) VALUES ({placeholders})"
+
+
+@dataclass
+class SelectQuery:
+    """Builder for SELECT statements with dialect awareness.
+
+    Examples:
+        >>> query = SelectQuery(
+        ...     table="users",
+        ...     columns=["id", "name"],
+        ...     where_conditions=["active = ?"],
+        ...     order_by="created_at DESC",
+        ...     limit=10
+        ... )
+        >>> sql = query.to_sql(adapter)
+    """
+
+    table: str
+    columns: List[str]
+    where_conditions: List[str] = field(default_factory=list)
+    order_by: Optional[str] = None
+    limit: Optional[int] = None
+
+    def to_sql(self, adapter: "DialectAdapter") -> str:
+        """Generate SQL for the target dialect.
+
+        Args:
+            adapter: Dialect adapter to use for SQL generation
+
+        Returns:
+            SQL string with ? placeholders (will be translated by cursor wrapper)
+        """
+        cols = ", ".join(self.columns)
+        sql = f"SELECT {cols} FROM {self.table}"
+
+        if self.where_conditions:
+            sql += " WHERE " + " AND ".join(self.where_conditions)
+
+        if self.order_by:
+            sql += f" ORDER BY {self.order_by}"
+
+        if self.limit:
+            sql += f" LIMIT {self.limit}"
+
+        return sql
+
+
+class QueryBuilder:
+    """Factory for creating dialect-aware queries.
+
+    This class provides high-level methods for common query patterns,
+    automatically handling dialect-specific SQL generation.
+
+    Examples:
+        >>> from ciris_engine.logic.persistence.db.dialect import get_adapter
+        >>> adapter = get_adapter()
+        >>> builder = QueryBuilder(adapter)
+        >>> sql = builder.insert_ignore_node()
+    """
+
+    def __init__(self, adapter: "DialectAdapter"):
+        """Initialize query builder with dialect adapter.
+
+        Args:
+            adapter: Dialect adapter for SQL generation
+        """
+        self.adapter = adapter
+
+    def insert(
+        self,
+        table: str,
+        columns: List[str],
+        conflict_resolution: ConflictResolution = ConflictResolution.ERROR,
+        conflict_columns: Optional[List[str]] = None,
+    ) -> InsertQuery:
+        """Create an INSERT query builder.
+
+        Args:
+            table: Table name
+            columns: List of column names
+            conflict_resolution: How to handle conflicts
+            conflict_columns: Columns that define uniqueness constraint
+
+        Returns:
+            InsertQuery builder
+        """
+        return InsertQuery(
+            table=table, columns=columns, conflict_resolution=conflict_resolution, conflict_columns=conflict_columns
+        )
+
+    def select(self, table: str, columns: List[str]) -> SelectQuery:
+        """Create a SELECT query builder.
+
+        Args:
+            table: Table name
+            columns: List of column names to select
+
+        Returns:
+            SelectQuery builder
+        """
+        return SelectQuery(table=table, columns=columns)
+
+    def insert_ignore_node(self) -> str:
+        """Create INSERT OR IGNORE for graph_nodes with proper conflict handling.
+
+        Handles the PRIMARY KEY constraint (node_id, scope) correctly for both dialects.
+
+        Returns:
+            SQL string ready for execution (with ? placeholders)
+        """
+        query = self.insert(
+            table="graph_nodes",
+            columns=[
+                "node_id",
+                "scope",
+                "node_type",
+                "attributes_json",
+                "version",
+                "updated_by",
+                "updated_at",
+                "created_at",
+            ],
+            conflict_resolution=ConflictResolution.IGNORE,
+            conflict_columns=["node_id", "scope"],  # PRIMARY KEY
+        )
+        return query.to_sql(self.adapter)
+
+    def insert_ignore_edge(self) -> str:
+        """Create INSERT OR IGNORE for graph_edges with proper conflict handling.
+
+        Handles the PRIMARY KEY constraint (edge_id) correctly for both dialects.
+
+        Returns:
+            SQL string ready for execution (with ? placeholders)
+        """
+        query = self.insert(
+            table="graph_edges",
+            columns=[
+                "edge_id",
+                "source_node_id",
+                "target_node_id",
+                "scope",
+                "relationship",
+                "weight",
+                "attributes_json",
+                "created_at",
+            ],
+            conflict_resolution=ConflictResolution.IGNORE,
+            conflict_columns=["edge_id"],  # PRIMARY KEY
+        )
+        return query.to_sql(self.adapter)

ciris_engine/logic/persistence/db/retry.py

@@ -0,0 +1,154 @@
+"""
+Database retry mechanism for handling SQLite busy errors.
+
+Provides a minimal, clean retry wrapper for database operations
+to handle concurrent access issues gracefully.
+"""
+
+import functools
+import logging
+import sqlite3
+import time
+from contextlib import contextmanager
+from typing import Any, Callable, Iterator, TypeVar
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# Default retry configuration
+DEFAULT_MAX_RETRIES = 3
+DEFAULT_BASE_DELAY = 0.1  # 100ms
+DEFAULT_MAX_DELAY = 1.0  # 1 second
+
+
+def is_retryable_error(error: Exception) -> bool:
+    """Check if an error is retryable (database busy/locked)."""
+    if isinstance(error, sqlite3.OperationalError):
+        error_msg = str(error).lower()
+        return "database is locked" in error_msg or "database table is locked" in error_msg
+    return False
+
+
+def with_retry(
+    func: Callable[..., T] | None = None,
+    *,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    base_delay: float = DEFAULT_BASE_DELAY,
+    max_delay: float = DEFAULT_MAX_DELAY,
+) -> Callable[..., T] | Callable[[Callable[..., T]], Callable[..., T]]:
+    """
+    Decorator for retrying database operations on busy errors.
+
+    Can be used as:
+        @with_retry
+        def my_func(): ...
+
+        @with_retry(max_retries=5, base_delay=0.5)
+        def my_func(): ...
+
+    Args:
+        func: Function to retry (None when used with parameters)
+        max_retries: Maximum number of retry attempts
+        base_delay: Initial delay between retries (exponential backoff)
+        max_delay: Maximum delay between retries
+
+    Returns:
+        Wrapped function with retry logic or decorator
+    """
+
+    def decorator(f: Callable[..., T]) -> Callable[..., T]:
+        @functools.wraps(f)
+        def wrapper(*args: Any, **kwargs: Any) -> T:
+            last_error = None
+
+            for attempt in range(max_retries + 1):  # +1 for initial attempt
+                try:
+                    return f(*args, **kwargs)
+                except Exception as e:
+                    if not is_retryable_error(e) or attempt == max_retries:
+                        # Not retryable or last attempt - re-raise
+                        raise
+
+                    last_error = e
+                    delay = min(base_delay * (2**attempt), max_delay)
+
+                    logger.debug(
+                        f"Database busy error on attempt {attempt + 1}/{max_retries + 1}, "
+                        f"retrying in {delay:.2f}s: {e}"
+                    )
+
+                    time.sleep(delay)
+
+            # Should not reach here, but just in case
+            raise last_error if last_error else RuntimeError("Unexpected retry loop exit")
+
+        return wrapper
+
+    # If func is provided, we're being used as @with_retry
+    if func is not None:
+        return decorator(func)
+
+    # Otherwise, we're being used as @with_retry(...) and need to return decorator
+    return decorator
+
+
+@contextmanager
+def get_db_connection_with_retry(db_path: str | None = None, busy_timeout: int = 5000) -> Iterator[sqlite3.Connection]:
+    """
+    Get a database connection with busy timeout configured.
+
+    Args:
+        db_path: Optional database path
+        busy_timeout: Busy timeout in milliseconds (default 5 seconds)
+
+    Yields:
+        Database connection with retry configuration
+    """
+    from .core import get_db_connection
+
+    conn = get_db_connection(db_path)
+    try:
+        # Enable WAL mode for better concurrent access
+        conn.execute("PRAGMA journal_mode=WAL")
+        # Set busy timeout to wait before failing
+        conn.execute(f"PRAGMA busy_timeout={busy_timeout}")
+        yield conn  # type: ignore[misc]
+    finally:
+        conn.close()
+
+
+def execute_with_retry(
+    operation: Callable[[sqlite3.Connection], T],
+    db_path: str | None = None,
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    base_delay: float = DEFAULT_BASE_DELAY,
+) -> T:
+    """
+    Execute a database operation with automatic retry on busy errors.
+
+    Args:
+        operation: Function that takes a connection and performs database operations
+        db_path: Optional database path
+        max_retries: Maximum number of retry attempts
+        base_delay: Initial delay between retries
+
+    Returns:
+        Result of the operation
+
+    Example:
+        def insert_task(conn):
+            cursor = conn.cursor()
+            cursor.execute("INSERT INTO tasks ...")
+            conn.commit()
+            return cursor.lastrowid
+
+        task_id = execute_with_retry(insert_task)
+    """
+
+    @with_retry(max_retries=max_retries, base_delay=base_delay)  # type: ignore
+    def _execute() -> T:
+        with get_db_connection_with_retry(db_path) as conn:
+            return operation(conn)
+
+    return _execute()

ciris_engine/logic/persistence/db/setup.py

@@ -0,0 +1,18 @@
+import logging
+import sqlite3
+
+from ciris_engine.logic.config import get_sqlite_db_full_path
+
+from .migration_runner import run_migrations
+
+logger = logging.getLogger(__name__)
+
+
+def initialize_database(db_path: str | None = None) -> None:
+    """Apply pending migrations to initialize or update the database."""
+    try:
+        run_migrations(db_path)
+        logger.info(f"Database migrations applied at {db_path or get_sqlite_db_full_path()}")
+    except sqlite3.Error as e:
+        logger.exception(f"Database error during initialization: {e}")
+        raise

ciris_engine/logic/persistence/migrations/postgres/001_initial_schema.sql

@@ -0,0 +1,4 @@
+-- Initial schema for CIRIS Engine (PostgreSQL)
+-- NOTE: Base tables are created by initialize_database() using tables_postgresql.py
+-- This migration is intentionally empty as the schema is already applied.
+-- Kept for migration tracking consistency with SQLite.

ciris_engine/logic/persistence/migrations/postgres/002_add_retry_status.sql

@@ -0,0 +1,3 @@
+-- Sample migration (post-beta): add retry_count column to tasks
+-- NOTE: This is just a sample since retry_count is already in base schema for pre-beta
+-- In actual post-beta usage, this would be: ALTER TABLE tasks ADD COLUMN some_new_feature INTEGER DEFAULT 0;

ciris_engine/logic/persistence/migrations/postgres/003_add_task_update_tracking.sql

@@ -0,0 +1,8 @@
+-- Migration 003: Add task update tracking fields (PostgreSQL)
+-- Adds fields to support detecting when new observations arrive for an active task
+
+-- Add updated_info_available flag
+ALTER TABLE tasks ADD COLUMN IF NOT EXISTS updated_info_available INTEGER DEFAULT 0;
+
+-- Add updated_info_content field
+ALTER TABLE tasks ADD COLUMN IF NOT EXISTS updated_info_content TEXT DEFAULT NULL;

ciris_engine/logic/persistence/migrations/postgres/004_add_occurrence_id.sql

@@ -0,0 +1,54 @@
+-- Migration 004: Add agent_occurrence_id for multi-instance deployment support (PostgreSQL)
+-- Enables running multiple API-only CIRIS agent instances against the same PostgreSQL database
+-- Each runtime instance processes only its own work while sharing a single agent identity
+
+-- Add agent_occurrence_id to tasks table
+ALTER TABLE tasks ADD COLUMN IF NOT EXISTS agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Add agent_occurrence_id to thoughts table
+ALTER TABLE thoughts ADD COLUMN IF NOT EXISTS agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Add agent_occurrence_id to scheduled_tasks table (if exists)
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'scheduled_tasks') THEN
+        ALTER TABLE scheduled_tasks ADD COLUMN IF NOT EXISTS agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+    END IF;
+END $$;
+
+-- Add agent_occurrence_id to service_correlations for per-instance telemetry
+ALTER TABLE service_correlations ADD COLUMN IF NOT EXISTS agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Create composite indexes for efficient occurrence-scoped queries
+
+-- Tasks indexes
+CREATE INDEX IF NOT EXISTS idx_tasks_occurrence_status
+    ON tasks(agent_occurrence_id, status, created_at);
+
+CREATE INDEX IF NOT EXISTS idx_tasks_occurrence_channel
+    ON tasks(agent_occurrence_id, channel_id, status);
+
+-- Thoughts indexes
+CREATE INDEX IF NOT EXISTS idx_thoughts_occurrence_status
+    ON thoughts(agent_occurrence_id, status, created_at);
+
+CREATE INDEX IF NOT EXISTS idx_thoughts_occurrence_task
+    ON thoughts(agent_occurrence_id, source_task_id);
+
+-- Scheduled tasks index (if table exists)
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'scheduled_tasks') THEN
+        EXECUTE 'CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_occurrence ON scheduled_tasks(agent_occurrence_id, status, next_trigger_at)';
+    END IF;
+END $$;
+
+-- Service correlations index
+CREATE INDEX IF NOT EXISTS idx_correlations_occurrence
+    ON service_correlations(agent_occurrence_id);
+
+-- Note: Existing rows automatically get 'default' value via DEFAULT clause
+-- Note: graph_nodes, graph_edges, audit_log, and wa_cert remain global (no occurrence_id)
+--       - Memory is shared across all occurrences (single agent identity/knowledge)
+--       - Audit trail is global for complete history
+--       - WA certificates are shared (single agent identity)

ciris_engine/logic/persistence/migrations/postgres/005_add_consolidation_locks.sql

@@ -0,0 +1,22 @@
+-- Migration 005: Add consolidation_locks table for TSDB consolidation coordination
+-- Enables multiple instances to coordinate consolidation work without conflicting
+-- Uses conditional UPDATE pattern for lock acquisition (like distributed job queue)
+
+CREATE TABLE IF NOT EXISTS consolidation_locks (
+    lock_key TEXT PRIMARY KEY,           -- e.g., "basic:2025-10-22T06:00:00+00:00"
+    locked_by TEXT,                      -- Hostname or instance ID holding the lock
+    locked_at TIMESTAMP,                 -- Timestamp when lock was acquired
+    lock_timeout_seconds INTEGER DEFAULT 300  -- Auto-expire locks after 5 minutes
+);
+
+-- Index for finding expired locks
+CREATE INDEX IF NOT EXISTS idx_consolidation_locks_expiry
+    ON consolidation_locks(locked_at);
+
+-- Note: Lock acquisition uses conditional UPDATE:
+--   UPDATE consolidation_locks
+--   SET locked_by = $1, locked_at = $2
+--   WHERE lock_key = $3 AND (locked_by IS NULL OR locked_at < $4)
+--
+-- If rowcount > 0, lock was acquired
+-- If rowcount = 0, lock is held by another instance

ciris_engine/logic/persistence/migrations/postgres/006_add_correlation_id_unique_index.sql

@@ -0,0 +1,16 @@
+-- Migration 006: Add unique index on (agent_occurrence_id, correlation_id)
+--
+-- Purpose: Prevent duplicate task creation for the same Reddit comment/post
+--
+-- Problem: Multiple observer polls can race and create duplicate tasks for the same
+--          correlation_id (Reddit post/comment ID), causing duplicate replies.
+--
+-- Solution: Create a unique index on (agent_occurrence_id, correlation_id) extracted
+--           from context_json to enforce uniqueness at the database level.
+--
+-- Note: PostgreSQL supports JSON extraction in indexes using -> and ->> operators
+--       context_json is JSONB, so we use ->> to extract as text
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_occurrence_correlation
+ON tasks(agent_occurrence_id, (context_json->>'correlation_id'))
+WHERE context_json->>'correlation_id' IS NOT NULL;

ciris_engine/logic/persistence/migrations/postgres/007_add_dsar_tickets.sql

@@ -0,0 +1,39 @@
+-- Migration 007: Add DSAR Tickets Table
+-- Purpose: Replace in-memory dict with persistent storage for GDPR compliance
+-- Impact: DSAR tickets survive server restarts, meeting 30-day response requirement
+
+CREATE TABLE IF NOT EXISTS dsar_tickets (
+    ticket_id TEXT PRIMARY KEY,
+    request_type TEXT NOT NULL CHECK(request_type IN ('access', 'delete', 'export', 'correct')),
+    email TEXT NOT NULL,
+    user_identifier TEXT,
+    details TEXT,
+    urgent BOOLEAN NOT NULL DEFAULT FALSE,
+    status TEXT NOT NULL CHECK(status IN ('pending_review', 'in_progress', 'completed', 'rejected')),
+    submitted_at TIMESTAMP NOT NULL,
+    estimated_completion TIMESTAMP NOT NULL,
+    last_updated TIMESTAMP NOT NULL,
+    notes TEXT,
+    automated BOOLEAN NOT NULL DEFAULT FALSE,
+    access_package_json TEXT,  -- JSON serialized DSARAccessPackage
+    export_package_json TEXT,  -- JSON serialized DSARExportPackage
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Index for status queries (admin filtering by status)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_status ON dsar_tickets(status);
+
+-- Index for email lookups (user checking their requests)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_email ON dsar_tickets(email);
+
+-- Index for user_identifier lookups (data deletion tracking)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_user_identifier ON dsar_tickets(user_identifier);
+
+-- Index for request type queries (metrics by type)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_request_type ON dsar_tickets(request_type);
+
+-- Index for timestamp queries (30-day response compliance)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_submitted_at ON dsar_tickets(submitted_at);
+
+-- Composite index for common admin query (status + submitted date)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_status_submitted ON dsar_tickets(status, submitted_at);

ciris_engine/logic/persistence/migrations/postgres/008_rename_to_tickets_add_sop.sql

@@ -0,0 +1,123 @@
+-- Migration 008: Rename dsar_tickets to tickets and add SOP architecture
+-- Purpose: Make tickets universal with DSAR as always-present default capability
+-- Impact: All agents have DSAR SOPs, agents can add custom ticket SOPs via templates
+--
+-- Architecture Decision:
+-- - DSAR is universal (GDPR compliance required for all agents)
+-- - Other ticket types (appointments, incidents) are agent-specific
+-- - SOP column links to agent template ticket configuration
+-- - Metadata JSON stores stage progress and results
+
+-- Step 1: Create new tickets table with SOP architecture
+CREATE TABLE IF NOT EXISTS tickets (
+    ticket_id TEXT PRIMARY KEY,
+    sop TEXT NOT NULL,  -- Standard Operating Procedure (e.g., "DSAR_ACCESS", "APPOINTMENT_SCHEDULE")
+    ticket_type TEXT NOT NULL,  -- Category (e.g., "dsar", "appointment", "incident")
+    status TEXT NOT NULL CHECK(status IN ('pending', 'in_progress', 'completed', 'cancelled', 'failed')),
+    priority INTEGER NOT NULL DEFAULT 5 CHECK(priority BETWEEN 1 AND 10),  -- 1=lowest, 10=urgent
+
+    -- Contact & identification
+    email TEXT NOT NULL,
+    user_identifier TEXT,
+
+    -- Lifecycle timestamps
+    submitted_at TIMESTAMP NOT NULL,
+    deadline TIMESTAMP,  -- Calculated from SOP config
+    last_updated TIMESTAMP NOT NULL,
+    completed_at TIMESTAMP,
+
+    -- Workflow state
+    metadata JSONB NOT NULL DEFAULT '{}'::jsonb,  -- Stage progress, results, SOP-specific data
+    notes TEXT,
+
+    -- Tracking
+    automated BOOLEAN NOT NULL DEFAULT FALSE,
+    correlation_id TEXT,  -- Links to tasks/thoughts processing this ticket
+
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Step 2: Migrate existing dsar_tickets to new tickets table
+INSERT INTO tickets (
+    ticket_id,
+    sop,
+    ticket_type,
+    status,
+    priority,
+    email,
+    user_identifier,
+    submitted_at,
+    deadline,
+    last_updated,
+    completed_at,
+    metadata,
+    notes,
+    automated,
+    created_at
+)
+SELECT
+    ticket_id,
+    -- Map request_type to SOP
+    CASE request_type
+        WHEN 'access' THEN 'DSAR_ACCESS'
+        WHEN 'delete' THEN 'DSAR_DELETE'
+        WHEN 'export' THEN 'DSAR_EXPORT'
+        WHEN 'correct' THEN 'DSAR_RECTIFY'
+        ELSE 'DSAR_ACCESS'  -- fallback
+    END as sop,
+    'dsar' as ticket_type,
+    -- Map old status to new status
+    CASE status
+        WHEN 'pending_review' THEN 'pending'
+        WHEN 'in_progress' THEN 'in_progress'
+        WHEN 'completed' THEN 'completed'
+        WHEN 'rejected' THEN 'cancelled'
+        ELSE 'pending'
+    END as status,
+    -- Map urgent flag to priority
+    CASE urgent
+        WHEN TRUE THEN 9  -- urgent = priority 9
+        ELSE 5  -- normal = priority 5
+    END as priority,
+    email,
+    user_identifier,
+    submitted_at,
+    estimated_completion as deadline,
+    last_updated,
+    CASE
+        WHEN status = 'completed' THEN last_updated
+        ELSE NULL
+    END as completed_at,
+    -- Build metadata JSONB from legacy fields
+    jsonb_build_object(
+        'legacy_request_type', request_type,
+        'legacy_details', COALESCE(details, ''),
+        'access_package', COALESCE(access_package_json::jsonb, 'null'::jsonb),
+        'export_package', COALESCE(export_package_json::jsonb, 'null'::jsonb),
+        'stages', '{}'::jsonb
+    ) as metadata,
+    notes,
+    automated,
+    created_at
+FROM dsar_tickets;
+
+-- Step 3: Drop old dsar_tickets table
+DROP TABLE dsar_tickets;
+
+-- Step 4: Create indexes for new tickets table
+CREATE INDEX IF NOT EXISTS idx_tickets_sop ON tickets(sop);
+CREATE INDEX IF NOT EXISTS idx_tickets_type ON tickets(ticket_type);
+CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
+CREATE INDEX IF NOT EXISTS idx_tickets_email ON tickets(email);
+CREATE INDEX IF NOT EXISTS idx_tickets_user_identifier ON tickets(user_identifier);
+CREATE INDEX IF NOT EXISTS idx_tickets_submitted_at ON tickets(submitted_at);
+CREATE INDEX IF NOT EXISTS idx_tickets_deadline ON tickets(deadline);
+CREATE INDEX IF NOT EXISTS idx_tickets_correlation_id ON tickets(correlation_id);
+
+-- Composite indexes for common queries
+CREATE INDEX IF NOT EXISTS idx_tickets_status_submitted ON tickets(status, submitted_at);
+CREATE INDEX IF NOT EXISTS idx_tickets_sop_status ON tickets(sop, status);
+CREATE INDEX IF NOT EXISTS idx_tickets_type_status ON tickets(ticket_type, status);
+
+-- GIN index for JSONB metadata queries (PostgreSQL specific)
+CREATE INDEX IF NOT EXISTS idx_tickets_metadata_gin ON tickets USING GIN (metadata);