ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/persistence/migrations/postgres/009_add_ticket_status_columns.sql

@@ -0,0 +1,39 @@
+-- Migration 009: Add ticket status handling for multi-occurrence coordination
+-- Purpose: Support comprehensive ticket lifecycle with atomic claiming
+-- Date: 2025-11-07
+--
+-- Changes:
+-- 1. Add agent_occurrence_id column for multi-occurrence coordination
+-- 2. Expand status CHECK constraint to support full lifecycle
+--
+-- New Status Values:
+-- - assigned: Ticket claimed by specific occurrence
+-- - blocked: Ticket requires external intervention (stops task generation)
+-- - deferred: Ticket postponed to future time or awaiting human response
+--
+-- Architecture:
+-- - PENDING tickets use agent_occurrence_id="__shared__" for atomic claiming
+-- - WorkProcessor atomically claims PENDING → ASSIGNED with occurrence_id
+-- - BLOCKED/DEFERRED tickets do not generate new tasks until state changes
+-- - COMPLETED/FAILED tickets are terminal states
+
+-- Step 1: Add agent_occurrence_id column with default '__shared__'
+ALTER TABLE tickets ADD COLUMN IF NOT EXISTS agent_occurrence_id TEXT NOT NULL DEFAULT '__shared__';
+
+-- Step 2: Drop old status CHECK constraint
+ALTER TABLE tickets DROP CONSTRAINT IF EXISTS tickets_status_check;
+
+-- Step 3: Add new status CHECK constraint with expanded values
+ALTER TABLE tickets ADD CONSTRAINT tickets_status_check CHECK(status IN (
+    'pending',      -- Created, awaiting assignment
+    'assigned',     -- Claimed by occurrence, not yet started
+    'in_progress',  -- Actively being processed
+    'blocked',      -- Needs external input (stops task generation)
+    'deferred',     -- Postponed to future time (stops task generation)
+    'completed',    -- Successfully finished (terminal)
+    'cancelled',    -- User/admin cancelled (terminal)
+    'failed'        -- Processing failed (terminal)
+));
+
+-- Step 4: Create new index for multi-occurrence coordination
+CREATE INDEX IF NOT EXISTS idx_tickets_occurrence_status ON tickets(agent_occurrence_id, status);

ciris_engine/logic/persistence/migrations/postgres/010_add_images_to_tasks.sql

@@ -0,0 +1,5 @@
+-- Migration: Add images_json column to tasks table for native multimodal vision support
+-- Images are stored as JSONB array of ImageContent objects on the TASK level
+-- All thoughts associated with a task inherit images from their source task
+
+ALTER TABLE tasks ADD COLUMN IF NOT EXISTS images_json JSONB;

ciris_engine/logic/persistence/migrations/sqlite/001_initial_schema.sql

@@ -0,0 +1,357 @@
+-- Initial schema for CIRIS Engine
+-- This migration creates all the core tables needed for the system
+
+-- Tasks table
+CREATE TABLE IF NOT EXISTS tasks (
+    task_id TEXT PRIMARY KEY,
+    channel_id TEXT NOT NULL,
+    description TEXT NOT NULL,
+    status TEXT NOT NULL,
+    priority INTEGER DEFAULT 0,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    parent_task_id TEXT,
+    context_json TEXT,
+    outcome_json TEXT,
+    retry_count INTEGER DEFAULT 0,
+    -- Task signing fields (from migration 003)
+    signed_by TEXT,
+    signature TEXT,
+    signed_at TEXT
+);
+
+-- Thoughts table with thought_depth (renamed from ponder_count)
+CREATE TABLE IF NOT EXISTS thoughts (
+    thought_id TEXT PRIMARY KEY,
+    source_task_id TEXT NOT NULL,
+    thought_type TEXT DEFAULT 'standard',
+    status TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    round_number INTEGER DEFAULT 0,
+    content TEXT NOT NULL,
+    context_json TEXT,
+    thought_depth INTEGER DEFAULT 0,
+    ponder_notes_json TEXT,
+    parent_thought_id TEXT,
+    final_action_json TEXT,
+    FOREIGN KEY (source_task_id) REFERENCES tasks(task_id)
+);
+
+-- Feedback mappings table
+CREATE TABLE IF NOT EXISTS feedback_mappings (
+    feedback_id TEXT PRIMARY KEY,
+    source_message_id TEXT,
+    target_thought_id TEXT,
+    feedback_type TEXT,  -- 'identity' or 'environment'
+    created_at TEXT NOT NULL
+);
+
+-- Graph nodes table
+CREATE TABLE IF NOT EXISTS graph_nodes (
+    node_id TEXT NOT NULL,
+    scope TEXT NOT NULL,
+    node_type TEXT NOT NULL,
+    attributes_json TEXT,
+    version INTEGER DEFAULT 1,
+    updated_by TEXT,
+    updated_at TEXT,
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (node_id, scope)
+);
+CREATE INDEX IF NOT EXISTS idx_graph_nodes_scope ON graph_nodes(scope);
+CREATE INDEX IF NOT EXISTS idx_graph_nodes_type ON graph_nodes(node_type);
+CREATE INDEX IF NOT EXISTS idx_graph_nodes_created ON graph_nodes(created_at);
+CREATE INDEX IF NOT EXISTS idx_graph_nodes_type_scope_created ON graph_nodes(node_type, scope, created_at);
+CREATE INDEX IF NOT EXISTS idx_graph_nodes_tsdb_lookup ON graph_nodes(node_type, scope, created_at DESC);
+
+-- Graph edges table
+CREATE TABLE IF NOT EXISTS graph_edges (
+    edge_id TEXT PRIMARY KEY,
+    source_node_id TEXT NOT NULL,
+    target_node_id TEXT NOT NULL,
+    scope TEXT NOT NULL,
+    relationship TEXT NOT NULL,
+    weight REAL DEFAULT 1.0,
+    attributes_json TEXT,
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (source_node_id, scope) REFERENCES graph_nodes(node_id, scope),
+    FOREIGN KEY (target_node_id, scope) REFERENCES graph_nodes(node_id, scope)
+);
+CREATE INDEX IF NOT EXISTS idx_graph_edges_scope ON graph_edges(scope);
+CREATE INDEX IF NOT EXISTS idx_graph_edges_source ON graph_edges(source_node_id);
+CREATE INDEX IF NOT EXISTS idx_graph_edges_target ON graph_edges(target_node_id);
+
+-- Service correlations table with TSDB fields
+CREATE TABLE IF NOT EXISTS service_correlations (
+    correlation_id TEXT PRIMARY KEY,
+    service_type TEXT NOT NULL,
+    handler_name TEXT NOT NULL,
+    action_type TEXT NOT NULL,
+    request_data TEXT,
+    response_data TEXT,
+    status TEXT NOT NULL DEFAULT 'pending',
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    -- TSDB fields for unified telemetry storage
+    correlation_type TEXT NOT NULL DEFAULT 'service_interaction',
+    timestamp TEXT, -- ISO8601 timestamp for time queries
+    metric_name TEXT, -- For metric correlations
+    metric_value REAL, -- For metric correlations
+    log_level TEXT, -- For log correlations
+    trace_id TEXT, -- For distributed tracing
+    span_id TEXT, -- For trace spans
+    parent_span_id TEXT, -- For trace hierarchy
+    tags TEXT, -- JSON object for flexible tagging
+    retention_policy TEXT NOT NULL DEFAULT 'raw' -- raw, hourly_summary, daily_summary
+);
+
+-- Core indexes
+CREATE INDEX IF NOT EXISTS idx_correlations_status ON service_correlations(status);
+CREATE INDEX IF NOT EXISTS idx_correlations_handler ON service_correlations(handler_name);
+
+-- TSDB indexes for efficient queries
+CREATE INDEX IF NOT EXISTS idx_correlations_type ON service_correlations(correlation_type);
+CREATE INDEX IF NOT EXISTS idx_correlations_timestamp ON service_correlations(timestamp);
+CREATE INDEX IF NOT EXISTS idx_correlations_metric_name ON service_correlations(metric_name);
+CREATE INDEX IF NOT EXISTS idx_correlations_log_level ON service_correlations(log_level);
+CREATE INDEX IF NOT EXISTS idx_correlations_trace_id ON service_correlations(trace_id);
+CREATE INDEX IF NOT EXISTS idx_correlations_span_id ON service_correlations(span_id);
+CREATE INDEX IF NOT EXISTS idx_correlations_retention ON service_correlations(retention_policy);
+
+-- Composite indexes for common TSDB query patterns
+CREATE INDEX IF NOT EXISTS idx_correlations_type_timestamp ON service_correlations(correlation_type, timestamp);
+CREATE INDEX IF NOT EXISTS idx_correlations_metric_timestamp ON service_correlations(metric_name, timestamp);
+CREATE INDEX IF NOT EXISTS idx_correlations_log_level_timestamp ON service_correlations(log_level, timestamp);
+
+-- Audit log table
+CREATE TABLE IF NOT EXISTS audit_log (
+    entry_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    event_id TEXT NOT NULL UNIQUE,              -- UUID for the event
+    event_timestamp TEXT NOT NULL,              -- ISO8601
+    event_type TEXT NOT NULL,
+    originator_id TEXT NOT NULL,
+    event_payload TEXT,                         -- JSON payload
+
+    -- Hash chain fields
+    sequence_number INTEGER NOT NULL,           -- Monotonic counter
+    previous_hash TEXT NOT NULL,                -- SHA-256 of previous entry
+    entry_hash TEXT NOT NULL,                   -- SHA-256 of this entry's content
+
+    -- Signature fields
+    signature TEXT NOT NULL,                    -- Base64 encoded signature
+    signing_key_id TEXT NOT NULL,               -- Key used to sign
+
+    -- Indexing
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    -- Constraints
+    UNIQUE(sequence_number),
+    CHECK(sequence_number > 0)
+);
+
+-- Create indexes for efficient querying
+CREATE INDEX IF NOT EXISTS idx_audit_event_type ON audit_log(event_type);
+CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_log(event_timestamp);
+CREATE INDEX IF NOT EXISTS idx_audit_originator ON audit_log(originator_id);
+CREATE INDEX IF NOT EXISTS idx_audit_sequence ON audit_log(sequence_number);
+
+-- Audit roots table
+CREATE TABLE IF NOT EXISTS audit_roots (
+    root_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    sequence_start INTEGER NOT NULL,
+    sequence_end INTEGER NOT NULL,
+    root_hash TEXT NOT NULL,                    -- Merkle root of entries
+    timestamp TEXT NOT NULL,
+    external_anchor TEXT,                       -- External timestamp proof
+
+    UNIQUE(sequence_start, sequence_end)
+);
+
+-- Create index for root lookup
+CREATE INDEX IF NOT EXISTS idx_audit_roots_range ON audit_roots(sequence_start, sequence_end);
+
+-- Audit signing keys table
+CREATE TABLE IF NOT EXISTS audit_signing_keys (
+    key_id TEXT PRIMARY KEY,
+    public_key TEXT NOT NULL,                   -- PEM format public key
+    algorithm TEXT NOT NULL DEFAULT 'rsa-pss',
+    key_size INTEGER NOT NULL DEFAULT 2048,
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    revoked_at TEXT,                           -- NULL if active
+
+    CHECK(algorithm IN ('rsa-pss', 'ed25519'))
+);
+
+-- Create index for active key lookup
+CREATE INDEX IF NOT EXISTS idx_audit_keys_active ON audit_signing_keys(created_at)
+WHERE revoked_at IS NULL;
+
+-- WA certificate table
+CREATE TABLE IF NOT EXISTS wa_cert (
+  wa_id              TEXT PRIMARY KEY,
+  name               TEXT NOT NULL,
+  role               TEXT CHECK(role IN ('root','authority','observer')),
+  pubkey             TEXT NOT NULL,              -- base64url Ed25519
+  jwt_kid            TEXT NOT NULL UNIQUE,
+  password_hash      TEXT,
+  api_key_hash       TEXT,
+  oauth_provider     TEXT,
+  oauth_external_id  TEXT,
+  veilid_id          TEXT,
+  auto_minted        INTEGER DEFAULT 0,          -- 1 = OAuth observer
+  parent_wa_id       TEXT,
+  parent_signature   TEXT,
+  scopes_json        TEXT NOT NULL,
+  adapter_id         TEXT,                       -- for adapter observers
+  token_type         TEXT DEFAULT 'standard',    -- 'channel'|'oauth'|'standard'
+  created            TEXT NOT NULL,
+  last_login         TEXT,
+  active             INTEGER DEFAULT 1,
+
+  -- Foreign key constraints
+  FOREIGN KEY (parent_wa_id) REFERENCES wa_cert(wa_id)
+);
+
+-- wa_cert indexes moved to end of file
+
+-- Tables from migration 004: Identity Root and Scheduled Tasks
+
+-- Table for scheduled tasks (integrates with DEFER system)
+CREATE TABLE IF NOT EXISTS scheduled_tasks (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    goal_description TEXT NOT NULL,
+    status TEXT NOT NULL CHECK(status IN ('PENDING', 'ACTIVE', 'COMPLETE', 'FAILED')),
+
+    -- Scheduling (integrates with time-based DEFER)
+    defer_until TEXT,  -- ISO 8601 timestamp for one-time execution
+    schedule_cron TEXT,  -- Cron expression for recurring tasks
+
+    -- Execution details
+    trigger_prompt TEXT NOT NULL,
+    origin_thought_id TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    last_triggered_at TEXT,
+    next_trigger_at TEXT,  -- Computed next execution time
+
+    -- Self-deferral tracking
+    deferral_count INTEGER DEFAULT 0,
+    deferral_history TEXT,  -- JSON array of deferral records
+
+    -- Indexes for efficient querying
+    created_by_agent TEXT,  -- Agent that created this task
+
+    FOREIGN KEY (origin_thought_id) REFERENCES thoughts(thought_id)
+);
+
+CREATE INDEX idx_scheduled_tasks_status ON scheduled_tasks(status);
+CREATE INDEX idx_scheduled_tasks_next_trigger ON scheduled_tasks(next_trigger_at);
+CREATE INDEX idx_scheduled_tasks_agent ON scheduled_tasks(created_by_agent);
+
+-- Table for agent creation ceremonies
+CREATE TABLE IF NOT EXISTS creation_ceremonies (
+    ceremony_id TEXT PRIMARY KEY,
+    timestamp TEXT NOT NULL,
+
+    -- Participants
+    creator_agent_id TEXT NOT NULL,
+    creator_human_id TEXT NOT NULL,
+    wise_authority_id TEXT NOT NULL,
+
+    -- New agent details
+    new_agent_id TEXT NOT NULL,
+    new_agent_name TEXT NOT NULL,
+    new_agent_purpose TEXT NOT NULL,
+    new_agent_description TEXT,
+
+    -- Ceremony record
+    creation_justification TEXT NOT NULL,
+    expected_capabilities TEXT,  -- JSON array
+    ethical_considerations TEXT NOT NULL,
+    template_profile_hash TEXT,
+
+    -- Result
+    ceremony_status TEXT NOT NULL
+);
+
+CREATE INDEX idx_ceremonies_timestamp ON creation_ceremonies(timestamp);
+CREATE INDEX idx_ceremonies_creator_agent ON creation_ceremonies(creator_agent_id);
+CREATE INDEX idx_ceremonies_new_agent ON creation_ceremonies(new_agent_id);
+
+-- Table for continuity awareness memories
+CREATE TABLE IF NOT EXISTS continuity_awareness (
+    id TEXT PRIMARY KEY,
+    agent_id TEXT NOT NULL,
+    shutdown_timestamp TEXT NOT NULL,
+
+    -- Shutdown context
+    is_terminal BOOLEAN NOT NULL,
+    shutdown_reason TEXT NOT NULL,
+    expected_reactivation TEXT,
+    initiated_by TEXT NOT NULL,
+
+    -- Agent's final state
+    final_thoughts TEXT NOT NULL,
+    unfinished_tasks TEXT,  -- JSON array of task IDs
+    reactivation_instructions TEXT,
+    deferred_goals TEXT,  -- JSON array of goals
+
+    -- Continuity
+    preservation_node_id TEXT NOT NULL,  -- Graph node ID for the memory
+    preservation_scope TEXT NOT NULL DEFAULT 'IDENTITY',  -- Graph node scope
+    reactivation_count INTEGER DEFAULT 0,
+
+    FOREIGN KEY (preservation_node_id, preservation_scope) REFERENCES graph_nodes(node_id, scope)
+);
+
+CREATE INDEX idx_preservation_agent ON continuity_awareness(agent_id);
+CREATE INDEX idx_preservation_timestamp ON continuity_awareness(shutdown_timestamp);
+
+-- View for active scheduled tasks (for scheduler service)
+CREATE VIEW IF NOT EXISTS active_scheduled_tasks AS
+SELECT
+    st.*,
+    t.content as thought_content,
+    t.task_id as associated_task_id
+FROM scheduled_tasks st
+LEFT JOIN thoughts t ON st.origin_thought_id = t.thought_id
+WHERE st.status IN ('PENDING', 'ACTIVE')
+  AND (st.next_trigger_at IS NULL OR st.next_trigger_at <= datetime('now', '+5 minutes'))
+ORDER BY st.next_trigger_at ASC;
+
+-- View for agent lineage tracking
+CREATE VIEW IF NOT EXISTS agent_lineage AS
+SELECT
+    cc.new_agent_id,
+    cc.new_agent_name,
+    cc.creator_agent_id,
+    cc.creator_human_id,
+    cc.wise_authority_id,
+    cc.timestamp as birth_timestamp,
+    cc.new_agent_purpose,
+    COUNT(DISTINCT cp.id) as lifetime_shutdowns
+FROM creation_ceremonies cc
+LEFT JOIN continuity_awareness cp ON cc.new_agent_id = cp.agent_id
+WHERE cc.ceremony_status = 'completed'
+GROUP BY cc.new_agent_id;
+
+-- All indexes moved to end to ensure tables exist first
+
+-- wa_cert indexes
+CREATE UNIQUE INDEX IF NOT EXISTS idx_wa_oauth ON wa_cert(oauth_provider, oauth_external_id)
+  WHERE oauth_provider IS NOT NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_wa_adapter ON wa_cert(adapter_id)
+  WHERE adapter_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_wa_pubkey ON wa_cert(pubkey);
+CREATE INDEX IF NOT EXISTS idx_wa_active ON wa_cert(active);
+CREATE INDEX IF NOT EXISTS idx_wa_jwt_kid ON wa_cert(jwt_kid);
+CREATE INDEX IF NOT EXISTS idx_wa_role ON wa_cert(role);
+CREATE INDEX IF NOT EXISTS idx_wa_parent ON wa_cert(parent_wa_id);
+CREATE INDEX IF NOT EXISTS idx_wa_created ON wa_cert(created DESC);
+CREATE INDEX IF NOT EXISTS idx_wa_token_type ON wa_cert(token_type);
+
+-- Tasks table indexes
+CREATE INDEX IF NOT EXISTS idx_tasks_signed_by ON tasks(signed_by);
+CREATE INDEX IF NOT EXISTS idx_tasks_channel_id ON tasks(channel_id);

ciris_engine/logic/persistence/migrations/sqlite/002_add_retry_status.sql

@@ -0,0 +1,3 @@
+-- Sample migration (post-beta): add retry_count column to tasks
+-- NOTE: This is just a sample since retry_count is already in base schema for pre-beta
+-- In actual post-beta usage, this would be: ALTER TABLE tasks ADD COLUMN some_new_feature INTEGER DEFAULT 0;

ciris_engine/logic/persistence/migrations/sqlite/003_add_task_update_tracking.sql

@@ -0,0 +1,8 @@
+-- Migration 003: Add task update tracking fields
+-- Adds fields to support detecting when new observations arrive for an active task
+
+-- Add updated_info_available flag
+ALTER TABLE tasks ADD COLUMN updated_info_available INTEGER DEFAULT 0;
+
+-- Add updated_info_content field
+ALTER TABLE tasks ADD COLUMN updated_info_content TEXT DEFAULT NULL;

ciris_engine/logic/persistence/migrations/sqlite/004_add_occurrence_id.sql

@@ -0,0 +1,45 @@
+-- Migration 004: Add agent_occurrence_id for multi-instance deployment support
+-- Enables running multiple API-only CIRIS agent instances against the same SQLite database
+-- Each runtime instance processes only its own work while sharing a single agent identity
+
+-- Add agent_occurrence_id to tasks table
+ALTER TABLE tasks ADD COLUMN agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Add agent_occurrence_id to thoughts table
+ALTER TABLE thoughts ADD COLUMN agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Add agent_occurrence_id to scheduled_tasks table
+ALTER TABLE scheduled_tasks ADD COLUMN agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Add agent_occurrence_id to service_correlations for per-instance telemetry
+ALTER TABLE service_correlations ADD COLUMN agent_occurrence_id TEXT NOT NULL DEFAULT 'default';
+
+-- Create composite indexes for efficient occurrence-scoped queries
+
+-- Tasks indexes
+CREATE INDEX IF NOT EXISTS idx_tasks_occurrence_status
+    ON tasks(agent_occurrence_id, status, created_at);
+
+CREATE INDEX IF NOT EXISTS idx_tasks_occurrence_channel
+    ON tasks(agent_occurrence_id, channel_id, status);
+
+-- Thoughts indexes
+CREATE INDEX IF NOT EXISTS idx_thoughts_occurrence_status
+    ON thoughts(agent_occurrence_id, status, created_at);
+
+CREATE INDEX IF NOT EXISTS idx_thoughts_occurrence_task
+    ON thoughts(agent_occurrence_id, source_task_id);
+
+-- Scheduled tasks index
+CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_occurrence
+    ON scheduled_tasks(agent_occurrence_id, status, next_trigger_at);
+
+-- Service correlations index
+CREATE INDEX IF NOT EXISTS idx_correlations_occurrence
+    ON service_correlations(agent_occurrence_id);
+
+-- Note: Existing rows automatically get 'default' value via DEFAULT clause
+-- Note: graph_nodes, graph_edges, audit_log, and wa_cert remain global (no occurrence_id)
+--       - Memory is shared across all occurrences (single agent identity/knowledge)
+--       - Audit trail is global for complete history
+--       - WA certificates are shared (single agent identity)

ciris_engine/logic/persistence/migrations/sqlite/005_add_consolidation_locks.sql

@@ -0,0 +1,22 @@
+-- Migration 005: Add consolidation_locks table for TSDB consolidation coordination
+-- Enables multiple instances to coordinate consolidation work without conflicting
+-- Uses conditional UPDATE pattern for lock acquisition (like distributed job queue)
+
+CREATE TABLE IF NOT EXISTS consolidation_locks (
+    lock_key TEXT PRIMARY KEY,           -- e.g., "basic:2025-10-22T06:00:00+00:00"
+    locked_by TEXT,                      -- Hostname or instance ID holding the lock
+    locked_at TEXT,                      -- ISO timestamp when lock was acquired
+    lock_timeout_seconds INTEGER DEFAULT 300  -- Auto-expire locks after 5 minutes
+);
+
+-- Index for finding expired locks
+CREATE INDEX IF NOT EXISTS idx_consolidation_locks_expiry
+    ON consolidation_locks(locked_at);
+
+-- Note: Lock acquisition uses conditional UPDATE:
+--   UPDATE consolidation_locks
+--   SET locked_by = ?, locked_at = ?
+--   WHERE lock_key = ? AND (locked_by IS NULL OR locked_at < ?)
+--
+-- If rowcount > 0, lock was acquired
+-- If rowcount = 0, lock is held by another instance

ciris_engine/logic/persistence/migrations/sqlite/006_add_correlation_id_unique_index.sql

@@ -0,0 +1,16 @@
+-- Migration 006: Add unique index on (agent_occurrence_id, correlation_id)
+--
+-- Purpose: Prevent duplicate task creation for the same Reddit comment/post
+--
+-- Problem: Multiple observer polls can race and create duplicate tasks for the same
+--          correlation_id (Reddit post/comment ID), causing duplicate replies.
+--
+-- Solution: Create a unique index on (agent_occurrence_id, correlation_id) extracted
+--           from context_json to enforce uniqueness at the database level.
+--
+-- Note: SQLite supports JSON extraction in indexes since version 3.9.0 (2015)
+--       Uses json_extract() function to pull correlation_id from context_json
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_occurrence_correlation
+ON tasks(agent_occurrence_id, json_extract(context_json, '$.correlation_id'))
+WHERE json_extract(context_json, '$.correlation_id') IS NOT NULL;

ciris_engine/logic/persistence/migrations/sqlite/007_add_dsar_tickets.sql

@@ -0,0 +1,39 @@
+-- Migration 007: Add DSAR Tickets Table
+-- Purpose: Replace in-memory dict with persistent storage for GDPR compliance
+-- Impact: DSAR tickets survive server restarts, meeting 30-day response requirement
+
+CREATE TABLE IF NOT EXISTS dsar_tickets (
+    ticket_id TEXT PRIMARY KEY,
+    request_type TEXT NOT NULL CHECK(request_type IN ('access', 'delete', 'export', 'correct')),
+    email TEXT NOT NULL,
+    user_identifier TEXT,
+    details TEXT,
+    urgent INTEGER NOT NULL DEFAULT 0,  -- 0 = false, 1 = true
+    status TEXT NOT NULL CHECK(status IN ('pending_review', 'in_progress', 'completed', 'rejected')),
+    submitted_at TEXT NOT NULL,  -- ISO8601 timestamp
+    estimated_completion TEXT NOT NULL,  -- ISO8601 timestamp
+    last_updated TEXT NOT NULL,  -- ISO8601 timestamp
+    notes TEXT,
+    automated INTEGER NOT NULL DEFAULT 0,  -- 0 = false, 1 = true
+    access_package_json TEXT,  -- JSON serialized DSARAccessPackage
+    export_package_json TEXT,  -- JSON serialized DSARExportPackage
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Index for status queries (admin filtering by status)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_status ON dsar_tickets(status);
+
+-- Index for email lookups (user checking their requests)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_email ON dsar_tickets(email);
+
+-- Index for user_identifier lookups (data deletion tracking)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_user_identifier ON dsar_tickets(user_identifier);
+
+-- Index for request type queries (metrics by type)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_request_type ON dsar_tickets(request_type);
+
+-- Index for timestamp queries (30-day response compliance)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_submitted_at ON dsar_tickets(submitted_at);
+
+-- Composite index for common admin query (status + submitted date)
+CREATE INDEX IF NOT EXISTS idx_dsar_tickets_status_submitted ON dsar_tickets(status, submitted_at);

ciris_engine/logic/persistence/migrations/sqlite/008_rename_to_tickets_add_sop.sql

@@ -0,0 +1,120 @@
+-- Migration 008: Rename dsar_tickets to tickets and add SOP architecture
+-- Purpose: Make tickets universal with DSAR as always-present default capability
+-- Impact: All agents have DSAR SOPs, agents can add custom ticket SOPs via templates
+--
+-- Architecture Decision:
+-- - DSAR is universal (GDPR compliance required for all agents)
+-- - Other ticket types (appointments, incidents) are agent-specific
+-- - SOP column links to agent template ticket configuration
+-- - Metadata JSON stores stage progress and results
+
+-- Step 1: Create new tickets table with SOP architecture
+CREATE TABLE IF NOT EXISTS tickets (
+    ticket_id TEXT PRIMARY KEY,
+    sop TEXT NOT NULL,  -- Standard Operating Procedure (e.g., "DSAR_ACCESS", "APPOINTMENT_SCHEDULE")
+    ticket_type TEXT NOT NULL,  -- Category (e.g., "dsar", "appointment", "incident")
+    status TEXT NOT NULL CHECK(status IN ('pending', 'in_progress', 'completed', 'cancelled', 'failed')),
+    priority INTEGER NOT NULL DEFAULT 5 CHECK(priority BETWEEN 1 AND 10),  -- 1=lowest, 10=urgent
+
+    -- Contact & identification
+    email TEXT NOT NULL,
+    user_identifier TEXT,
+
+    -- Lifecycle timestamps
+    submitted_at TEXT NOT NULL,  -- ISO8601 timestamp
+    deadline TEXT,  -- ISO8601 timestamp (calculated from SOP config)
+    last_updated TEXT NOT NULL,  -- ISO8601 timestamp
+    completed_at TEXT,  -- ISO8601 timestamp
+
+    -- Workflow state
+    metadata TEXT NOT NULL DEFAULT '{}',  -- JSON: stage progress, results, SOP-specific data
+    notes TEXT,
+
+    -- Tracking
+    automated INTEGER NOT NULL DEFAULT 0,  -- 0 = false, 1 = true
+    correlation_id TEXT,  -- Links to tasks/thoughts processing this ticket
+
+    created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Step 2: Migrate existing dsar_tickets to new tickets table
+INSERT INTO tickets (
+    ticket_id,
+    sop,
+    ticket_type,
+    status,
+    priority,
+    email,
+    user_identifier,
+    submitted_at,
+    deadline,
+    last_updated,
+    completed_at,
+    metadata,
+    notes,
+    automated,
+    created_at
+)
+SELECT
+    ticket_id,
+    -- Map request_type to SOP
+    CASE request_type
+        WHEN 'access' THEN 'DSAR_ACCESS'
+        WHEN 'delete' THEN 'DSAR_DELETE'
+        WHEN 'export' THEN 'DSAR_EXPORT'
+        WHEN 'correct' THEN 'DSAR_RECTIFY'
+        ELSE 'DSAR_ACCESS'  -- fallback
+    END as sop,
+    'dsar' as ticket_type,
+    -- Map old status to new status
+    CASE status
+        WHEN 'pending_review' THEN 'pending'
+        WHEN 'in_progress' THEN 'in_progress'
+        WHEN 'completed' THEN 'completed'
+        WHEN 'rejected' THEN 'cancelled'
+        ELSE 'pending'
+    END as status,
+    -- Map urgent flag to priority
+    CASE urgent
+        WHEN 1 THEN 9  -- urgent = priority 9
+        ELSE 5  -- normal = priority 5
+    END as priority,
+    email,
+    user_identifier,
+    submitted_at,
+    estimated_completion as deadline,
+    last_updated,
+    CASE
+        WHEN status = 'completed' THEN last_updated
+        ELSE NULL
+    END as completed_at,
+    -- Build metadata JSON from legacy fields
+    json_object(
+        'legacy_request_type', request_type,
+        'legacy_details', COALESCE(details, ''),
+        'access_package', COALESCE(access_package_json, 'null'),
+        'export_package', COALESCE(export_package_json, 'null'),
+        'stages', json_object()
+    ) as metadata,
+    notes,
+    automated,
+    created_at
+FROM dsar_tickets;
+
+-- Step 3: Drop old dsar_tickets table
+DROP TABLE dsar_tickets;
+
+-- Step 4: Create indexes for new tickets table
+CREATE INDEX IF NOT EXISTS idx_tickets_sop ON tickets(sop);
+CREATE INDEX IF NOT EXISTS idx_tickets_type ON tickets(ticket_type);
+CREATE INDEX IF NOT EXISTS idx_tickets_status ON tickets(status);
+CREATE INDEX IF NOT EXISTS idx_tickets_email ON tickets(email);
+CREATE INDEX IF NOT EXISTS idx_tickets_user_identifier ON tickets(user_identifier);
+CREATE INDEX IF NOT EXISTS idx_tickets_submitted_at ON tickets(submitted_at);
+CREATE INDEX IF NOT EXISTS idx_tickets_deadline ON tickets(deadline);
+CREATE INDEX IF NOT EXISTS idx_tickets_correlation_id ON tickets(correlation_id);
+
+-- Composite indexes for common queries
+CREATE INDEX IF NOT EXISTS idx_tickets_status_submitted ON tickets(status, submitted_at);
+CREATE INDEX IF NOT EXISTS idx_tickets_sop_status ON tickets(sop, status);
+CREATE INDEX IF NOT EXISTS idx_tickets_type_status ON tickets(ticket_type, status);