omni-cortex 1.17.1__py3-none-any.whl → 1.17.3__py3-none-any.whl

omni_cortex/database/migrations.py

@@ -0,0 +1,210 @@
+"""Database migration management for Omni Cortex."""
+
+import sqlite3
+from typing import Optional
+from pathlib import Path
+
+from .schema import SCHEMA_VERSION, get_schema_sql
+from .connection import get_connection
+from ..utils.timestamps import now_iso
+
+
+# Migration definitions: version -> SQL
+MIGRATIONS: dict[str, str] = {
+    # Command analytics columns for slash command/skill tracking
+    "1.1": """
+        -- Add command analytics columns to activities table
+        ALTER TABLE activities ADD COLUMN command_name TEXT;
+        ALTER TABLE activities ADD COLUMN command_scope TEXT;
+        ALTER TABLE activities ADD COLUMN mcp_server TEXT;
+        ALTER TABLE activities ADD COLUMN skill_name TEXT;
+
+        -- Create indexes for new columns
+        CREATE INDEX IF NOT EXISTS idx_activities_command ON activities(command_name);
+        CREATE INDEX IF NOT EXISTS idx_activities_mcp ON activities(mcp_server);
+        CREATE INDEX IF NOT EXISTS idx_activities_skill ON activities(skill_name);
+    """,
+    # Natural language summary columns for activity display
+    "1.2": """
+        -- Add natural language summary columns to activities table
+        ALTER TABLE activities ADD COLUMN summary TEXT;
+        ALTER TABLE activities ADD COLUMN summary_detail TEXT;
+    """,
+    # Duration tracking columns for concrete time analysis
+    "1.3": """
+        -- Add duration tracking to sessions table
+        ALTER TABLE sessions ADD COLUMN duration_ms INTEGER;
+
+        -- Add duration tracking to session_summaries table
+        ALTER TABLE session_summaries ADD COLUMN duration_ms INTEGER;
+        ALTER TABLE session_summaries ADD COLUMN tool_duration_breakdown TEXT;
+
+        -- Create index for duration queries
+        CREATE INDEX IF NOT EXISTS idx_activities_duration ON activities(duration_ms);
+        CREATE INDEX IF NOT EXISTS idx_sessions_duration ON sessions(duration_ms);
+    """,
+    # User message tracking for style analysis
+    "1.4": """
+        -- User messages table for tracking all user prompts
+        CREATE TABLE IF NOT EXISTS user_messages (
+            id TEXT PRIMARY KEY,                          -- msg_{timestamp}_{random}
+            session_id TEXT,
+            timestamp TEXT NOT NULL,                      -- ISO 8601
+            content TEXT NOT NULL,                        -- The full user message
+            word_count INTEGER,
+            char_count INTEGER,
+            line_count INTEGER,
+            has_code_blocks INTEGER DEFAULT 0,
+            has_questions INTEGER DEFAULT 0,
+            has_commands INTEGER DEFAULT 0,               -- Starts with /
+            tone_indicators TEXT,                         -- JSON: detected tone markers
+            project_path TEXT,
+            metadata TEXT,                                -- JSON for extensibility
+            FOREIGN KEY (session_id) REFERENCES sessions(id)
+        );
+
+        -- User style profile for aggregated style analysis
+        CREATE TABLE IF NOT EXISTS user_style_profiles (
+            id TEXT PRIMARY KEY,                          -- profile_{timestamp}_{random}
+            project_path TEXT,                            -- NULL for global profile
+            total_messages INTEGER DEFAULT 0,
+            avg_word_count REAL,
+            avg_char_count REAL,
+            common_phrases TEXT,                          -- JSON array of frequent phrases
+            vocabulary_richness REAL,                     -- Type-token ratio
+            formality_score REAL,                         -- 0-100 scale
+            question_frequency REAL,                      -- % of messages with questions
+            command_frequency REAL,                       -- % of messages starting with /
+            code_block_frequency REAL,                    -- % with code blocks
+            punctuation_style TEXT,                       -- JSON: punctuation patterns
+            greeting_patterns TEXT,                       -- JSON: how user starts conversations
+            instruction_style TEXT,                       -- JSON: how user gives instructions
+            sample_messages TEXT,                         -- JSON array of representative samples
+            created_at TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            metadata TEXT
+        );
+
+        -- Indexes for user message queries
+        CREATE INDEX IF NOT EXISTS idx_user_messages_session ON user_messages(session_id);
+        CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp ON user_messages(timestamp DESC);
+        CREATE INDEX IF NOT EXISTS idx_user_messages_project ON user_messages(project_path);
+        CREATE INDEX IF NOT EXISTS idx_user_style_project ON user_style_profiles(project_path);
+    """,
+}
+
+
+def get_current_version(conn: sqlite3.Connection) -> Optional[str]:
+    """Get the current schema version from the database."""
+    try:
+        cursor = conn.execute(
+            "SELECT version FROM schema_migrations ORDER BY applied_at DESC LIMIT 1"
+        )
+        row = cursor.fetchone()
+        return row[0] if row else None
+    except sqlite3.OperationalError:
+        # Table doesn't exist yet
+        return None
+
+
+def strip_sql_comments(sql: str) -> str:
+    """Strip SQL comments from a statement while preserving inline comments in strings."""
+    import re
+    lines = sql.split('\n')
+    cleaned_lines = []
+    for line in lines:
+        # Remove line comments (-- ...) but only if not inside a string
+        # Simple approach: just strip everything after -- if not in CREATE TABLE definition
+        # For CREATE TABLE, we need to keep the structure
+        stripped = line.strip()
+        if stripped.startswith('--'):
+            continue  # Skip pure comment lines
+        # Keep lines that have SQL content (even if they have trailing comments)
+        cleaned_lines.append(line)
+    return '\n'.join(cleaned_lines)
+
+
+def apply_migration(conn: sqlite3.Connection, version: str, sql: str) -> None:
+    """Apply a single migration.
+
+    Handles ALTER TABLE ADD COLUMN gracefully - if column exists, skips it.
+    """
+    # First, strip pure comment lines from the SQL
+    sql = strip_sql_comments(sql)
+
+    # Split into individual statements and apply each
+    for statement in sql.strip().split(';'):
+        statement = statement.strip()
+        if not statement:
+            continue
+
+        # Skip if the remaining statement is just whitespace or pure comments
+        non_comment_content = '\n'.join(
+            line for line in statement.split('\n')
+            if line.strip() and not line.strip().startswith('--')
+        ).strip()
+        if not non_comment_content:
+            continue
+
+        try:
+            conn.execute(statement)
+            conn.commit()  # Commit each statement so next one sees the change
+        except sqlite3.OperationalError as e:
+            error_msg = str(e).lower()
+            # Ignore "duplicate column" errors - column already exists
+            if "duplicate column" in error_msg:
+                continue
+            # Ignore "index already exists" errors
+            if "already exists" in error_msg:
+                continue
+            # Ignore "no such column" for indexes on columns that may not exist yet
+            # (will be created in a later migration)
+            if "no such column" in error_msg and "CREATE INDEX" in statement.upper():
+                continue
+            # Ignore "table already exists" errors
+            if "already exists" in error_msg:
+                continue
+            raise
+
+    conn.execute(
+        "INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)",
+        (version, now_iso())
+    )
+    conn.commit()
+
+
+def migrate(db_path: Optional[Path] = None, is_global: bool = False) -> str:
+    """Run all pending migrations.
+
+    Returns:
+        The final schema version
+    """
+    conn = get_connection(db_path, is_global)
+    current = get_current_version(conn)
+
+    if current is None:
+        # Fresh database - apply full schema
+        conn.executescript(get_schema_sql())
+        conn.execute(
+            "INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)",
+            (SCHEMA_VERSION, now_iso())
+        )
+        conn.commit()
+        return SCHEMA_VERSION
+
+    # Apply pending migrations in order
+    versions = sorted(MIGRATIONS.keys())
+    for version in versions:
+        if version > current:
+            apply_migration(conn, version, MIGRATIONS[version])
+            current = version
+
+    return current
+
+
+def needs_migration(conn: sqlite3.Connection) -> bool:
+    """Check if database needs migration."""
+    current = get_current_version(conn)
+    if current is None:
+        return True
+    return any(v > current for v in MIGRATIONS.keys())

omni_cortex/database/schema.py

@@ -0,0 +1,212 @@
+"""Database schema definitions for Omni Cortex."""
+
+SCHEMA_VERSION = "1.0"
+
+# Main schema SQL
+SCHEMA_SQL = """
+-- ============================================
+-- OMNI CORTEX MCP DATABASE SCHEMA v1.0
+-- ============================================
+
+-- Sessions Table
+CREATE TABLE IF NOT EXISTS sessions (
+    id TEXT PRIMARY KEY,                          -- sess_{timestamp}_{random}
+    project_path TEXT NOT NULL,
+    started_at TEXT NOT NULL,                     -- ISO 8601
+    ended_at TEXT,
+    summary TEXT,
+    tags TEXT,                                    -- JSON array
+    metadata TEXT                                 -- JSON object
+);
+
+-- Agents Table
+CREATE TABLE IF NOT EXISTS agents (
+    id TEXT PRIMARY KEY,                          -- Agent ID from Claude Code
+    name TEXT,
+    type TEXT NOT NULL DEFAULT 'main',            -- main, subagent, tool
+    first_seen TEXT NOT NULL,
+    last_seen TEXT NOT NULL,
+    total_activities INTEGER DEFAULT 0,
+    metadata TEXT
+);
+
+-- Activities Table (Layer 1)
+CREATE TABLE IF NOT EXISTS activities (
+    id TEXT PRIMARY KEY,                          -- act_{timestamp}_{random}
+    session_id TEXT,
+    agent_id TEXT,
+    timestamp TEXT NOT NULL,                      -- ISO 8601 with timezone
+    event_type TEXT NOT NULL,                     -- pre_tool_use, post_tool_use, etc.
+    tool_name TEXT,
+    tool_input TEXT,                              -- JSON (truncated to 10KB)
+    tool_output TEXT,                             -- JSON (truncated to 10KB)
+    duration_ms INTEGER,
+    success INTEGER DEFAULT 1,
+    error_message TEXT,
+    project_path TEXT,
+    file_path TEXT,
+    metadata TEXT,
+    -- Command analytics columns (v1.1)
+    command_name TEXT,
+    command_scope TEXT,
+    mcp_server TEXT,
+    skill_name TEXT,
+    -- Natural language summaries (v1.2)
+    summary TEXT,
+    summary_detail TEXT,
+    FOREIGN KEY (session_id) REFERENCES sessions(id),
+    FOREIGN KEY (agent_id) REFERENCES agents(id)
+);
+
+-- Memories Table (Layer 2)
+CREATE TABLE IF NOT EXISTS memories (
+    id TEXT PRIMARY KEY,                          -- mem_{timestamp}_{random}
+    content TEXT NOT NULL,
+    type TEXT NOT NULL DEFAULT 'general',
+    tags TEXT,                                    -- JSON array
+    context TEXT,
+
+    -- Timestamps
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    last_accessed TEXT NOT NULL,
+    last_verified TEXT,
+
+    -- Usage
+    access_count INTEGER DEFAULT 0,
+
+    -- Importance/Decay
+    importance_score REAL DEFAULT 50.0,           -- 0-100
+    manual_importance INTEGER,                    -- User override
+
+    -- Freshness
+    status TEXT DEFAULT 'fresh',                  -- fresh, needs_review, outdated, archived
+
+    -- Attribution
+    source_session_id TEXT,
+    source_agent_id TEXT,
+    source_activity_id TEXT,
+
+    -- Project
+    project_path TEXT,
+    file_context TEXT,                            -- JSON array
+
+    -- Embedding
+    has_embedding INTEGER DEFAULT 0,
+
+    metadata TEXT,
+
+    FOREIGN KEY (source_session_id) REFERENCES sessions(id),
+    FOREIGN KEY (source_agent_id) REFERENCES agents(id),
+    FOREIGN KEY (source_activity_id) REFERENCES activities(id)
+);
+
+-- FTS5 for Full-Text Search
+CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+    content, context, tags,
+    content=memories,
+    content_rowid=rowid,
+    tokenize='porter unicode61'
+);
+
+-- Triggers to keep FTS in sync with memories table
+CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+    INSERT INTO memories_fts(rowid, content, context, tags)
+    VALUES (NEW.rowid, NEW.content, NEW.context, NEW.tags);
+END;
+
+CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, content, context, tags)
+    VALUES ('delete', OLD.rowid, OLD.content, OLD.context, OLD.tags);
+END;
+
+CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, content, context, tags)
+    VALUES ('delete', OLD.rowid, OLD.content, OLD.context, OLD.tags);
+    INSERT INTO memories_fts(rowid, content, context, tags)
+    VALUES (NEW.rowid, NEW.content, NEW.context, NEW.tags);
+END;
+
+-- Memory Relationships
+CREATE TABLE IF NOT EXISTS memory_relationships (
+    id TEXT PRIMARY KEY,
+    source_memory_id TEXT NOT NULL,
+    target_memory_id TEXT NOT NULL,
+    relationship_type TEXT NOT NULL,              -- related_to, supersedes, derived_from, contradicts
+    strength REAL DEFAULT 1.0,
+    created_at TEXT NOT NULL,
+    metadata TEXT,
+    FOREIGN KEY (source_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
+    FOREIGN KEY (target_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
+    UNIQUE(source_memory_id, target_memory_id, relationship_type)
+);
+
+-- Activity-Memory Links
+CREATE TABLE IF NOT EXISTS activity_memory_links (
+    activity_id TEXT NOT NULL,
+    memory_id TEXT NOT NULL,
+    link_type TEXT NOT NULL,                      -- created, accessed, updated, referenced
+    created_at TEXT NOT NULL,
+    PRIMARY KEY (activity_id, memory_id, link_type),
+    FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE,
+    FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
+);
+
+-- Embeddings
+CREATE TABLE IF NOT EXISTS embeddings (
+    id TEXT PRIMARY KEY,
+    memory_id TEXT NOT NULL UNIQUE,
+    model_name TEXT NOT NULL,                     -- 'all-MiniLM-L6-v2'
+    vector BLOB NOT NULL,                         -- float32 array
+    dimensions INTEGER NOT NULL,                  -- 384
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
+);
+
+-- Session Summaries
+CREATE TABLE IF NOT EXISTS session_summaries (
+    id TEXT PRIMARY KEY,
+    session_id TEXT NOT NULL UNIQUE,
+    key_learnings TEXT,                           -- JSON array
+    key_decisions TEXT,                           -- JSON array
+    key_errors TEXT,                              -- JSON array
+    files_modified TEXT,                          -- JSON array
+    tools_used TEXT,                              -- JSON object
+    total_activities INTEGER DEFAULT 0,
+    total_memories_created INTEGER DEFAULT 0,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (session_id) REFERENCES sessions(id) ON DELETE CASCADE
+);
+
+-- Configuration
+CREATE TABLE IF NOT EXISTS config (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL,
+    updated_at TEXT NOT NULL
+);
+
+-- Schema Migrations
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version TEXT PRIMARY KEY,
+    applied_at TEXT NOT NULL
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_activities_session ON activities(session_id);
+CREATE INDEX IF NOT EXISTS idx_activities_agent ON activities(agent_id);
+CREATE INDEX IF NOT EXISTS idx_activities_timestamp ON activities(timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_activities_tool ON activities(tool_name);
+CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
+CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
+CREATE INDEX IF NOT EXISTS idx_memories_project ON memories(project_path);
+CREATE INDEX IF NOT EXISTS idx_memories_importance ON memories(importance_score DESC);
+CREATE INDEX IF NOT EXISTS idx_memories_accessed ON memories(last_accessed DESC);
+CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_relationships_source ON memory_relationships(source_memory_id);
+CREATE INDEX IF NOT EXISTS idx_relationships_target ON memory_relationships(target_memory_id);
+"""
+
+
+def get_schema_sql() -> str:
+    """Get the complete schema SQL."""
+    return SCHEMA_SQL