specmem-hardwicksoftware 3.5.99

package/dist/db/embeddingOverflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddingOverflow.js","sourceRoot":"","sources":["../../src/db/embeddingOverflow.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAuC,MAAM,iCAAiC,CAAC;AAE3G,+EAA+E;AAC/E,kCAAkC;AAClC,+EAA+E;AAE/E;;GAEG;AACH,MAAM,OAAO,wBAAwB;IAC3B,EAAE,CAAkB;IACpB,WAAW,GAAG,KAAK,CAAC;IACpB,gBAAgB,GAAsB,IAAI,CAAC;IAEnD,YAAY,EAAmB;QAC7B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,+BAA+B;QAC/B,IAAI,CAAC;YACH,IAAI,CAAC,gBAAgB,GAAG,mBAAmB,CAAC,EAAE,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,IAAI,CAAC,gBAAgB,GAAG,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,kBAAkB;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,uDAAuD;YACvD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,yBAAyB,EAAE,CAAC;YAC/D,IAAI,SAAS,KAAK,IAAI,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBACxC,OAAO,UAAU,SAAS,GAAG,CAAC;YAChC,CAAC;YACD,mDAAmD;YACnD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,oEAAoE,CAAC,CAAC;YAC9F,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,SAAmB;QAChD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACpC,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;YAC5F,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,CAAC,KAAK,CAAC;oBACX,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,WAAW,EAAE,SAAS,CAAC,MAAM;oBAC7B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM;iBAClC,EAAE,kDAAkD,CAAC,CAAC;YACzD,CAAC;YACD,OAAO,IAAI,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACpC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,IAAI,CAAC;YACH,4CAA4C;YAC5C,uDAAuD;YACvD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAClD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;sBAGJ,SAAS;;;;;;;OAOxB,CAAC,CAAC;YAEH,sDAAsD;YACtD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;OAGnB,CAAC,CAAC;YAEH,4CAA4C;YAC5C,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;OAGnB,CAAC,CAAC;YAEH,mDAAmD;YACnD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;;OAIzC,CAAC,CAAC;YACH,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,sCAAsC,CAAC,CAAC;YAC9F,CAAC;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,+CAA+C,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,OAA+B;QAClD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEnC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,IAAI,CAAC;YACH,mCAAmC;YACnC,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;gBACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,IAAI,CAAC;wBACH,yDAAyD;wBACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAE9D,2CAA2C;wBAC3C,MAAM,MAAM,CAAC,KAAK,CAAC;;;;;;;;;aASlB,EAAE;4BACD,KAAK,CAAC,GAAG;4BACT,YAAY;4BACZ,KAAK,CAAC,WAAW;4BACjB,KAAK,CAAC,YAAY;4BAClB,KAAK,CAAC,SAAS;4BACf,KAAK,CAAC,IAAI;yBACX,CAAC,CAAC;wBAEH,UAAU,EAAE,CAAC;oBACf,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,EAAE,6BAA6B,CAAC,CAAC;oBACxE,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,sCAAsC,CAAC,CAAC;QACjG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,sCAAsC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC;QACd,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,IAAc;QACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,GAAG,EAAE,CAAC;QAExC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAE3C,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;;OAIvC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;YAEX,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;oBAClB,gDAAgD;oBAChD,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;oBACrD,IAAI,SAAS,EAAE,CAAC;wBACd,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;oBACjC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,kDAAkD;YAClD,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;;SAInB,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,sBAAsB,CAAC,CAAC;QACvF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,yCAAyC,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC3E,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,wBAAwB,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,gCAAgC,CAAC,CAAC;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QAMZ,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;;;;;OAOlC,CAAC,CAAC;YAEH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC3B,OAAO;gBACL,YAAY,EAAE,QAAQ,CAAC,GAAG,EAAE,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC;gBACrD,SAAS,EAAE,QAAQ,CAAC,GAAG,EAAE,UAAU,IAAI,GAAG,EAAE,EAAE,CAAC;gBAC/C,WAAW,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI;gBAClE,WAAW,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI;aACnE,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,8BAA8B,CAAC,CAAC;YACxD,OAAO;gBACL,YAAY,EAAE,CAAC;gBACf,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE,IAAI;gBACjB,WAAW,EAAE,IAAI;aAClB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,aAAqB,KAAK;QACtC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExB,IAAI,CAAC;YACH,uBAAuB;YACvB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACrC,wDAAwD,CACzD,CAAC;YACF,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAE9D,IAAI,KAAK,IAAI,UAAU;gBAAE,OAAO,CAAC,CAAC;YAElC,mCAAmC;YACnC,MAAM,QAAQ,GAAG,KAAK,GAAG,UAAU,CAAC;YACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC;;;;;;;OAOlC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;YAEf,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,6BAA6B,CAAC,CAAC;YACpE,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,4BAA4B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAc;QACnC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,sDAAsD;gBACtD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gBACjD,OAAO,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,wEAAwE;gBACxE,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,2BAA2B,CAAC,CAAC;gBACnF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAAC,EAAmB;IAChE,OAAO,IAAI,wBAAwB,CAAC,EAAE,CAAC,CAAC;AAC1C,CAAC"}

package/dist/db/embeddingOverflow.sql

@@ -0,0 +1,221 @@
+-- Embedding Overflow Database Schema
+-- Stores violation/security data when AI models are paused
+-- Max 10GB storage, 90-day retention (configurable)
+--
+-- =============================================================================
+-- EMBEDDING DIMENSION NOTE
+-- =============================================================================
+-- DEPRECATED: SPECMEM_EMBEDDING_DIMENSIONS is no longer used.
+--
+-- Embedding dimensions are now AUTO-DETECTED from the database pgvector column.
+-- The database pg_attribute table is the single source of truth for dimensions.
+-- The Frankenstein embedding model outputs 384-dim embeddings natively.
+--
+-- The system auto-migrates when dimension mismatch is detected at startup.
+-- WARNING: Changing dimensions requires clearing existing embeddings!
+-- =============================================================================
+
+CREATE TABLE IF NOT EXISTS embedding_overflow (
+    id BIGSERIAL PRIMARY KEY,
+
+    -- Data identification
+    data_hash VARCHAR(64) NOT NULL UNIQUE,  -- SHA256 hash for ACK checking
+    data_type VARCHAR(50) NOT NULL,         -- 'violation', 'security_event', etc.
+
+    -- Content (the actual data to be embedded)
+    content TEXT NOT NULL,
+    content_json JSONB,                      -- Original structured data
+
+    -- Metadata
+    source_ip INET,
+    user_agent TEXT,
+    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    priority INTEGER DEFAULT 2,              -- 0=critical, 1=high, 2=normal, 3=low
+
+    -- Processing status
+    status VARCHAR(20) DEFAULT 'pending',    -- pending, processing, completed, failed
+    -- NOTE: Dimension is auto-detected at runtime, unbounded initially
+    embedding_vector vector,                  -- pgvector for semantic search (lightweight for overflow)
+    processed_at TIMESTAMPTZ,
+    error_message TEXT,
+    retry_count INTEGER DEFAULT 0,
+
+    -- Storage management
+    data_size_bytes INTEGER,                 -- Track size for 10GB limit
+    expires_at TIMESTAMPTZ,                  -- Auto-expire old data
+
+    -- Indexes for performance
+    created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_overflow_status ON embedding_overflow(status, priority, created_at);
+CREATE INDEX IF NOT EXISTS idx_overflow_hash ON embedding_overflow(data_hash);
+CREATE INDEX IF NOT EXISTS idx_overflow_expires ON embedding_overflow(expires_at);
+CREATE INDEX IF NOT EXISTS idx_overflow_type ON embedding_overflow(data_type, status);
+CREATE INDEX IF NOT EXISTS idx_overflow_timestamp ON embedding_overflow(timestamp DESC);
+
+-- Embedding vector index for semantic search (when embeddings are generated)
+CREATE INDEX IF NOT EXISTS idx_overflow_embedding
+ON embedding_overflow
+USING ivfflat (embedding_vector vector_cosine_ops)
+WITH (lists = 100);
+
+-- Configuration table for overflow system
+CREATE TABLE IF NOT EXISTS embedding_overflow_config (
+    id INTEGER PRIMARY KEY DEFAULT 1,
+
+    -- Storage limits
+    max_storage_gb DECIMAL(5,2) DEFAULT 10.0,
+    current_storage_gb DECIMAL(5,2) DEFAULT 0.0,
+
+    -- Retention policy
+    retention_days INTEGER DEFAULT 90,
+    auto_purge_enabled BOOLEAN DEFAULT true,
+    purge_batch_size INTEGER DEFAULT 1000,
+
+    -- Processing limits
+    catchup_batch_size INTEGER DEFAULT 50,
+    catchup_rps_limit DECIMAL(4,2) DEFAULT 2.0,
+
+    -- Statistics
+    total_queued BIGINT DEFAULT 0,
+    total_processed BIGINT DEFAULT 0,
+    total_failed BIGINT DEFAULT 0,
+    total_purged BIGINT DEFAULT 0,
+
+    last_catchup_at TIMESTAMPTZ,
+    last_purge_at TIMESTAMPTZ,
+
+    updated_at TIMESTAMPTZ DEFAULT NOW(),
+
+    CONSTRAINT only_one_config CHECK (id = 1)
+);
+
+-- Initialize config
+INSERT INTO embedding_overflow_config (id)
+VALUES (1)
+ON CONFLICT (id) DO NOTHING;
+
+-- Function to calculate total storage size
+CREATE OR REPLACE FUNCTION update_overflow_storage_size()
+RETURNS DECIMAL AS $$
+DECLARE
+    total_bytes BIGINT;
+    total_gb DECIMAL;
+BEGIN
+    SELECT COALESCE(SUM(data_size_bytes), 0) INTO total_bytes
+    FROM embedding_overflow;
+
+    total_gb := total_bytes / 1024.0 / 1024.0 / 1024.0;
+
+    UPDATE embedding_overflow_config
+    SET current_storage_gb = total_gb,
+        updated_at = NOW()
+    WHERE id = 1;
+
+    RETURN total_gb;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to purge old data
+CREATE OR REPLACE FUNCTION purge_old_overflow_data()
+RETURNS INTEGER AS $$
+DECLARE
+    config_row embedding_overflow_config%ROWTYPE;
+    deleted_count INTEGER;
+    cutoff_date TIMESTAMPTZ;
+BEGIN
+    -- Get config
+    SELECT * INTO config_row FROM embedding_overflow_config WHERE id = 1;
+
+    IF NOT config_row.auto_purge_enabled THEN
+        RETURN 0;
+    END IF;
+
+    -- Calculate cutoff date
+    cutoff_date := NOW() - (config_row.retention_days || ' days')::INTERVAL;
+
+    -- Delete old completed/failed records
+    WITH deleted AS (
+        DELETE FROM embedding_overflow
+        WHERE status IN ('completed', 'failed')
+        AND created_at < cutoff_date
+        RETURNING id
+    )
+    SELECT COUNT(*) INTO deleted_count FROM deleted;
+
+    -- Update stats
+    UPDATE embedding_overflow_config
+    SET total_purged = total_purged + deleted_count,
+        last_purge_at = NOW(),
+        updated_at = NOW()
+    WHERE id = 1;
+
+    -- Update storage size
+    PERFORM update_overflow_storage_size();
+
+    RETURN deleted_count;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to check if storage limit exceeded and purge if needed
+CREATE OR REPLACE FUNCTION enforce_storage_limit()
+RETURNS BOOLEAN AS $$
+DECLARE
+    config_row embedding_overflow_config%ROWTYPE;
+    current_gb DECIMAL;
+    deleted_count INTEGER;
+BEGIN
+    SELECT * INTO config_row FROM embedding_overflow_config WHERE id = 1;
+    current_gb := update_overflow_storage_size();
+
+    IF current_gb > config_row.max_storage_gb THEN
+        -- Emergency purge: delete oldest completed/failed records
+        WITH deleted AS (
+            DELETE FROM embedding_overflow
+            WHERE status IN ('completed', 'failed')
+            AND id IN (
+                SELECT id FROM embedding_overflow
+                WHERE status IN ('completed', 'failed')
+                ORDER BY created_at ASC
+                LIMIT config_row.purge_batch_size
+            )
+            RETURNING id
+        )
+        SELECT COUNT(*) INTO deleted_count FROM deleted;
+
+        -- Update stats
+        UPDATE embedding_overflow_config
+        SET total_purged = total_purged + deleted_count,
+            updated_at = NOW()
+        WHERE id = 1;
+
+        PERFORM update_overflow_storage_size();
+
+        RETURN true;  -- Purge performed
+    END IF;
+
+    RETURN false;  -- No purge needed
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger to update storage size on insert/delete
+CREATE OR REPLACE FUNCTION trigger_update_storage()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM update_overflow_storage_size();
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER update_storage_on_change
+AFTER INSERT OR DELETE ON embedding_overflow
+FOR EACH STATEMENT
+EXECUTE FUNCTION trigger_update_storage();
+
+-- Comments
+COMMENT ON TABLE embedding_overflow IS 'Stores security/violation data waiting for AI model embedding. Max 10GB, auto-purge after 90 days.';
+COMMENT ON COLUMN embedding_overflow.data_hash IS 'SHA256 hash for ACK checking - prevents duplicate embeddings';
+COMMENT ON COLUMN embedding_overflow.status IS 'pending=waiting for model, processing=being embedded, completed=done, failed=error';
+COMMENT ON TABLE embedding_overflow_config IS 'Configuration for overflow system - editable from SpecMem dashboard';

package/dist/db/findThatShit.d.ts

@@ -0,0 +1,145 @@
+import { ConnectionPoolGoBrrr } from './connectionPoolGoBrrr.js';
+import { Memory, MemoryType, ImportanceLevelType, SearchResult, RecallMemoryParams } from '../types/index.js';
+interface RawMemoryRow {
+    id: string;
+    content: string;
+    content_hash: string;
+    memory_type: MemoryType;
+    importance: ImportanceLevelType;
+    tags: string[];
+    metadata: Record<string, unknown>;
+    embedding: string | null;
+    image_data: Buffer | null;
+    image_mime_type: string | null;
+    created_at: Date;
+    updated_at: Date;
+    access_count: number;
+    last_accessed_at: Date | null;
+    expires_at: Date | null;
+    consolidated_from: string[] | null;
+    similarity?: number;
+    text_rank?: number;
+}
+interface VectorSearchOpts {
+    embedding: number[];
+    limit?: number;
+    threshold?: number;
+    memoryTypes?: MemoryType[];
+    tags?: string[];
+    importance?: ImportanceLevelType[];
+    includeExpired?: boolean;
+    dateRange?: {
+        start?: string;
+        end?: string;
+    };
+    allProjects?: boolean;
+    projectPath?: string;
+}
+interface TextSearchOpts {
+    query: string;
+    limit?: number;
+    memoryTypes?: MemoryType[];
+    tags?: string[];
+    includeExpired?: boolean;
+    allProjects?: boolean;
+    projectPath?: string;
+}
+/**
+ * BigBrainSearchEngine - finds memories FAST
+ *
+ * search modes that absolutely SLAP:
+ * - vector search: semantic similarity with cosine distance
+ * - text search: full-text with ranking
+ * - hybrid: combines both for best results
+ * - tag search: filter by tags with AND/OR
+ * - find by id: basic lookup
+ * - find similar: related memories
+ * - find duplicates: content deduplication
+ */
+export declare class BigBrainSearchEngine {
+    private pool;
+    private searchCount;
+    private cacheHits;
+    constructor(pool: ConnectionPoolGoBrrr);
+    getPool(): ConnectionPoolGoBrrr;
+    vectorSearch(opts: VectorSearchOpts): Promise<SearchResult[]>;
+    textSearch(opts: TextSearchOpts): Promise<SearchResult[]>;
+    hybridSearch(query: string, embedding: number[], opts?: {
+        limit?: number;
+        vectorWeight?: number;
+        memoryTypes?: MemoryType[];
+        tags?: string[];
+        includeExpired?: boolean;
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<SearchResult[]>;
+    tagSearch(tags: string[], mode?: 'AND' | 'OR', opts?: {
+        limit?: number;
+        includeExpired?: boolean;
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<Memory[]>;
+    findById(id: string, opts?: {
+        crossProject?: boolean;
+    }): Promise<Memory | null>;
+    findByIds(ids: string[], opts?: {
+        crossProject?: boolean;
+    }): Promise<Memory[]>;
+    findSimilarToMemory(memoryId: string, opts?: {
+        limit?: number;
+        threshold?: number;
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<SearchResult[]>;
+    findDuplicates(threshold?: number, opts?: {
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<Array<{
+        memory1: Memory;
+        memory2: Memory;
+        similarity: number;
+    }>>;
+    recall(params: RecallMemoryParams & {
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<{
+        memories: Memory[];
+        total: number;
+        hasMore: boolean;
+    }>;
+    findRelated(memoryId: string, opts?: {
+        depth?: number;
+        relationType?: string;
+        limit?: number;
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<Memory[]>;
+    getAllTags(limit?: number, opts?: {
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<Array<{
+        name: string;
+        count: number;
+    }>>;
+    countMemories(filters?: {
+        memoryType?: MemoryType;
+        importance?: ImportanceLevelType;
+        tags?: string[];
+        includeExpired?: boolean;
+        allProjects?: boolean;
+        projectPath?: string;
+    }): Promise<number>;
+    private touchMemories;
+    private rowToMemory;
+    private parseEmbedding;
+    private extractHighlights;
+    getStats(): {
+        totalSearches: number;
+        cacheHits: number;
+    };
+}
+export declare function getBigBrain(pool?: ConnectionPoolGoBrrr, projectPath?: string): BigBrainSearchEngine;
+export declare function resetBigBrain(projectPath?: string): void;
+export declare function resetAllBigBrains(): void;
+export type { VectorSearchOpts, TextSearchOpts, RawMemoryRow };
+//# sourceMappingURL=findThatShit.d.ts.map

package/dist/db/findThatShit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"findThatShit.d.ts","sourceRoot":"","sources":["../../src/db/findThatShit.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAIjE,OAAO,EACL,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,YAAY,EAEZ,kBAAkB,EACnB,MAAM,mBAAmB,CAAC;AAkB3B,UAAU,YAAY;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,UAAU,CAAC;IACxB,UAAU,EAAE,mBAAmB,CAAC;IAChC,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,IAAI,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,IAAI,GAAG,IAAI,CAAC;IACxB,iBAAiB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAGD,UAAU,gBAAgB;IACxB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;IAEF,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAGD,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,IAAI,CAAuB;IACnC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,oBAAoB;IAKtC,OAAO,IAAI,oBAAoB;IAKzB,YAAY,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqI7D,UAAU,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IA+EzD,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,GAAE;QACJ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,cAAc,CAAC,EAAE,OAAO,CAAC;QAEzB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;KACjB,GACL,OAAO,CAAC,YAAY,EAAE,CAAC;IAkHpB,SAAS,CACb,IAAI,EAAE,MAAM,EAAE,EACd,IAAI,GAAE,KAAK,GAAG,IAAW,EACzB,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACnG,OAAO,CAAC,MAAM,EAAE,CAAC;IA0Dd,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,GAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAyBnF,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAsBlF,mBAAmB,CACvB,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAC7F,OAAO,CAAC,YAAY,EAAE,CAAC;IA0CpB,cAAc,CAClB,SAAS,SAAO,EAChB,IAAI,GAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GACzD,OAAO,CAAC,KAAK,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC,CAAC;IA2EG,MAAM,CACV,MAAM,EAAE,kBAAkB,GAAG;QAAE,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAC3E,OAAO,CAAC;QACT,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC;IAmFI,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAChH,OAAO,CAAC,MAAM,EAAE,CAAC;IAmEd,UAAU,CAAC,KAAK,SAAO,EAAE,IAAI,GAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAmCrI,aAAa,CAAC,OAAO,CAAC,EAAE;QAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,UAAU,CAAC,EAAE,mBAAmB,CAAC;QACjC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,cAAc,CAAC,EAAE,OAAO,CAAC;QAEzB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC;YA+CL,aAAa;IAa3B,OAAO,CAAC,WAAW;IAqBnB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,iBAAiB;IAoBzB,QAAQ;;;;CAMT;AAaD,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,oBAAoB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,oBAAoB,CAcnG;AAED,wBAAgB,aAAa,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAUxD;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAGxC;AAGD,YAAY,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC"}