specmem-hardwicksoftware 3.5.99

package/embedding-sandbox/deploy-frankenstein.sh

@@ -0,0 +1,240 @@
+#!/bin/bash
+#
+# Deploy Frankenstein Embeddings v3 - ULTIMATE Dynamic Dimension System
+#
+# Features:
+# - TRULY DYNAMIC dimensions 256↔20,000 based on data & query complexity
+# - Dimension EXPANSION: Go BEYOND native model dims!
+# - Dimension COMPRESSION: PCA for efficient reduction
+# - Self-training transforms that learn from YOUR data
+# - RAM guard: Auto-throttles to stay under 4GB
+# - Stats endpoint for monitoring
+#
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SPECMEM_HOME="${SPECMEM_HOME:-$(dirname "$SCRIPT_DIR")}"
+SPECMEM_RUN_DIR="${SPECMEM_RUN_DIR:-${SPECMEM_HOME}/run}"
+SPECMEM_SOCKET_DIR="${SPECMEM_SOCKET_DIR:-${SPECMEM_RUN_DIR}}"
+
+SOCKET_PATH="${SPECMEM_SOCKET_DIR}/embeddings.sock"
+LOG_FILE="/tmp/frankenstein-embeddings.log"
+PID_FILE="/tmp/frankenstein-embeddings.pid"
+
+echo "========================================================================"
+echo "🧟 FRANKENSTEIN EMBEDDINGS v3 - ULTIMATE DEPLOYMENT"
+echo "========================================================================"
+echo ""
+echo "Features:"
+echo "  ⚡ TRULY DYNAMIC dimensions: 256 → 20,000!"
+echo "  📈 Dimension EXPANSION: Go BEYOND native model dims!"
+echo "  📉 Dimension COMPRESSION: PCA for efficient reduction"
+echo "  🧠 Self-training transforms: Learns from YOUR data"
+echo "  🔒 RAM guard: Up to 4GB with auto-throttling"
+echo "  📊 Stats endpoint: Send {\"stats\": true}"
+echo ""
+
+# Check Python
+if ! command -v python3 &> /dev/null; then
+    echo "❌ Error: Python3 not found"
+    exit 1
+fi
+
+# Check if running from correct directory
+if [ ! -f "$SCRIPT_DIR/frankenstein-embeddings.py" ]; then
+    echo "❌ Error: frankenstein-embeddings.py not found in $SCRIPT_DIR"
+    exit 1
+fi
+
+# Install Python dependencies
+echo "📦 Installing Python dependencies..."
+pip3 install -q sentence-transformers scikit-learn torch numpy 2>/dev/null || {
+    echo "  Installing with --user flag..."
+    pip3 install --user -q sentence-transformers scikit-learn torch numpy
+}
+
+# Download model (will be cached)
+echo ""
+echo "📥 Downloading embedding model..."
+echo "   Model: all-MiniLM-L6-v2 (~80MB)"
+python3 << 'DOWNLOAD'
+import sys
+try:
+    from sentence_transformers import SentenceTransformer
+    print("  Loading model (will cache for future runs)...")
+    model = SentenceTransformer(
+        "sentence-transformers/all-MiniLM-L6-v2",
+        device='cpu',
+        cache_folder="/tmp/frankenstein-models"
+    )
+    dims = model.get_sentence_embedding_dimension()
+    print(f"  ✅ Model loaded! Native dims: {dims}")
+except Exception as e:
+    print(f"  ❌ Error: {e}", file=sys.stderr)
+    sys.exit(1)
+DOWNLOAD
+
+# Stop existing server
+echo ""
+echo "🛑 Stopping existing embedding server..."
+pkill -f "frankenstein-embeddings.py" 2>/dev/null || true
+if [ -f "$PID_FILE" ]; then
+    kill $(cat "$PID_FILE") 2>/dev/null || true
+    rm -f "$PID_FILE"
+fi
+sleep 2
+
+# Remove old socket
+rm -f "$SOCKET_PATH"
+
+# Get database config from environment
+DB_HOST="${SPECMEM_DB_HOST:-localhost}"
+DB_PORT="${SPECMEM_DB_PORT:-5432}"
+DB_NAME="${SPECMEM_DB_NAME:-specmem_westayunprofessional}"
+DB_USER="${SPECMEM_DB_USER:-specmem_westayunprofessional}"
+DB_PASSWORD="${SPECMEM_DB_PASSWORD:-${SPECMEM_PASSWORD:-specmem_westayunprofessional}}"
+
+# Launch Frankenstein server with QQMS throttling
+echo ""
+echo "🚀 Launching Frankenstein v3 ULTIMATE server with QQMS throttling..."
+python3 "$SCRIPT_DIR/frankenstein-embeddings.py" \
+    --socket "$SOCKET_PATH" \
+    --db-host "$DB_HOST" \
+    --db-port "$DB_PORT" \
+    --db-name "$DB_NAME" \
+    --db-user "$DB_USER" \
+    --db-password "$DB_PASSWORD" \
+    --max-rps "${SPECMEM_EMBEDDING_MAX_RPS:-10}" \
+    --base-delay "${SPECMEM_EMBEDDING_BASE_DELAY:-100}" \
+    --cpu-threshold "${SPECMEM_EMBEDDING_CPU_THRESHOLD:-60}" \
+    > "$LOG_FILE" 2>&1 &
+
+FRANK_PID=$!
+echo "$FRANK_PID" > "$PID_FILE"
+echo "   PID: $FRANK_PID"
+
+# Wait for socket
+echo ""
+echo "⏳ Waiting for server to start..."
+for i in {1..30}; do
+    if [ -S "$SOCKET_PATH" ]; then
+        echo "   ✅ Socket ready after ${i}s!"
+        break
+    fi
+    if ! kill -0 $FRANK_PID 2>/dev/null; then
+        echo "   ❌ Process died! Check logs:"
+        tail -20 "$LOG_FILE"
+        exit 1
+    fi
+    sleep 1
+done
+
+if [ ! -S "$SOCKET_PATH" ]; then
+    echo "   ❌ Socket not created after 30s!"
+    tail -20 "$LOG_FILE"
+    exit 1
+fi
+
+# Test the server
+echo ""
+echo "🧪 Testing Frankenstein v2..."
+python3 << 'TEST'
+import socket
+import json
+import sys
+
+sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+try:
+    sock.connect('${SPECMEM_SOCKET_DIR}/embeddings.sock')
+    sock.settimeout(30)
+
+    # Test 1: Simple semantic query (should use 256 dims)
+    request = {'text': 'find all memories about testing'}
+    sock.sendall(json.dumps(request).encode('utf-8') + b'\n')
+    data = sock.recv(65536)
+    response = json.loads(data.decode('utf-8'))
+
+    if 'embedding' in response:
+        print(f"  ✅ Simple query: {response['dimensions']}D ({response.get('query_type', '?')})")
+    else:
+        print(f"  ❌ Error: {response}")
+        sys.exit(1)
+
+    sock.close()
+
+    # Test 2: Code query (should use 384 dims)
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.connect('${SPECMEM_SOCKET_DIR}/embeddings.sock')
+    sock.settimeout(30)
+
+    request = {'text': 'async function handleRequest() { return await database.query(); }'}
+    sock.sendall(json.dumps(request).encode('utf-8') + b'\n')
+    data = sock.recv(65536)
+    response = json.loads(data.decode('utf-8'))
+
+    if 'embedding' in response:
+        print(f"  ✅ Code query: {response['dimensions']}D ({response.get('query_type', '?')})")
+    else:
+        print(f"  ❌ Error: {response}")
+
+    sock.close()
+
+    # Test 3: Get stats
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.connect('${SPECMEM_SOCKET_DIR}/embeddings.sock')
+    sock.settimeout(30)
+
+    request = {'stats': True}
+    sock.sendall(json.dumps(request).encode('utf-8') + b'\n')
+    data = sock.recv(65536)
+    response = json.loads(data.decode('utf-8'))
+
+    if 'stats' in response:
+        stats = response['stats']
+        print(f"  ✅ Stats: {stats['total_embeddings']} total, {stats['current_dims']}D mode, {stats['ram_usage_mb']}MB RAM")
+
+    sock.close()
+
+except Exception as e:
+    print(f"  ❌ Test error: {e}")
+    sys.exit(1)
+TEST
+
+if [ $? -ne 0 ]; then
+    echo "❌ Tests failed!"
+    exit 1
+fi
+
+echo ""
+echo "========================================================================"
+echo "✅ FRANKENSTEIN v3 ULTIMATE DEPLOYED!"
+echo "========================================================================"
+echo ""
+echo "Server Info:"
+echo "  PID: $FRANK_PID"
+echo "  Socket: $SOCKET_PATH"
+echo "  Logs: $LOG_FILE"
+echo "  PID file: $PID_FILE"
+echo ""
+echo "TRULY DYNAMIC Dimensions (256 → 20,000!):"
+echo "  • Simple queries      → 256D  (ultra-fast)"
+echo "  • Technical queries   → 512-1024D (balanced)"
+echo "  • Code queries        → 1024-1536D (quality)"
+echo "  • Scientific queries  → 1536-3072D (high quality)"
+echo "  • Maximum complexity  → up to 20,000D (MAXIMUM!)"
+echo ""
+echo "Features:"
+echo "  • Dimension EXPANSION: Go BEYOND native model dims!"
+echo "  • Dimension COMPRESSION: PCA for efficient reduction"
+echo "  • RAM Guard: Up to 4GB with auto-throttling"
+echo ""
+echo "Commands:"
+echo "  Monitor: tail -f $LOG_FILE"
+echo "  Stop: kill \$(cat $PID_FILE)"
+echo "  Stats: echo '{\"stats\":true}' | nc -U $SOCKET_PATH"
+echo ""
+
+# Show recent logs
+echo "Recent logs:"
+tail -10 "$LOG_FILE"

package/embedding-sandbox/docker-compose.yml

@@ -0,0 +1,60 @@
+# SANDBOXED EMBEDDING CONTAINER
+# =============================
+# This container is AIR-GAPPED with ZERO network access
+#
+# Security features:
+# - network_mode: "none" - ZERO network access
+# - cap_drop: ALL - No special capabilities
+# - read_only: true - Cannot modify filesystem
+# - security_opt: no-new-privileges - Cannot escalate
+# - mem_limit: 512m - Max 512MB RAM (model needs ~150MB)
+#
+# Communication: Unix socket at ${SPECMEM_SOCKET_DIR}/embeddings.sock (or ${SPECMEM_HOME}/run)
+
+version: '3.8'
+
+services:
+  embedding-sandbox:
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+    # CRITICAL: NO NETWORK ACCESS
+    network_mode: "none"
+
+    # Security hardening
+    cap_drop:
+      - ALL
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+
+    # Resource limits
+    mem_limit: 512m
+    mem_reservation: 256m
+    cpus: 1.0
+
+    # Temp directories for Python (read-only root needs these)
+    tmpfs:
+      - /tmp:size=64m,mode=1777
+
+    # Unix socket mount - uses dynamic path for project isolation
+    volumes:
+      - ${SPECMEM_SOCKET_DIR:-${SPECMEM_HOME:-/specmem}/run}:/sockets:rw
+
+    # Environment
+    environment:
+      - SOCKET_PATH=/sockets/embeddings.sock
+      - TRANSFORMERS_OFFLINE=1
+      - HF_DATASETS_OFFLINE=1
+
+    # Restart policy
+    restart: unless-stopped
+
+    # Health check via socket (using Node.js since container runs Node)
+    healthcheck:
+      test: ["CMD", "node", "-e", "const net=require('net');const s=net.createConnection('/sockets/embeddings.sock',()=>{s.write('{\"type\":\"health\"}\\n')});s.on('data',d=>{console.log(d.toString());s.destroy();process.exit(0)});s.on('error',e=>{process.exit(1)});setTimeout(()=>process.exit(1),5000)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s

package/embedding-sandbox/docker-manager.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+"""
+Frankenstein Docker Manager
+
+Manages the Frankenstein embeddings Docker container with:
+- Auto-pause when idle (no requests for X seconds)
+- Warm-start with overflow queue injection
+- CPU monitoring and enforcement
+- Health checks
+- Project isolation for multi-instance support
+"""
+
+import os
+
+SPECMEM_HOME = os.environ.get('SPECMEM_HOME', os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+SPECMEM_RUN_DIR = os.environ.get('SPECMEM_RUN_DIR', os.path.join(SPECMEM_HOME, 'run'))
+SPECMEM_SOCKET_DIR = os.environ.get('SPECMEM_SOCKET_DIR', SPECMEM_RUN_DIR)
+
+import subprocess
+import time
+import sys
+import json
+import socket
+import threading
+from datetime import datetime
+from typing import Optional
+from overflow_queue import get_overflow_queue
+
+# Import project isolation utilities - centralized module for multi-instance support
+# All project-scoped paths and names come from this shared module
+from project_isolation import (
+    get_project_path,
+    get_project_hash,
+    get_project_container_name,
+    get_project_socket_dir,
+    get_project_socket_path,
+    get_project_overflow_dir,
+    get_project_overflow_db,
+    ensure_project_dirs
+)
+
+# =============================================================================
+# CONFIGURATION (project-scoped via project_isolation module)
+# =============================================================================
+# Container name format: frankenstein-{8char_hash} (e.g., frankenstein-a1b2c3d4)
+# Each project gets its own isolated container, socket, and overflow directories.
+# Hash is SHA256 of SPECMEM_PROJECT_PATH (fallback: cwd), first 8 chars.
+
+CONTAINER_NAME = get_project_container_name()
+SOCKET_PATH = get_project_socket_path()
+IDLE_TIMEOUT_SECONDS = 60  # Pause after 60 seconds of no activity
+CHECK_INTERVAL_SECONDS = 10
+MAX_CPU_PERCENT = 20
+OVERFLOW_DB = get_project_overflow_db()
+
+class DockerManager:
+    """Manages the Frankenstein Docker container lifecycle"""
+
+    def __init__(self):
+        self.last_request_time = time.time()
+        self.is_paused = False
+        self.container_running = False
+        self.queue = get_overflow_queue()  # Uses PostgreSQL, path not needed
+        self._lock = threading.Lock()
+
+    def docker_cmd(self, *args) -> tuple:
+        """Run a docker command and return (success, output)"""
+        try:
+            result = subprocess.run(
+                ["docker"] + list(args),
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+            return result.returncode == 0, result.stdout.strip()
+        except Exception as e:
+            return False, str(e)
+
+    def is_container_running(self) -> bool:
+        """Check if container is running"""
+        success, output = self.docker_cmd(
+            "inspect", "-f", "{{.State.Running}}", CONTAINER_NAME
+        )
+        return success and output == "true"
+
+    def is_container_paused(self) -> bool:
+        """Check if container is paused"""
+        success, output = self.docker_cmd(
+            "inspect", "-f", "{{.State.Paused}}", CONTAINER_NAME
+        )
+        return success and output == "true"
+
+    def start_container(self):
+        """Start the Frankenstein container with CPU limits"""
+        # Log project isolation info
+        project_path = get_project_path()
+        project_hash = get_project_hash()
+        print(f"Project: {project_path}")
+        print(f"Container: {CONTAINER_NAME}")
+        print(f"Starting Frankenstein container with {MAX_CPU_PERCENT}% CPU limit...")
+
+        # Remove old container if exists
+        self.docker_cmd("rm", "-f", CONTAINER_NAME)
+
+        # Create project-scoped directories using shared utility
+        dirs = ensure_project_dirs()
+        socket_dir = dirs['socket_dir']
+        overflow_dir = dirs['overflow_dir']
+
+        # Start with CPU limit and project-scoped volumes
+        success, output = self.docker_cmd(
+            "run", "-d",
+            "--name", CONTAINER_NAME,
+            f"--cpus={MAX_CPU_PERCENT / 100}",  # 0.2 = 20%
+            "--memory=2g",
+            "-v", f"{socket_dir}:/sockets",
+            "-v", f"{overflow_dir}:/overflow",
+            "-e", f"SOCKET_PATH=/sockets/frankenstein.sock",
+            "-e", f"OVERFLOW_DB=/overflow/queue.db",
+            "-e", f"SPECMEM_PROJECT_PATH={project_path}",
+            "-e", f"SPECMEM_PROJECT_HASH={project_hash}",
+            "-e", "MAX_RPS=3",
+            "-e", "BASE_DELAY=200",
+            "--label", f"specmem.project={project_hash}",
+            "--label", f"specmem.created={int(time.time())}",
+            "--label", f"specmem.path={project_path}",
+            "frankenstein-embeddings"
+        )
+
+        if success:
+            print(f"Container started: {output[:12]}")
+            self.container_running = True
+            self.is_paused = False
+            self.last_request_time = time.time()
+            return True
+        else:
+            print(f"Failed to start container: {output}")
+            return False
+
+    def pause_container(self):
+        """Pause the container to save CPU"""
+        if self.is_paused:
+            return True
+
+        print(f"⏸️  Pausing container (idle for {IDLE_TIMEOUT_SECONDS}s)...")
+        success, output = self.docker_cmd("pause", CONTAINER_NAME)
+
+        if success:
+            self.is_paused = True
+            print("✅ Container paused - CPU usage: 0%")
+            return True
+        else:
+            print(f"❌ Failed to pause: {output}")
+            return False
+
+    def unpause_container(self):
+        """Unpause the container (warm start)"""
+        if not self.is_paused:
+            return True
+
+        print("▶️  Unpausing container (warm start)...")
+        success, output = self.docker_cmd("unpause", CONTAINER_NAME)
+
+        if success:
+            self.is_paused = False
+            self.last_request_time = time.time()
+            print("✅ Container unpaused")
+            # Process overflow queue
+            self.process_overflow_queue()
+            return True
+        else:
+            print(f"❌ Failed to unpause: {output}")
+            return False
+
+    def process_overflow_queue(self):
+        """Process queued requests after warm start"""
+        pending = self.queue.get_pending_count()
+        if pending == 0:
+            return
+
+        print(f"📦 Processing {pending} queued requests...")
+
+        # Get pending requests
+        requests = self.queue.dequeue(limit=100)
+        if not requests:
+            return
+
+        # Mark as processing
+        ids = [r.id for r in requests]
+        self.queue.mark_processing(ids)
+
+        # Send to container via socket
+        for req in requests:
+            try:
+                # Connect to socket and send request
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                sock.connect(SOCKET_PATH)
+
+                message = json.dumps({
+                    'action': 'embed_batch' if req.request_type == 'batch' else 'embed',
+                    'texts': req.texts,
+                    'priority': req.priority,
+                    'dimensions': req.dimensions
+                })
+                sock.sendall(message.encode() + b'\n')
+
+                # Read response
+                response = b''
+                while True:
+                    chunk = sock.recv(4096)
+                    if not chunk:
+                        break
+                    response += chunk
+                    if b'\n' in chunk:
+                        break
+
+                sock.close()
+
+                result = json.loads(response.decode())
+                self.queue.mark_completed(req.id, result=result)
+
+            except Exception as e:
+                self.queue.mark_completed(req.id, error=str(e))
+
+        print(f"✅ Processed {len(requests)} queued requests")
+
+    def record_request(self):
+        """Record that a request was received"""
+        with self._lock:
+            self.last_request_time = time.time()
+            if self.is_paused:
+                self.unpause_container()
+
+    def queue_request(self, texts: list, priority: int = 2, dimensions: int = None) -> int:
+        """Queue a request when container is paused"""
+        return self.queue.enqueue(
+            texts=texts,
+            request_type='batch' if len(texts) > 1 else 'single',
+            priority=priority,
+            dimensions=dimensions
+        )
+
+    def run_idle_monitor(self):
+        """Monitor for idle and pause when appropriate"""
+        print(f"👀 Starting idle monitor (timeout: {IDLE_TIMEOUT_SECONDS}s)")
+
+        while True:
+            time.sleep(CHECK_INTERVAL_SECONDS)
+
+            if not self.container_running:
+                continue
+
+            if self.is_paused:
+                continue
+
+            with self._lock:
+                idle_time = time.time() - self.last_request_time
+
+            if idle_time > IDLE_TIMEOUT_SECONDS:
+                self.pause_container()
+
+    def get_status(self) -> dict:
+        """Get current status"""
+        with self._lock:
+            idle_time = time.time() - self.last_request_time
+
+        return {
+            'container_running': self.container_running,
+            'is_paused': self.is_paused,
+            'idle_seconds': round(idle_time, 1),
+            'max_cpu_percent': MAX_CPU_PERCENT,
+            'queue_pending': self.queue.get_pending_count(),
+            'queue_stats': self.queue.get_stats(),
+            # Project isolation info
+            'project_path': get_project_path(),
+            'project_hash': get_project_hash(),
+            'container_name': CONTAINER_NAME,
+            'socket_path': SOCKET_PATH,
+            'overflow_db': OVERFLOW_DB
+        }
+
+
+def main():
+    """Main entry point"""
+    manager = DockerManager()
+
+    # Check if container image exists
+    success, _ = manager.docker_cmd("image", "inspect", "frankenstein-embeddings")
+    if not success:
+        print("❌ Docker image 'frankenstein-embeddings' not found!")
+        print("   Build it first: docker build -f Dockerfile.frankenstein -t frankenstein-embeddings .")
+        sys.exit(1)
+
+    # Start container
+    if not manager.start_container():
+        sys.exit(1)
+
+    # Start idle monitor in background
+    monitor_thread = threading.Thread(target=manager.run_idle_monitor, daemon=True)
+    monitor_thread.start()
+
+    # Keep running
+    print(f"Frankenstein Docker Manager running")
+    print(f"   Project: {get_project_path()}")
+    print(f"   Container: {CONTAINER_NAME}")
+    print(f"   Hash: {get_project_hash()}")
+    print(f"   CPU cap: {MAX_CPU_PERCENT}%")
+    print(f"   Idle timeout: {IDLE_TIMEOUT_SECONDS}s")
+    print(f"   Socket: {SOCKET_PATH}")
+    print(f"   Overflow: {OVERFLOW_DB}")
+
+    try:
+        while True:
+            time.sleep(30)
+            status = manager.get_status()
+            print(f"📊 Status: {'PAUSED' if status['is_paused'] else 'RUNNING'} | "
+                  f"Idle: {status['idle_seconds']}s | "
+                  f"Queue: {status['queue_pending']}")
+    except KeyboardInterrupt:
+        print("\n👋 Shutting down...")
+        manager.docker_cmd("stop", CONTAINER_NAME)
+
+
+if __name__ == "__main__":
+    main()