remdb 0.3.0__py3-none-any.whl → 0.3.127__py3-none-any.whl

rem/sql/migrations/004_cache_system.sql

@@ -0,0 +1,548 @@
+-- REM Cache System
+-- Description: Self-healing cache for UNLOGGED tables (kv_store)
+-- Version: 1.0.0
+-- Date: 2025-11-29
+--
+-- This migration adds:
+--   1. cache_system_state table for debouncing and API secret storage
+--   2. maybe_trigger_kv_rebuild() function for async rebuild triggering
+--   3. Updated rem_lookup/fuzzy/traverse with self-healing on empty cache
+--
+-- Self-Healing Flow:
+--   Query returns 0 results → Check if kv_store empty → Trigger async rebuild
+--   Priority: pg_net (if available) → dblink (always available)
+
+-- ============================================================================
+-- REQUIRED EXTENSION
+-- ============================================================================
+-- pgcrypto is needed for gen_random_bytes() to generate API secrets
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+-- ============================================================================
+-- CACHE SYSTEM STATE TABLE
+-- ============================================================================
+-- Stores:
+--   - Last rebuild trigger timestamp (for debouncing)
+--   - API secret for internal endpoint authentication
+--   - Rebuild statistics
+
+CREATE TABLE IF NOT EXISTS cache_system_state (
+    id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1),  -- Single row table
+    api_secret TEXT NOT NULL,                          -- Secret for internal API auth
+    last_triggered_at TIMESTAMPTZ,                     -- Debounce: last trigger time
+    last_rebuild_at TIMESTAMPTZ,                       -- Last successful rebuild
+    triggered_by TEXT,                                 -- What triggered last rebuild
+    trigger_count INTEGER DEFAULT 0,                   -- Total trigger count
+    rebuild_count INTEGER DEFAULT 0,                   -- Total successful rebuilds
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Generate initial secret if table is empty
+INSERT INTO cache_system_state (id, api_secret)
+SELECT 1, encode(gen_random_bytes(32), 'hex')
+WHERE NOT EXISTS (SELECT 1 FROM cache_system_state WHERE id = 1);
+
+COMMENT ON TABLE cache_system_state IS
+'Single-row table storing cache system state: API secret for internal auth and debounce tracking';
+
+-- ============================================================================
+-- HELPER: Check if extension exists
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_extension_exists(p_extension TEXT)
+RETURNS BOOLEAN AS $$
+BEGIN
+    RETURN EXISTS (SELECT 1 FROM pg_extension WHERE extname = p_extension);
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- HELPER: Check if kv_store is empty for user
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_kv_store_empty(p_user_id TEXT)
+RETURNS BOOLEAN AS $$
+BEGIN
+    -- Quick existence check - very fast with index
+    RETURN NOT EXISTS (
+        SELECT 1 FROM kv_store
+        WHERE user_id = p_user_id
+        LIMIT 1
+    );
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- MAIN: Maybe trigger KV rebuild (async, non-blocking)
+-- ============================================================================
+-- Called when a query returns 0 results and kv_store appears empty.
+-- Uses pg_net (if available) to call API, falls back to dblink.
+-- Includes debouncing to prevent request storms.
+
+CREATE OR REPLACE FUNCTION maybe_trigger_kv_rebuild(
+    p_user_id TEXT,
+    p_triggered_by TEXT DEFAULT 'query'
+)
+RETURNS VOID AS $$
+DECLARE
+    v_has_pgnet BOOLEAN;
+    v_has_dblink BOOLEAN;
+    v_last_trigger TIMESTAMPTZ;
+    v_api_secret TEXT;
+    v_debounce_seconds CONSTANT INTEGER := 30;
+    v_api_url TEXT := 'http://rem-api.rem.svc.cluster.local:8000/api/admin/internal/rebuild-kv';
+    v_request_id BIGINT;
+BEGIN
+    -- Quick check: is kv_store actually empty for this user?
+    IF NOT rem_kv_store_empty(p_user_id) THEN
+        RETURN;  -- Cache has data, nothing to do
+    END IF;
+
+    -- Try to acquire advisory lock (non-blocking, transaction-scoped)
+    -- This prevents multiple concurrent triggers
+    IF NOT pg_try_advisory_xact_lock(2147483646) THEN
+        RETURN;  -- Another session is handling it
+    END IF;
+
+    -- Check debounce: was rebuild triggered recently?
+    SELECT last_triggered_at, api_secret
+    INTO v_last_trigger, v_api_secret
+    FROM cache_system_state
+    WHERE id = 1;
+
+    IF v_last_trigger IS NOT NULL
+       AND v_last_trigger > (CURRENT_TIMESTAMP - (v_debounce_seconds || ' seconds')::INTERVAL) THEN
+        RETURN;  -- Triggered recently, skip
+    END IF;
+
+    -- Update state (so concurrent callers see it)
+    UPDATE cache_system_state
+    SET last_triggered_at = CURRENT_TIMESTAMP,
+        triggered_by = p_triggered_by,
+        trigger_count = trigger_count + 1,
+        updated_at = CURRENT_TIMESTAMP
+    WHERE id = 1;
+
+    -- Check available extensions
+    v_has_pgnet := rem_extension_exists('pg_net');
+    v_has_dblink := rem_extension_exists('dblink');
+
+    -- Priority 1: pg_net (async HTTP to API - supports S3 restore)
+    IF v_has_pgnet THEN
+        BEGIN
+            SELECT net.http_post(
+                url := v_api_url,
+                headers := jsonb_build_object(
+                    'Content-Type', 'application/json',
+                    'X-Internal-Secret', v_api_secret
+                ),
+                body := jsonb_build_object(
+                    'user_id', p_user_id,
+                    'triggered_by', 'pg_net_' || p_triggered_by,
+                    'timestamp', CURRENT_TIMESTAMP
+                )
+            ) INTO v_request_id;
+
+            RAISE DEBUG 'kv_rebuild triggered via pg_net (request_id: %)', v_request_id;
+            RETURN;
+        EXCEPTION WHEN OTHERS THEN
+            RAISE WARNING 'pg_net trigger failed: %, falling back to dblink', SQLERRM;
+        END;
+    END IF;
+
+    -- Priority 2: dblink (async SQL - direct rebuild)
+    IF v_has_dblink THEN
+        BEGIN
+            -- Connect to self (same database)
+            PERFORM dblink_connect(
+                'kv_rebuild_conn',
+                format('dbname=%s', current_database())
+            );
+
+            -- Send async query (returns immediately)
+            PERFORM dblink_send_query(
+                'kv_rebuild_conn',
+                'SELECT rebuild_kv_store()'
+            );
+
+            -- Don't disconnect - query continues in background
+            -- Connection auto-closes when session ends
+
+            RAISE DEBUG 'kv_rebuild triggered via dblink';
+            RETURN;
+        EXCEPTION WHEN OTHERS THEN
+            -- Clean up failed connection
+            BEGIN
+                PERFORM dblink_disconnect('kv_rebuild_conn');
+            EXCEPTION WHEN OTHERS THEN
+                NULL;
+            END;
+            RAISE WARNING 'dblink trigger failed: %', SQLERRM;
+        END;
+    END IF;
+
+    -- No async method available - log warning but don't block query
+    RAISE WARNING 'No async rebuild method available (pg_net or dblink). Cache rebuild skipped.';
+
+EXCEPTION WHEN OTHERS THEN
+    -- Never fail the calling query
+    RAISE WARNING 'maybe_trigger_kv_rebuild failed: %', SQLERRM;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION maybe_trigger_kv_rebuild IS
+'Async trigger for kv_store rebuild. Uses pg_net (API) or dblink (SQL). Includes debouncing.';
+
+-- ============================================================================
+-- UPDATED: rem_lookup with self-healing
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_lookup(
+    p_entity_key VARCHAR(255),
+    p_tenant_id VARCHAR(100),
+    p_user_id VARCHAR(100)
+)
+RETURNS TABLE(
+    entity_type VARCHAR(100),
+    data JSONB
+) AS $$
+DECLARE
+    entity_table VARCHAR(100);
+    query_sql TEXT;
+    effective_user_id VARCHAR(100);
+    v_result_count INTEGER := 0;
+BEGIN
+    effective_user_id := COALESCE(p_user_id, p_tenant_id);
+
+    -- First lookup in KV store to get entity_type (table name)
+    SELECT kv.entity_type INTO entity_table
+    FROM kv_store kv
+    WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
+    AND kv.entity_key = p_entity_key
+    LIMIT 1;
+
+    -- If not found, check if cache is empty and maybe trigger rebuild
+    IF entity_table IS NULL THEN
+        -- SELF-HEALING: Check if this is because cache is empty
+        IF rem_kv_store_empty(effective_user_id) THEN
+            PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_lookup');
+        END IF;
+        RETURN;
+    END IF;
+
+    -- Fetch raw record from underlying table as JSONB
+    query_sql := format('
+        SELECT
+            %L::VARCHAR(100) AS entity_type,
+            row_to_json(t)::jsonb AS data
+        FROM %I t
+        WHERE (t.user_id = $1 OR t.user_id IS NULL)
+        AND t.name = $2
+        AND t.deleted_at IS NULL
+    ', entity_table, entity_table);
+
+    RETURN QUERY EXECUTE query_sql USING effective_user_id, p_entity_key;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- UPDATED: rem_fuzzy with self-healing
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_fuzzy(
+    p_query TEXT,
+    p_tenant_id VARCHAR(100),
+    p_threshold REAL DEFAULT 0.3,
+    p_limit INTEGER DEFAULT 10,
+    p_user_id VARCHAR(100) DEFAULT NULL
+)
+RETURNS TABLE(
+    entity_type VARCHAR(100),
+    similarity_score REAL,
+    data JSONB
+) AS $$
+DECLARE
+    kv_matches RECORD;
+    entities_by_table JSONB := '{}'::jsonb;
+    table_keys JSONB;
+    effective_user_id VARCHAR(100);
+    v_found_any BOOLEAN := FALSE;
+BEGIN
+    effective_user_id := COALESCE(p_user_id, p_tenant_id);
+
+    -- Find matching keys in KV store
+    FOR kv_matches IN
+        SELECT
+            kv.entity_key,
+            kv.entity_type,
+            similarity(kv.entity_key, p_query) AS sim_score
+        FROM kv_store kv
+        WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
+        AND kv.entity_key % p_query
+        AND similarity(kv.entity_key, p_query) >= p_threshold
+        ORDER BY sim_score DESC
+        LIMIT p_limit
+    LOOP
+        v_found_any := TRUE;
+        -- Build JSONB mapping {table: [keys]}
+        IF entities_by_table ? kv_matches.entity_type THEN
+            table_keys := entities_by_table->kv_matches.entity_type;
+            entities_by_table := jsonb_set(
+                entities_by_table,
+                ARRAY[kv_matches.entity_type],
+                table_keys || jsonb_build_array(kv_matches.entity_key)
+            );
+        ELSE
+            entities_by_table := jsonb_set(
+                entities_by_table,
+                ARRAY[kv_matches.entity_type],
+                jsonb_build_array(kv_matches.entity_key)
+            );
+        END IF;
+    END LOOP;
+
+    -- SELF-HEALING: If no matches and cache is empty, trigger rebuild
+    IF NOT v_found_any AND rem_kv_store_empty(effective_user_id) THEN
+        PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_fuzzy');
+    END IF;
+
+    -- Fetch full records
+    RETURN QUERY
+    SELECT
+        f.entity_type::VARCHAR(100),
+        similarity(f.entity_key, p_query) AS similarity_score,
+        f.entity_record AS data
+    FROM rem_fetch(entities_by_table, effective_user_id) f
+    ORDER BY similarity_score DESC;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- UPDATED: rem_traverse with self-healing
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_traverse(
+    p_entity_key VARCHAR(255),
+    p_tenant_id VARCHAR(100),
+    p_user_id VARCHAR(100),
+    p_max_depth INTEGER DEFAULT 1,
+    p_rel_type VARCHAR(100) DEFAULT NULL,
+    p_keys_only BOOLEAN DEFAULT FALSE
+)
+RETURNS TABLE(
+    depth INTEGER,
+    entity_key VARCHAR(255),
+    entity_type VARCHAR(100),
+    entity_id UUID,
+    rel_type VARCHAR(100),
+    rel_weight REAL,
+    path TEXT[],
+    entity_record JSONB
+) AS $$
+DECLARE
+    graph_keys RECORD;
+    entities_by_table JSONB := '{}'::jsonb;
+    table_keys JSONB;
+    effective_user_id VARCHAR(100);
+    v_found_start BOOLEAN := FALSE;
+BEGIN
+    effective_user_id := COALESCE(p_user_id, p_tenant_id);
+
+    -- Check if start entity exists in kv_store
+    SELECT TRUE INTO v_found_start
+    FROM kv_store kv
+    WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
+    AND kv.entity_key = p_entity_key
+    LIMIT 1;
+
+    -- SELF-HEALING: If start not found and cache is empty, trigger rebuild
+    IF NOT COALESCE(v_found_start, FALSE) THEN
+        IF rem_kv_store_empty(effective_user_id) THEN
+            PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_traverse');
+        END IF;
+        RETURN;
+    END IF;
+
+    -- Original traverse logic
+    FOR graph_keys IN
+        WITH RECURSIVE graph_traversal AS (
+            SELECT
+                0 AS depth,
+                kv.entity_key,
+                kv.entity_type,
+                kv.entity_id,
+                NULL::VARCHAR(100) AS rel_type,
+                NULL::REAL AS rel_weight,
+                ARRAY[kv.entity_key]::TEXT[] AS path
+            FROM kv_store kv
+            WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
+            AND kv.entity_key = p_entity_key
+
+            UNION ALL
+
+            SELECT
+                gt.depth + 1,
+                target_kv.entity_key,
+                target_kv.entity_type,
+                target_kv.entity_id,
+                (edge->>'rel_type')::VARCHAR(100) AS rel_type,
+                COALESCE((edge->>'weight')::REAL, 1.0) AS rel_weight,
+                gt.path || target_kv.entity_key AS path
+            FROM graph_traversal gt
+            JOIN kv_store source_kv ON source_kv.entity_key = gt.entity_key
+                AND (source_kv.user_id = effective_user_id OR source_kv.user_id IS NULL)
+            CROSS JOIN LATERAL jsonb_array_elements(COALESCE(source_kv.graph_edges, '[]'::jsonb)) AS edge
+            JOIN kv_store target_kv ON target_kv.entity_key = (edge->>'dst')::VARCHAR(255)
+                AND (target_kv.user_id = effective_user_id OR target_kv.user_id IS NULL)
+            WHERE gt.depth < p_max_depth
+            AND (p_rel_type IS NULL OR (edge->>'rel_type')::VARCHAR(100) = p_rel_type)
+            AND NOT (target_kv.entity_key = ANY(gt.path))
+        )
+        SELECT DISTINCT ON (gt.entity_key)
+            gt.depth,
+            gt.entity_key,
+            gt.entity_type,
+            gt.entity_id,
+            gt.rel_type,
+            gt.rel_weight,
+            gt.path
+        FROM graph_traversal gt
+        WHERE gt.depth > 0
+        ORDER BY gt.entity_key, gt.depth
+    LOOP
+        IF p_keys_only THEN
+            depth := graph_keys.depth;
+            entity_key := graph_keys.entity_key;
+            entity_type := graph_keys.entity_type;
+            entity_id := graph_keys.entity_id;
+            rel_type := graph_keys.rel_type;
+            rel_weight := graph_keys.rel_weight;
+            path := graph_keys.path;
+            entity_record := NULL;
+            RETURN NEXT;
+        ELSE
+            IF entities_by_table ? graph_keys.entity_type THEN
+                table_keys := entities_by_table->graph_keys.entity_type;
+                entities_by_table := jsonb_set(
+                    entities_by_table,
+                    ARRAY[graph_keys.entity_type],
+                    table_keys || jsonb_build_array(graph_keys.entity_key)
+                );
+            ELSE
+                entities_by_table := jsonb_set(
+                    entities_by_table,
+                    ARRAY[graph_keys.entity_type],
+                    jsonb_build_array(graph_keys.entity_key)
+                );
+            END IF;
+        END IF;
+    END LOOP;
+
+    IF NOT p_keys_only THEN
+        RETURN QUERY
+        SELECT
+            g.depth,
+            g.entity_key,
+            g.entity_type,
+            g.entity_id,
+            g.rel_type,
+            g.rel_weight,
+            g.path,
+            f.entity_record
+        FROM (
+            SELECT * FROM rem_traverse(p_entity_key, p_tenant_id, effective_user_id, p_max_depth, p_rel_type, TRUE)
+        ) g
+        LEFT JOIN rem_fetch(entities_by_table, effective_user_id) f
+            ON g.entity_key = f.entity_key;
+    END IF;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- HELPER: Get API secret for validation
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_get_cache_api_secret()
+RETURNS TEXT AS $$
+DECLARE
+    v_secret TEXT;
+BEGIN
+    SELECT api_secret INTO v_secret FROM cache_system_state WHERE id = 1;
+    RETURN v_secret;
+END;
+$$ LANGUAGE plpgsql STABLE SECURITY DEFINER;
+
+-- Only allow rem user to execute
+REVOKE ALL ON FUNCTION rem_get_cache_api_secret() FROM PUBLIC;
+
+-- ============================================================================
+-- HELPER: Record successful rebuild
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_record_cache_rebuild(p_triggered_by TEXT DEFAULT 'api')
+RETURNS VOID AS $$
+BEGIN
+    UPDATE cache_system_state
+    SET last_rebuild_at = CURRENT_TIMESTAMP,
+        rebuild_count = rebuild_count + 1,
+        updated_at = CURRENT_TIMESTAMP
+    WHERE id = 1;
+END;
+$$ LANGUAGE plpgsql;
+
+-- ============================================================================
+-- RECORD INSTALLATION
+-- ============================================================================
+
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'rem_migrations') THEN
+        INSERT INTO rem_migrations (name, type, version)
+        VALUES ('004_cache_system.sql', 'install', '1.0.0')
+        ON CONFLICT (name) DO UPDATE
+        SET applied_at = CURRENT_TIMESTAMP,
+            applied_by = CURRENT_USER;
+    END IF;
+END $$;
+
+-- ============================================================================
+-- COMPLETION
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_has_pgnet BOOLEAN;
+    v_has_dblink BOOLEAN;
+BEGIN
+    v_has_pgnet := rem_extension_exists('pg_net');
+    v_has_dblink := rem_extension_exists('dblink');
+
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE 'Cache System Installation Complete';
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE '';
+    RAISE NOTICE 'Tables:';
+    RAISE NOTICE '  cache_system_state - Debounce tracking and API secret';
+    RAISE NOTICE '';
+    RAISE NOTICE 'Functions:';
+    RAISE NOTICE '  maybe_trigger_kv_rebuild() - Async rebuild trigger';
+    RAISE NOTICE '  rem_lookup() - Updated with self-healing';
+    RAISE NOTICE '  rem_fuzzy() - Updated with self-healing';
+    RAISE NOTICE '  rem_traverse() - Updated with self-healing';
+    RAISE NOTICE '';
+    RAISE NOTICE 'Async Methods Available:';
+    IF v_has_pgnet THEN
+        RAISE NOTICE '  [x] pg_net - HTTP POST to API (preferred)';
+    ELSE
+        RAISE NOTICE '  [ ] pg_net - Not installed';
+    END IF;
+    IF v_has_dblink THEN
+        RAISE NOTICE '  [x] dblink - Async SQL (fallback)';
+    ELSE
+        RAISE NOTICE '  [ ] dblink - Not installed';
+    END IF;
+    RAISE NOTICE '';
+    RAISE NOTICE 'Self-Healing: Queries will auto-trigger rebuild on empty cache';
+    RAISE NOTICE '============================================================';
+END $$;

rem/utils/__init__.py

@@ -5,6 +5,7 @@ Utility functions and helpers for the REM system:
 - sql_types: Pydantic to PostgreSQL type mapping
 - embeddings: Vector embeddings generation using requests library
 - user_id: Deterministic UUID generation from email addresses
+- sql_paths: SQL file path resolution for packages and user migrations
 """
 
 from .embeddings import (
@@ -24,6 +25,15 @@ from .user_id import (
     is_valid_uuid,
     user_id_to_uuid,
 )
+from .sql_paths import (
+    USER_SQL_DIR_CONVENTION,
+    get_package_sql_dir,
+    get_package_migrations_dir,
+    get_user_sql_dir,
+    list_package_migrations,
+    list_user_migrations,
+    list_all_migrations,
+)
 
 __all__ = [
     # SQL Types
@@ -40,4 +50,12 @@ __all__ = [
     "email_to_user_id",
     "user_id_to_uuid",
     "is_valid_uuid",
+    # SQL Paths
+    "USER_SQL_DIR_CONVENTION",
+    "get_package_sql_dir",
+    "get_package_migrations_dir",
+    "get_user_sql_dir",
+    "list_package_migrations",
+    "list_user_migrations",
+    "list_all_migrations",
 ]

rem/utils/constants.py

@@ -0,0 +1,97 @@
+"""
+Centralized constants for the REM system.
+
+All magic numbers and commonly-used values should be defined here
+to ensure consistency and make tuning easier.
+"""
+
+# =============================================================================
+# Embedding Model Constants
+# =============================================================================
+
+# OpenAI embedding dimensions by model
+OPENAI_EMBEDDING_DIMS_SMALL = 1536  # text-embedding-3-small
+OPENAI_EMBEDDING_DIMS_LARGE = 3072  # text-embedding-3-large
+OPENAI_EMBEDDING_DIMS_ADA = 1536  # text-embedding-ada-002
+
+# Default embedding dimension (text-embedding-3-small)
+DEFAULT_EMBEDDING_DIMS = 1536
+
+# Voyage AI embedding dimensions
+VOYAGE_EMBEDDING_DIMS = 1024  # voyage-2
+
+# =============================================================================
+# HTTP/API Timeouts (seconds)
+# =============================================================================
+
+HTTP_TIMEOUT_DEFAULT = 30.0  # Standard API calls
+HTTP_TIMEOUT_LONG = 60.0  # Vision/embedding APIs
+HTTP_TIMEOUT_VERY_LONG = 300.0  # Subprocess/batch operations
+
+# Request timeout for httpx AsyncClient
+ASYNC_CLIENT_TIMEOUT = 300.0
+
+# =============================================================================
+# Audio Processing Constants
+# =============================================================================
+
+# Minimum valid WAV file size (header only)
+WAV_HEADER_MIN_BYTES = 44
+
+# OpenAI Whisper API cost per minute (USD)
+WHISPER_COST_PER_MINUTE = 0.006
+
+# Audio chunking parameters
+AUDIO_CHUNK_TARGET_SECONDS = 60.0  # Target chunk duration
+AUDIO_CHUNK_WINDOW_SECONDS = 2.0  # Window for silence detection
+SILENCE_THRESHOLD_DB = -40.0  # Silence detection threshold
+MIN_SILENCE_MS = 500  # Minimum silence duration to split on
+
+# =============================================================================
+# File Processing Constants
+# =============================================================================
+
+# Subprocess timeout for document parsing
+SUBPROCESS_TIMEOUT_SECONDS = 300  # 5 minutes
+
+# Maximum file sizes
+MAX_AUDIO_FILE_SIZE_MB = 25  # Whisper API limit
+
+# =============================================================================
+# Database/Query Constants
+# =============================================================================
+
+# Default batch sizes
+DEFAULT_BATCH_SIZE = 100
+EMBEDDING_BATCH_SIZE = 50
+
+# Default pagination limits
+DEFAULT_PAGE_SIZE = 20
+MAX_PAGE_SIZE = 100
+
+# =============================================================================
+# Rate Limiting
+# =============================================================================
+
+# Default retry settings
+DEFAULT_MAX_RETRIES = 3
+RETRY_BACKOFF_MULTIPLIER = 1
+RETRY_BACKOFF_MIN = 1
+RETRY_BACKOFF_MAX = 60
+
+# =============================================================================
+# S3/Storage Constants
+# =============================================================================
+
+S3_URI_PREFIX = "s3://"
+FILE_URI_PREFIX = "file://"
+
+# =============================================================================
+# LLM Constants
+# =============================================================================
+
+# Default max tokens for vision analysis
+VISION_MAX_TOKENS = 2048
+
+# Default temperature
+DEFAULT_TEMPERATURE = 0.0