remdb 0.3.242__py3-none-any.whl

rem/sql/migrations/003_optional_extensions.sql

@@ -0,0 +1,326 @@
+-- REM Optional Extensions
+-- Description: Optional PostgreSQL extensions that enhance functionality but are not required
+-- Version: 1.0.0
+-- Date: 2025-11-29
+--
+-- These extensions are installed with try/catch - failures are logged but don't break the install.
+-- This allows the same migration to work on:
+--   - Custom images with extensions baked in (percolationlabs/rem-pg:18)
+--   - Standard PostgreSQL images (extensions will be skipped)
+--
+-- Extensions:
+--   - pg_net: Async HTTP/HTTPS requests from triggers and functions (Supabase)
+
+-- ============================================================================
+-- pg_net: Async HTTP Extension
+-- ============================================================================
+-- Enables PostgreSQL to make non-blocking HTTP requests from triggers and functions.
+-- Requires: Custom image with pg_net compiled, shared_preload_libraries='pg_net'
+--
+-- Use cases:
+--   - Webhook notifications on data changes
+--   - Async event publishing to external APIs
+--   - Background HTTP requests from triggers
+
+DO $$
+BEGIN
+    -- Attempt to create pg_net extension
+    CREATE EXTENSION IF NOT EXISTS pg_net;
+    RAISE NOTICE '  pg_net extension installed successfully';
+EXCEPTION
+    WHEN OTHERS THEN
+        RAISE NOTICE '  pg_net extension not available (this is OK if using standard PostgreSQL image)';
+        RAISE NOTICE '  Error: %', SQLERRM;
+END $$;
+
+-- ============================================================================
+-- pg_net Helper Functions (only created if extension exists)
+-- ============================================================================
+-- Wrapper functions for common HTTP operations with sensible defaults
+
+DO $$
+BEGIN
+    -- Only create helpers if pg_net is available
+    IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_net') THEN
+
+        -- Helper: POST JSON to a URL with standard headers
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_http_post(
+            p_url TEXT,
+            p_body JSONB,
+            p_headers JSONB DEFAULT '{}'::jsonb
+        )
+        RETURNS BIGINT AS $inner$
+        DECLARE
+            merged_headers JSONB;
+            request_id BIGINT;
+        BEGIN
+            -- Merge default headers with provided headers
+            merged_headers := '{"Content-Type": "application/json"}'::jsonb || p_headers;
+
+            SELECT net.http_post(
+                url := p_url,
+                headers := merged_headers,
+                body := p_body
+            ) INTO request_id;
+
+            RETURN request_id;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_http_post helper function created';
+
+        -- Helper: GET from a URL
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_http_get(
+            p_url TEXT,
+            p_headers JSONB DEFAULT '{}'::jsonb
+        )
+        RETURNS BIGINT AS $inner$
+        DECLARE
+            request_id BIGINT;
+        BEGIN
+            SELECT net.http_get(
+                url := p_url,
+                headers := p_headers
+            ) INTO request_id;
+
+            RETURN request_id;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_http_get helper function created';
+
+        -- ====================================================================
+        -- REM Query Function
+        -- ====================================================================
+        -- Executes REM queries via the REM API using pg_net
+        --
+        -- Default API host: rem-api (works in K8s same namespace)
+        -- For local Docker testing: Add "host.docker.internal rem-api" to /etc/hosts
+        -- Or override with p_api_host parameter
+        --
+        -- Example:
+        --   SELECT rem_query('LOOKUP sarah-chen', 'user123');
+        --   SELECT rem_query('SEARCH resources ''API design'' LIMIT 5', 'user123');
+
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_query(
+            p_query TEXT,
+            p_user_id TEXT,
+            p_api_host TEXT DEFAULT 'rem-api',
+            p_api_port INTEGER DEFAULT 8000,
+            p_mode TEXT DEFAULT 'rem-dialect'
+        )
+        RETURNS BIGINT AS $inner$
+        DECLARE
+            api_url TEXT;
+            request_body JSONB;
+            request_headers JSONB;
+            request_id BIGINT;
+        BEGIN
+            -- Build API URL
+            -- Default: http://rem-api:8000/api/v1/query (K8s same namespace)
+            api_url := format('http://%s:%s/api/v1/query', p_api_host, p_api_port);
+
+            -- Build request body
+            request_body := jsonb_build_object(
+                'query', p_query,
+                'mode', p_mode
+            );
+
+            -- Build headers with user ID
+            request_headers := jsonb_build_object(
+                'Content-Type', 'application/json',
+                'X-User-Id', p_user_id
+            );
+
+            -- Make async HTTP POST request
+            SELECT net.http_post(
+                url := api_url,
+                headers := request_headers,
+                body := request_body
+            ) INTO request_id;
+
+            RETURN request_id;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_query() function created';
+
+        -- Helper to get query results (waits for async response)
+        -- NOTE: pg_net is async by design. This function polls for the response.
+        -- For best results, use rem_query() and check results later, or use longer timeouts.
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_query_result(
+            p_request_id BIGINT,
+            p_timeout_ms INTEGER DEFAULT 10000
+        )
+        RETURNS JSONB AS $inner$
+        DECLARE
+            v_status_code INTEGER;
+            v_content TEXT;
+            v_found BOOLEAN;
+            start_time TIMESTAMP;
+            elapsed_ms INTEGER;
+        BEGIN
+            start_time := clock_timestamp();
+
+            -- Poll for response with timeout
+            -- Each iteration starts a new query to see committed data from background worker
+            LOOP
+                -- Check if response exists (background worker commits independently)
+                SELECT true, status_code, content::text
+                INTO v_found, v_status_code, v_content
+                FROM net._http_response
+                WHERE id = p_request_id;
+
+                -- Found response
+                IF v_found THEN
+                    IF v_status_code = 200 THEN
+                        RETURN v_content::jsonb;
+                    ELSE
+                        RETURN jsonb_build_object(
+                            'error', true,
+                            'status_code', v_status_code,
+                            'content', v_content
+                        );
+                    END IF;
+                END IF;
+
+                -- Check timeout
+                elapsed_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000;
+                IF elapsed_ms >= p_timeout_ms THEN
+                    RETURN jsonb_build_object(
+                        'error', true,
+                        'message', 'Request timeout - pg_net is async, response may arrive later',
+                        'request_id', p_request_id,
+                        'hint', 'Check net._http_response table or increase timeout'
+                    );
+                END IF;
+
+                -- Wait 500ms before next poll (pg_net worker runs every 100ms)
+                PERFORM pg_sleep(0.5);
+            END LOOP;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_query_result() function created';
+
+        -- Convenience function: execute query and wait for result
+        -- WARNING: Due to PostgreSQL transaction isolation, this may timeout even when
+        -- the request succeeds. The background worker commits separately and the polling
+        -- loop may not see the response. Use rem_query() + check net._http_response for
+        -- more reliable async operation.
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_query_sync(
+            p_query TEXT,
+            p_user_id TEXT,
+            p_api_host TEXT DEFAULT 'rem-api',
+            p_api_port INTEGER DEFAULT 8000,
+            p_mode TEXT DEFAULT 'rem-dialect',
+            p_timeout_ms INTEGER DEFAULT 10000
+        )
+        RETURNS JSONB AS $inner$
+        DECLARE
+            request_id BIGINT;
+            v_status_code INTEGER;
+            v_content TEXT;
+            v_found BOOLEAN := false;
+            start_time TIMESTAMP;
+            elapsed_ms INTEGER;
+        BEGIN
+            -- Execute query - this queues the HTTP request
+            request_id := rem_query(p_query, p_user_id, p_api_host, p_api_port, p_mode);
+
+            -- Wait for response with explicit snapshot refresh attempts
+            start_time := clock_timestamp();
+            LOOP
+                -- Query in separate subtransaction-like context
+                SELECT true, status_code, content::text
+                INTO v_found, v_status_code, v_content
+                FROM net._http_response
+                WHERE id = request_id;
+
+                IF v_found THEN
+                    IF v_status_code = 200 THEN
+                        RETURN v_content::jsonb;
+                    ELSE
+                        RETURN jsonb_build_object('error', true, 'status_code', v_status_code, 'content', v_content);
+                    END IF;
+                END IF;
+
+                elapsed_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000;
+                IF elapsed_ms >= p_timeout_ms THEN
+                    -- Return info about the async request so caller can check later
+                    RETURN jsonb_build_object(
+                        'pending', true,
+                        'request_id', request_id,
+                        'message', 'Request queued but response not yet visible due to transaction isolation',
+                        'hint', 'Query net._http_response WHERE id = ' || request_id || ' after this transaction commits'
+                    );
+                END IF;
+
+                PERFORM pg_sleep(0.3);
+            END LOOP;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_query_sync() function created (async pattern recommended)';
+
+    ELSE
+        RAISE NOTICE '  Skipping pg_net helper functions (extension not installed)';
+    END IF;
+END $$;
+
+-- ============================================================================
+-- RECORD INSTALLATION
+-- ============================================================================
+
+DO $$
+BEGIN
+    -- Only record if migrations table exists
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'rem_migrations') THEN
+        INSERT INTO rem_migrations (name, type, version)
+        VALUES ('003_optional_extensions.sql', 'install', '1.0.0')
+        ON CONFLICT (name) DO UPDATE
+        SET applied_at = CURRENT_TIMESTAMP,
+            applied_by = CURRENT_USER;
+    END IF;
+END $$;
+
+-- ============================================================================
+-- COMPLETION
+-- ============================================================================
+
+DO $$
+DECLARE
+    pg_net_installed BOOLEAN;
+BEGIN
+    SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_net') INTO pg_net_installed;
+
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE 'Optional Extensions Installation Complete';
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE '';
+    IF pg_net_installed THEN
+        RAISE NOTICE 'Installed:';
+        RAISE NOTICE '  pg_net (async HTTP/HTTPS requests)';
+        RAISE NOTICE '  rem_http_post() - POST JSON to URL';
+        RAISE NOTICE '  rem_http_get() - GET from URL';
+        RAISE NOTICE '  rem_query() - Execute REM query (async)';
+        RAISE NOTICE '  rem_query_result() - Get async query result';
+        RAISE NOTICE '  rem_query_sync() - Execute and wait for result';
+    ELSE
+        RAISE NOTICE 'Skipped (not available in this PostgreSQL image):';
+        RAISE NOTICE '  pg_net';
+        RAISE NOTICE '';
+        RAISE NOTICE 'To enable pg_net, use the custom image: percolationlabs/rem-pg:18';
+    END IF;
+    RAISE NOTICE '============================================================';
+END $$;

rem/sql/migrations/004_cache_system.sql

@@ -0,0 +1,282 @@
+-- REM Cache System
+-- Description: Cache management helpers for UNLOGGED tables (kv_store)
+-- Version: 1.0.0
+-- Date: 2025-11-29
+--
+-- This migration adds:
+--   1. cache_system_state table for debouncing and API secret storage
+--   2. maybe_trigger_kv_rebuild() function for async rebuild triggering
+--   3. Helper functions for cache management
+--
+-- NOTE: Core functions (rem_lookup, rem_fuzzy, rem_traverse) are defined in 001_install.sql
+-- This file only provides cache-specific infrastructure.
+
+-- ============================================================================
+-- REQUIRED EXTENSION
+-- ============================================================================
+-- pgcrypto is needed for gen_random_bytes() to generate API secrets
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+-- ============================================================================
+-- CACHE SYSTEM STATE TABLE
+-- ============================================================================
+-- Stores:
+--   - Last rebuild trigger timestamp (for debouncing)
+--   - API secret for internal endpoint authentication
+--   - Rebuild statistics
+
+CREATE TABLE IF NOT EXISTS cache_system_state (
+    id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1),  -- Single row table
+    api_secret TEXT NOT NULL,                          -- Secret for internal API auth
+    last_triggered_at TIMESTAMPTZ,                     -- Debounce: last trigger time
+    last_rebuild_at TIMESTAMPTZ,                       -- Last successful rebuild
+    triggered_by TEXT,                                 -- What triggered last rebuild
+    trigger_count INTEGER DEFAULT 0,                   -- Total trigger count
+    rebuild_count INTEGER DEFAULT 0,                   -- Total successful rebuilds
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Generate initial secret if table is empty
+INSERT INTO cache_system_state (id, api_secret)
+SELECT 1, encode(gen_random_bytes(32), 'hex')
+WHERE NOT EXISTS (SELECT 1 FROM cache_system_state WHERE id = 1);
+
+COMMENT ON TABLE cache_system_state IS
+'Single-row table storing cache system state: API secret for internal auth and debounce tracking';
+
+-- ============================================================================
+-- HELPER: Check if extension exists
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_extension_exists(p_extension TEXT)
+RETURNS BOOLEAN AS $$
+BEGIN
+    RETURN EXISTS (SELECT 1 FROM pg_extension WHERE extname = p_extension);
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- HELPER: Check if kv_store is empty for user
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_kv_store_empty(p_user_id TEXT)
+RETURNS BOOLEAN AS $$
+BEGIN
+    -- Quick existence check - very fast with index
+    -- Check for user-specific OR public (NULL user_id) entries
+    -- This ensures self-healing triggers correctly for public ontologies
+    RETURN NOT EXISTS (
+        SELECT 1 FROM kv_store
+        WHERE user_id = p_user_id OR user_id IS NULL
+        LIMIT 1
+    );
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+-- ============================================================================
+-- MAIN: Maybe trigger KV rebuild (async, non-blocking)
+-- ============================================================================
+-- Called when a query returns 0 results and kv_store appears empty.
+-- Uses pg_net (if available) to call API, falls back to dblink.
+-- Includes debouncing to prevent request storms.
+
+CREATE OR REPLACE FUNCTION maybe_trigger_kv_rebuild(
+    p_user_id TEXT,
+    p_triggered_by TEXT DEFAULT 'query'
+)
+RETURNS VOID AS $$
+DECLARE
+    v_has_pgnet BOOLEAN;
+    v_has_dblink BOOLEAN;
+    v_last_trigger TIMESTAMPTZ;
+    v_api_secret TEXT;
+    v_debounce_seconds CONSTANT INTEGER := 30;
+    v_api_url TEXT := 'http://rem-api.rem.svc.cluster.local:8000/api/admin/internal/rebuild-kv';
+    v_request_id BIGINT;
+BEGIN
+    -- Quick check: is kv_store actually empty for this user?
+    IF NOT rem_kv_store_empty(p_user_id) THEN
+        RETURN;  -- Cache has data, nothing to do
+    END IF;
+
+    -- Try to acquire advisory lock (non-blocking, transaction-scoped)
+    -- This prevents multiple concurrent triggers
+    IF NOT pg_try_advisory_xact_lock(2147483646) THEN
+        RETURN;  -- Another session is handling it
+    END IF;
+
+    -- Check debounce: was rebuild triggered recently?
+    SELECT last_triggered_at, api_secret
+    INTO v_last_trigger, v_api_secret
+    FROM cache_system_state
+    WHERE id = 1;
+
+    IF v_last_trigger IS NOT NULL
+       AND v_last_trigger > (CURRENT_TIMESTAMP - (v_debounce_seconds || ' seconds')::INTERVAL) THEN
+        RETURN;  -- Triggered recently, skip
+    END IF;
+
+    -- Update state (so concurrent callers see it)
+    UPDATE cache_system_state
+    SET last_triggered_at = CURRENT_TIMESTAMP,
+        triggered_by = p_triggered_by,
+        trigger_count = trigger_count + 1,
+        updated_at = CURRENT_TIMESTAMP
+    WHERE id = 1;
+
+    -- Check available extensions
+    v_has_pgnet := rem_extension_exists('pg_net');
+    v_has_dblink := rem_extension_exists('dblink');
+
+    -- Priority 1: pg_net (async HTTP to API - supports S3 restore)
+    IF v_has_pgnet THEN
+        BEGIN
+            SELECT net.http_post(
+                url := v_api_url,
+                headers := jsonb_build_object(
+                    'Content-Type', 'application/json',
+                    'X-Internal-Secret', v_api_secret
+                ),
+                body := jsonb_build_object(
+                    'user_id', p_user_id,
+                    'triggered_by', 'pg_net_' || p_triggered_by,
+                    'timestamp', CURRENT_TIMESTAMP
+                )
+            ) INTO v_request_id;
+
+            RAISE DEBUG 'kv_rebuild triggered via pg_net (request_id: %)', v_request_id;
+            RETURN;
+        EXCEPTION WHEN OTHERS THEN
+            RAISE WARNING 'pg_net trigger failed: %, falling back to dblink', SQLERRM;
+        END;
+    END IF;
+
+    -- Priority 2: dblink (async SQL - direct rebuild)
+    IF v_has_dblink THEN
+        BEGIN
+            -- Connect to self (same database)
+            PERFORM dblink_connect(
+                'kv_rebuild_conn',
+                format('dbname=%s', current_database())
+            );
+
+            -- Send async query (returns immediately)
+            PERFORM dblink_send_query(
+                'kv_rebuild_conn',
+                'SELECT rebuild_kv_store()'
+            );
+
+            -- Don't disconnect - query continues in background
+            -- Connection auto-closes when session ends
+
+            RAISE DEBUG 'kv_rebuild triggered via dblink';
+            RETURN;
+        EXCEPTION WHEN OTHERS THEN
+            -- Clean up failed connection
+            BEGIN
+                PERFORM dblink_disconnect('kv_rebuild_conn');
+            EXCEPTION WHEN OTHERS THEN
+                NULL;
+            END;
+            RAISE WARNING 'dblink trigger failed: %', SQLERRM;
+        END;
+    END IF;
+
+    -- No async method available - log warning but don't block query
+    RAISE WARNING 'No async rebuild method available (pg_net or dblink). Cache rebuild skipped.';
+
+EXCEPTION WHEN OTHERS THEN
+    -- Never fail the calling query
+    RAISE WARNING 'maybe_trigger_kv_rebuild failed: %', SQLERRM;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION maybe_trigger_kv_rebuild IS
+'Async trigger for kv_store rebuild. Uses pg_net (API) or dblink (SQL). Includes debouncing.';
+
+-- ============================================================================
+-- HELPER: Get API secret for validation
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_get_cache_api_secret()
+RETURNS TEXT AS $$
+DECLARE
+    v_secret TEXT;
+BEGIN
+    SELECT api_secret INTO v_secret FROM cache_system_state WHERE id = 1;
+    RETURN v_secret;
+END;
+$$ LANGUAGE plpgsql STABLE SECURITY DEFINER;
+
+-- Only allow rem user to execute
+REVOKE ALL ON FUNCTION rem_get_cache_api_secret() FROM PUBLIC;
+
+-- ============================================================================
+-- HELPER: Record successful rebuild
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION rem_record_cache_rebuild(p_triggered_by TEXT DEFAULT 'api')
+RETURNS VOID AS $$
+BEGIN
+    UPDATE cache_system_state
+    SET last_rebuild_at = CURRENT_TIMESTAMP,
+        rebuild_count = rebuild_count + 1,
+        updated_at = CURRENT_TIMESTAMP
+    WHERE id = 1;
+END;
+$$ LANGUAGE plpgsql;
+
+-- ============================================================================
+-- RECORD INSTALLATION
+-- ============================================================================
+
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'rem_migrations') THEN
+        INSERT INTO rem_migrations (name, type, version)
+        VALUES ('004_cache_system.sql', 'install', '1.0.0')
+        ON CONFLICT (name) DO UPDATE
+        SET applied_at = CURRENT_TIMESTAMP,
+            applied_by = CURRENT_USER;
+    END IF;
+END $$;
+
+-- ============================================================================
+-- COMPLETION
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_has_pgnet BOOLEAN;
+    v_has_dblink BOOLEAN;
+BEGIN
+    v_has_pgnet := rem_extension_exists('pg_net');
+    v_has_dblink := rem_extension_exists('dblink');
+
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE 'Cache System Installation Complete';
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE '';
+    RAISE NOTICE 'Tables:';
+    RAISE NOTICE '  cache_system_state - Debounce tracking and API secret';
+    RAISE NOTICE '';
+    RAISE NOTICE 'Functions:';
+    RAISE NOTICE '  maybe_trigger_kv_rebuild() - Async rebuild trigger';
+    RAISE NOTICE '  rem_kv_store_empty() - Check if cache is empty';
+    RAISE NOTICE '  rem_get_cache_api_secret() - Get API secret';
+    RAISE NOTICE '  rem_record_cache_rebuild() - Record rebuild completion';
+    RAISE NOTICE '';
+    RAISE NOTICE 'Async Methods Available:';
+    IF v_has_pgnet THEN
+        RAISE NOTICE '  [x] pg_net - HTTP POST to API (preferred)';
+    ELSE
+        RAISE NOTICE '  [ ] pg_net - Not installed';
+    END IF;
+    IF v_has_dblink THEN
+        RAISE NOTICE '  [x] dblink - Async SQL (fallback)';
+    ELSE
+        RAISE NOTICE '  [ ] dblink - Not installed';
+    END IF;
+    RAISE NOTICE '============================================================';
+END $$;