remdb 0.3.0__py3-none-any.whl → 0.3.114__py3-none-any.whl

rem/sql/migrations/003_optional_extensions.sql

@@ -0,0 +1,326 @@
+-- REM Optional Extensions
+-- Description: Optional PostgreSQL extensions that enhance functionality but are not required
+-- Version: 1.0.0
+-- Date: 2025-11-29
+--
+-- These extensions are installed with try/catch - failures are logged but don't break the install.
+-- This allows the same migration to work on:
+--   - Custom images with extensions baked in (percolationlabs/rem-pg:18)
+--   - Standard PostgreSQL images (extensions will be skipped)
+--
+-- Extensions:
+--   - pg_net: Async HTTP/HTTPS requests from triggers and functions (Supabase)
+
+-- ============================================================================
+-- pg_net: Async HTTP Extension
+-- ============================================================================
+-- Enables PostgreSQL to make non-blocking HTTP requests from triggers and functions.
+-- Requires: Custom image with pg_net compiled, shared_preload_libraries='pg_net'
+--
+-- Use cases:
+--   - Webhook notifications on data changes
+--   - Async event publishing to external APIs
+--   - Background HTTP requests from triggers
+
+DO $$
+BEGIN
+    -- Attempt to create pg_net extension
+    CREATE EXTENSION IF NOT EXISTS pg_net;
+    RAISE NOTICE '  pg_net extension installed successfully';
+EXCEPTION
+    WHEN OTHERS THEN
+        RAISE NOTICE '  pg_net extension not available (this is OK if using standard PostgreSQL image)';
+        RAISE NOTICE '  Error: %', SQLERRM;
+END $$;
+
+-- ============================================================================
+-- pg_net Helper Functions (only created if extension exists)
+-- ============================================================================
+-- Wrapper functions for common HTTP operations with sensible defaults
+
+DO $$
+BEGIN
+    -- Only create helpers if pg_net is available
+    IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_net') THEN
+
+        -- Helper: POST JSON to a URL with standard headers
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_http_post(
+            p_url TEXT,
+            p_body JSONB,
+            p_headers JSONB DEFAULT '{}'::jsonb
+        )
+        RETURNS BIGINT AS $inner$
+        DECLARE
+            merged_headers JSONB;
+            request_id BIGINT;
+        BEGIN
+            -- Merge default headers with provided headers
+            merged_headers := '{"Content-Type": "application/json"}'::jsonb || p_headers;
+
+            SELECT net.http_post(
+                url := p_url,
+                headers := merged_headers,
+                body := p_body
+            ) INTO request_id;
+
+            RETURN request_id;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_http_post helper function created';
+
+        -- Helper: GET from a URL
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_http_get(
+            p_url TEXT,
+            p_headers JSONB DEFAULT '{}'::jsonb
+        )
+        RETURNS BIGINT AS $inner$
+        DECLARE
+            request_id BIGINT;
+        BEGIN
+            SELECT net.http_get(
+                url := p_url,
+                headers := p_headers
+            ) INTO request_id;
+
+            RETURN request_id;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_http_get helper function created';
+
+        -- ====================================================================
+        -- REM Query Function
+        -- ====================================================================
+        -- Executes REM queries via the REM API using pg_net
+        --
+        -- Default API host: rem-api (works in K8s same namespace)
+        -- For local Docker testing: Add "host.docker.internal rem-api" to /etc/hosts
+        -- Or override with p_api_host parameter
+        --
+        -- Example:
+        --   SELECT rem_query('LOOKUP sarah-chen', 'user123');
+        --   SELECT rem_query('SEARCH resources ''API design'' LIMIT 5', 'user123');
+
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_query(
+            p_query TEXT,
+            p_user_id TEXT,
+            p_api_host TEXT DEFAULT 'rem-api',
+            p_api_port INTEGER DEFAULT 8000,
+            p_mode TEXT DEFAULT 'rem-dialect'
+        )
+        RETURNS BIGINT AS $inner$
+        DECLARE
+            api_url TEXT;
+            request_body JSONB;
+            request_headers JSONB;
+            request_id BIGINT;
+        BEGIN
+            -- Build API URL
+            -- Default: http://rem-api:8000/api/v1/query (K8s same namespace)
+            api_url := format('http://%s:%s/api/v1/query', p_api_host, p_api_port);
+
+            -- Build request body
+            request_body := jsonb_build_object(
+                'query', p_query,
+                'mode', p_mode
+            );
+
+            -- Build headers with user ID
+            request_headers := jsonb_build_object(
+                'Content-Type', 'application/json',
+                'X-User-Id', p_user_id
+            );
+
+            -- Make async HTTP POST request
+            SELECT net.http_post(
+                url := api_url,
+                headers := request_headers,
+                body := request_body
+            ) INTO request_id;
+
+            RETURN request_id;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_query() function created';
+
+        -- Helper to get query results (waits for async response)
+        -- NOTE: pg_net is async by design. This function polls for the response.
+        -- For best results, use rem_query() and check results later, or use longer timeouts.
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_query_result(
+            p_request_id BIGINT,
+            p_timeout_ms INTEGER DEFAULT 10000
+        )
+        RETURNS JSONB AS $inner$
+        DECLARE
+            v_status_code INTEGER;
+            v_content TEXT;
+            v_found BOOLEAN;
+            start_time TIMESTAMP;
+            elapsed_ms INTEGER;
+        BEGIN
+            start_time := clock_timestamp();
+
+            -- Poll for response with timeout
+            -- Each iteration starts a new query to see committed data from background worker
+            LOOP
+                -- Check if response exists (background worker commits independently)
+                SELECT true, status_code, content::text
+                INTO v_found, v_status_code, v_content
+                FROM net._http_response
+                WHERE id = p_request_id;
+
+                -- Found response
+                IF v_found THEN
+                    IF v_status_code = 200 THEN
+                        RETURN v_content::jsonb;
+                    ELSE
+                        RETURN jsonb_build_object(
+                            'error', true,
+                            'status_code', v_status_code,
+                            'content', v_content
+                        );
+                    END IF;
+                END IF;
+
+                -- Check timeout
+                elapsed_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000;
+                IF elapsed_ms >= p_timeout_ms THEN
+                    RETURN jsonb_build_object(
+                        'error', true,
+                        'message', 'Request timeout - pg_net is async, response may arrive later',
+                        'request_id', p_request_id,
+                        'hint', 'Check net._http_response table or increase timeout'
+                    );
+                END IF;
+
+                -- Wait 500ms before next poll (pg_net worker runs every 100ms)
+                PERFORM pg_sleep(0.5);
+            END LOOP;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_query_result() function created';
+
+        -- Convenience function: execute query and wait for result
+        -- WARNING: Due to PostgreSQL transaction isolation, this may timeout even when
+        -- the request succeeds. The background worker commits separately and the polling
+        -- loop may not see the response. Use rem_query() + check net._http_response for
+        -- more reliable async operation.
+        EXECUTE $func$
+        CREATE OR REPLACE FUNCTION rem_query_sync(
+            p_query TEXT,
+            p_user_id TEXT,
+            p_api_host TEXT DEFAULT 'rem-api',
+            p_api_port INTEGER DEFAULT 8000,
+            p_mode TEXT DEFAULT 'rem-dialect',
+            p_timeout_ms INTEGER DEFAULT 10000
+        )
+        RETURNS JSONB AS $inner$
+        DECLARE
+            request_id BIGINT;
+            v_status_code INTEGER;
+            v_content TEXT;
+            v_found BOOLEAN := false;
+            start_time TIMESTAMP;
+            elapsed_ms INTEGER;
+        BEGIN
+            -- Execute query - this queues the HTTP request
+            request_id := rem_query(p_query, p_user_id, p_api_host, p_api_port, p_mode);
+
+            -- Wait for response with explicit snapshot refresh attempts
+            start_time := clock_timestamp();
+            LOOP
+                -- Query in separate subtransaction-like context
+                SELECT true, status_code, content::text
+                INTO v_found, v_status_code, v_content
+                FROM net._http_response
+                WHERE id = request_id;
+
+                IF v_found THEN
+                    IF v_status_code = 200 THEN
+                        RETURN v_content::jsonb;
+                    ELSE
+                        RETURN jsonb_build_object('error', true, 'status_code', v_status_code, 'content', v_content);
+                    END IF;
+                END IF;
+
+                elapsed_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000;
+                IF elapsed_ms >= p_timeout_ms THEN
+                    -- Return info about the async request so caller can check later
+                    RETURN jsonb_build_object(
+                        'pending', true,
+                        'request_id', request_id,
+                        'message', 'Request queued but response not yet visible due to transaction isolation',
+                        'hint', 'Query net._http_response WHERE id = ' || request_id || ' after this transaction commits'
+                    );
+                END IF;
+
+                PERFORM pg_sleep(0.3);
+            END LOOP;
+        END;
+        $inner$ LANGUAGE plpgsql;
+        $func$;
+
+        RAISE NOTICE '  rem_query_sync() function created (async pattern recommended)';
+
+    ELSE
+        RAISE NOTICE '  Skipping pg_net helper functions (extension not installed)';
+    END IF;
+END $$;
+
+-- ============================================================================
+-- RECORD INSTALLATION
+-- ============================================================================
+
+DO $$
+BEGIN
+    -- Only record if migrations table exists
+    IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'rem_migrations') THEN
+        INSERT INTO rem_migrations (name, type, version)
+        VALUES ('003_optional_extensions.sql', 'install', '1.0.0')
+        ON CONFLICT (name) DO UPDATE
+        SET applied_at = CURRENT_TIMESTAMP,
+            applied_by = CURRENT_USER;
+    END IF;
+END $$;
+
+-- ============================================================================
+-- COMPLETION
+-- ============================================================================
+
+DO $$
+DECLARE
+    pg_net_installed BOOLEAN;
+BEGIN
+    SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_net') INTO pg_net_installed;
+
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE 'Optional Extensions Installation Complete';
+    RAISE NOTICE '============================================================';
+    RAISE NOTICE '';
+    IF pg_net_installed THEN
+        RAISE NOTICE 'Installed:';
+        RAISE NOTICE '  pg_net (async HTTP/HTTPS requests)';
+        RAISE NOTICE '  rem_http_post() - POST JSON to URL';
+        RAISE NOTICE '  rem_http_get() - GET from URL';
+        RAISE NOTICE '  rem_query() - Execute REM query (async)';
+        RAISE NOTICE '  rem_query_result() - Get async query result';
+        RAISE NOTICE '  rem_query_sync() - Execute and wait for result';
+    ELSE
+        RAISE NOTICE 'Skipped (not available in this PostgreSQL image):';
+        RAISE NOTICE '  pg_net';
+        RAISE NOTICE '';
+        RAISE NOTICE 'To enable pg_net, use the custom image: percolationlabs/rem-pg:18';
+    END IF;
+    RAISE NOTICE '============================================================';
+END $$;

rem/utils/constants.py

@@ -0,0 +1,97 @@
+"""
+Centralized constants for the REM system.
+
+All magic numbers and commonly-used values should be defined here
+to ensure consistency and make tuning easier.
+"""
+
+# =============================================================================
+# Embedding Model Constants
+# =============================================================================
+
+# OpenAI embedding dimensions by model
+OPENAI_EMBEDDING_DIMS_SMALL = 1536  # text-embedding-3-small
+OPENAI_EMBEDDING_DIMS_LARGE = 3072  # text-embedding-3-large
+OPENAI_EMBEDDING_DIMS_ADA = 1536  # text-embedding-ada-002
+
+# Default embedding dimension (text-embedding-3-small)
+DEFAULT_EMBEDDING_DIMS = 1536
+
+# Voyage AI embedding dimensions
+VOYAGE_EMBEDDING_DIMS = 1024  # voyage-2
+
+# =============================================================================
+# HTTP/API Timeouts (seconds)
+# =============================================================================
+
+HTTP_TIMEOUT_DEFAULT = 30.0  # Standard API calls
+HTTP_TIMEOUT_LONG = 60.0  # Vision/embedding APIs
+HTTP_TIMEOUT_VERY_LONG = 300.0  # Subprocess/batch operations
+
+# Request timeout for httpx AsyncClient
+ASYNC_CLIENT_TIMEOUT = 300.0
+
+# =============================================================================
+# Audio Processing Constants
+# =============================================================================
+
+# Minimum valid WAV file size (header only)
+WAV_HEADER_MIN_BYTES = 44
+
+# OpenAI Whisper API cost per minute (USD)
+WHISPER_COST_PER_MINUTE = 0.006
+
+# Audio chunking parameters
+AUDIO_CHUNK_TARGET_SECONDS = 60.0  # Target chunk duration
+AUDIO_CHUNK_WINDOW_SECONDS = 2.0  # Window for silence detection
+SILENCE_THRESHOLD_DB = -40.0  # Silence detection threshold
+MIN_SILENCE_MS = 500  # Minimum silence duration to split on
+
+# =============================================================================
+# File Processing Constants
+# =============================================================================
+
+# Subprocess timeout for document parsing
+SUBPROCESS_TIMEOUT_SECONDS = 300  # 5 minutes
+
+# Maximum file sizes
+MAX_AUDIO_FILE_SIZE_MB = 25  # Whisper API limit
+
+# =============================================================================
+# Database/Query Constants
+# =============================================================================
+
+# Default batch sizes
+DEFAULT_BATCH_SIZE = 100
+EMBEDDING_BATCH_SIZE = 50
+
+# Default pagination limits
+DEFAULT_PAGE_SIZE = 20
+MAX_PAGE_SIZE = 100
+
+# =============================================================================
+# Rate Limiting
+# =============================================================================
+
+# Default retry settings
+DEFAULT_MAX_RETRIES = 3
+RETRY_BACKOFF_MULTIPLIER = 1
+RETRY_BACKOFF_MIN = 1
+RETRY_BACKOFF_MAX = 60
+
+# =============================================================================
+# S3/Storage Constants
+# =============================================================================
+
+S3_URI_PREFIX = "s3://"
+FILE_URI_PREFIX = "file://"
+
+# =============================================================================
+# LLM Constants
+# =============================================================================
+
+# Default max tokens for vision analysis
+VISION_MAX_TOKENS = 2048
+
+# Default temperature
+DEFAULT_TEMPERATURE = 0.0

rem/utils/date_utils.py

@@ -0,0 +1,228 @@
+"""
+Centralized datetime utilities for consistent UTC-naive datetime handling.
+
+IMPORTANT: REM uses UTC-naive datetimes throughout the codebase.
+PostgreSQL stores TIMESTAMP WITHOUT TIME ZONE, so all Python datetime
+operations should use UTC-naive datetimes to avoid comparison errors.
+
+Convention:
+- All timestamps are implicitly UTC
+- Use utc_now() instead of datetime.utcnow() or datetime.now(timezone.utc)
+- Use parse_iso() to parse ISO format strings (handles "Z" suffix)
+- Use to_iso() to format datetimes as ISO strings
+
+See CLAUDE.md Section 1 (Datetime Convention) for details.
+"""
+
+from datetime import UTC, datetime, timedelta
+from typing import Optional
+
+
+def utc_now() -> datetime:
+    """
+    Get current UTC time as a naive datetime.
+
+    Returns:
+        UTC-naive datetime representing current time.
+
+    Example:
+        >>> now = utc_now()
+        >>> now.tzinfo is None
+        True
+    """
+    return datetime.now(UTC).replace(tzinfo=None)
+
+
+def to_iso(dt: datetime) -> str:
+    """
+    Convert datetime to ISO 8601 format string.
+
+    Args:
+        dt: Datetime to format (should be UTC-naive)
+
+    Returns:
+        ISO format string (e.g., "2024-01-15T10:30:00")
+
+    Example:
+        >>> dt = datetime(2024, 1, 15, 10, 30, 0)
+        >>> to_iso(dt)
+        '2024-01-15T10:30:00'
+    """
+    return dt.isoformat()
+
+
+def to_iso_with_z(dt: datetime) -> str:
+    """
+    Convert datetime to ISO 8601 format with Z suffix.
+
+    Use this when interfacing with external APIs that expect
+    the Z suffix to indicate UTC.
+
+    Args:
+        dt: Datetime to format (should be UTC-naive)
+
+    Returns:
+        ISO format string with Z suffix (e.g., "2024-01-15T10:30:00Z")
+    """
+    return dt.isoformat() + "Z"
+
+
+def parse_iso(iso_string: str) -> datetime:
+    """
+    Parse ISO 8601 format string to UTC-naive datetime.
+
+    Handles:
+    - Standard ISO format: "2024-01-15T10:30:00"
+    - Z suffix: "2024-01-15T10:30:00Z"
+    - Timezone offset: "2024-01-15T10:30:00+00:00" (converts to naive)
+    - Microseconds: "2024-01-15T10:30:00.123456"
+
+    Args:
+        iso_string: ISO format datetime string
+
+    Returns:
+        UTC-naive datetime
+
+    Raises:
+        ValueError: If string cannot be parsed
+
+    Example:
+        >>> parse_iso("2024-01-15T10:30:00Z")
+        datetime.datetime(2024, 1, 15, 10, 30)
+        >>> parse_iso("2024-01-15T10:30:00+00:00")
+        datetime.datetime(2024, 1, 15, 10, 30)
+    """
+    # Handle Z suffix (replace with +00:00 for fromisoformat)
+    if iso_string.endswith("Z"):
+        iso_string = iso_string[:-1] + "+00:00"
+
+    # Parse the ISO string
+    dt = datetime.fromisoformat(iso_string)
+
+    # Convert to naive UTC if timezone-aware
+    if dt.tzinfo is not None:
+        # Convert to UTC and strip timezone
+        from datetime import timezone
+        dt = dt.astimezone(timezone.utc).replace(tzinfo=None)
+
+    return dt
+
+
+def parse_iso_safe(iso_string: Optional[str], default: Optional[datetime] = None) -> Optional[datetime]:
+    """
+    Safely parse ISO string, returning default on failure.
+
+    Args:
+        iso_string: ISO format string or None
+        default: Default value if parsing fails
+
+    Returns:
+        Parsed datetime or default value
+    """
+    if not iso_string:
+        return default
+    try:
+        return parse_iso(iso_string)
+    except (ValueError, TypeError):
+        return default
+
+
+def format_timestamp(dt: Optional[datetime] = None) -> str:
+    """
+    Format datetime for display/logging.
+
+    Args:
+        dt: Datetime to format (defaults to current UTC time)
+
+    Returns:
+        Formatted string like "2024-01-15 10:30:00 UTC"
+    """
+    if dt is None:
+        dt = utc_now()
+    return dt.strftime("%Y-%m-%d %H:%M:%S") + " UTC"
+
+
+def format_timestamp_compact(dt: Optional[datetime] = None) -> str:
+    """
+    Format datetime as compact string for filenames/IDs.
+
+    Args:
+        dt: Datetime to format (defaults to current UTC time)
+
+    Returns:
+        Formatted string like "20240115_103000"
+    """
+    if dt is None:
+        dt = utc_now()
+    return dt.strftime("%Y%m%d_%H%M%S")
+
+
+def format_timestamp_for_experiment(dt: Optional[datetime] = None) -> str:
+    """
+    Format datetime for experiment names.
+
+    Args:
+        dt: Datetime to format (defaults to current UTC time)
+
+    Returns:
+        Formatted string like "20240115-103000"
+    """
+    if dt is None:
+        dt = utc_now()
+    return dt.strftime("%Y%m%d-%H%M%S")
+
+
+def days_ago(days: int) -> datetime:
+    """
+    Get datetime N days ago from now.
+
+    Args:
+        days: Number of days ago
+
+    Returns:
+        UTC-naive datetime
+    """
+    return utc_now() - timedelta(days=days)
+
+
+def hours_ago(hours: int) -> datetime:
+    """
+    Get datetime N hours ago from now.
+
+    Args:
+        hours: Number of hours ago
+
+    Returns:
+        UTC-naive datetime
+    """
+    return utc_now() - timedelta(hours=hours)
+
+
+def is_within_hours(dt: datetime, hours: int) -> bool:
+    """
+    Check if datetime is within N hours of now.
+
+    Args:
+        dt: Datetime to check (should be UTC-naive)
+        hours: Number of hours
+
+    Returns:
+        True if dt is within the time window
+    """
+    cutoff = hours_ago(hours)
+    return dt >= cutoff
+
+
+def is_within_days(dt: datetime, days: int) -> bool:
+    """
+    Check if datetime is within N days of now.
+
+    Args:
+        dt: Datetime to check (should be UTC-naive)
+        days: Number of days
+
+    Returns:
+        True if dt is within the time window
+    """
+    cutoff = days_ago(days)
+    return dt >= cutoff

rem/utils/embeddings.py

@@ -20,7 +20,6 @@ Usage:
     embeddings = generate_embeddings("openai:text-embedding-3-small", texts)
 """
 
-import os
 from typing import Any, cast
 
 import requests
@@ -31,6 +30,16 @@ from tenacity import (
     wait_exponential,
 )
 
+from rem.utils.constants import (
+    HTTP_TIMEOUT_LONG,
+    OPENAI_EMBEDDING_DIMS_SMALL,
+    OPENAI_EMBEDDING_DIMS_LARGE,
+    VOYAGE_EMBEDDING_DIMS,
+    RETRY_BACKOFF_MULTIPLIER,
+    RETRY_BACKOFF_MIN,
+    RETRY_BACKOFF_MAX,
+)
+
 
 class EmbeddingError(Exception):
     """Base exception for embedding generation errors."""
@@ -166,7 +175,11 @@ def _create_retry_decorator(max_retries: int):
     return retry(
         retry=retry_if_exception_type(RateLimitError),
         stop=stop_after_attempt(max_retries),
-        wait=wait_exponential(multiplier=1, min=1, max=60),
+        wait=wait_exponential(
+            multiplier=RETRY_BACKOFF_MULTIPLIER,
+            min=RETRY_BACKOFF_MIN,
+            max=RETRY_BACKOFF_MAX,
+        ),
         reraise=True,
     )
 
@@ -234,7 +247,7 @@ def _generate_openai_embeddings(
     }
 
     try:
-        response = requests.post(url, json=payload, headers=headers, timeout=60)
+        response = requests.post(url, json=payload, headers=headers, timeout=HTTP_TIMEOUT_LONG)
 
         # Handle rate limits
         if response.status_code == 429:
@@ -334,7 +347,7 @@ def _generate_voyage_embeddings(
     }
 
     try:
-        response = requests.post(url, json=payload, headers=headers, timeout=60)
+        response = requests.post(url, json=payload, headers=headers, timeout=HTTP_TIMEOUT_LONG)
 
         # Handle rate limits
         if response.status_code == 429: