remdb 0.2.6__py3-none-any.whl → 0.3.103__py3-none-any.whl

rem/settings.py

@@ -15,7 +15,7 @@ Example .env file:
     API__LOG_LEVEL=info
 
     # LLM
-    LLM__DEFAULT_MODEL=anthropic:claude-sonnet-4-5-20250929
+    LLM__DEFAULT_MODEL=openai:gpt-4.1
     LLM__DEFAULT_TEMPERATURE=0.5
     LLM__MAX_RETRIES=10
     LLM__OPENAI_API_KEY=sk-...
@@ -57,8 +57,10 @@ Example .env file:
 """
 
 import os
-from pydantic import Field, field_validator
+import hashlib
+from pydantic import Field, field_validator, FieldValidationInfo
 from pydantic_settings import BaseSettings, SettingsConfigDict
+from loguru import logger
 
 
 class LLMSettings(BaseSettings):
@@ -72,7 +74,7 @@ class LLMSettings(BaseSettings):
         LLM__EVALUATOR_MODEL or EVALUATOR_MODEL - Model for LLM-as-judge evaluation
         LLM__OPENAI_API_KEY or OPENAI_API_KEY - OpenAI API key
         LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
-        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai, cohere, jina, etc.)
+        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
         LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
     """
 
@@ -84,7 +86,7 @@ class LLMSettings(BaseSettings):
     )
 
     default_model: str = Field(
-        default="anthropic:claude-sonnet-4-5-20250929",
+        default="openai:gpt-4.1",
         description="Default LLM model (format: provider:model-id)",
     )
 
@@ -127,7 +129,7 @@ class LLMSettings(BaseSettings):
 
     embedding_provider: str = Field(
         default="openai",
-        description="Default embedding provider (openai, cohere, jina, etc.)",
+        description="Default embedding provider (currently only openai supported)",
     )
 
     embedding_model: str = Field(
@@ -359,10 +361,16 @@ class AuthSettings(BaseSettings):
     - Custom OIDC provider
 
     Environment variables:
-        AUTH__ENABLED - Enable authentication (default: false)
+        AUTH__ENABLED - Enable authentication (default: true)
+        AUTH__ALLOW_ANONYMOUS - Allow rate-limited anonymous access (default: true)
         AUTH__SESSION_SECRET - Secret for session cookie signing
         AUTH__GOOGLE__* - Google OAuth settings
         AUTH__MICROSOFT__* - Microsoft OAuth settings
+
+    Access modes:
+    - enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
+    - enabled=true, allow_anonymous=false: Auth required for all requests
+    - enabled=false: No auth, all requests treated as default user (dev mode)
     """
 
     model_config = SettingsConfigDict(
@@ -373,8 +381,26 @@ class AuthSettings(BaseSettings):
     )
 
     enabled: bool = Field(
-        default=False,
-        description="Enable authentication (disabled by default for testing)",
+        default=True,
+        description="Enable authentication (OAuth endpoints and middleware)",
+    )
+
+    allow_anonymous: bool = Field(
+        default=True,
+        description=(
+            "Allow anonymous (unauthenticated) access with rate limits. "
+            "When true, requests without auth get ANONYMOUS tier rate limits. "
+            "When false, all requests require authentication."
+        ),
+    )
+
+    mcp_requires_auth: bool = Field(
+        default=True,
+        description=(
+            "Require authentication for MCP endpoints. "
+            "MCP is a protected service and should always require login in production. "
+            "Set to false only for local development/testing."
+        ),
     )
 
     session_secret: str = Field(
@@ -386,6 +412,22 @@ class AuthSettings(BaseSettings):
     google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
     microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)
 
+    @field_validator("session_secret", mode="before")
+    @classmethod
+    def generate_dev_secret(cls, v: str | None, info: FieldValidationInfo) -> str:
+        # Only generate if not already set and not in production
+        if not v and info.data.get("environment") != "production":
+            # Deterministic secret for development
+            seed_string = f"{info.data.get('team', 'rem')}-{info.data.get('environment', 'development')}-auth-secret-salt"
+            logger.warning(
+                "AUTH__SESSION_SECRET not set. Generating deterministic secret for non-production environment. "
+                "DO NOT use in production."
+            )
+            return hashlib.sha256(seed_string.encode()).hexdigest()
+        elif not v and info.data.get("environment") == "production":
+            raise ValueError("AUTH__SESSION_SECRET must be set in production environment.")
+        return v
+
 
 class PostgresSettings(BaseSettings):
     """
@@ -962,6 +1004,54 @@ class APISettings(BaseSettings):
     )
 
 
+class SchemaSettings(BaseSettings):
+    """
+    Schema search path settings for agent and evaluator schemas.
+
+    Allows extending REM's schema search with custom directories.
+    Custom paths are searched BEFORE built-in package schemas.
+
+    Environment variables:
+        SCHEMA__PATHS - Semicolon-separated list of directories to search
+                        Example: "/app/schemas;/shared/agents;./local-schemas"
+
+    Search Order:
+    1. Exact path (if file exists)
+    2. Custom paths from SCHEMA__PATHS (in order)
+    3. Built-in package schemas (schemas/agents/, schemas/evaluators/, etc.)
+    4. Database LOOKUP (if enabled)
+
+    Example:
+        # In .env or environment
+        SCHEMA__PATHS=/app/custom-agents;/shared/evaluators
+
+        # Then in code
+        from rem.utils.schema_loader import load_agent_schema
+        schema = load_agent_schema("my-custom-agent")  # Found in /app/custom-agents/
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="SCHEMA__",
+        extra="ignore",
+    )
+
+    paths: str = Field(
+        default="",
+        description=(
+            "Semicolon-separated list of directories to search for schemas. "
+            "These paths are searched BEFORE built-in package schemas. "
+            "Example: '/app/schemas;/shared/agents'"
+        ),
+    )
+
+    @property
+    def path_list(self) -> list[str]:
+        """Get paths as a list, filtering empty strings."""
+        if not self.paths:
+            return []
+        return [p.strip() for p in self.paths.split(";") if p.strip()]
+
+
 class GitSettings(BaseSettings):
     """
     Git repository provider settings for versioned schema/experiment syncing.
@@ -1166,6 +1256,11 @@ class Settings(BaseSettings):
         extra="ignore",
     )
 
+    app_name: str = Field(
+        default="REM",
+        description="Application/API name used in docs, titles, and user-facing text",
+    )
+
     team: str = Field(
         default="rem",
         description="Team or project name for observability",
@@ -1207,20 +1302,23 @@ class Settings(BaseSettings):
     sqs: SQSSettings = Field(default_factory=SQSSettings)
     chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
     content: ContentSettings = Field(default_factory=ContentSettings)
+    schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
     test: TestSettings = Field(default_factory=TestSettings)
 
 
 # Load configuration from ~/.rem/config.yaml before initializing settings
 # This allows user configuration to be merged with environment variables
-try:
-    from rem.config import load_config, merge_config_to_env
-
-    _config = load_config()
-    if _config:
-        merge_config_to_env(_config)
-except ImportError:
-    # config module not available (e.g., during initial setup)
-    pass
+# Set REM_SKIP_CONFIG_FILE=true to disable (useful for development with .env)
+if not os.getenv("REM_SKIP_CONFIG_FILE", "").lower() in ("true", "1", "yes"):
+    try:
+        from rem.config import load_config, merge_config_to_env
+
+        _config = load_config()
+        if _config:
+            merge_config_to_env(_config)
+    except ImportError:
+        # config module not available (e.g., during initial setup)
+        pass
 
 # Global settings singleton
 settings = Settings()

rem/sql/background_indexes.sql

@@ -16,6 +16,11 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
 ON embeddings_moments
 USING hnsw (embedding vector_cosine_ops);
 
+-- HNSW vector index for embeddings_sessions
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_sessions_vector_hnsw
+ON embeddings_sessions
+USING hnsw (embedding vector_cosine_ops);
+
 -- HNSW vector index for embeddings_resources
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_resources_vector_hnsw
 ON embeddings_resources
@@ -36,6 +41,11 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontology_configs_vector_h
 ON embeddings_ontology_configs
 USING hnsw (embedding vector_cosine_ops);
 
+-- HNSW vector index for embeddings_domain_resources
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_domain_resources_vector_hnsw
+ON embeddings_domain_resources
+USING hnsw (embedding vector_cosine_ops);
+
 -- HNSW vector index for embeddings_schemas
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_schemas_vector_hnsw
 ON embeddings_schemas

rem/sql/migrations/001_install.sql

@@ -381,8 +381,140 @@ $$ LANGUAGE plpgsql STABLE;
 COMMENT ON FUNCTION rem_fuzzy IS
 'REM FUZZY query: Fuzzy text search using pg_trgm. Returns raw entity data as JSONB for LLM consumption. tenant_id parameter exists for backward compatibility but filtering uses user_id.';
 
--- REM TRAVERSE: Moved to 002_install_models.sql (after entity tables are created)
--- See 002_install_models.sql for the full rem_traverse function with keys_only parameter
+-- ============================================================================
+-- REM TRAVERSE (Graph Traversal)
+-- ============================================================================
+
+-- REM TRAVERSE: Recursive graph traversal following edges
+-- Explores graph_edges starting from entity_key up to max_depth
+-- Uses cached kv_store.graph_edges for fast traversal (no polymorphic view!)
+-- When keys_only=false, automatically fetches full entity records
+CREATE OR REPLACE FUNCTION rem_traverse(
+    p_entity_key VARCHAR(255),
+    p_tenant_id VARCHAR(100),  -- Backward compat parameter (not used for filtering)
+    p_user_id VARCHAR(100),
+    p_max_depth INTEGER DEFAULT 1,
+    p_rel_type VARCHAR(100) DEFAULT NULL,
+    p_keys_only BOOLEAN DEFAULT FALSE
+)
+RETURNS TABLE(
+    depth INTEGER,
+    entity_key VARCHAR(255),
+    entity_type VARCHAR(100),
+    entity_id UUID,
+    rel_type VARCHAR(100),
+    rel_weight REAL,
+    path TEXT[],
+    entity_record JSONB
+) AS $$
+DECLARE
+    graph_keys RECORD;
+    entities_by_table JSONB := '{}'::jsonb;
+    table_keys JSONB;
+BEGIN
+    -- First, build graph structure from KV store
+    FOR graph_keys IN
+        WITH RECURSIVE graph_traversal AS (
+            -- Base case: Find starting entity
+            SELECT
+                0 AS depth,
+                kv.entity_key,
+                kv.entity_type,
+                kv.entity_id,
+                NULL::VARCHAR(100) AS rel_type,
+                NULL::REAL AS rel_weight,
+                ARRAY[kv.entity_key]::TEXT[] AS path
+            FROM kv_store kv
+            WHERE kv.user_id = p_user_id
+            AND kv.entity_key = p_entity_key
+
+            UNION ALL
+
+            -- Recursive case: Follow outbound edges from discovered entities
+            SELECT
+                gt.depth + 1,
+                target_kv.entity_key,
+                target_kv.entity_type,
+                target_kv.entity_id,
+                (edge->>'rel_type')::VARCHAR(100) AS rel_type,
+                COALESCE((edge->>'weight')::REAL, 1.0) AS rel_weight,
+                gt.path || target_kv.entity_key AS path
+            FROM graph_traversal gt
+            -- Join to KV store to get source entity (with cached graph_edges!)
+            JOIN kv_store source_kv ON source_kv.entity_key = gt.entity_key
+                AND source_kv.user_id = p_user_id
+            -- Extract edges directly from cached kv_store.graph_edges (NO polymorphic view!)
+            CROSS JOIN LATERAL jsonb_array_elements(COALESCE(source_kv.graph_edges, '[]'::jsonb)) AS edge
+            -- Lookup target entity in KV store
+            JOIN kv_store target_kv ON target_kv.entity_key = (edge->>'dst')::VARCHAR(255)
+                AND target_kv.user_id = p_user_id
+            WHERE gt.depth < p_max_depth
+            -- Filter by relationship type if specified
+            AND (p_rel_type IS NULL OR (edge->>'rel_type')::VARCHAR(100) = p_rel_type)
+            -- Prevent cycles by checking path
+            AND NOT (target_kv.entity_key = ANY(gt.path))
+        )
+        SELECT DISTINCT ON (entity_key)
+            gt.depth,
+            gt.entity_key,
+            gt.entity_type,
+            gt.entity_id,
+            gt.rel_type,
+            gt.rel_weight,
+            gt.path
+        FROM graph_traversal gt
+        WHERE gt.depth > 0  -- Exclude starting entity
+        ORDER BY gt.entity_key, gt.depth
+    LOOP
+        IF p_keys_only THEN
+            -- Return just graph structure (no entity_record)
+            depth := graph_keys.depth;
+            entity_key := graph_keys.entity_key;
+            entity_type := graph_keys.entity_type;
+            entity_id := graph_keys.entity_id;
+            rel_type := graph_keys.rel_type;
+            rel_weight := graph_keys.rel_weight;
+            path := graph_keys.path;
+            entity_record := NULL;
+            RETURN NEXT;
+        ELSE
+            -- Build JSONB mapping {table: [keys]} for batch fetch
+            IF entities_by_table ? graph_keys.entity_type THEN
+                table_keys := entities_by_table->graph_keys.entity_type;
+                entities_by_table := jsonb_set(
+                    entities_by_table,
+                    ARRAY[graph_keys.entity_type],
+                    table_keys || jsonb_build_array(graph_keys.entity_key)
+                );
+            ELSE
+                entities_by_table := jsonb_set(
+                    entities_by_table,
+                    ARRAY[graph_keys.entity_type],
+                    jsonb_build_array(graph_keys.entity_key)
+                );
+            END IF;
+        END IF;
+    END LOOP;
+
+    -- If keys_only=false, fetch full records using rem_fetch
+    IF NOT p_keys_only AND entities_by_table != '{}'::jsonb THEN
+        RETURN QUERY
+        SELECT
+            NULL::INTEGER AS depth,
+            f.entity_key::VARCHAR(255),
+            f.entity_type::VARCHAR(100),
+            NULL::UUID AS entity_id,
+            NULL::VARCHAR(100) AS rel_type,
+            NULL::REAL AS rel_weight,
+            NULL::TEXT[] AS path,
+            f.entity_record
+        FROM rem_fetch(entities_by_table, p_user_id) f;
+    END IF;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION rem_traverse IS
+'REM TRAVERSE query: Recursive graph traversal using cached kv_store.graph_edges. When keys_only=false (default), automatically fetches full entity records via rem_fetch.';
 
 -- REM SEARCH: Vector similarity search using embeddings
 -- Joins to embeddings table for semantic search
@@ -464,6 +596,24 @@ $$ LANGUAGE plpgsql;
 COMMENT ON FUNCTION migration_status() IS
 'Get summary of applied migrations by type';
 
+-- ============================================================================
+-- RATE LIMITS (UNLOGGED for performance)
+-- ============================================================================
+-- High-performance rate limiting table. Uses UNLOGGED for speed - counts may
+-- be lost on database crash/restart, which is acceptable (fail-open on error).
+
+CREATE UNLOGGED TABLE IF NOT EXISTS rate_limits (
+    key VARCHAR(512) PRIMARY KEY,
+    count INTEGER NOT NULL DEFAULT 1,
+    expires_at TIMESTAMP NOT NULL,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_rate_limits_expires ON rate_limits (expires_at);
+
+COMMENT ON TABLE rate_limits IS
+'UNLOGGED rate limiting table. Counts may be lost on crash (acceptable for rate limiting).';
+
 -- ============================================================================
 -- RECORD INSTALLATION
 -- ============================================================================