remdb 0.2.6__py3-none-any.whl → 0.3.118__py3-none-any.whl

rem/services/rem/parser.py

@@ -50,9 +50,36 @@ class RemQueryParser:
         params: Dict[str, Any] = {}
         positional_args: List[str] = []
 
-        # Process remaining tokens
-        for token in tokens[1:]:
-            if "=" in token:
+        # For SQL queries, preserve the raw query (keywords like LIMIT are SQL keywords)
+        if query_type == QueryType.SQL:
+            # Everything after "SQL" is the raw SQL query
+            raw_sql = query_string[3:].strip()  # Skip "SQL" prefix
+            params["raw_query"] = raw_sql
+            return query_type, params
+
+        # Process remaining tokens, handling REM keywords
+        i = 1
+        while i < len(tokens):
+            token = tokens[i]
+            token_upper = token.upper()
+
+            # Handle REM keywords that take a value
+            if token_upper in ("LIMIT", "DEPTH", "THRESHOLD", "TYPE", "FROM", "WITH"):
+                if i + 1 < len(tokens):
+                    keyword_map = {
+                        "LIMIT": "limit",
+                        "DEPTH": "max_depth",
+                        "THRESHOLD": "threshold",
+                        "TYPE": "edge_types",
+                        "FROM": "initial_query",
+                        "WITH": "initial_query",
+                    }
+                    key = keyword_map[token_upper]
+                    value = tokens[i + 1]
+                    params[key] = self._convert_value(key, value)
+                    i += 2
+                    continue
+            elif "=" in token:
                 # It's a keyword argument
                 key, value = token.split("=", 1)
                 # Handle parameter aliases
@@ -61,6 +88,7 @@ class RemQueryParser:
             else:
                 # It's a positional argument part
                 positional_args.append(token)
+            i += 1
 
         # Map positional arguments to specific fields based on QueryType
         self._map_positional_args(query_type, positional_args, params)
@@ -133,13 +161,20 @@ class RemQueryParser:
             params["query_text"] = combined_value
 
         elif query_type == QueryType.SEARCH:
-            params["query_text"] = combined_value
+            # SEARCH expects: SEARCH <table> <query_text> [LIMIT n]
+            # First positional arg is table name, rest is query text
+            if len(positional_args) >= 2:
+                params["table_name"] = positional_args[0]
+                params["query_text"] = " ".join(positional_args[1:])
+            elif len(positional_args) == 1:
+                # Could be table name or query text - assume query text if no table
+                params["query_text"] = positional_args[0]
+            # If no positional args, params stays empty
 
         elif query_type == QueryType.TRAVERSE:
             params["initial_query"] = combined_value
         
-        # SQL typically requires named arguments (table=...), but if we supported 
-        # SQL SELECT * FROM ..., we might handle it differently. 
-        # For now, RemService expects table=...
-        # If there are positional args for SQL, we might ignore or raise, 
-        # but current service doesn't use them.
+        elif query_type == QueryType.SQL:
+            # SQL with positional args means "SQL SELECT * FROM ..." form
+            # Treat the combined positional args as the raw SQL query
+            params["raw_query"] = combined_value

rem/services/rem/service.py

@@ -13,6 +13,31 @@ Design:
 - All queries pushed down to Postgres for performance
 - Model schema inspection for validation only
 - Exceptions for missing fields/embeddings
+
+TODO: Staged Plan Execution
+- Implement execute_staged_plan() method for multi-stage query execution
+- Each stage can be:
+  1. Static query (query field): Execute REM dialect directly
+  2. Dynamic query (intent field): LLM interprets intent + previous results to build query
+- Flow for dynamic stages:
+  1. Gather results from depends_on stages (from previous_results or current execution)
+  2. Pass intent + previous results to LLM (like ask_rem but with context)
+  3. LLM generates REM query based on what it learned from previous stages
+  4. Execute generated query
+  5. Store results in stage_results for client to use in continuation
+- Multi-turn continuation:
+  - Client passes previous_results back from response's stage_results
+  - Client sets resume_from_stage to skip already-executed stages
+  - Server uses previous_results as context for depends_on lookups
+- Use cases:
+  - LOOKUP "Sarah" → intent: "find her team members" (LLM sees Sarah's graph_edges, builds TRAVERSE)
+  - SEARCH "API docs" → intent: "get authors" (LLM extracts author refs, builds LOOKUP)
+  - Complex graph exploration with LLM-driven navigation
+- API: POST /api/v1/query with:
+  - mode="staged-plan"
+  - plan=[{stage, query|intent, name, depends_on}]
+  - previous_results=[{stage, name, query_executed, results, count}] (for continuation)
+  - resume_from_stage=N (to skip completed stages)
 """
 
 from typing import Any
@@ -309,17 +334,26 @@ class RemService:
         )
 
         # Execute vector search via rem_search() PostgreSQL function
+        min_sim = params.min_similarity if params.min_similarity is not None else 0.3
+        limit = params.limit or 10
         query_params = get_search_params(
             query_embedding,
             table_name,
             field_name,
             tenant_id,
             provider,
-            params.min_similarity or 0.7,
-            params.limit or 10,
+            min_sim,
+            limit,
             tenant_id, # Use tenant_id (query.user_id) as user_id
         )
+        logger.debug(
+            f"SEARCH params: table={table_name}, field={field_name}, "
+            f"tenant_id={tenant_id}, provider={provider}, "
+            f"min_similarity={min_sim}, limit={limit}, "
+            f"embedding_dims={len(query_embedding)}"
+        )
         results = await self.db.execute(SEARCH_QUERY, query_params)
+        logger.debug(f"SEARCH results: {len(results)} rows")
 
         return {
             "query_type": "SEARCH",

rem/services/session/compression.py

@@ -14,6 +14,21 @@ from typing import Any
 
 from loguru import logger
 
+# Max length for entity keys (kv_store.entity_key is varchar(255))
+MAX_ENTITY_KEY_LENGTH = 255
+
+
+def truncate_key(key: str, max_length: int = MAX_ENTITY_KEY_LENGTH) -> str:
+    """Truncate a key to max length, preserving useful suffix if possible."""
+    if len(key) <= max_length:
+        return key
+    # Keep first part and add hash suffix for uniqueness
+    import hashlib
+    hash_suffix = hashlib.md5(key.encode()).hexdigest()[:8]
+    truncated = key[:max_length - 9] + "-" + hash_suffix
+    logger.warning(f"Truncated key from {len(key)} to {len(truncated)} chars: {key[:50]}...")
+    return truncated
+
 from rem.models.entities import Message
 from rem.services.postgres import PostgresService, Repository
 from rem.settings import settings
@@ -151,7 +166,8 @@ class SessionMessageStore:
             return f"msg-{message_index}"
 
         # Create entity key for REM LOOKUP: session-{session_id}-msg-{index}
-        entity_key = f"session-{session_id}-msg-{message_index}"
+        # Truncate to avoid exceeding kv_store.entity_key varchar(255) limit
+        entity_key = truncate_key(f"session-{session_id}-msg-{message_index}")
 
         # Create Message entity for assistant response
         msg = Message(

rem/services/session/reload.py

@@ -65,7 +65,7 @@ async def reload_session(
             session_id=session_id, user_id=user_id, decompress=decompress_messages
         )
 
-        logger.info(
+        logger.debug(
             f"Reloaded {len(messages)} messages for session {session_id} "
             f"(decompressed={decompress_messages})"
         )

rem/services/user_service.py

@@ -0,0 +1,98 @@
+"""
+User Service - User account management.
+
+Handles user creation, profile updates, and session linking.
+"""
+
+from datetime import datetime
+from typing import Optional
+
+from loguru import logger
+
+from ..models.entities.user import User, UserTier
+from .postgres.repository import Repository
+from .postgres.service import PostgresService
+
+
+class UserService:
+    """
+    Service for managing user accounts and sessions.
+    """
+
+    def __init__(self, db: PostgresService):
+        self.db = db
+        self.repo = Repository(User, "users", db=db)
+
+    async def get_or_create_user(
+        self,
+        email: str,
+        tenant_id: str = "default",
+        name: str = "New User",
+        avatar_url: Optional[str] = None,
+    ) -> User:
+        """
+        Get existing user by email or create a new one.
+        """
+        users = await self.repo.find(filters={"email": email}, limit=1)
+        
+        if users:
+            user = users[0]
+            # Update profile if needed (e.g., name/avatar from OAuth)
+            updated = False
+            if name and user.name == "New User": # Only update if placeholder
+                user.name = name
+                updated = True
+            
+            # Store avatar in metadata if provided
+            if avatar_url:
+                user.metadata = user.metadata or {}
+                if user.metadata.get("avatar_url") != avatar_url:
+                    user.metadata["avatar_url"] = avatar_url
+                    updated = True
+            
+            if updated:
+                user.updated_at = datetime.utcnow()
+                await self.repo.upsert(user)
+            
+            return user
+        
+        # Create new user
+        user = User(
+            tenant_id=tenant_id,
+            user_id=email, # Use email as user_id for now? Or UUID?
+            # The User model has 'user_id' field but also 'id' UUID.
+            # Usually user_id is the external ID or email.
+            name=name,
+            email=email,
+            tier=UserTier.FREE,
+            created_at=datetime.utcnow(),
+            updated_at=datetime.utcnow(),
+            metadata={"avatar_url": avatar_url} if avatar_url else {},
+        )
+        await self.repo.upsert(user)
+        logger.info(f"Created new user: {email}")
+        return user
+
+    async def link_anonymous_session(self, user: User, anon_id: str) -> None:
+        """
+        Link an anonymous session ID to a user account.
+        
+        This allows merging history from the anonymous session into the user's profile.
+        """
+        if not anon_id:
+            return
+
+        # Check if already linked
+        if anon_id in user.anonymous_ids:
+            return
+
+        # Add to list
+        user.anonymous_ids.append(anon_id)
+        user.updated_at = datetime.utcnow()
+        
+        # Save
+        await self.repo.upsert(user)
+        logger.info(f"Linked anonymous session {anon_id} to user {user.email}")
+        
+        # TODO: Migrate/Merge actual data (rate limit counts, history) if needed.
+        # For now, we just link the IDs so future queries can include data from this anon_id.

rem/settings.py

@@ -15,7 +15,7 @@ Example .env file:
     API__LOG_LEVEL=info
 
     # LLM
-    LLM__DEFAULT_MODEL=anthropic:claude-sonnet-4-5-20250929
+    LLM__DEFAULT_MODEL=openai:gpt-4.1
     LLM__DEFAULT_TEMPERATURE=0.5
     LLM__MAX_RETRIES=10
     LLM__OPENAI_API_KEY=sk-...
@@ -57,8 +57,10 @@ Example .env file:
 """
 
 import os
-from pydantic import Field, field_validator
+import hashlib
+from pydantic import Field, field_validator, ValidationInfo
 from pydantic_settings import BaseSettings, SettingsConfigDict
+from loguru import logger
 
 
 class LLMSettings(BaseSettings):
@@ -72,7 +74,7 @@ class LLMSettings(BaseSettings):
         LLM__EVALUATOR_MODEL or EVALUATOR_MODEL - Model for LLM-as-judge evaluation
         LLM__OPENAI_API_KEY or OPENAI_API_KEY - OpenAI API key
         LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
-        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai, cohere, jina, etc.)
+        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
         LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
     """
 
@@ -84,7 +86,7 @@ class LLMSettings(BaseSettings):
     )
 
     default_model: str = Field(
-        default="anthropic:claude-sonnet-4-5-20250929",
+        default="openai:gpt-4.1",
         description="Default LLM model (format: provider:model-id)",
     )
 
@@ -127,7 +129,7 @@ class LLMSettings(BaseSettings):
 
     embedding_provider: str = Field(
         default="openai",
-        description="Default embedding provider (openai, cohere, jina, etc.)",
+        description="Default embedding provider (currently only openai supported)",
     )
 
     embedding_model: str = Field(
@@ -359,10 +361,16 @@ class AuthSettings(BaseSettings):
     - Custom OIDC provider
 
     Environment variables:
-        AUTH__ENABLED - Enable authentication (default: false)
+        AUTH__ENABLED - Enable authentication (default: true)
+        AUTH__ALLOW_ANONYMOUS - Allow rate-limited anonymous access (default: true)
         AUTH__SESSION_SECRET - Secret for session cookie signing
         AUTH__GOOGLE__* - Google OAuth settings
         AUTH__MICROSOFT__* - Microsoft OAuth settings
+
+    Access modes:
+    - enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
+    - enabled=true, allow_anonymous=false: Auth required for all requests
+    - enabled=false: No auth, all requests treated as default user (dev mode)
     """
 
     model_config = SettingsConfigDict(
@@ -373,8 +381,26 @@ class AuthSettings(BaseSettings):
     )
 
     enabled: bool = Field(
-        default=False,
-        description="Enable authentication (disabled by default for testing)",
+        default=True,
+        description="Enable authentication (OAuth endpoints and middleware)",
+    )
+
+    allow_anonymous: bool = Field(
+        default=True,
+        description=(
+            "Allow anonymous (unauthenticated) access with rate limits. "
+            "When true, requests without auth get ANONYMOUS tier rate limits. "
+            "When false, all requests require authentication."
+        ),
+    )
+
+    mcp_requires_auth: bool = Field(
+        default=True,
+        description=(
+            "Require authentication for MCP endpoints. "
+            "MCP is a protected service and should always require login in production. "
+            "Set to false only for local development/testing."
+        ),
     )
 
     session_secret: str = Field(
@@ -386,6 +412,22 @@ class AuthSettings(BaseSettings):
     google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
     microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)
 
+    @field_validator("session_secret", mode="before")
+    @classmethod
+    def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
+        # Only generate if not already set and not in production
+        if not v and info.data.get("environment") != "production":
+            # Deterministic secret for development
+            seed_string = f"{info.data.get('team', 'rem')}-{info.data.get('environment', 'development')}-auth-secret-salt"
+            logger.warning(
+                "AUTH__SESSION_SECRET not set. Generating deterministic secret for non-production environment. "
+                "DO NOT use in production."
+            )
+            return hashlib.sha256(seed_string.encode()).hexdigest()
+        elif not v and info.data.get("environment") == "production":
+            raise ValueError("AUTH__SESSION_SECRET must be set in production environment.")
+        return v
+
 
 class PostgresSettings(BaseSettings):
     """
@@ -962,6 +1004,107 @@ class APISettings(BaseSettings):
     )
 
 
+class ModelsSettings(BaseSettings):
+    """
+    Custom model registration settings for downstream applications.
+
+    Allows downstream apps to specify Python modules containing custom models
+    that should be imported (and thus registered) before schema generation.
+
+    This enables `rem db schema generate` to discover models registered with
+    `@rem.register_model` in downstream applications.
+
+    Environment variables:
+        MODELS__IMPORT_MODULES - Semicolon-separated list of Python modules to import
+                                 Example: "models;myapp.entities;myapp.custom_models"
+
+    Example:
+        # In downstream app's .env
+        MODELS__IMPORT_MODULES=models
+
+        # In downstream app's models/__init__.py
+        import rem
+        from rem.models.core import CoreModel
+
+        @rem.register_model
+        class MyCustomEntity(CoreModel):
+            name: str
+
+        # Then run schema generation
+        rem db schema generate  # Includes MyCustomEntity
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="MODELS__",
+        extra="ignore",
+    )
+
+    import_modules: str = Field(
+        default="",
+        description=(
+            "Semicolon-separated list of Python modules to import for model registration. "
+            "These modules are imported before schema generation to ensure custom models "
+            "decorated with @rem.register_model are discovered. "
+            "Example: 'models;myapp.entities'"
+        ),
+    )
+
+    @property
+    def module_list(self) -> list[str]:
+        """Get modules as a list, filtering empty strings."""
+        if not self.import_modules:
+            return []
+        return [m.strip() for m in self.import_modules.split(";") if m.strip()]
+
+
+class SchemaSettings(BaseSettings):
+    """
+    Schema search path settings for agent and evaluator schemas.
+
+    Allows extending REM's schema search with custom directories.
+    Custom paths are searched BEFORE built-in package schemas.
+
+    Environment variables:
+        SCHEMA__PATHS - Semicolon-separated list of directories to search
+                        Example: "/app/schemas;/shared/agents;./local-schemas"
+
+    Search Order:
+    1. Exact path (if file exists)
+    2. Custom paths from SCHEMA__PATHS (in order)
+    3. Built-in package schemas (schemas/agents/, schemas/evaluators/, etc.)
+    4. Database LOOKUP (if enabled)
+
+    Example:
+        # In .env or environment
+        SCHEMA__PATHS=/app/custom-agents;/shared/evaluators
+
+        # Then in code
+        from rem.utils.schema_loader import load_agent_schema
+        schema = load_agent_schema("my-custom-agent")  # Found in /app/custom-agents/
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="SCHEMA__",
+        extra="ignore",
+    )
+
+    paths: str = Field(
+        default="",
+        description=(
+            "Semicolon-separated list of directories to search for schemas. "
+            "These paths are searched BEFORE built-in package schemas. "
+            "Example: '/app/schemas;/shared/agents'"
+        ),
+    )
+
+    @property
+    def path_list(self) -> list[str]:
+        """Get paths as a list, filtering empty strings."""
+        if not self.paths:
+            return []
+        return [p.strip() for p in self.paths.split(";") if p.strip()]
+
+
 class GitSettings(BaseSettings):
     """
     Git repository provider settings for versioned schema/experiment syncing.
@@ -1166,6 +1309,11 @@ class Settings(BaseSettings):
         extra="ignore",
     )
 
+    app_name: str = Field(
+        default="REM",
+        description="Application/API name used in docs, titles, and user-facing text",
+    )
+
     team: str = Field(
         default="rem",
         description="Team or project name for observability",
@@ -1186,16 +1334,12 @@ class Settings(BaseSettings):
         description="Root path for reverse proxy (e.g., /rem for ALB routing)",
     )
 
-    sql_dir: str = Field(
-        default="src/rem/sql",
-        description="Directory for SQL files and migrations",
-    )
-
     # Nested settings groups
     api: APISettings = Field(default_factory=APISettings)
     chat: ChatSettings = Field(default_factory=ChatSettings)
     llm: LLMSettings = Field(default_factory=LLMSettings)
     mcp: MCPSettings = Field(default_factory=MCPSettings)
+    models: ModelsSettings = Field(default_factory=ModelsSettings)
     otel: OTELSettings = Field(default_factory=OTELSettings)
     phoenix: PhoenixSettings = Field(default_factory=PhoenixSettings)
     auth: AuthSettings = Field(default_factory=AuthSettings)
@@ -1207,20 +1351,23 @@ class Settings(BaseSettings):
     sqs: SQSSettings = Field(default_factory=SQSSettings)
     chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
     content: ContentSettings = Field(default_factory=ContentSettings)
+    schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
     test: TestSettings = Field(default_factory=TestSettings)
 
 
 # Load configuration from ~/.rem/config.yaml before initializing settings
 # This allows user configuration to be merged with environment variables
-try:
-    from rem.config import load_config, merge_config_to_env
-
-    _config = load_config()
-    if _config:
-        merge_config_to_env(_config)
-except ImportError:
-    # config module not available (e.g., during initial setup)
-    pass
+# Set REM_SKIP_CONFIG_FILE=true to disable (useful for development with .env)
+if not os.getenv("REM_SKIP_CONFIG_FILE", "").lower() in ("true", "1", "yes"):
+    try:
+        from rem.config import load_config, merge_config_to_env
+
+        _config = load_config()
+        if _config:
+            merge_config_to_env(_config)
+    except ImportError:
+        # config module not available (e.g., during initial setup)
+        pass
 
 # Global settings singleton
 settings = Settings()

rem/sql/background_indexes.sql

@@ -1,9 +1,9 @@
 -- Background index creation
 -- Run AFTER initial data load to avoid blocking writes
 
--- HNSW vector index for embeddings_users
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_users_vector_hnsw
-ON embeddings_users
+-- HNSW vector index for embeddings_files
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_files_vector_hnsw
+ON embeddings_files
 USING hnsw (embedding vector_cosine_ops);
 
 -- HNSW vector index for embeddings_image_resources
@@ -11,24 +11,14 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_image_resources_vector_hn
 ON embeddings_image_resources
 USING hnsw (embedding vector_cosine_ops);
 
--- HNSW vector index for embeddings_moments
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
-ON embeddings_moments
-USING hnsw (embedding vector_cosine_ops);
-
--- HNSW vector index for embeddings_resources
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_resources_vector_hnsw
-ON embeddings_resources
-USING hnsw (embedding vector_cosine_ops);
-
 -- HNSW vector index for embeddings_messages
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_messages_vector_hnsw
 ON embeddings_messages
 USING hnsw (embedding vector_cosine_ops);
 
--- HNSW vector index for embeddings_files
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_files_vector_hnsw
-ON embeddings_files
+-- HNSW vector index for embeddings_moments
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
+ON embeddings_moments
 USING hnsw (embedding vector_cosine_ops);
 
 -- HNSW vector index for embeddings_ontology_configs
@@ -36,7 +26,22 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontology_configs_vector_h
 ON embeddings_ontology_configs
 USING hnsw (embedding vector_cosine_ops);
 
+-- HNSW vector index for embeddings_resources
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_resources_vector_hnsw
+ON embeddings_resources
+USING hnsw (embedding vector_cosine_ops);
+
 -- HNSW vector index for embeddings_schemas
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_schemas_vector_hnsw
 ON embeddings_schemas
 USING hnsw (embedding vector_cosine_ops);
+
+-- HNSW vector index for embeddings_sessions
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_sessions_vector_hnsw
+ON embeddings_sessions
+USING hnsw (embedding vector_cosine_ops);
+
+-- HNSW vector index for embeddings_users
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_users_vector_hnsw
+ON embeddings_users
+USING hnsw (embedding vector_cosine_ops);