remdb 0.2.6__py3-none-any.whl → 0.3.103__py3-none-any.whl

rem/utils/schema_loader.py

@@ -9,7 +9,7 @@ Design Pattern:
 - Support short names: "contract-analyzer" → "schemas/agents/contract-analyzer.yaml"
 - Support relative/absolute paths
 - Consistent error messages and logging
-i
+
 Usage:
     # From API
     schema = load_agent_schema("rem")
@@ -20,6 +20,26 @@ Usage:
     # From agent factory
     schema = load_agent_schema("contract-analyzer")
 
+TODO: Git FS Integration
+    The schema loader currently uses importlib.resources for package schemas
+    and direct filesystem access for custom paths. The FS abstraction layer
+    (rem.services.fs.FS) could be used to abstract storage backends:
+
+    - Local filesystem (current)
+    - Git repositories (GitService)
+    - S3 (via FS provider)
+
+    This would enable loading schemas from versioned Git repos or S3 buckets
+    without changing the API. The FS provider pattern already exists and just
+    needs integration testing with the schema loader.
+
+    Example future usage:
+        # Load from Git at specific version
+        schema = load_agent_schema("git://rem/schemas/agents/rem.yaml?ref=v1.0.0")
+
+        # Load from S3
+        schema = load_agent_schema("s3://rem-schemas/agents/cv-parser.yaml")
+
 Schema Caching Status:
 
     ✅ IMPLEMENTED: Filesystem Schema Caching (2025-11-22)
@@ -71,13 +91,14 @@ import yaml
 from loguru import logger
 
 
-# Standard search paths for agent schemas (in priority order)
+# Standard search paths for agent/evaluator schemas (in priority order)
 SCHEMA_SEARCH_PATHS = [
     "schemas/agents/{name}.yaml",          # Top-level agents (e.g., rem.yaml)
     "schemas/agents/core/{name}.yaml",     # Core system agents
     "schemas/agents/examples/{name}.yaml", # Example agents
-    "schemas/evaluators/{name}.yaml",
-    "schemas/{name}.yaml",
+    "schemas/evaluators/{name}.yaml",      # Nested evaluators (e.g., hello-world/default)
+    "schemas/evaluators/rem/{name}.yaml",  # REM evaluators (e.g., lookup-correctness)
+    "schemas/{name}.yaml",                 # Generic schemas
 ]
 
 # In-memory cache for filesystem schemas (no TTL - immutable)
@@ -89,7 +110,88 @@ _fs_schema_cache: dict[str, dict[str, Any]] = {}
 # _db_schema_ttl: int = 300  # 5 minutes in seconds
 
 
-def load_agent_schema(schema_name_or_path: str, use_cache: bool = True) -> dict[str, Any]:
+def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any] | None:
+    """
+    Load schema from database using LOOKUP query.
+
+    This function is synchronous but calls async database operations.
+    It's designed to be called from load_agent_schema() which is sync.
+
+    Args:
+        schema_name: Schema name to lookup
+        user_id: User ID for data scoping
+
+    Returns:
+        Schema spec (dict) if found, None otherwise
+
+    Raises:
+        RuntimeError: If database connection fails
+    """
+    import asyncio
+
+    # Check if we're already in an async context
+    try:
+        loop = asyncio.get_running_loop()
+        # We're in an async context - can't use asyncio.run()
+        # This shouldn't happen in normal usage since load_agent_schema is called from sync contexts
+        logger.warning(
+            "Database schema lookup called from async context. "
+            "This may cause issues. Consider using async version of load_agent_schema."
+        )
+        return None
+    except RuntimeError:
+        # Not in async context - safe to use asyncio.run()
+        pass
+
+    async def _async_lookup():
+        """Async helper to query database."""
+        from rem.services.postgres import get_postgres_service
+
+        db = get_postgres_service()
+        if not db:
+            logger.debug("PostgreSQL service not available for schema lookup")
+            return None
+
+        try:
+            await db.connect()
+
+            # Query schemas table directly by name
+            # Note: Schema name lookup is case-insensitive for user convenience
+            query = """
+                SELECT spec FROM schemas
+                WHERE LOWER(name) = LOWER($1)
+                AND (user_id = $2 OR user_id = 'system')
+                LIMIT 1
+            """
+            logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id}")
+
+            row = await db.fetchrow(query, schema_name, user_id)
+
+            if row:
+                spec = row.get("spec")
+                if spec and isinstance(spec, dict):
+                    logger.debug(f"Found schema in database: {schema_name}")
+                    return spec
+
+            logger.debug(f"Schema not found in database: {schema_name}")
+            return None
+
+        except Exception as e:
+            logger.debug(f"Database schema lookup error: {e}")
+            return None
+        finally:
+            await db.disconnect()
+
+    # Run async lookup in new event loop
+    return asyncio.run(_async_lookup())
+
+
+def load_agent_schema(
+    schema_name_or_path: str,
+    use_cache: bool = True,
+    user_id: str | None = None,
+    enable_db_fallback: bool = True,
+) -> dict[str, Any]:
     """
     Load agent schema from YAML file with unified search logic and caching.
 
@@ -107,22 +209,26 @@ def load_agent_schema(schema_name_or_path: str, use_cache: bool = True) -> dict[
     Search Order:
     1. Check cache (if use_cache=True and schema found in FS cache)
     2. Exact path if it exists (absolute or relative)
-    3. Package resources: schemas/agents/{name}.yaml (top-level)
-    4. Package resources: schemas/agents/core/{name}.yaml
-    5. Package resources: schemas/agents/examples/{name}.yaml
-    6. Package resources: schemas/evaluators/{name}.yaml
-    7. Package resources: schemas/{name}.yaml
+    3. Custom paths from rem.register_schema_path() and SCHEMA__PATHS env var
+    4. Package resources: schemas/agents/{name}.yaml (top-level)
+    5. Package resources: schemas/agents/core/{name}.yaml
+    6. Package resources: schemas/agents/examples/{name}.yaml
+    7. Package resources: schemas/evaluators/{name}.yaml
+    8. Package resources: schemas/{name}.yaml
+    9. Database LOOKUP: schemas table (if enable_db_fallback=True and user_id provided)
 
     Args:
         schema_name_or_path: Schema name or file path
             Examples: "rem-query-agent", "contract-analyzer", "./my-schema.yaml"
         use_cache: If True, uses in-memory cache for filesystem schemas
+        user_id: User ID for database schema lookup (required for DB fallback)
+        enable_db_fallback: If True, falls back to database LOOKUP when file not found
 
     Returns:
         Agent schema as dictionary
 
     Raises:
-        FileNotFoundError: If schema not found in any search location
+        FileNotFoundError: If schema not found in any search location (filesystem + database)
         yaml.YAMLError: If schema file is invalid YAML
 
     Examples:
@@ -134,6 +240,9 @@ def load_agent_schema(schema_name_or_path: str, use_cache: bool = True) -> dict[
         >>>
         >>> # Load evaluator schema (cached)
         >>> schema = load_agent_schema("rem-lookup-correctness")
+        >>>
+        >>> # Load custom user schema from database
+        >>> schema = load_agent_schema("my-custom-agent", user_id="user-123")
     """
     # Normalize the name for cache key
     cache_key = str(schema_name_or_path).replace('agents/', '').replace('schemas/', '').replace('evaluators/', '').replace('core/', '').replace('examples/', '')
@@ -160,7 +269,28 @@ def load_agent_schema(schema_name_or_path: str, use_cache: bool = True) -> dict[
     # 2. Normalize name for package resource search
     base_name = cache_key
 
-    # 3. Try package resources with standard search paths
+    # 3. Try custom schema paths (from registry + SCHEMA__PATHS env var)
+    from ..registry import get_schema_paths
+
+    custom_paths = get_schema_paths()
+    for custom_dir in custom_paths:
+        # Try various patterns within each custom directory
+        for pattern in [
+            f"{base_name}.yaml",
+            f"{base_name}.yml",
+            f"agents/{base_name}.yaml",
+            f"evaluators/{base_name}.yaml",
+        ]:
+            custom_path = Path(custom_dir) / pattern
+            if custom_path.exists():
+                logger.debug(f"Loading schema from custom path: {custom_path}")
+                with open(custom_path, "r") as f:
+                    schema = yaml.safe_load(f)
+                logger.debug(f"Loaded schema with keys: {list(schema.keys())}")
+                # Don't cache custom paths (they may change during development)
+                return cast(dict[str, Any], schema)
+
+    # 4. Try package resources with standard search paths
     for search_pattern in SCHEMA_SEARCH_PATHS:
         search_path = search_pattern.format(name=base_name)
 
@@ -185,16 +315,145 @@ def load_agent_schema(schema_name_or_path: str, use_cache: bool = True) -> dict[
             logger.debug(f"Could not load from {search_path}: {e}")
             continue
 
-    # 4. Schema not found in any location
+    # 5. Try database LOOKUP fallback (if enabled and user_id provided)
+    if enable_db_fallback and user_id:
+        try:
+            logger.debug(f"Attempting database LOOKUP for schema: {base_name} (user_id={user_id})")
+            db_schema = _load_schema_from_database(base_name, user_id)
+            if db_schema:
+                logger.info(f"✅ Loaded schema from database: {base_name} (user_id={user_id})")
+                return db_schema
+        except Exception as e:
+            logger.debug(f"Database schema lookup failed: {e}")
+            # Fall through to error below
+
+    # 6. Schema not found in any location
     searched_paths = [pattern.format(name=base_name) for pattern in SCHEMA_SEARCH_PATHS]
+
+    custom_paths_note = ""
+    if custom_paths:
+        custom_paths_note = f"\n  - Custom paths: {', '.join(custom_paths)}"
+
+    db_search_note = ""
+    if enable_db_fallback:
+        if user_id:
+            db_search_note = f"\n  - Database: LOOKUP '{base_name}' FROM schemas WHERE user_id='{user_id}' (no match)"
+        else:
+            db_search_note = "\n  - Database: (skipped - no user_id provided)"
+
     raise FileNotFoundError(
         f"Schema not found: {schema_name_or_path}\n"
         f"Searched locations:\n"
-        f"  - Exact path: {path}\n"
+        f"  - Exact path: {path}"
+        f"{custom_paths_note}\n"
         f"  - Package resources: {', '.join(searched_paths)}"
+        f"{db_search_note}"
     )
 
 
+async def load_agent_schema_async(
+    schema_name_or_path: str,
+    user_id: str | None = None,
+    db=None,
+) -> dict[str, Any]:
+    """
+    Async version of load_agent_schema for use in async contexts.
+
+    This version accepts an existing database connection to avoid creating new connections.
+
+    Args:
+        schema_name_or_path: Schema name or file path
+        user_id: User ID for database schema lookup
+        db: Optional existing PostgresService connection (if None, will create one)
+
+    Returns:
+        Agent schema as dictionary
+
+    Raises:
+        FileNotFoundError: If schema not found
+    """
+    # First try filesystem search (sync operations are fine)
+    path = Path(schema_name_or_path)
+
+    # Normalize the name for cache key
+    cache_key = str(schema_name_or_path).replace('agents/', '').replace('schemas/', '').replace('evaluators/', '').replace('core/', '').replace('examples/', '')
+    if cache_key.endswith('.yaml') or cache_key.endswith('.yml'):
+        cache_key = cache_key.rsplit('.', 1)[0]
+
+    is_custom_path = path.exists() or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
+
+    # Check cache
+    if not is_custom_path and cache_key in _fs_schema_cache:
+        logger.debug(f"Loading schema from cache: {cache_key}")
+        return _fs_schema_cache[cache_key]
+
+    # Try exact path
+    if path.exists():
+        logger.debug(f"Loading schema from exact path: {path}")
+        with open(path, "r") as f:
+            schema = yaml.safe_load(f)
+        return cast(dict[str, Any], schema)
+
+    base_name = cache_key
+
+    # Try custom schema paths
+    from ..registry import get_schema_paths
+    custom_paths = get_schema_paths()
+    for custom_dir in custom_paths:
+        for pattern in [f"{base_name}.yaml", f"{base_name}.yml", f"agents/{base_name}.yaml"]:
+            custom_path = Path(custom_dir) / pattern
+            if custom_path.exists():
+                with open(custom_path, "r") as f:
+                    schema = yaml.safe_load(f)
+                return cast(dict[str, Any], schema)
+
+    # Try package resources
+    for search_pattern in SCHEMA_SEARCH_PATHS:
+        search_path = search_pattern.format(name=base_name)
+        try:
+            schema_ref = importlib.resources.files("rem") / search_path
+            schema_path = Path(str(schema_ref))
+            if schema_path.exists():
+                with open(schema_path, "r") as f:
+                    schema = yaml.safe_load(f)
+                _fs_schema_cache[cache_key] = schema
+                return cast(dict[str, Any], schema)
+        except Exception:
+            continue
+
+    # Try database lookup
+    if user_id:
+        from rem.services.postgres import get_postgres_service
+
+        should_disconnect = False
+        if db is None:
+            db = get_postgres_service()
+            if db:
+                await db.connect()
+                should_disconnect = True
+
+        if db:
+            try:
+                query = """
+                    SELECT spec FROM schemas
+                    WHERE LOWER(name) = LOWER($1)
+                    AND (user_id = $2 OR user_id = 'system')
+                    LIMIT 1
+                """
+                row = await db.fetchrow(query, base_name, user_id)
+                if row:
+                    spec = row.get("spec")
+                    if spec and isinstance(spec, dict):
+                        logger.info(f"✅ Loaded schema from database: {base_name}")
+                        return spec
+            finally:
+                if should_disconnect:
+                    await db.disconnect()
+
+    # Not found
+    raise FileNotFoundError(f"Schema not found: {schema_name_or_path}")
+
+
 def validate_agent_schema(schema: dict[str, Any]) -> bool:
     """
     Validate agent schema structure.

rem/utils/sql_types.py

@@ -16,6 +16,7 @@ Best Practices:
 - UUID for identifiers in Union types
 """
 
+import types
 from datetime import date, datetime, time
 from typing import Any, Union, get_args, get_origin
 from uuid import UUID
@@ -78,8 +79,9 @@ def get_sql_type(field_info: FieldInfo, field_name: str) -> str:
         return "TEXT"
 
     # Handle Union types (including Optional[T] which is Union[T, None])
+    # Also handles Python 3.10+ `X | None` syntax which uses types.UnionType
     origin = get_origin(annotation)
-    if origin is Union:
+    if origin is Union or isinstance(annotation, types.UnionType):
         args = get_args(annotation)
         # Filter out NoneType
         non_none_args = [arg for arg in args if arg is not type(None)]

rem/utils/vision.py

@@ -11,7 +11,6 @@ markdown descriptions of images.
 """
 
 import base64
-import os
 from enum import Enum
 from pathlib import Path
 from typing import Optional
@@ -19,6 +18,9 @@ from typing import Optional
 import requests
 from loguru import logger
 
+from rem.utils.constants import HTTP_TIMEOUT_LONG, VISION_MAX_TOKENS
+from rem.utils.mime_types import EXTENSION_TO_MIME
+
 
 class VisionProvider(str, Enum):
     """Supported vision providers."""
@@ -141,14 +143,7 @@ class ImageAnalyzer:
 
         # Detect media type
         suffix = image_path.suffix.lower()
-        media_type_map = {
-            ".png": "image/png",
-            ".jpg": "image/jpeg",
-            ".jpeg": "image/jpeg",
-            ".gif": "image/gif",
-            ".webp": "image/webp",
-        }
-        media_type = media_type_map.get(suffix, "image/png")
+        media_type = EXTENSION_TO_MIME.get(suffix, "image/png")
 
         logger.info(f"Analyzing {image_path.name} with {self.provider.value} ({self.model})")
 
@@ -190,7 +185,7 @@ class ImageAnalyzer:
 
         body = {
             "model": self.model,
-            "max_tokens": 2048,
+            "max_tokens": VISION_MAX_TOKENS,
             "messages": [
                 {
                     "role": "user",
@@ -216,7 +211,7 @@ class ImageAnalyzer:
             "https://api.anthropic.com/v1/messages",
             headers=headers,
             json=body,
-            timeout=60.0,
+            timeout=HTTP_TIMEOUT_LONG,
         )
 
         if response.status_code != 200:
@@ -261,7 +256,7 @@ class ImageAnalyzer:
             url,
             params=params,
             json=body,
-            timeout=60.0,
+            timeout=HTTP_TIMEOUT_LONG,
         )
 
         if response.status_code != 200:
@@ -311,14 +306,14 @@ class ImageAnalyzer:
                     ],
                 }
             ],
-            "max_tokens": 2048,
+            "max_tokens": VISION_MAX_TOKENS,
         }
 
         response = requests.post(
             url,
             headers=headers,
             json=body,
-            timeout=60.0,
+            timeout=HTTP_TIMEOUT_LONG,
         )
 
         if response.status_code != 200:

rem/workers/README.md

@@ -207,7 +207,7 @@ Reads recent activity to generate comprehensive user profiles.
 
 **CLI:**
 ```bash
-rem-dreaming user-model --tenant-id=tenant-123
+rem-dreaming user-model 
 ```
 
 **Frequency:** Daily (runs as part of full workflow)
@@ -235,13 +235,13 @@ Extracts temporal narratives from resources.
 **CLI:**
 ```bash
 # Process last 24 hours
-rem-dreaming moments --tenant-id=tenant-123
+rem-dreaming moments 
 
 # Custom lookback
-rem-dreaming moments --tenant-id=tenant-123 --lookback-hours=48
+rem-dreaming moments  --lookback-hours=48
 
 # Limit resources processed
-rem-dreaming moments --tenant-id=tenant-123 --limit=100
+rem-dreaming moments  --limit=100
 ```
 
 **Frequency:** Daily or on-demand
@@ -283,13 +283,13 @@ Builds semantic relationships between resources.
 **CLI:**
 ```bash
 # Semantic mode (fast, cheap)
-rem-dreaming affinity --tenant-id=tenant-123
+rem-dreaming affinity 
 
 # LLM mode (intelligent, expensive)
-rem-dreaming affinity --tenant-id=tenant-123 --use-llm --limit=100
+rem-dreaming affinity  --use-llm --limit=100
 
 # Custom lookback
-rem-dreaming affinity --tenant-id=tenant-123 --lookback-hours=168
+rem-dreaming affinity  --lookback-hours=168
 ```
 
 **Frequency:**
@@ -308,13 +308,13 @@ Runs all operations in sequence.
 **CLI:**
 ```bash
 # Single tenant
-rem-dreaming full --tenant-id=tenant-123
+rem-dreaming full 
 
 # All active tenants (daily cron)
 rem-dreaming full --all-tenants
 
 # Use LLM affinity mode
-rem-dreaming full --tenant-id=tenant-123 --use-llm-affinity
+rem-dreaming full  --use-llm-affinity
 ```
 
 **Frequency:** Daily at 3 AM UTC
@@ -455,16 +455,16 @@ export REM_API_URL=http://localhost:8000
 export OPENAI_API_KEY=sk-...
 
 # Run user model update
-python -m rem.cli.dreaming user-model --tenant-id=tenant-test
+python -m rem.cli.dreaming user-model 
 
 # Run moment construction
-python -m rem.cli.dreaming moments --tenant-id=tenant-test --lookback-hours=24
+python -m rem.cli.dreaming moments  --lookback-hours=24
 
 # Run affinity (semantic mode)
-python -m rem.cli.dreaming affinity --tenant-id=tenant-test
+python -m rem.cli.dreaming affinity 
 
 # Run full workflow
-python -m rem.cli.dreaming full --tenant-id=tenant-test
+python -m rem.cli.dreaming full 
 ```
 
 ### Testing with Docker
@@ -478,7 +478,7 @@ docker run --rm \
   -e REM_API_URL=http://host.docker.internal:8000 \
   -e OPENAI_API_KEY=$OPENAI_API_KEY \
   rem-stack:latest \
-  python -m rem.cli.dreaming full --tenant-id=tenant-test
+  python -m rem.cli.dreaming full 
 ```
 
 ## Architecture Decisions

rem/workers/db_maintainer.py

@@ -0,0 +1,74 @@
+"""
+Database Maintainer Worker.
+
+Handles background maintenance tasks for PostgreSQL:
+1. Cleaning up expired rate limit counters (UNLOGGED table).
+2. Refreshing materialized views (if any).
+3. Vacuuming specific tables (if needed).
+
+Usage:
+    python -m rem.workers.db_maintainer
+    
+    # Or via docker-compose:
+    # command: python -m rem.workers.db_maintainer
+"""
+
+import asyncio
+import signal
+from loguru import logger
+
+from ..services.postgres.service import PostgresService
+from ..services.rate_limit import RateLimitService
+
+class DatabaseMaintainer:
+    def __init__(self):
+        self.running = False
+        self.db = PostgresService()
+        self.rate_limiter = RateLimitService(self.db)
+
+    async def start(self):
+        """Start maintenance loop."""
+        self.running = True
+        logger.info("Starting Database Maintainer Worker")
+        
+        await self.db.connect()
+        
+        try:
+            while self.running:
+                await self._run_maintenance_cycle()
+                # Sleep for 5 minutes
+                await asyncio.sleep(300)
+        finally:
+            await self.db.disconnect()
+
+    async def _run_maintenance_cycle(self):
+        """Execute maintenance tasks."""
+        logger.debug("Running maintenance cycle...")
+        
+        try:
+            # 1. Cleanup Rate Limits
+            await self.rate_limiter.cleanup_expired()
+            
+            # 2. (Future) Refresh Views
+            # await self.db.execute("REFRESH MATERIALIZED VIEW ...")
+            
+        except Exception as e:
+            logger.error(f"Maintenance cycle failed: {e}")
+
+    def stop(self):
+        """Stop worker gracefully."""
+        self.running = False
+        logger.info("Stopping Database Maintainer Worker...")
+
+async def main():
+    worker = DatabaseMaintainer()
+    
+    # Handle signals
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        loop.add_signal_handler(sig, worker.stop)
+        
+    await worker.start()
+
+if __name__ == "__main__":
+    asyncio.run(main())