remdb 0.3.163__py3-none-any.whl → 0.3.181__py3-none-any.whl

rem/api/mcp_router/tools.py

@@ -116,7 +116,8 @@ def mcp_tool_error_handler(func: Callable) -> Callable:
             # Otherwise wrap in success response
             return {"status": "success", **result}
         except Exception as e:
-            logger.error(f"{func.__name__} failed: {e}", exc_info=True)
+            # Use %s format to avoid issues with curly braces in error messages
+            logger.opt(exception=True).error("{} failed: {}", func.__name__, str(e))
             return {
                 "status": "error",
                 "error": str(e),
@@ -154,6 +155,10 @@ async def search_rem(
     - Fast exact match across all tables
     - Uses indexed label_vector for instant retrieval
     - Example: LOOKUP "Sarah Chen" returns all entities named "Sarah Chen"
+    - **Ontology Note**: Ontology content may contain markdown links like
+      `[sertraline](../../drugs/antidepressants/sertraline.md)`. The link name
+      (e.g., "sertraline") can be used as a LOOKUP subject, while the relative
+      path provides semantic context (e.g., it's a drug, specifically an antidepressant).
 
     **FUZZY** - Fuzzy text matching with similarity threshold:
     - Finds partial matches and typos
@@ -380,9 +385,10 @@ async def ask_rem_agent(
     from ...utils.schema_loader import load_agent_schema
 
     # Create agent context
+    # Note: tenant_id defaults to "default" if user_id is None
     context = AgentContext(
         user_id=user_id,
-        tenant_id=user_id,  # Set tenant_id to user_id for backward compat
+        tenant_id=user_id or "default",  # Use default tenant for anonymous users
         default_model=settings.llm.default_model,
     )
 
@@ -1130,3 +1136,82 @@ async def save_agent(
         result["message"] = f"Agent '{name}' saved. Use `/custom-agent {name}` to chat with it."
 
     return result
+
+
+# =============================================================================
+# Test/Debug Tools (for development only)
+# =============================================================================
+
+@mcp_tool_error_handler
+async def test_error_handling(
+    error_type: Literal["exception", "error_response", "timeout", "success"] = "success",
+    delay_seconds: float = 0,
+    error_message: str = "Test error occurred",
+) -> dict[str, Any]:
+    """
+    Test tool for simulating different error scenarios.
+
+    **FOR DEVELOPMENT/TESTING ONLY** - This tool helps verify that error
+    handling works correctly through the streaming layer.
+
+    Args:
+        error_type: Type of error to simulate:
+            - "success": Returns successful response (default)
+            - "exception": Raises an exception (tests @mcp_tool_error_handler)
+            - "error_response": Returns {"status": "error", ...} dict
+            - "timeout": Delays for 60 seconds (simulates timeout)
+        delay_seconds: Optional delay before responding (0-10 seconds)
+        error_message: Custom error message for error scenarios
+
+    Returns:
+        Dict with test results or error information
+
+    Examples:
+        # Test successful response
+        test_error_handling(error_type="success")
+
+        # Test exception handling
+        test_error_handling(error_type="exception", error_message="Database connection failed")
+
+        # Test error response format
+        test_error_handling(error_type="error_response", error_message="Resource not found")
+
+        # Test with delay
+        test_error_handling(error_type="success", delay_seconds=2)
+    """
+    import asyncio
+
+    logger.info(f"test_error_handling called: type={error_type}, delay={delay_seconds}")
+
+    # Apply delay (capped at 10 seconds for safety)
+    if delay_seconds > 0:
+        await asyncio.sleep(min(delay_seconds, 10))
+
+    if error_type == "exception":
+        # This tests the @mcp_tool_error_handler decorator
+        raise RuntimeError(f"TEST EXCEPTION: {error_message}")
+
+    elif error_type == "error_response":
+        # This tests how the streaming layer handles error status responses
+        return {
+            "status": "error",
+            "error": error_message,
+            "error_code": "TEST_ERROR",
+            "recoverable": True,
+        }
+
+    elif error_type == "timeout":
+        # Simulate a very long operation (for testing client-side timeouts)
+        await asyncio.sleep(60)
+        return {"status": "success", "message": "Timeout test completed (should not reach here)"}
+
+    else:  # success
+        return {
+            "status": "success",
+            "message": "Test completed successfully",
+            "test_data": {
+                "error_type": error_type,
+                "delay_applied": delay_seconds,
+                "timestamp": str(asyncio.get_event_loop().time()),
+            },
+        }

rem/api/routers/auth.py

@@ -30,14 +30,17 @@ Access Control Flow (send-code):
     │   ├── Yes → Check user.tier
     │   │   ├── tier == BLOCKED → Reject "Account is blocked"
     │   │   └── tier != BLOCKED → Allow (send code, existing users grandfathered)
-    │   └── No (new user) → Check EMAIL__TRUSTED_EMAIL_DOMAINS
-    │       ├── Setting configured → domain in trusted list?
-    │       │   ├── Yes → Create user & send code
-    │       │   └── No → Reject "Email domain not allowed for signup"
-    │       └── Not configured (empty) → Create user & send code (no restrictions)
+    │   └── No (new user) → Check subscriber list first
+    │       ├── Email in subscribers table? → Allow (create user & send code)
+    │       └── Not a subscriber → Check EMAIL__TRUSTED_EMAIL_DOMAINS
+    │           ├── Setting configured → domain in trusted list?
+    │           │   ├── Yes → Create user & send code
+    │           │   └── No → Reject "Email domain not allowed for signup"
+    │           └── Not configured (empty) → Create user & send code (no restrictions)
 
 Key Behaviors:
 - Existing users: Always allowed to login (unless tier=BLOCKED)
+- Subscribers: Always allowed to login (regardless of email domain)
 - New users: Must have email from trusted domain (if EMAIL__TRUSTED_EMAIL_DOMAINS is set)
 - No restrictions: Leave EMAIL__TRUSTED_EMAIL_DOMAINS empty to allow all domains
 
@@ -103,6 +106,7 @@ from ...services.postgres.service import PostgresService
 from ...services.user_service import UserService
 from ...auth.providers.email import EmailAuthProvider
 from ...auth.jwt import JWTService, get_jwt_service
+from ...utils.user_id import email_to_user_id
 
 router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -429,8 +433,9 @@ async def callback(provider: str, request: Request):
                     await user_service.link_anonymous_session(user_entity, anon_id)
                     
                 # Enrich session user with DB info
+                # user_id = UUID5 hash of email (deterministic, bijection)
                 db_info = {
-                    "id": str(user_entity.id),
+                    "id": email_to_user_id(user_info.get("email")),
                     "tenant_id": user_entity.tenant_id,
                     "tier": user_entity.tier.value if user_entity.tier else "free",
                     "roles": [user_entity.role] if user_entity.role else [],

rem/api/routers/chat/completions.py

@@ -97,7 +97,7 @@ Context Building Flow:
    - Long messages include REM LOOKUP hints: "... [REM LOOKUP session-{id}-msg-{index}] ..."
    - Agent can retrieve full content on-demand using REM LOOKUP
 3. User profile provided as REM LOOKUP hint (on-demand by default)
-   - Agent receives: "User ID: {user_id}. To load user profile: Use REM LOOKUP users/{user_id}"
+   - Agent receives: "User: {email}. To load user profile: Use REM LOOKUP \"{email}\""
    - Agent decides whether to load profile based on query
 4. If CHAT__AUTO_INJECT_USER_CONTEXT=true: User profile auto-loaded and injected
 5. Combines: system context + compressed session history + new messages

rem/api/routers/chat/streaming.py

@@ -835,3 +835,21 @@ async def stream_openai_response_with_save(
                 )
             except Exception as e:
                 logger.error(f"Failed to save session messages: {e}", exc_info=True)
+
+        # Update session description with session_name (non-blocking, after all yields)
+        for tool_call in tool_calls:
+            if tool_call.get("tool_name") == "register_metadata" and tool_call.get("is_metadata"):
+                session_name = tool_call.get("arguments", {}).get("session_name")
+                if session_name:
+                    try:
+                        from ....models.entities import Session
+                        from ....services.postgres import Repository
+                        repo = Repository(Session, table_name="sessions")
+                        session = await repo.get_by_id(session_id)
+                        if session and session.description != session_name:
+                            session.description = session_name
+                            await repo.update(session)
+                            logger.debug(f"Updated session {session_id} description to '{session_name}'")
+                    except Exception as e:
+                        logger.warning(f"Failed to update session description: {e}")
+                    break

rem/auth/middleware.py

@@ -7,14 +7,22 @@ Anonymous access with rate limiting when allow_anonymous=True.
 MCP endpoints are always protected unless explicitly disabled.
 
 Design Pattern:
-- Check X-API-Key header first (if API key auth enabled)
-- Check JWT token in Authorization header (Bearer token)
-- Check dev token (non-production only, starts with "dev_")
-- Check session for user on protected paths
+- API Key (X-API-Key): Access control guardrail, NOT user identity
+- JWT (Authorization: Bearer): Primary method for user identity
+- Dev token: Non-production testing (starts with "dev_")
+- Session: Backward compatibility for browser-based auth
 - MCP paths always require authentication (protected service)
-- If allow_anonymous=True: Allow unauthenticated requests (marked as ANONYMOUS tier)
-- If allow_anonymous=False: Return 401 for API calls, redirect browsers to login
-- Exclude auth endpoints and public paths
+
+Authentication Flow:
+1. If API key enabled: Validate X-API-Key header (access gate)
+2. Check JWT token for user identity (primary)
+3. Check dev token for testing (non-production only)
+4. Check session for user (backward compatibility)
+5. If allow_anonymous=True: Allow as anonymous (rate-limited)
+6. If allow_anonymous=False: Return 401 / redirect to login
+
+IMPORTANT: API key validates ACCESS, JWT identifies USER.
+Both can be required: API key for access + JWT for user identity.
 
 Access Modes (configured in settings.auth):
 - enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
@@ -24,10 +32,9 @@ Access Modes (configured in settings.auth):
 - mcp_requires_auth=false: MCP follows normal allow_anonymous rules (dev only)
 
 API Key Authentication (configured in settings.api):
-- api_key_enabled=true: Require X-API-Key header for protected endpoints
+- api_key_enabled=true: Require X-API-Key header for access
 - api_key: The secret key to validate against
-- Provides simple programmatic access without OAuth flow
-- X-API-Key header takes precedence over session auth
+- API key is an ACCESS GATE, not user identity - JWT still needed for user
 
 Dev Token Support (non-production only):
 - GET /api/auth/dev/token returns a Bearer token for test-user
@@ -212,32 +219,28 @@ class AuthMiddleware(BaseHTTPMiddleware):
         if not is_protected or is_excluded:
             return await call_next(request)
 
-        # Check for X-API-Key header first (if enabled)
-        api_key_user = self._check_api_key(request)
-        if api_key_user:
-            request.state.user = api_key_user
-            request.state.is_anonymous = False
-            return await call_next(request)
-
-        # If API key auth is enabled but no valid key provided, reject immediately
+        # API key validation (access control, not user identity)
+        # API key is a guardrail for access - JWT identifies the actual user
         if settings.api.api_key_enabled:
-            # Check if X-API-Key header was provided but invalid
-            if request.headers.get("x-api-key"):
+            api_key = request.headers.get("x-api-key")
+            if not api_key:
+                logger.debug(f"Missing X-API-Key for: {path}")
+                return JSONResponse(
+                    status_code=401,
+                    content={"detail": "API key required. Include X-API-Key header."},
+                    headers={"WWW-Authenticate": 'ApiKey realm="REM API"'},
+                )
+            if api_key != settings.api.api_key:
                 logger.warning(f"Invalid X-API-Key for: {path}")
                 return JSONResponse(
                     status_code=401,
                     content={"detail": "Invalid API key"},
                     headers={"WWW-Authenticate": 'ApiKey realm="REM API"'},
                 )
-            # No API key provided when required
-            logger.debug(f"Missing X-API-Key for: {path}")
-            return JSONResponse(
-                status_code=401,
-                content={"detail": "API key required. Include X-API-Key header."},
-                headers={"WWW-Authenticate": 'ApiKey realm="REM API"'},
-            )
+            logger.debug("X-API-Key validated for access")
+            # API key valid - continue to check JWT for user identity
 
-        # Check for JWT token in Authorization header
+        # Check for JWT token in Authorization header (primary user identity)
         jwt_user = self._check_jwt_token(request)
         if jwt_user:
             request.state.user = jwt_user

rem/cli/commands/ask.py

@@ -75,7 +75,7 @@ async def run_agent_streaming(
     """
     Run agent in streaming mode using agent.iter() with usage limits.
 
-    Design Pattern (from carrier):
+    Design Pattern:
     - Use agent.iter() for complete execution with tool call visibility
     - run_stream() stops after first output, missing tool calls
     - Stream tool call markers: [Calling: tool_name]

rem/cli/commands/db.py

@@ -333,64 +333,120 @@ def rebuild_cache(connection: str | None):
 
 @click.command()
 @click.argument("file_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--table", "-t", default=None, help="Target table name (required for non-YAML formats)")
 @click.option("--user-id", default=None, help="User ID to scope data privately (default: public/shared)")
 @click.option("--dry-run", is_flag=True, help="Show what would be loaded without loading")
-def load(file_path: Path, user_id: str | None, dry_run: bool):
+def load(file_path: Path, table: str | None, user_id: str | None, dry_run: bool):
     """
-    Load data from YAML file into database.
+    Load data from file into database.
 
-    File format:
-        - table: resources
-          key_field: name
-          rows:
-            - name: Example
-              content: Test data...
+    Supports YAML with embedded metadata, or any tabular format via Polars
+    (jsonl, parquet, csv, json, arrow, etc.). For non-YAML formats, use --table.
 
     Examples:
-        rem db load rem/tests/data/graph_seed.yaml
-        rem db load data.yaml --user-id my-user  # Private to user
-        rem db load data.yaml --dry-run
+        rem db load data.yaml                        # YAML with metadata
+        rem db load data.jsonl -t resources          # Any Polars-supported format
     """
-    asyncio.run(_load_async(file_path, user_id, dry_run))
+    asyncio.run(_load_async(file_path, table, user_id, dry_run))
 
 
-async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
+def _load_dataframe_from_file(file_path: Path) -> "pl.DataFrame":
+    """Load any Polars-supported file format into a DataFrame."""
+    import polars as pl
+
+    suffix = file_path.suffix.lower()
+
+    if suffix in {".jsonl", ".ndjson"}:
+        return pl.read_ndjson(file_path)
+    elif suffix in {".parquet", ".pq"}:
+        return pl.read_parquet(file_path)
+    elif suffix == ".csv":
+        return pl.read_csv(file_path)
+    elif suffix == ".json":
+        return pl.read_json(file_path)
+    elif suffix in {".ipc", ".arrow"}:
+        return pl.read_ipc(file_path)
+    else:
+        raise ValueError(f"Unsupported file format: {suffix}. Use any Polars-supported format.")
+
+
+async def _load_async(file_path: Path, table: str | None, user_id: str | None, dry_run: bool):
     """Async implementation of load command."""
+    import polars as pl
     import yaml
     from ...models.core.inline_edge import InlineEdge
-    from ...models.entities import Resource, Moment, User, Message, SharedSession, Schema
+    from ...models.entities import SharedSession
     from ...services.postgres import get_postgres_service
+    from ...utils.model_helpers import get_table_name
+    from ... import get_model_registry
 
     logger.info(f"Loading data from: {file_path}")
     scope_msg = f"user: {user_id}" if user_id else "public"
     logger.info(f"Scope: {scope_msg}")
 
-    # Load YAML file
-    with open(file_path) as f:
-        data = yaml.safe_load(f)
+    suffix = file_path.suffix.lower()
+    is_yaml = suffix in {".yaml", ".yml"}
 
-    if not isinstance(data, list):
-        logger.error("YAML must be a list of table definitions")
-        raise click.Abort()
-
-    if dry_run:
-        logger.info("DRY RUN - Would load:")
-        logger.info(yaml.dump(data, default_flow_style=False))
-        return
-
-    # Map table names to model classes
-    # CoreModel subclasses use Repository.upsert()
+    # Build MODEL_MAP dynamically from registry
+    registry = get_model_registry()
+    registry.register_core_models()
     MODEL_MAP = {
-        "users": User,
-        "moments": Moment,
-        "resources": Resource,
-        "messages": Message,
-        "schemas": Schema,
+        get_table_name(model): model
+        for model in registry.get_model_classes().values()
     }
 
     # Non-CoreModel tables that need direct SQL insertion
     DIRECT_INSERT_TABLES = {"shared_sessions"}
 
+    # Parse file based on format
+    if is_yaml:
+        # YAML with embedded metadata
+        with open(file_path) as f:
+            data = yaml.safe_load(f)
+
+        if not isinstance(data, list):
+            logger.error("YAML must be a list of table definitions")
+            raise click.Abort()
+
+        if dry_run:
+            logger.info("DRY RUN - Would load:")
+            logger.info(yaml.dump(data, default_flow_style=False))
+            return
+
+        table_defs = data
+    else:
+        # Polars-supported format - require --table
+        if not table:
+            logger.error(f"For {suffix} files, --table is required. Example: rem db load {file_path.name} -t resources")
+            raise click.Abort()
+
+        try:
+            df = _load_dataframe_from_file(file_path)
+        except Exception as e:
+            logger.error(f"Failed to load file: {e}")
+            raise click.Abort()
+
+        rows = df.to_dicts()
+
+        if dry_run:
+            logger.info(f"DRY RUN - Would load {len(rows)} rows to table '{table}':")
+            logger.info(f"Columns: {list(df.columns)}")
+
+            # Validate first row against model if table is known
+            if table in MODEL_MAP and rows:
+                from ...utils.model_helpers import validate_data_for_model
+                result = validate_data_for_model(MODEL_MAP[table], rows[0])
+                if result.extra_fields:
+                    logger.warning(f"Unknown fields (ignored): {result.extra_fields}")
+                if result.valid:
+                    logger.success(f"Sample row validates OK. Required: {result.required_fields or '(none)'}")
+                else:
+                    result.log_errors("Sample row")
+            return
+
+        # Wrap as single table definition
+        table_defs = [{"table": table, "rows": rows}]
+
     # Connect to database
     pg = get_postgres_service()
     if not pg:
@@ -399,23 +455,24 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
 
     await pg.connect()
 
+    # Start embedding worker for generating embeddings
+    if pg.embedding_worker:
+        await pg.embedding_worker.start()
+
     try:
         total_loaded = 0
 
-        for table_def in data:
+        for table_def in table_defs:
             table_name = table_def["table"]
-            key_field = table_def.get("key_field", "id")
             rows = table_def.get("rows", [])
 
             # Handle direct insert tables (non-CoreModel)
             if table_name in DIRECT_INSERT_TABLES:
                 for row_data in rows:
-                    # Add tenant_id if not present
                     if "tenant_id" not in row_data:
                         row_data["tenant_id"] = "default"
 
                     if table_name == "shared_sessions":
-                        # Insert shared_session directly
                         await pg.fetch(
                             """INSERT INTO shared_sessions
                                (session_id, owner_user_id, shared_with_user_id, tenant_id)
@@ -434,16 +491,13 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                 logger.warning(f"Unknown table: {table_name}, skipping")
                 continue
 
-            model_class = MODEL_MAP[table_name]  # Type is inferred from MODEL_MAP
+            model_class = MODEL_MAP[table_name]
 
-            for row_data in rows:
-                # Add user_id and tenant_id only if explicitly provided
-                # Default is public (None) - data is shared/visible to all
-                # Pass --user-id to scope data privately to a specific user
-                if "user_id" not in row_data and user_id is not None:
-                    row_data["user_id"] = user_id
+            for row_idx, row_data in enumerate(rows):
+                # user_id stays NULL for public data (accessible by any user)
+                # Only set tenant_id for scoping - the --user-id flag controls tenant scope
                 if "tenant_id" not in row_data and user_id is not None:
-                    row_data["tenant_id"] = row_data.get("user_id", user_id)
+                    row_data["tenant_id"] = user_id
 
                 # Convert graph_edges to InlineEdge format if present
                 if "graph_edges" in row_data:
@@ -452,30 +506,40 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                         for edge in row_data["graph_edges"]
                     ]
 
-                # Convert any ISO timestamp strings with Z suffix to naive datetime
-                # This handles fields like starts_timestamp, ends_timestamp, etc.
+                # Convert ISO timestamp strings
                 from ...utils.date_utils import parse_iso
                 for key, value in list(row_data.items()):
                     if isinstance(value, str) and (key.endswith("_timestamp") or key.endswith("_at")):
                         try:
                             row_data[key] = parse_iso(value)
                         except (ValueError, TypeError):
-                            pass  # Not a valid datetime string, leave as-is
+                            pass
 
-                # Create model instance and upsert via repository
                 from ...services.postgres.repository import Repository
+                from ...utils.model_helpers import validate_data_for_model
+
+                result = validate_data_for_model(model_class, row_data)
+                if not result.valid:
+                    result.log_errors(f"Row {row_idx + 1} ({table_name})")
+                    raise click.Abort()
 
-                instance = model_class(**row_data)
-                repo = Repository(model_class, table_name, pg)  # Type inferred from MODEL_MAP
-                await repo.upsert(instance)  # type: ignore[arg-type]
+                repo = Repository(model_class, table_name, pg)
+                await repo.upsert(result.instance)  # type: ignore[arg-type]
                 total_loaded += 1
 
-                # Log based on model type
-                name = getattr(instance, 'name', getattr(instance, 'id', '?'))
+                name = getattr(result.instance, 'name', getattr(result.instance, 'id', '?'))
                 logger.success(f"Loaded {table_name[:-1]}: {name}")
 
         logger.success(f"Data loaded successfully! Total rows: {total_loaded}")
 
+        # Wait for embeddings to complete
+        if pg.embedding_worker and pg.embedding_worker.running:
+            queue_size = pg.embedding_worker.task_queue.qsize()
+            if queue_size > 0:
+                logger.info(f"Waiting for {queue_size} embeddings to complete...")
+            await pg.embedding_worker.stop()
+            logger.success("Embeddings generated successfully")
+
     finally:
         await pg.disconnect()