remdb 0.2.6__py3-none-any.whl → 0.3.103__py3-none-any.whl

rem/api/README.md

@@ -158,9 +158,70 @@ The dreaming worker runs periodically to build user models:
 - User profile automatically loaded and injected into system message
 - Simpler for basic chatbots that always need context
 
+## Authentication
+
+### Production Authentication
+
+When `AUTH__ENABLED=true`, users authenticate via OAuth (Google or Microsoft). The OAuth flow:
+
+1. User visits `/api/auth/google/login` or `/api/auth/microsoft/login`
+2. User authenticates with provider
+3. Callback stores user in session cookie
+4. Subsequent requests use session cookie
+
+### Development Token (Non-Production Only)
+
+For local development and testing, you can use a dev token instead of OAuth. This endpoint is available at `/api/dev/token` whenever `ENVIRONMENT != "production"`, regardless of whether auth is enabled.
+
+**Get Token:**
+```bash
+curl http://localhost:8000/api/dev/token
+```
+
+**Response:**
+```json
+{
+  "token": "dev_89737a19376332bfd9a4a06db8b79fd1",
+  "type": "Bearer",
+  "user": {
+    "id": "test-user",
+    "email": "test@rem.local",
+    "name": "Test User"
+  },
+  "usage": "curl -H \"Authorization: Bearer dev_...\" http://localhost:8000/api/v1/...",
+  "warning": "This token is for development/testing only and will not work in production."
+}
+```
+
+**Use Token:**
+```bash
+# Get the token
+TOKEN=$(curl -s http://localhost:8000/api/dev/token | jq -r .token)
+
+# Use it in requests
+curl -H "Authorization: Bearer $TOKEN" \
+     -H "X-Tenant-Id: default" \
+     http://localhost:8000/api/v1/shared-with-me
+```
+
+**Security Notes:**
+- Only available when `ENVIRONMENT != "production"`
+- Token is HMAC-signed using session secret
+- Authenticates as `test-user` with `pro` tier and `admin` role
+- Token is deterministic per environment (same secret = same token)
+
+### Anonymous Access
+
+When `AUTH__ALLOW_ANONYMOUS=true` (default in development):
+- Requests without authentication are allowed
+- Anonymous users get rate-limited access
+- MCP endpoints still require auth unless `AUTH__MCP_REQUIRES_AUTH=false`
+
 ## Usage Examples
 
-**Note on Authentication**: By default, authentication is disabled (`AUTH__ENABLED=false`) for local development and testing. The examples below work without an `Authorization` header. If authentication is enabled in your environment, add: `-H "Authorization: Bearer your_jwt_token"` to cURL requests or `"Authorization": "Bearer your_jwt_token"` to Python headers.
+**Note on Authentication**: By default, authentication is disabled (`AUTH__ENABLED=false`) for local development and testing. The examples below work without an `Authorization` header. If authentication is enabled, use either:
+- **Dev token**: `-H "Authorization: Bearer $(curl -s http://localhost:8000/api/dev/token | jq -r .token)"`
+- **Session cookie**: Login via OAuth first, then use cookies
 
 ### cURL: Simple Chat
 
@@ -363,6 +424,157 @@ data: {"id":"chatcmpl-abc123","choices":[{"delta":{},"finish_reason":"stop","ind
 data: [DONE]
 ```
 
+## Extended SSE Event Protocol
+
+REM uses OpenAI-compatible format for text content streaming, plus custom named SSE events for rich UI interactions.
+
+### Event Types
+
+| Event Type | Format | Purpose | UI Display |
+|------------|--------|---------|------------|
+| (text content) | `data:` (OpenAI format) | Content chunks | Main response area |
+| `reasoning` | `event:` | Model thinking | Collapsible "thinking" section |
+| `progress` | `event:` | Step indicators | Progress bar/stepper |
+| `tool_call` | `event:` | Tool invocations | Tool status panel |
+| `action_request` | `event:` | User input solicitation | Buttons, forms, modals |
+| `metadata` | `event:` | System info | Hidden or badge display |
+| `error` | `event:` | Error notification | Error toast/alert |
+| `done` | `event:` | Stream completion | Cleanup signal |
+
+### Event Format
+
+**Text content (OpenAI-compatible `data:` format):**
+```
+data: {"id":"chatcmpl-abc123","object":"chat.completion.chunk","created":1732748123,"model":"gpt-4","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello "},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-abc123","object":"chat.completion.chunk","created":1732748123,"model":"gpt-4","choices":[{"index":0,"delta":{"content":"world!"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-abc123","object":"chat.completion.chunk","created":1732748123,"model":"gpt-4","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+**Named events (use `event:` prefix):**
+```
+event: reasoning
+data: {"type": "reasoning", "content": "Analyzing the request...", "step": 1}
+
+event: progress
+data: {"type": "progress", "step": 1, "total_steps": 3, "label": "Searching", "status": "in_progress"}
+
+event: tool_call
+data: {"type": "tool_call", "tool_name": "search_rem", "status": "started", "arguments": {"query": "..."}}
+
+event: action_request
+data: {"type": "action_request", "card": {"id": "feedback-1", "prompt": "Was this helpful?", "actions": [...]}}
+
+event: metadata
+data: {"type": "metadata", "confidence": 0.95, "sources": ["doc1.md"], "hidden": false}
+
+event: done
+data: {"type": "done", "reason": "stop"}
+```
+
+### Action Request Cards (Adaptive Cards-inspired)
+
+Action requests solicit user input using a schema inspired by [Microsoft Adaptive Cards](https://adaptivecards.io/):
+
+```json
+{
+  "type": "action_request",
+  "card": {
+    "id": "confirm-delete-123",
+    "prompt": "Are you sure you want to delete this item?",
+    "display_style": "modal",
+    "actions": [
+      {
+        "type": "Action.Submit",
+        "id": "confirm",
+        "title": "Delete",
+        "style": "destructive",
+        "data": {"action": "delete", "item_id": "123"}
+      },
+      {
+        "type": "Action.Submit",
+        "id": "cancel",
+        "title": "Cancel",
+        "style": "secondary",
+        "data": {"action": "cancel"}
+      }
+    ],
+    "inputs": [
+      {
+        "type": "Input.Text",
+        "id": "reason",
+        "label": "Reason (optional)",
+        "placeholder": "Why are you deleting this?"
+      }
+    ],
+    "timeout_ms": 30000
+  }
+}
+```
+
+**Action Types:**
+- `Action.Submit` - Send data to server
+- `Action.OpenUrl` - Navigate to URL
+- `Action.ShowCard` - Reveal nested content
+
+**Input Types:**
+- `Input.Text` - Text field (single or multiline)
+- `Input.ChoiceSet` - Dropdown/radio selection
+- `Input.Toggle` - Checkbox/toggle
+
+### SSE Simulator Endpoint
+
+For frontend development and testing, use the simulator which generates all event types without LLM costs:
+
+```bash
+curl -X POST http://localhost:8000/api/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "X-Agent-Schema: simulator" \
+  -d '{"messages": [{"role": "user", "content": "demo"}], "stream": true}'
+```
+
+The simulator produces a scripted sequence demonstrating:
+1. Reasoning events (4 steps)
+2. Progress indicators
+3. Simulated tool calls
+4. Rich markdown content
+5. Metadata with confidence
+6. Action request for feedback
+
+See `rem/agentic/agents/sse_simulator.py` for implementation details.
+
+### Frontend Integration
+
+```typescript
+// Parse SSE events in React/TypeScript
+const eventSource = new EventSource('/api/v1/chat/completions');
+
+eventSource.onmessage = (e) => {
+  // Default handler for data-only events (text_delta)
+  const event = JSON.parse(e.data);
+  if (event.type === 'text_delta') {
+    appendContent(event.content);
+  }
+};
+
+eventSource.addEventListener('reasoning', (e) => {
+  const event = JSON.parse(e.data);
+  appendReasoning(event.content);
+});
+
+eventSource.addEventListener('action_request', (e) => {
+  const event = JSON.parse(e.data);
+  showActionCard(event.card);
+});
+
+eventSource.addEventListener('done', () => {
+  eventSource.close();
+});
+```
+
 ## Architecture
 
 ### Middleware Ordering
@@ -392,6 +604,29 @@ Middleware runs in reverse order of addition:
 
 ## Error Responses
 
+### 429 - Rate Limit Exceeded
+
+When a user exceeds their rate limit (based on their tier), the API returns a 429 status code with a structured error body. The frontend should intercept this error to prompt the user to sign in or upgrade.
+
+```json
+{
+  "error": {
+    "code": "rate_limit_exceeded",
+    "message": "You have exceeded your rate limit. Please sign in or upgrade to continue.",
+    "details": {
+      "limit": 50,
+      "tier": "anonymous",
+      "retry_after": 60
+    }
+  }
+}
+```
+
+**Handling Strategy:**
+1.  **Intercept 429s:** API client should listen for `status === 429`.
+2.  **Check Code:** If `error.code === 'rate_limit_exceeded'` AND `error.details.tier === 'anonymous'`, trigger "Login / Sign Up" flow.
+3.  **Authenticated Users:** If `tier !== 'anonymous'`, prompt to upgrade plan.
+
 ### 500 - Agent Schema Not Found
 
 ```json
@@ -414,6 +649,8 @@ Middleware runs in reverse order of addition:
 ## Related Documentation
 
 - [Chat Router](routers/chat/completions.py) - Chat completions implementation
+- [SSE Events](routers/chat/sse_events.py) - SSE event type definitions
+- [SSE Simulator](../../agentic/agents/sse_simulator.py) - Event simulator for testing
 - [MCP Router](mcp_router/server.py) - MCP server implementation
 - [Agent Schemas](../../schemas/agents/) - Available agent schemas
 - [Session Compression](../../services/session/compression.py) - Compression implementation

rem/api/deps.py

@@ -0,0 +1,255 @@
+"""
+Shared FastAPI dependencies for authentication and authorization.
+
+Provides dependency injection utilities for:
+- Extracting current user from session
+- Requiring authentication
+- Requiring specific roles (admin, user)
+- User-scoped data filtering with admin override
+
+Design Pattern:
+- Use as FastAPI dependencies via Depends()
+- Middleware sets request.state.user and request.state.is_anonymous
+- Dependencies extract and validate from request.state
+- Admin users can access any user's data via filters
+
+Roles:
+- "admin": Full access to all data across all users
+- "user": Default role, access limited to own data
+- Anonymous: Rate-limited access, no persistent data
+
+Usage:
+    from rem.api.deps import require_auth, require_admin, get_user_filter
+
+    @router.get("/items")
+    async def list_items(user: dict = Depends(require_auth)):
+        # user is guaranteed to be authenticated
+        ...
+
+    @router.post("/admin/action")
+    async def admin_action(user: dict = Depends(require_admin)):
+        # user is guaranteed to have admin role
+        ...
+
+    @router.get("/sessions/{session_id}")
+    async def get_session(
+        session_id: str,
+        filters: dict = Depends(get_user_filter),
+    ):
+        # filters includes user_id constraint (unless admin)
+        ...
+"""
+
+from typing import Any
+
+from fastapi import Depends, HTTPException, Request
+from loguru import logger
+
+
+class AuthError(HTTPException):
+    """Authentication/Authorization error."""
+
+    def __init__(self, detail: str, status_code: int = 401):
+        super().__init__(status_code=status_code, detail=detail)
+
+
+def get_current_user(request: Request) -> dict | None:
+    """
+    Get current user from request state (set by AuthMiddleware).
+
+    Returns None if no user authenticated.
+    Use require_auth() if authentication is mandatory.
+
+    Args:
+        request: FastAPI request
+
+    Returns:
+        User dict from session or None
+    """
+    return getattr(request.state, "user", None)
+
+
+def get_is_anonymous(request: Request) -> bool:
+    """
+    Check if current request is anonymous.
+
+    Args:
+        request: FastAPI request
+
+    Returns:
+        True if anonymous, False if authenticated
+    """
+    return getattr(request.state, "is_anonymous", True)
+
+
+def require_auth(request: Request) -> dict:
+    """
+    Require authenticated user.
+
+    Use as FastAPI dependency to enforce authentication.
+
+    Args:
+        request: FastAPI request
+
+    Returns:
+        User dict from session
+
+    Raises:
+        HTTPException 401 if not authenticated
+    """
+    user = get_current_user(request)
+    if not user:
+        raise AuthError("Authentication required", status_code=401)
+    return user
+
+
+def require_admin(request: Request) -> dict:
+    """
+    Require authenticated user with admin role.
+
+    Use as FastAPI dependency to protect admin-only endpoints.
+
+    Args:
+        request: FastAPI request
+
+    Returns:
+        User dict from session
+
+    Raises:
+        HTTPException 401 if not authenticated
+        HTTPException 403 if not admin
+    """
+    user = require_auth(request)
+    roles = user.get("roles", [])
+
+    if "admin" not in roles:
+        logger.warning(f"Admin access denied for user {user.get('email')}")
+        raise AuthError("Admin access required", status_code=403)
+
+    return user
+
+
+def is_admin(user: dict | None) -> bool:
+    """
+    Check if user has admin role.
+
+    Args:
+        user: User dict or None
+
+    Returns:
+        True if user is admin
+    """
+    if not user:
+        return False
+    return "admin" in user.get("roles", [])
+
+
+async def get_user_filter(
+    request: Request,
+    x_user_id: str | None = None,
+    x_tenant_id: str = "default",
+) -> dict[str, Any]:
+    """
+    Get user-scoped filter dict for database queries.
+
+    For regular users: Always filters by their own user_id.
+    For admin users: Can filter by any user_id (or no filter for all users).
+
+    Args:
+        request: FastAPI request
+        x_user_id: Optional user_id filter (admin only for cross-user)
+        x_tenant_id: Tenant ID for multi-tenancy
+
+    Returns:
+        Filter dict with appropriate user_id constraint
+
+    Usage:
+        @router.get("/items")
+        async def list_items(filters: dict = Depends(get_user_filter)):
+            return await repo.find(filters)
+    """
+    user = get_current_user(request)
+    filters: dict[str, Any] = {"tenant_id": x_tenant_id}
+
+    if is_admin(user):
+        # Admin can filter by any user or see all
+        if x_user_id:
+            filters["user_id"] = x_user_id
+        # If no user_id specified, admin sees all (no user_id filter)
+        logger.debug(f"Admin access: filters={filters}")
+    elif user:
+        # Regular authenticated user: always filter by own user_id
+        filters["user_id"] = user.get("id")
+        if x_user_id and x_user_id != user.get("id"):
+            logger.warning(
+                f"User {user.get('email')} attempted to filter by user_id={x_user_id}"
+            )
+    else:
+        # Anonymous: could use anonymous tracking ID or restrict access
+        # For now, anonymous can't access user-scoped data
+        anon_id = getattr(request.state, "anon_id", None)
+        if anon_id:
+            filters["user_id"] = f"anon:{anon_id}"
+        else:
+            filters["user_id"] = "anonymous"
+
+    return filters
+
+
+async def require_owner_or_admin(
+    request: Request,
+    resource_user_id: str,
+) -> dict:
+    """
+    Require that current user owns the resource or is admin.
+
+    Use for parametric endpoints (GET /resource/{id}) where
+    only the owner or admin should access.
+
+    Args:
+        request: FastAPI request
+        resource_user_id: The user_id of the resource being accessed
+
+    Returns:
+        User dict from session
+
+    Raises:
+        HTTPException 401 if not authenticated
+        HTTPException 403 if not owner and not admin
+    """
+    user = require_auth(request)
+
+    if is_admin(user):
+        return user
+
+    if user.get("id") != resource_user_id:
+        logger.warning(
+            f"Access denied: user {user.get('email')} tried to access "
+            f"resource owned by {resource_user_id}"
+        )
+        raise AuthError("Access denied: not owner", status_code=403)
+
+    return user
+
+
+def get_user_id_from_request(request: Request) -> str:
+    """
+    Get effective user_id for creating resources.
+
+    Returns authenticated user's ID or anonymous tracking ID.
+
+    Args:
+        request: FastAPI request
+
+    Returns:
+        User ID string
+    """
+    user = get_current_user(request)
+    if user:
+        return user.get("id", "unknown")
+
+    anon_id = getattr(request.state, "anon_id", None)
+    if anon_id:
+        return f"anon:{anon_id}"
+
+    return "anonymous"

rem/api/main.py

@@ -163,7 +163,22 @@ async def lifespan(app: FastAPI):
 
 def create_app() -> FastAPI:
     """
-    Create and configure the FastAPI application.
+    Create and configure the FastAPI application with MCP server.
+
+    The returned app exposes `app.mcp_server` (FastMCP instance) for adding
+    custom tools, resources, and prompts:
+
+        app = create_app()
+
+        @app.mcp_server.tool()
+        async def my_tool(query: str) -> dict:
+            '''Custom MCP tool.'''
+            return {"result": query}
+
+        @app.mcp_server.resource("custom://data")
+        async def my_resource() -> str:
+            '''Custom resource.'''
+            return '{"data": "value"}'
 
     Design Pattern:
     1. Create MCP server
@@ -174,9 +189,10 @@ def create_app() -> FastAPI:
     6. Define health endpoints
     7. Register API routers
     8. Mount MCP app
+    9. Expose mcp_server on app for extension
 
     Returns:
-        Configured FastAPI application
+        Configured FastAPI application with .mcp_server attribute
     """
     # Create MCP server and get HTTP app
     # path="/" creates routes at root, then mount at /api/v1/mcp
@@ -198,15 +214,42 @@ def create_app() -> FastAPI:
                 yield
 
     app = FastAPI(
-        title="REM API",
-        description="Resources Entities Moments system for agentic AI",
+        title=f"{settings.app_name} API",
+        description=f"{settings.app_name} - Resources Entities Moments system for agentic AI",
         version="0.1.0",
         lifespan=combined_lifespan,
         root_path=settings.root_path if settings.root_path else "",
         redirect_slashes=False,  # Don't redirect /mcp/ -> /mcp
     )
 
+    # Add request logging middleware
+    app.add_middleware(RequestLoggingMiddleware)
+
+    # Add SSE buffering middleware (for MCP SSE transport)
+    app.add_middleware(SSEBufferingMiddleware)
+
+    # Add Anonymous Tracking & Rate Limiting (Runs AFTER Auth if Auth is enabled)
+    # Must be added BEFORE AuthMiddleware in code to be INNER in the stack
+    from .middleware.tracking import AnonymousTrackingMiddleware
+    app.add_middleware(AnonymousTrackingMiddleware)
+
+    # Add authentication middleware
+    # Always load middleware for dev token support, but allow anonymous when auth disabled
+    from ..auth.middleware import AuthMiddleware
+
+    app.add_middleware(
+        AuthMiddleware,
+        protected_paths=["/api/v1"],
+        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth"],
+        # Allow anonymous when auth is disabled, otherwise use setting
+        allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,
+        # MCP requires auth only when auth is fully enabled
+        mcp_requires_auth=settings.auth.enabled and settings.auth.mcp_requires_auth,
+    )
+
     # Add session middleware for OAuth state management
+    # Must be added AFTER AuthMiddleware in code so it runs BEFORE (middleware runs in reverse)
+    # AuthMiddleware needs request.session to be available
     session_secret = settings.auth.session_secret or secrets.token_hex(32)
     if not settings.auth.session_secret:
         logger.warning(
@@ -223,27 +266,12 @@ def create_app() -> FastAPI:
         https_only=settings.environment == "production",
     )
 
-    # Add request logging middleware
-    app.add_middleware(RequestLoggingMiddleware)
-
-    # Add SSE buffering middleware (for MCP SSE transport)
-    app.add_middleware(SSEBufferingMiddleware)
-
-    # Add authentication middleware (if enabled)
-    if settings.auth.enabled:
-        from ..auth.middleware import AuthMiddleware
-
-        app.add_middleware(
-            AuthMiddleware,
-            protected_paths=["/api/v1"],
-            excluded_paths=["/api/auth", "/api/v1/mcp/auth"],
-        )
-
     # Add CORS middleware LAST (runs first in middleware chain)
     # Must expose mcp-session-id header for MCP session management
     CORS_ORIGIN_WHITELIST = [
-        "http://localhost:5173",  # Local development (Vite)
         "http://localhost:3000",  # Local development (React)
+        "http://localhost:5000",  # Local development (Flask/other)
+        "http://localhost:5173",  # Local development (Vite)
     ]
 
     app.add_middleware(
@@ -261,7 +289,7 @@ def create_app() -> FastAPI:
         """API information endpoint."""
         # TODO: If auth enabled and no user, return 401 with WWW-Authenticate
         return {
-            "name": "REM API",
+            "name": f"{settings.app_name} API",
             "version": "0.1.0",
             "mcp_endpoint": "/api/v1/mcp",
             "docs": "/docs",
@@ -275,8 +303,18 @@ def create_app() -> FastAPI:
 
     # Register API routers
     from .routers.chat import router as chat_router
+    from .routers.models import router as models_router
+    from .routers.messages import router as messages_router
+    from .routers.feedback import router as feedback_router
+    from .routers.admin import router as admin_router
+    from .routers.shared_sessions import router as shared_sessions_router
 
     app.include_router(chat_router)
+    app.include_router(models_router)
+    app.include_router(messages_router)
+    app.include_router(feedback_router)
+    app.include_router(admin_router)
+    app.include_router(shared_sessions_router)
 
     # Register auth router (if enabled)
     if settings.auth.enabled:
@@ -284,6 +322,12 @@ def create_app() -> FastAPI:
 
         app.include_router(auth_router)
 
+    # Register dev router (non-production only)
+    if settings.environment != "production":
+        from .routers.dev import router as dev_router
+
+        app.include_router(dev_router)
+
     # TODO: Register additional routers
     # from .routers.query import router as query_router
     # from .routers.resources import router as resources_router
@@ -305,6 +349,10 @@ def create_app() -> FastAPI:
     # Mount MCP app at /api/v1/mcp
     app.mount("/api/v1/mcp", mcp_app)
 
+    # Expose MCP server on app for extension
+    # Users can add tools/resources/prompts via app.mcp_server
+    app.mcp_server = mcp_server  # type: ignore[attr-defined]
+
     return app
 
 

rem/api/mcp_router/server.py

@@ -165,11 +165,18 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     )
 
     # Register REM tools
-    from .tools import ask_rem_agent, ingest_into_rem, read_resource, search_rem
+    from .tools import (
+        ask_rem_agent,
+        ingest_into_rem,
+        read_resource,
+        register_metadata,
+        search_rem,
+    )
 
     mcp.tool()(search_rem)
     mcp.tool()(ask_rem_agent)
     mcp.tool()(read_resource)
+    mcp.tool()(register_metadata)
 
     # File ingestion tool (with local path support for local servers)
     # Wrap to inject is_local parameter