remdb 0.3.14__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/agentic/schema.py

@@ -13,7 +13,7 @@ The schema protocol serves as:
 """
 
 from typing import Any, Literal
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 
 class MCPToolReference(BaseModel):
@@ -23,11 +23,21 @@ class MCPToolReference(BaseModel):
     Tools are functions that agents can call during execution to
     interact with external systems, retrieve data, or perform actions.
 
-    Example:
+    Two usage patterns:
+    1. With mcp_servers config: Just declare name + description, tools loaded from MCP servers
+    2. Explicit MCP server: Specify mcp_server to load tool from specific server
+
+    Example (declarative with mcp_servers):
+        {
+            "name": "search_rem",
+            "description": "Execute REM queries for entity lookup and search"
+        }
+
+    Example (explicit server):
         {
             "name": "lookup_entity",
             "mcp_server": "rem",
-            "description": "Lookup entities by exact key with O(1) performance"
+            "description": "Lookup entities by exact key"
         }
     """
 
@@ -38,20 +48,20 @@ class MCPToolReference(BaseModel):
         )
     )
 
-    mcp_server: str = Field(
+    mcp_server: str | None = Field(
+        default=None,
         description=(
-            "MCP server identifier. Resolved via environment variable: "
-            "MCP_SERVER_{NAME} or MCP__{NAME}__URL. "
-            "Common values: 'rem' (REM knowledge graph), 'filesystem', 'web'."
+            "MCP server identifier (optional when using mcp_servers config). "
+            "If not specified, tool is expected from configured mcp_servers. "
+            "Resolved via environment variable: MCP_SERVER_{NAME} or MCP__{NAME}__URL."
         )
     )
 
     description: str | None = Field(
         default=None,
         description=(
-            "Optional description override. If provided, replaces the tool's "
-            "description from the MCP server in the agent's context. "
-            "Use this to provide agent-specific guidance on tool usage."
+            "Tool description for the agent. Explains what the tool does "
+            "and when to use it. This is visible to the LLM."
         ),
     )
 
@@ -63,29 +73,92 @@ class MCPResourceReference(BaseModel):
     Resources are data sources that can be read by agents, such as
     knowledge graph entities, files, or API endpoints.
 
-    Example:
+    Two formats supported:
+    1. uri: Exact URI or URI with query params
+    2. uri_pattern: Regex pattern for flexible matching
+
+    Example (exact URI):
+        {
+            "uri": "rem://schemas",
+            "name": "Agent Schemas",
+            "description": "List all available agent schemas"
+        }
+
+    Example (pattern):
         {
             "uri_pattern": "rem://resources/.*",
             "mcp_server": "rem"
         }
     """
 
-    uri_pattern: str = Field(
+    # Support both exact URI and pattern
+    uri: str | None = Field(
+        default=None,
+        description=(
+            "Exact resource URI or URI with query parameters. "
+            "Examples: 'rem://schemas', 'rem://resources?category=drug.*'"
+        )
+    )
+
+    uri_pattern: str | None = Field(
+        default=None,
         description=(
             "Regex pattern matching resource URIs. "
-            "Examples: "
-            "'rem://resources/.*' (all resources), "
-            "'rem://moments/.*' (all moments), "
-            "'file:///data/.*' (local files). "
-            "Supports full regex syntax for flexible matching."
+            "Examples: 'rem://resources/.*' (all resources). "
+            "Use uri for exact URIs, uri_pattern for regex matching."
+        )
+    )
+
+    name: str | None = Field(
+        default=None,
+        description="Human-readable name for the resource."
+    )
+
+    description: str | None = Field(
+        default=None,
+        description="Description of what the resource provides."
+    )
+
+    mcp_server: str | None = Field(
+        default=None,
+        description=(
+            "MCP server identifier (optional when using mcp_servers config). "
+            "Resolved via environment variable MCP_SERVER_{NAME}."
+        )
+    )
+
+
+class MCPServerConfig(BaseModel):
+    """
+    MCP server configuration for in-process tool loading.
+
+    Example:
+        {
+            "type": "local",
+            "module": "rem.mcp_server",
+            "id": "rem-local"
+        }
+    """
+
+    type: Literal["local"] = Field(
+        default="local",
+        description="Server type. Currently only 'local' (in-process) is supported.",
+    )
+
+    module: str = Field(
+        description=(
+            "Python module path containing the MCP server. "
+            "The module must export an 'mcp' object that supports get_tools(). "
+            "Example: 'rem.mcp_server'"
         )
     )
 
-    mcp_server: str = Field(
+    id: str = Field(
+        default="mcp-server",
         description=(
-            "MCP server identifier that provides these resources. "
-            "Resolved via environment variable MCP_SERVER_{NAME}. "
-            "The server must expose resources matching the uri_pattern."
+            "Server identifier for logging and debugging. "
+            "Defaults to 'mcp-server' if not specified. "
+            "Example: 'rem-local'"
         )
     )
 
@@ -130,6 +203,38 @@ class AgentSchemaMetadata(BaseModel):
         ),
     )
 
+    # System prompt override (takes precedence over description when present)
+    system_prompt: str | None = Field(
+        default=None,
+        description=(
+            "Custom system prompt that overrides or extends the schema description. "
+            "When present, this is combined with the main schema.description field "
+            "to form the complete system prompt. Use this for detailed instructions "
+            "that you don't want in the public schema description."
+        ),
+    )
+
+    # Structured output toggle
+    structured_output: bool = Field(
+        default=True,
+        description=(
+            "Whether to enforce structured JSON output. "
+            "When False, the agent produces free-form text and schema properties "
+            "are converted to prompt guidance instead. Default: True (JSON output)."
+        ),
+    )
+
+    # MCP server configurations (for dynamic tool loading)
+    mcp_servers: list[MCPServerConfig] = Field(
+        default_factory=list,
+        description=(
+            "MCP server configurations for dynamic tool loading. "
+            "Servers are loaded in-process at agent creation time. "
+            "All tools from configured servers become available to the agent. "
+            "If not specified, defaults to rem.mcp_server (REM's built-in tools)."
+        ),
+    )
+
     tools: list[MCPToolReference] = Field(
         default_factory=list,
         description=(
@@ -394,3 +499,238 @@ def create_agent_schema(
         json_schema_extra=metadata.model_dump(),
         **kwargs,
     )
+
+
+# =============================================================================
+# YAML and Database Serialization
+# =============================================================================
+
+
+def schema_to_dict(schema: AgentSchema, exclude_none: bool = True) -> dict[str, Any]:
+    """
+    Serialize AgentSchema to a dictionary suitable for YAML or database storage.
+
+    This produces the canonical format used in:
+    - YAML files (schemas/agents/*.yaml)
+    - Database spec column (schemas table)
+    - API responses
+
+    Args:
+        schema: AgentSchema instance to serialize
+        exclude_none: If True, omit None values from output
+
+    Returns:
+        Dictionary representation of the schema
+
+    Example:
+        >>> schema = AgentSchema(
+        ...     description="System prompt...",
+        ...     properties={"answer": {"type": "string"}},
+        ...     json_schema_extra={"name": "my-agent", "structured_output": False}
+        ... )
+        >>> d = schema_to_dict(schema)
+        >>> d["json_schema_extra"]["name"]
+        "my-agent"
+    """
+    return schema.model_dump(exclude_none=exclude_none)
+
+
+def schema_from_dict(data: dict[str, Any]) -> AgentSchema:
+    """
+    Deserialize a dictionary to AgentSchema.
+
+    This handles:
+    - YAML files loaded with yaml.safe_load()
+    - Database spec column (JSON)
+    - API request bodies
+
+    Args:
+        data: Dictionary containing schema data
+
+    Returns:
+        Validated AgentSchema instance
+
+    Raises:
+        ValidationError: If data doesn't match schema structure
+
+    Example:
+        >>> data = {"type": "object", "description": "...", "properties": {}, "json_schema_extra": {"name": "test"}}
+        >>> schema = schema_from_dict(data)
+        >>> schema.json_schema_extra["name"]
+        "test"
+    """
+    return AgentSchema.model_validate(data)
+
+
+def schema_to_yaml(schema: AgentSchema) -> str:
+    """
+    Serialize AgentSchema to YAML string.
+
+    The output format matches the canonical schema file format:
+    ```yaml
+    type: object
+    description: |
+      System prompt here...
+    properties:
+      answer:
+        type: string
+    json_schema_extra:
+      name: my-agent
+      system_prompt: |
+        Extended prompt here...
+    ```
+
+    Args:
+        schema: AgentSchema instance to serialize
+
+    Returns:
+        YAML string representation
+
+    Example:
+        >>> schema = create_agent_schema(
+        ...     description="You are a test agent",
+        ...     properties={"answer": {"type": "string"}},
+        ...     required=["answer"],
+        ...     name="test-agent"
+        ... )
+        >>> yaml_str = schema_to_yaml(schema)
+        >>> "test-agent" in yaml_str
+        True
+    """
+    import yaml
+
+    return yaml.dump(
+        schema_to_dict(schema),
+        default_flow_style=False,
+        allow_unicode=True,
+        sort_keys=False,
+    )
+
+
+def schema_from_yaml(yaml_content: str) -> AgentSchema:
+    """
+    Deserialize YAML string to AgentSchema.
+
+    Args:
+        yaml_content: YAML string containing schema definition
+
+    Returns:
+        Validated AgentSchema instance
+
+    Raises:
+        yaml.YAMLError: If YAML parsing fails
+        ValidationError: If schema structure is invalid
+
+    Example:
+        >>> yaml_str = '''
+        ... type: object
+        ... description: Test agent
+        ... properties:
+        ...   answer:
+        ...     type: string
+        ... json_schema_extra:
+        ...   name: test
+        ... '''
+        >>> schema = schema_from_yaml(yaml_str)
+        >>> schema.json_schema_extra["name"]
+        "test"
+    """
+    import yaml
+
+    data = yaml.safe_load(yaml_content)
+    return schema_from_dict(data)
+
+
+def schema_from_yaml_file(file_path: str) -> AgentSchema:
+    """
+    Load AgentSchema from a YAML file.
+
+    Args:
+        file_path: Path to YAML file
+
+    Returns:
+        Validated AgentSchema instance
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        yaml.YAMLError: If YAML parsing fails
+        ValidationError: If schema structure is invalid
+
+    Example:
+        >>> schema = schema_from_yaml_file("schemas/agents/rem.yaml")
+        >>> schema.json_schema_extra["name"]
+        "rem"
+    """
+    with open(file_path, "r") as f:
+        return schema_from_yaml(f.read())
+
+
+def get_system_prompt(schema: AgentSchema | dict[str, Any]) -> str:
+    """
+    Extract the complete system prompt from a schema.
+
+    Combines:
+    1. schema.description (base system prompt / public description)
+    2. json_schema_extra.system_prompt (extended instructions if present)
+
+    Args:
+        schema: AgentSchema instance or raw dict
+
+    Returns:
+        Complete system prompt string
+
+    Example:
+        >>> schema = AgentSchema(
+        ...     description="Base description",
+        ...     properties={},
+        ...     json_schema_extra={"name": "test", "system_prompt": "Extended instructions"}
+        ... )
+        >>> prompt = get_system_prompt(schema)
+        >>> "Base description" in prompt and "Extended instructions" in prompt
+        True
+    """
+    if isinstance(schema, dict):
+        base = schema.get("description", "")
+        extra = schema.get("json_schema_extra", {})
+        custom = extra.get("system_prompt") if isinstance(extra, dict) else None
+    else:
+        base = schema.description
+        extra = schema.json_schema_extra
+        if isinstance(extra, dict):
+            custom = extra.get("system_prompt")
+        elif isinstance(extra, AgentSchemaMetadata):
+            custom = extra.system_prompt
+        else:
+            custom = None
+
+    if custom:
+        return f"{base}\n\n{custom}" if base else custom
+    return base
+
+
+def get_metadata(schema: AgentSchema | dict[str, Any]) -> AgentSchemaMetadata:
+    """
+    Extract and validate metadata from a schema.
+
+    Args:
+        schema: AgentSchema instance or raw dict
+
+    Returns:
+        Validated AgentSchemaMetadata instance
+
+    Example:
+        >>> schema = {"json_schema_extra": {"name": "test", "system_prompt": "hello"}}
+        >>> meta = get_metadata(schema)
+        >>> meta.name
+        "test"
+        >>> meta.system_prompt
+        "hello"
+    """
+    if isinstance(schema, dict):
+        extra = schema.get("json_schema_extra", {})
+    else:
+        extra = schema.json_schema_extra
+
+    if isinstance(extra, AgentSchemaMetadata):
+        return extra
+    return AgentSchemaMetadata.model_validate(extra)

rem/agentic/tools/rem_tools.py

@@ -162,10 +162,10 @@ async def search_rem_tool(
             return {"status": "error", "error": f"Unknown query_type: {query_type}"}
 
         # Execute query
-        logger.info(f"Executing REM query: {query_type} for user {user_id}")
+        logger.debug(f"Executing REM query: {query_type} for user {user_id}")
         result = await rem_service.execute_query(query)
 
-        logger.info(f"Query completed: {query_type}")
+        logger.debug(f"Query completed: {query_type}")
         return {
             "status": "success",
             "query_type": query_type,
@@ -212,7 +212,7 @@ async def ingest_file_tool(
             is_local_server=is_local_server,
         )
 
-        logger.info(
+        logger.debug(
             f"File ingestion complete: {result['file_name']} "
             f"(status: {result['processing_status']}, "
             f"resources: {result['resources_created']})"

rem/api/README.md

@@ -158,9 +158,70 @@ The dreaming worker runs periodically to build user models:
 - User profile automatically loaded and injected into system message
 - Simpler for basic chatbots that always need context
 
+## Authentication
+
+### Production Authentication
+
+When `AUTH__ENABLED=true`, users authenticate via OAuth (Google or Microsoft). The OAuth flow:
+
+1. User visits `/api/auth/google/login` or `/api/auth/microsoft/login`
+2. User authenticates with provider
+3. Callback stores user in session cookie
+4. Subsequent requests use session cookie
+
+### Development Token (Non-Production Only)
+
+For local development and testing, you can use a dev token instead of OAuth. This endpoint is available at `/api/dev/token` whenever `ENVIRONMENT != "production"`, regardless of whether auth is enabled.
+
+**Get Token:**
+```bash
+curl http://localhost:8000/api/dev/token
+```
+
+**Response:**
+```json
+{
+  "token": "dev_89737a19376332bfd9a4a06db8b79fd1",
+  "type": "Bearer",
+  "user": {
+    "id": "test-user",
+    "email": "test@rem.local",
+    "name": "Test User"
+  },
+  "usage": "curl -H \"Authorization: Bearer dev_...\" http://localhost:8000/api/v1/...",
+  "warning": "This token is for development/testing only and will not work in production."
+}
+```
+
+**Use Token:**
+```bash
+# Get the token
+TOKEN=$(curl -s http://localhost:8000/api/dev/token | jq -r .token)
+
+# Use it in requests
+curl -H "Authorization: Bearer $TOKEN" \
+     -H "X-Tenant-Id: default" \
+     http://localhost:8000/api/v1/shared-with-me
+```
+
+**Security Notes:**
+- Only available when `ENVIRONMENT != "production"`
+- Token is HMAC-signed using session secret
+- Authenticates as `test-user` with `pro` tier and `admin` role
+- Token is deterministic per environment (same secret = same token)
+
+### Anonymous Access
+
+When `AUTH__ALLOW_ANONYMOUS=true` (default in development):
+- Requests without authentication are allowed
+- Anonymous users get rate-limited access
+- MCP endpoints still require auth unless `AUTH__MCP_REQUIRES_AUTH=false`
+
 ## Usage Examples
 
-**Note on Authentication**: By default, authentication is disabled (`AUTH__ENABLED=false`) for local development and testing. The examples below work without an `Authorization` header. If authentication is enabled in your environment, add: `-H "Authorization: Bearer your_jwt_token"` to cURL requests or `"Authorization": "Bearer your_jwt_token"` to Python headers.
+**Note on Authentication**: By default, authentication is disabled (`AUTH__ENABLED=false`) for local development and testing. The examples below work without an `Authorization` header. If authentication is enabled, use either:
+- **Dev token**: `-H "Authorization: Bearer $(curl -s http://localhost:8000/api/dev/token | jq -r .token)"`
+- **Session cookie**: Login via OAuth first, then use cookies
 
 ### cURL: Simple Chat
 
@@ -363,6 +424,157 @@ data: {"id":"chatcmpl-abc123","choices":[{"delta":{},"finish_reason":"stop","ind
 data: [DONE]
 ```
 
+## Extended SSE Event Protocol
+
+REM uses OpenAI-compatible format for text content streaming, plus custom named SSE events for rich UI interactions.
+
+### Event Types
+
+| Event Type | Format | Purpose | UI Display |
+|------------|--------|---------|------------|
+| (text content) | `data:` (OpenAI format) | Content chunks | Main response area |
+| `reasoning` | `event:` | Model thinking | Collapsible "thinking" section |
+| `progress` | `event:` | Step indicators | Progress bar/stepper |
+| `tool_call` | `event:` | Tool invocations | Tool status panel |
+| `action_request` | `event:` | User input solicitation | Buttons, forms, modals |
+| `metadata` | `event:` | System info | Hidden or badge display |
+| `error` | `event:` | Error notification | Error toast/alert |
+| `done` | `event:` | Stream completion | Cleanup signal |
+
+### Event Format
+
+**Text content (OpenAI-compatible `data:` format):**
+```
+data: {"id":"chatcmpl-abc123","object":"chat.completion.chunk","created":1732748123,"model":"gpt-4","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello "},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-abc123","object":"chat.completion.chunk","created":1732748123,"model":"gpt-4","choices":[{"index":0,"delta":{"content":"world!"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-abc123","object":"chat.completion.chunk","created":1732748123,"model":"gpt-4","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+**Named events (use `event:` prefix):**
+```
+event: reasoning
+data: {"type": "reasoning", "content": "Analyzing the request...", "step": 1}
+
+event: progress
+data: {"type": "progress", "step": 1, "total_steps": 3, "label": "Searching", "status": "in_progress"}
+
+event: tool_call
+data: {"type": "tool_call", "tool_name": "search_rem", "status": "started", "arguments": {"query": "..."}}
+
+event: action_request
+data: {"type": "action_request", "card": {"id": "feedback-1", "prompt": "Was this helpful?", "actions": [...]}}
+
+event: metadata
+data: {"type": "metadata", "confidence": 0.95, "sources": ["doc1.md"], "hidden": false}
+
+event: done
+data: {"type": "done", "reason": "stop"}
+```
+
+### Action Request Cards (Adaptive Cards-inspired)
+
+Action requests solicit user input using a schema inspired by [Microsoft Adaptive Cards](https://adaptivecards.io/):
+
+```json
+{
+  "type": "action_request",
+  "card": {
+    "id": "confirm-delete-123",
+    "prompt": "Are you sure you want to delete this item?",
+    "display_style": "modal",
+    "actions": [
+      {
+        "type": "Action.Submit",
+        "id": "confirm",
+        "title": "Delete",
+        "style": "destructive",
+        "data": {"action": "delete", "item_id": "123"}
+      },
+      {
+        "type": "Action.Submit",
+        "id": "cancel",
+        "title": "Cancel",
+        "style": "secondary",
+        "data": {"action": "cancel"}
+      }
+    ],
+    "inputs": [
+      {
+        "type": "Input.Text",
+        "id": "reason",
+        "label": "Reason (optional)",
+        "placeholder": "Why are you deleting this?"
+      }
+    ],
+    "timeout_ms": 30000
+  }
+}
+```
+
+**Action Types:**
+- `Action.Submit` - Send data to server
+- `Action.OpenUrl` - Navigate to URL
+- `Action.ShowCard` - Reveal nested content
+
+**Input Types:**
+- `Input.Text` - Text field (single or multiline)
+- `Input.ChoiceSet` - Dropdown/radio selection
+- `Input.Toggle` - Checkbox/toggle
+
+### SSE Simulator Endpoint
+
+For frontend development and testing, use the simulator which generates all event types without LLM costs:
+
+```bash
+curl -X POST http://localhost:8000/api/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "X-Agent-Schema: simulator" \
+  -d '{"messages": [{"role": "user", "content": "demo"}], "stream": true}'
+```
+
+The simulator produces a scripted sequence demonstrating:
+1. Reasoning events (4 steps)
+2. Progress indicators
+3. Simulated tool calls
+4. Rich markdown content
+5. Metadata with confidence
+6. Action request for feedback
+
+See `rem/agentic/agents/sse_simulator.py` for implementation details.
+
+### Frontend Integration
+
+```typescript
+// Parse SSE events in React/TypeScript
+const eventSource = new EventSource('/api/v1/chat/completions');
+
+eventSource.onmessage = (e) => {
+  // Default handler for data-only events (text_delta)
+  const event = JSON.parse(e.data);
+  if (event.type === 'text_delta') {
+    appendContent(event.content);
+  }
+};
+
+eventSource.addEventListener('reasoning', (e) => {
+  const event = JSON.parse(e.data);
+  appendReasoning(event.content);
+});
+
+eventSource.addEventListener('action_request', (e) => {
+  const event = JSON.parse(e.data);
+  showActionCard(event.card);
+});
+
+eventSource.addEventListener('done', () => {
+  eventSource.close();
+});
+```
+
 ## Architecture
 
 ### Middleware Ordering
@@ -437,6 +649,8 @@ When a user exceeds their rate limit (based on their tier), the API returns a 42
 ## Related Documentation
 
 - [Chat Router](routers/chat/completions.py) - Chat completions implementation
+- [SSE Events](routers/chat/sse_events.py) - SSE event type definitions
+- [SSE Simulator](../../agentic/agents/sse_simulator.py) - Event simulator for testing
 - [MCP Router](mcp_router/server.py) - MCP server implementation
 - [Agent Schemas](../../schemas/agents/) - Available agent schemas
 - [Session Compression](../../services/session/compression.py) - Compression implementation