remdb 0.3.118__py3-none-any.whl → 0.3.146__py3-none-any.whl

rem/agentic/providers/pydantic_ai.py

@@ -553,7 +553,7 @@ async def create_agent(
     if agent_schema:
         system_prompt = get_system_prompt(agent_schema)
         metadata = get_metadata(agent_schema)
-        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') else []
+        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') and metadata.mcp_servers else []
         resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
 
         if metadata.system_prompt:
@@ -564,6 +564,38 @@ async def create_agent(
         mcp_server_configs = []
         resource_configs = []
 
+    # Auto-detect local MCP server if not explicitly configured
+    # This makes mcp_servers config optional - agents get tools automatically
+    if not mcp_server_configs:
+        import importlib
+        import os
+        import sys
+
+        # Ensure current working directory is in sys.path for local imports
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+
+        # Try common local MCP server module paths first
+        auto_detect_modules = [
+            "tools.mcp_server",  # Convention: tools/mcp_server.py
+            "mcp_server",        # Alternative: mcp_server.py in root
+        ]
+        for module_path in auto_detect_modules:
+            try:
+                mcp_module = importlib.import_module(module_path)
+                if hasattr(mcp_module, "mcp"):
+                    logger.info(f"Auto-detected local MCP server: {module_path}")
+                    mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
+                    break
+            except ImportError:
+                continue
+
+        # Fall back to REM's default MCP server if no local server found
+        if not mcp_server_configs:
+            logger.debug("No local MCP server found, using REM default")
+            mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
+
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
     if metadata:
         temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
@@ -608,50 +640,97 @@ async def create_agent(
             search_rem_suffix += f"Example: `SEARCH \"your query\" FROM {default_table} LIMIT 10`"
 
     # Add tools from MCP server (in-process, no subprocess)
-    if mcp_server_configs:
-        for server_config in mcp_server_configs:
-            server_type = server_config.get("type")
-            server_id = server_config.get("id", "mcp-server")
-
-            if server_type == "local":
-                # Import MCP server directly (in-process)
-                module_path = server_config.get("module", "rem.mcp_server")
-
-                try:
-                    # Dynamic import of MCP server module
-                    import importlib
-                    mcp_module = importlib.import_module(module_path)
-                    mcp_server = mcp_module.mcp
-
-                    # Extract tools from MCP server (get_tools is async)
-                    from ..mcp.tool_wrapper import create_mcp_tool_wrapper
-
-                    # Await async get_tools() call
-                    mcp_tools_dict = await mcp_server.get_tools()
-
-                    for tool_name, tool_func in mcp_tools_dict.items():
-                        # Add description suffix to search_rem tool if schema specifies a default table
-                        tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
-
-                        wrapped_tool = create_mcp_tool_wrapper(
-                            tool_name,
-                            tool_func,
-                            user_id=context.user_id if context else None,
-                            description_suffix=tool_suffix,
-                        )
-                        tools.append(wrapped_tool)
-                        logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
-
-                    logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
-
-                except Exception as e:
-                    logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
-            else:
-                logger.warning(f"Unsupported MCP server type: {server_type}")
+    # Track loaded MCP servers for resource resolution
+    loaded_mcp_server = None
+
+    for server_config in mcp_server_configs:
+        server_type = server_config.get("type")
+        server_id = server_config.get("id", "mcp-server")
+
+        if server_type == "local":
+            # Import MCP server directly (in-process)
+            module_path = server_config.get("module", "rem.mcp_server")
+
+            try:
+                # Dynamic import of MCP server module
+                import importlib
+                mcp_module = importlib.import_module(module_path)
+                mcp_server = mcp_module.mcp
+
+                # Store the loaded server for resource resolution
+                loaded_mcp_server = mcp_server
+
+                # Extract tools from MCP server (get_tools is async)
+                from ..mcp.tool_wrapper import create_mcp_tool_wrapper
+
+                # Await async get_tools() call
+                mcp_tools_dict = await mcp_server.get_tools()
+
+                for tool_name, tool_func in mcp_tools_dict.items():
+                    # Add description suffix to search_rem tool if schema specifies a default table
+                    tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
 
+                    wrapped_tool = create_mcp_tool_wrapper(
+                        tool_name,
+                        tool_func,
+                        user_id=context.user_id if context else None,
+                        description_suffix=tool_suffix,
+                    )
+                    tools.append(wrapped_tool)
+                    logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
+
+                logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
+
+            except Exception as e:
+                logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
+        else:
+            logger.warning(f"Unsupported MCP server type: {server_type}")
+
+    # Convert resources to tools (MCP convenience syntax)
+    # Resources declared in agent YAML become callable tools - eliminates
+    # the artificial MCP distinction between tools and resources
+    #
+    # Supports both concrete and template URIs:
+    # - Concrete: "rem://schemas" -> no-param tool
+    # - Template: "patient-profile://field/{field_key}" -> tool with field_key param
+    from ..mcp.tool_wrapper import create_resource_tool
+
+    # Collect all resource URIs from both resources section AND tools section
+    resource_uris = []
+
+    # From resources section (legacy format)
     if resource_configs:
-        # TODO: Convert resources to tools (MCP convenience syntax)
-        pass
+        for resource_config in resource_configs:
+            if hasattr(resource_config, 'uri'):
+                uri = resource_config.uri
+                usage = resource_config.description or ""
+            else:
+                uri = resource_config.get("uri", "")
+                usage = resource_config.get("description", "")
+            if uri:
+                resource_uris.append((uri, usage))
+
+    # From tools section - detect URIs (anything with ://)
+    # This allows unified syntax: resources as tools
+    tool_configs = metadata.tools if metadata and hasattr(metadata, 'tools') else []
+    for tool_config in tool_configs:
+        if hasattr(tool_config, 'name'):
+            tool_name = tool_config.name
+            tool_desc = tool_config.description or ""
+        else:
+            tool_name = tool_config.get("name", "")
+            tool_desc = tool_config.get("description", "")
+
+        # Auto-detect resource URIs (anything with :// scheme)
+        if "://" in tool_name:
+            resource_uris.append((tool_name, tool_desc))
+
+    # Create tools from collected resource URIs
+    # Pass the loaded MCP server so resources can be resolved from it
+    for uri, usage in resource_uris:
+        resource_tool = create_resource_tool(uri, usage, mcp_server=loaded_mcp_server)
+        tools.append(resource_tool)
+        logger.debug(f"Loaded resource as tool: {uri}")
 
     # Create dynamic result_type from schema if not provided
     # Note: use_structured_output is set earlier from metadata.structured_output

rem/agentic/schema.py

@@ -154,8 +154,10 @@ class MCPServerConfig(BaseModel):
     )
 
     id: str = Field(
+        default="mcp-server",
         description=(
             "Server identifier for logging and debugging. "
+            "Defaults to 'mcp-server' if not specified. "
             "Example: 'rem-local'"
         )
     )
@@ -228,7 +230,8 @@ class AgentSchemaMetadata(BaseModel):
         description=(
             "MCP server configurations for dynamic tool loading. "
             "Servers are loaded in-process at agent creation time. "
-            "All tools from configured servers become available to the agent."
+            "All tools from configured servers become available to the agent. "
+            "If not specified, defaults to rem.mcp_server (REM's built-in tools)."
         ),
     )
 

rem/api/mcp_router/tools.py

@@ -606,7 +606,9 @@ async def register_metadata(
     references: list[str] | None = None,
     sources: list[str] | None = None,
     flags: list[str] | None = None,
-    # Risk assessment fields (used by mental health agents like Siggy)
+    # Session naming
+    session_name: str | None = None,
+    # Risk assessment fields (used by specialized agents)
     risk_level: str | None = None,
     risk_score: int | None = None,
     risk_reasoning: str | None = None,
@@ -639,6 +641,11 @@ async def register_metadata(
         flags: Optional flags for the response (e.g., "needs_review",
             "uncertain", "incomplete", "crisis_alert").
 
+        session_name: Short 1-3 phrase name describing the session topic.
+            Used by the UI to label conversations in the sidebar.
+            Examples: "Prescription Drug Questions", "AWS Setup Help",
+            "Python Code Review", "Travel Planning".
+
         risk_level: Risk level indicator (e.g., "green", "orange", "red").
             Used by mental health agents for C-SSRS style assessment.
         risk_score: Numeric risk score (e.g., 0-6 for C-SSRS).
@@ -663,7 +670,7 @@ async def register_metadata(
             sources=["REM database lookup"]
         )
 
-        # Mental health risk assessment (Siggy-style)
+        # Risk assessment example
         register_metadata(
             confidence=0.9,
             risk_level="green",
@@ -706,6 +713,10 @@ async def register_metadata(
         "flags": flags,
     }
 
+    # Add session name if provided
+    if session_name is not None:
+        result["session_name"] = session_name
+
     # Add risk assessment fields if provided
     if risk_level is not None:
         result["risk_level"] = risk_level

rem/api/routers/chat/completions.py

@@ -1,13 +1,94 @@
 """
 OpenAI-compatible chat completions router for REM.
 
-Design Pattern:
-- Headers map to AgentContext (X-User-Id, X-Tenant-Id, X-Session-Id, X-Agent-Schema)
+Quick Start (Local Development)
+===============================
+
+NOTE: Local dev uses LOCAL databases (Postgres via Docker Compose on port 5050).
+      Do NOT port-forward databases. Only port-forward observability services.
+
+IMPORTANT: Session IDs MUST be UUIDs. Non-UUID session IDs will cause message
+           storage issues and feedback will not work correctly.
+
+1. Port Forwarding (REQUIRED for trace capture and Phoenix sync):
+
+    # Terminal 1: OTEL Collector (HTTP) - sends traces to Phoenix
+    kubectl port-forward -n observability svc/otel-collector-collector 4318:4318
+
+    # Terminal 2: Phoenix UI - view traces at http://localhost:6006
+    kubectl port-forward -n siggy svc/phoenix 6006:6006
+
+2. Get Phoenix API Key (REQUIRED for feedback->Phoenix sync):
+
+    export PHOENIX_API_KEY=$(kubectl get secret -n siggy rem-phoenix-api-key \\
+      -o jsonpath='{.data.PHOENIX_API_KEY}' | base64 -d)
+
+3. Start API with OTEL and Phoenix enabled:
+
+    cd /path/to/remstack/rem
+    source .venv/bin/activate
+    OTEL__ENABLED=true \\
+    PHOENIX__ENABLED=true \\
+    PHOENIX_API_KEY="$PHOENIX_API_KEY" \\
+    uvicorn rem.api.main:app --host 0.0.0.0 --port 8000 --app-dir src
+
+4. Test Chat Request (session_id MUST be a UUID):
+
+    SESSION_ID=$(python3 -c "import uuid; print(uuid.uuid4())")
+    curl -s -N -X POST http://localhost:8000/api/v1/chat/completions \\
+      -H 'Content-Type: application/json' \\
+      -H "X-Session-Id: $SESSION_ID" \\
+      -H 'X-Agent-Schema: rem' \\
+      -d '{"messages": [{"role": "user", "content": "Hello"}], "stream": true}'
+
+    # Note: Use 'rem' agent schema (default) for real LLM responses.
+    # The 'simulator' agent is for testing SSE events without LLM calls.
+
+5. Submit Feedback on Response:
+
+    The metadata SSE event contains message_id and trace_id for feedback:
+        event: metadata
+        data: {"message_id": "728882f8-...", "trace_id": "e53c701c...", ...}
+
+    Use session_id (UUID you generated) and message_id to submit feedback:
+
+    curl -X POST http://localhost:8000/api/v1/messages/feedback \\
+      -H 'Content-Type: application/json' \\
+      -H 'X-Tenant-Id: default' \\
+      -d '{
+        "session_id": "<your-uuid-session-id>",
+        "message_id": "<message-id-from-metadata>",
+        "rating": 1,
+        "categories": ["helpful"],
+        "comment": "Good response"
+      }'
+
+    Expected response (201 = synced to Phoenix):
+        {"phoenix_synced": true, "trace_id": "e53c701c...", "span_id": "6432d497..."}
+
+OTEL Architecture
+=================
+
+    REM API --[OTLP/HTTP]--> OTEL Collector --[relay]--> Phoenix
+             (port 4318)    (k8s: observability)         (k8s: siggy)
+
+Environment Variables:
+    OTEL__ENABLED=true              Enable OTEL tracing (required for trace capture)
+    PHOENIX__ENABLED=true           Enable Phoenix integration (required for feedback sync)
+    PHOENIX_API_KEY=<jwt>           Phoenix API key (required for feedback->Phoenix sync)
+    OTEL__COLLECTOR_ENDPOINT        Default: http://localhost:4318
+    OTEL__PROTOCOL                  Default: http (use port 4318, not gRPC 4317)
+
+Design Pattern
+==============
+
+- Headers map to AgentContext (X-User-Id, X-Tenant-Id, X-Session-Id, X-Agent-Schema, X-Is-Eval)
 - ContextBuilder centralizes message construction with user profile + session history
 - Body.model is the LLM model for Pydantic AI
 - X-Agent-Schema header specifies which agent schema to use (defaults to 'rem')
 - Support for streaming (SSE) and non-streaming modes
 - Response format control (text vs json_object)
+- OpenAI-compatible body fields: metadata, store, reasoning_effort, etc.
 
 Context Building Flow:
 1. ContextBuilder.build_from_headers() extracts user_id, session_id from headers
@@ -25,9 +106,10 @@ Context Building Flow:
 Headers Mapping
     X-User-Id        → AgentContext.user_id
     X-Tenant-Id      → AgentContext.tenant_id
-    X-Session-Id     → AgentContext.session_id
+    X-Session-Id     → AgentContext.session_id (use UUID for new sessions)
     X-Model-Name     → AgentContext.default_model (overrides body.model)
     X-Agent-Schema   → AgentContext.agent_schema_uri (defaults to 'rem')
+    X-Is-Eval        → AgentContext.is_eval (sets session mode to EVALUATION)
 
 Default Agent:
     If X-Agent-Schema header is not provided, the system loads 'rem' schema,
@@ -42,6 +124,7 @@ Example Request:
     POST /api/v1/chat/completions
     X-Tenant-Id: acme-corp
     X-User-Id: user123
+    X-Session-Id: a1b2c3d4-e5f6-7890-abcd-ef1234567890  # UUID
     X-Agent-Schema: rem  # Optional, this is the default
 
     {
@@ -67,7 +150,9 @@ from loguru import logger
 from ....agentic.context import AgentContext
 from ....agentic.context_builder import ContextBuilder
 from ....agentic.providers.pydantic_ai import create_agent
+from ....models.entities.session import Session, SessionMode
 from ....services.audio.transcriber import AudioTranscriber
+from ....services.postgres.repository import Repository
 from ....services.session import SessionMessageStore, reload_session
 from ....settings import settings
 from ....utils.schema_loader import load_agent_schema, load_agent_schema_async
@@ -87,6 +172,105 @@ router = APIRouter(prefix="/api/v1", tags=["chat"])
 DEFAULT_AGENT_SCHEMA = "rem"
 
 
+def get_current_trace_context() -> tuple[str | None, str | None]:
+    """Get trace_id and span_id from current OTEL context.
+
+    Returns:
+        Tuple of (trace_id, span_id) as hex strings, or (None, None) if not available.
+    """
+    try:
+        from opentelemetry import trace
+        span = trace.get_current_span()
+        if span and span.get_span_context().is_valid:
+            ctx = span.get_span_context()
+            trace_id = format(ctx.trace_id, '032x')
+            span_id = format(ctx.span_id, '016x')
+            return trace_id, span_id
+    except Exception:
+        pass
+    return None, None
+
+
+def get_tracer():
+    """Get the OpenTelemetry tracer for chat completions."""
+    try:
+        from opentelemetry import trace
+        return trace.get_tracer("rem.chat.completions")
+    except Exception:
+        return None
+
+
+async def ensure_session_with_metadata(
+    session_id: str,
+    user_id: str | None,
+    tenant_id: str,
+    is_eval: bool,
+    request_metadata: dict[str, str] | None,
+    agent_schema: str | None = None,
+) -> None:
+    """
+    Ensure session exists and update with metadata/mode.
+
+    If X-Is-Eval header is true, sets session mode to EVALUATION.
+    Merges request metadata with existing session metadata.
+
+    Args:
+        session_id: Session identifier (maps to Session.name)
+        user_id: User identifier
+        tenant_id: Tenant identifier
+        is_eval: Whether this is an evaluation session
+        request_metadata: Metadata from request body to merge
+        agent_schema: Optional agent schema being used
+    """
+    if not settings.postgres.enabled:
+        return
+
+    try:
+        repo = Repository(Session, table_name="sessions")
+
+        # Try to load existing session by name (session_id is the name field)
+        existing_list = await repo.find(
+            filters={"name": session_id, "tenant_id": tenant_id},
+            limit=1,
+        )
+        existing = existing_list[0] if existing_list else None
+
+        if existing:
+            # Merge metadata if provided
+            merged_metadata = existing.metadata or {}
+            if request_metadata:
+                merged_metadata.update(request_metadata)
+
+            # Update session if eval flag or new metadata
+            needs_update = False
+            if is_eval and existing.mode != SessionMode.EVALUATION:
+                existing.mode = SessionMode.EVALUATION
+                needs_update = True
+            if request_metadata:
+                existing.metadata = merged_metadata
+                needs_update = True
+
+            if needs_update:
+                await repo.upsert(existing)
+                logger.debug(f"Updated session {session_id} (eval={is_eval}, metadata keys={list(merged_metadata.keys())})")
+        else:
+            # Create new session
+            session = Session(
+                name=session_id,
+                mode=SessionMode.EVALUATION if is_eval else SessionMode.NORMAL,
+                user_id=user_id,
+                tenant_id=tenant_id,
+                agent_schema_uri=agent_schema,
+                metadata=request_metadata or {},
+            )
+            await repo.upsert(session)
+            logger.info(f"Created session {session_id} (eval={is_eval})")
+
+    except Exception as e:
+        # Non-critical - log but don't fail the request
+        logger.error(f"Failed to ensure session metadata: {e}", exc_info=True)
+
+
 @router.post("/chat/completions", response_model=None)
 async def chat_completions(body: ChatCompletionRequest, request: Request):
     """
@@ -102,6 +286,17 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
     | X-Tenant-Id         | Tenant identifier (multi-tenancy)    | AgentContext.tenant_id         | "default"     |
     | X-Session-Id        | Session/conversation identifier      | AgentContext.session_id        | None          |
     | X-Agent-Schema      | Agent schema name                    | AgentContext.agent_schema_uri  | "rem"         |
+    | X-Is-Eval           | Mark as evaluation session           | AgentContext.is_eval           | false         |
+
+    Additional OpenAI-compatible Body Fields:
+    - metadata: Key-value pairs merged with session metadata (max 16 keys)
+    - store: Whether to store for distillation/evaluation
+    - max_completion_tokens: Max tokens to generate (replaces max_tokens)
+    - seed: Seed for deterministic sampling
+    - top_p: Nucleus sampling probability
+    - logprobs: Return log probabilities
+    - reasoning_effort: low/medium/high for o-series models
+    - service_tier: auto/flex/priority/default
 
     Example Models:
     - anthropic:claude-sonnet-4-5-20250929 (Claude 4.5 Sonnet)
@@ -127,6 +322,12 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
     - If CHAT__AUTO_INJECT_USER_CONTEXT=true: User profile auto-loaded and injected
     - New messages saved to database with compression for session continuity
     - When Postgres is disabled, session management is skipped
+
+    Evaluation Sessions:
+    - Set X-Is-Eval: true header to mark session as evaluation
+    - Session mode will be set to EVALUATION
+    - Request metadata is merged with session metadata
+    - Useful for A/B testing, model comparison, and feedback collection
     """
     # Load agent schema: use header value from context or default
     # Extract AgentContext first to get schema name
@@ -151,6 +352,17 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
             new_messages=new_messages,
         )
 
+        # Ensure session exists with metadata and eval mode if applicable
+        if context.session_id:
+            await ensure_session_with_metadata(
+                session_id=context.session_id,
+                user_id=context.user_id,
+                tenant_id=context.tenant_id,
+                is_eval=context.is_eval,
+                request_metadata=body.metadata,
+                agent_schema="simulator",
+            )
+
         # Get the last user message as prompt
         prompt = body.messages[-1].content if body.messages else "demo"
         request_id = f"sim-{uuid.uuid4().hex[:24]}"
@@ -301,6 +513,17 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
 
     logger.info(f"Built context with {len(messages)} total messages (includes history + user context)")
 
+    # Ensure session exists with metadata and eval mode if applicable
+    if context.session_id:
+        await ensure_session_with_metadata(
+            session_id=context.session_id,
+            user_id=context.user_id,
+            tenant_id=context.tenant_id,
+            is_eval=context.is_eval,
+            request_metadata=body.metadata,
+            agent_schema=schema_name,
+        )
+
     # Create agent with schema and model override
     agent = await create_agent(
         context=context,
@@ -351,7 +574,26 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
         )
 
     # Non-streaming mode
-    result = await agent.run(prompt)
+    # Create a parent span to capture trace context for message storage
+    trace_id, span_id = None, None
+    tracer = get_tracer()
+
+    if tracer:
+        with tracer.start_as_current_span(
+            "chat_completion",
+            attributes={
+                "session.id": context.session_id or "",
+                "user.id": context.user_id or "",
+                "model": body.model,
+                "agent.schema": context.agent_schema_uri or DEFAULT_AGENT_SCHEMA,
+            }
+        ) as span:
+            # Capture trace context from the span we just created
+            trace_id, span_id = get_current_trace_context()
+            result = await agent.run(prompt)
+    else:
+        # No tracer available, run without tracing
+        result = await agent.run(prompt)
 
     # Determine content format based on response_format request
     if body.response_format and body.response_format.type == "json_object":
@@ -374,12 +616,16 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
             "role": "user",
             "content": body.messages[-1].content if body.messages else "",
             "timestamp": datetime.utcnow().isoformat(),
+            "trace_id": trace_id,
+            "span_id": span_id,
         }
 
         assistant_message = {
             "role": "assistant",
             "content": content,
             "timestamp": datetime.utcnow().isoformat(),
+            "trace_id": trace_id,
+            "span_id": span_id,
         }
 
         try: