remdb 0.3.114__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/agentic/providers/pydantic_ai.py

@@ -96,6 +96,35 @@ TODO:
 
     Priority: HIGH (blocks production scaling beyond 50 req/sec)
 
+    4. Response Format Control (structured_output enhancement):
+       - Current: structured_output is bool (True=strict schema, False=free-form text)
+       - Missing: OpenAI JSON mode (valid JSON without strict schema enforcement)
+       - Missing: Completions API support (some models only support completions, not chat)
+
+       Proposed schema field values for `structured_output`:
+         - True (default): Strict structured output using provider's native schema support
+         - False: Free-form text response (properties converted to prompt guidance)
+         - "json": JSON mode - ensures valid JSON but no schema enforcement
+                   (OpenAI: response_format={"type": "json_object"})
+         - "text": Explicit free-form text (alias for False)
+
+       Implementation:
+         a) Update AgentSchemaMetadata.structured_output type:
+            structured_output: bool | Literal["json", "text"] = True
+         b) In create_agent(), handle each mode:
+            - True: Use output_type with Pydantic model (current behavior)
+            - False/"text": Convert properties to prompt guidance (current behavior)
+            - "json": Use provider's JSON mode without strict schema
+         c) Provider-specific JSON mode:
+            - OpenAI: model_settings={"response_format": {"type": "json_object"}}
+            - Anthropic: Not supported natively, use prompt guidance
+            - Others: Fallback to prompt guidance with JSON instruction
+
+       Related: Some providers (Cerebras) have completions-only models where
+       structured output isn't available. Consider model capability detection.
+
+       Priority: MEDIUM (enables more flexible output control)
+
 Example Agent Schema:
 {
   "type": "object",
@@ -550,10 +579,18 @@ async def create_agent(
     # Extract schema fields using typed helpers
     from ..schema import get_system_prompt, get_metadata
 
+    # Track whether mcp_servers was explicitly configured (even if empty)
+    mcp_servers_explicitly_set = False
+
     if agent_schema:
         system_prompt = get_system_prompt(agent_schema)
         metadata = get_metadata(agent_schema)
-        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') else []
+        # Check if mcp_servers was explicitly set (could be empty list to disable)
+        if hasattr(metadata, 'mcp_servers') and metadata.mcp_servers is not None:
+            mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers]
+            mcp_servers_explicitly_set = True
+        else:
+            mcp_server_configs = []
         resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
 
         if metadata.system_prompt:
@@ -564,15 +601,49 @@ async def create_agent(
         mcp_server_configs = []
         resource_configs = []
 
+    # Auto-detect local MCP server if not explicitly configured
+    # This makes mcp_servers config optional - agents get tools automatically
+    # But if mcp_servers: [] is explicitly set, respect that (no auto-detection)
+    if not mcp_server_configs and not mcp_servers_explicitly_set:
+        import importlib
+        import os
+        import sys
+
+        # Ensure current working directory is in sys.path for local imports
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+
+        # Try common local MCP server module paths first
+        auto_detect_modules = [
+            "tools.mcp_server",  # Convention: tools/mcp_server.py
+            "mcp_server",        # Alternative: mcp_server.py in root
+        ]
+        for module_path in auto_detect_modules:
+            try:
+                mcp_module = importlib.import_module(module_path)
+                if hasattr(mcp_module, "mcp"):
+                    logger.info(f"Auto-detected local MCP server: {module_path}")
+                    mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
+                    break
+            except ImportError:
+                continue
+
+        # Fall back to REM's default MCP server if no local server found
+        if not mcp_server_configs:
+            logger.debug("No local MCP server found, using REM default")
+            mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
+
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
     if metadata:
         temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
         max_iterations = metadata.override_max_iterations if metadata.override_max_iterations is not None else settings.llm.default_max_iterations
-        use_structured_output = metadata.structured_output
+        # Use schema-level structured_output if set, otherwise fall back to global setting
+        use_structured_output = metadata.structured_output if metadata.structured_output is not None else settings.llm.default_structured_output
     else:
         temperature = settings.llm.default_temperature
         max_iterations = settings.llm.default_max_iterations
-        use_structured_output = True
+        use_structured_output = settings.llm.default_structured_output
 
     # Build list of tools - start with built-in tools
     tools = _get_builtin_tools()
@@ -591,43 +662,114 @@ async def create_agent(
 
         set_agent_resource_attributes(agent_schema=agent_schema)
 
+    # Extract schema metadata for search_rem tool description suffix
+    # This allows entity schemas to add context-specific notes to the search_rem tool
+    search_rem_suffix = None
+    if metadata:
+        # Check for default_search_table in metadata (set by entity schemas)
+        extra = agent_schema.get("json_schema_extra", {}) if agent_schema else {}
+        default_table = extra.get("default_search_table")
+        has_embeddings = extra.get("has_embeddings", False)
+
+        if default_table:
+            # Build description suffix for search_rem
+            search_rem_suffix = f"\n\nFor this schema, use `search_rem` to query `{default_table}`. "
+            if has_embeddings:
+                search_rem_suffix += f"SEARCH works well on {default_table} (has embeddings). "
+            search_rem_suffix += f"Example: `SEARCH \"your query\" FROM {default_table} LIMIT 10`"
+
     # Add tools from MCP server (in-process, no subprocess)
-    if mcp_server_configs:
-        for server_config in mcp_server_configs:
-            server_type = server_config.get("type")
-            server_id = server_config.get("id", "mcp-server")
+    # Track loaded MCP servers for resource resolution
+    loaded_mcp_server = None
 
-            if server_type == "local":
-                # Import MCP server directly (in-process)
-                module_path = server_config.get("module", "rem.mcp_server")
+    for server_config in mcp_server_configs:
+        server_type = server_config.get("type")
+        server_id = server_config.get("id", "mcp-server")
 
-                try:
-                    # Dynamic import of MCP server module
-                    import importlib
-                    mcp_module = importlib.import_module(module_path)
-                    mcp_server = mcp_module.mcp
+        if server_type == "local":
+            # Import MCP server directly (in-process)
+            module_path = server_config.get("module", "rem.mcp_server")
 
-                    # Extract tools from MCP server (get_tools is async)
-                    from ..mcp.tool_wrapper import create_mcp_tool_wrapper
+            try:
+                # Dynamic import of MCP server module
+                import importlib
+                mcp_module = importlib.import_module(module_path)
+                mcp_server = mcp_module.mcp
 
-                    # Await async get_tools() call
-                    mcp_tools_dict = await mcp_server.get_tools()
+                # Store the loaded server for resource resolution
+                loaded_mcp_server = mcp_server
 
-                    for tool_name, tool_func in mcp_tools_dict.items():
-                        wrapped_tool = create_mcp_tool_wrapper(tool_name, tool_func, user_id=context.user_id if context else None)
-                        tools.append(wrapped_tool)
-                        logger.debug(f"Loaded MCP tool: {tool_name}")
+                # Extract tools from MCP server (get_tools is async)
+                from ..mcp.tool_wrapper import create_mcp_tool_wrapper
 
-                    logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
+                # Await async get_tools() call
+                mcp_tools_dict = await mcp_server.get_tools()
 
-                except Exception as e:
-                    logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
-            else:
-                logger.warning(f"Unsupported MCP server type: {server_type}")
+                for tool_name, tool_func in mcp_tools_dict.items():
+                    # Add description suffix to search_rem tool if schema specifies a default table
+                    tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
+
+                    wrapped_tool = create_mcp_tool_wrapper(
+                        tool_name,
+                        tool_func,
+                        user_id=context.user_id if context else None,
+                        description_suffix=tool_suffix,
+                    )
+                    tools.append(wrapped_tool)
+                    logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
 
+                logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
+
+            except Exception as e:
+                logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
+        else:
+            logger.warning(f"Unsupported MCP server type: {server_type}")
+
+    # Convert resources to tools (MCP convenience syntax)
+    # Resources declared in agent YAML become callable tools - eliminates
+    # the artificial MCP distinction between tools and resources
+    #
+    # Supports both concrete and template URIs:
+    # - Concrete: "rem://schemas" -> no-param tool
+    # - Template: "patient-profile://field/{field_key}" -> tool with field_key param
+    from ..mcp.tool_wrapper import create_resource_tool
+
+    # Collect all resource URIs from both resources section AND tools section
+    resource_uris = []
+
+    # From resources section (legacy format)
     if resource_configs:
-        # TODO: Convert resources to tools (MCP convenience syntax)
-        pass
+        for resource_config in resource_configs:
+            if hasattr(resource_config, 'uri'):
+                uri = resource_config.uri
+                usage = resource_config.description or ""
+            else:
+                uri = resource_config.get("uri", "")
+                usage = resource_config.get("description", "")
+            if uri:
+                resource_uris.append((uri, usage))
+
+    # From tools section - detect URIs (anything with ://)
+    # This allows unified syntax: resources as tools
+    tool_configs = metadata.tools if metadata and hasattr(metadata, 'tools') else []
+    for tool_config in tool_configs:
+        if hasattr(tool_config, 'name'):
+            tool_name = tool_config.name
+            tool_desc = tool_config.description or ""
+        else:
+            tool_name = tool_config.get("name", "")
+            tool_desc = tool_config.get("description", "")
+
+        # Auto-detect resource URIs (anything with :// scheme)
+        if "://" in tool_name:
+            resource_uris.append((tool_name, tool_desc))
+
+    # Create tools from collected resource URIs
+    # Pass the loaded MCP server so resources can be resolved from it
+    for uri, usage in resource_uris:
+        resource_tool = create_resource_tool(uri, usage, mcp_server=loaded_mcp_server)
+        tools.append(resource_tool)
+        logger.debug(f"Loaded resource as tool: {uri}")
 
     # Create dynamic result_type from schema if not provided
     # Note: use_structured_output is set earlier from metadata.structured_output

rem/agentic/schema.py

@@ -154,8 +154,10 @@ class MCPServerConfig(BaseModel):
     )
 
     id: str = Field(
+        default="mcp-server",
         description=(
             "Server identifier for logging and debugging. "
+            "Defaults to 'mcp-server' if not specified. "
             "Example: 'rem-local'"
         )
     )
@@ -213,12 +215,13 @@ class AgentSchemaMetadata(BaseModel):
     )
 
     # Structured output toggle
-    structured_output: bool = Field(
-        default=True,
+    structured_output: bool | None = Field(
+        default=None,
         description=(
             "Whether to enforce structured JSON output. "
             "When False, the agent produces free-form text and schema properties "
-            "are converted to prompt guidance instead. Default: True (JSON output)."
+            "are converted to prompt guidance instead. "
+            "Default: None (uses LLM__DEFAULT_STRUCTURED_OUTPUT setting, which defaults to False)."
         ),
     )
 
@@ -228,7 +231,8 @@ class AgentSchemaMetadata(BaseModel):
         description=(
             "MCP server configurations for dynamic tool loading. "
             "Servers are loaded in-process at agent creation time. "
-            "All tools from configured servers become available to the agent."
+            "All tools from configured servers become available to the agent. "
+            "If not specified, defaults to rem.mcp_server (REM's built-in tools)."
         ),
     )
 

rem/api/deps.py

@@ -147,7 +147,6 @@ def is_admin(user: dict | None) -> bool:
 async def get_user_filter(
     request: Request,
     x_user_id: str | None = None,
-    x_tenant_id: str = "default",
 ) -> dict[str, Any]:
     """
     Get user-scoped filter dict for database queries.
@@ -158,7 +157,6 @@ async def get_user_filter(
     Args:
         request: FastAPI request
         x_user_id: Optional user_id filter (admin only for cross-user)
-        x_tenant_id: Tenant ID for multi-tenancy
 
     Returns:
         Filter dict with appropriate user_id constraint
@@ -169,7 +167,7 @@ async def get_user_filter(
             return await repo.find(filters)
     """
     user = get_current_user(request)
-    filters: dict[str, Any] = {"tenant_id": x_tenant_id}
+    filters: dict[str, Any] = {}
 
     if is_admin(user):
         # Admin can filter by any user or see all
@@ -185,8 +183,8 @@ async def get_user_filter(
                 f"User {user.get('email')} attempted to filter by user_id={x_user_id}"
             )
     else:
-        # Anonymous: could use anonymous tracking ID or restrict access
-        # For now, anonymous can't access user-scoped data
+        # Anonymous: use anonymous tracking ID
+        # Note: user_id should come from JWT, not from parameters
         anon_id = getattr(request.state, "anon_id", None)
         if anon_id:
             filters["user_id"] = f"anon:{anon_id}"

rem/api/main.py

@@ -149,19 +149,38 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
         client_host = request.client.host if request.client else "unknown"
         user_agent = request.headers.get('user-agent', 'unknown')[:100]
 
+        # Extract auth info for logging (first 8 chars of token for debugging)
+        auth_header = request.headers.get('authorization', '')
+        auth_preview = ""
+        if auth_header.startswith('Bearer '):
+            token = auth_header[7:]
+            auth_preview = f"Bearer {token[:8]}..." if len(token) > 8 else f"Bearer {token}"
+
         # Process request
         response = await call_next(request)
 
+        # Extract user info set by auth middleware (after processing)
+        user = getattr(request.state, "user", None)
+        user_id = user.get("id", "none")[:12] if user else "anon"
+        user_email = user.get("email", "") if user else ""
+
         # Determine log level based on path AND response status
         duration_ms = (time.time() - start_time) * 1000
         use_debug = self._should_log_at_debug(path, response.status_code)
         log_fn = logger.debug if use_debug else logger.info
 
-        # Log request and response together
+        # Build user info string
+        user_info = f"user={user_id}"
+        if user_email:
+            user_info += f" ({user_email})"
+        if auth_preview:
+            user_info += f" | auth={auth_preview}"
+
+        # Log request and response together with auth info
         log_fn(
             f"→ REQUEST: {request.method} {path} | "
             f"Client: {client_host} | "
-            f"User-Agent: {user_agent}"
+            f"{user_info}"
         )
         log_fn(
             f"← RESPONSE: {request.method} {path} | "
@@ -304,7 +323,7 @@ def create_app() -> FastAPI:
     app.add_middleware(
         AuthMiddleware,
         protected_paths=["/api/v1"],
-        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth"],
+        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth", "/api/v1/slack"],
         # Allow anonymous when auth is disabled, otherwise use setting
         allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,
         # MCP requires auth only when auth is fully enabled
@@ -376,10 +395,13 @@ def create_app() -> FastAPI:
 
     app.include_router(chat_router)
     app.include_router(models_router)
+    # shared_sessions_router MUST be before messages_router
+    # because messages_router has /sessions/{session_id} which would match
+    # before the more specific /sessions/shared-with-me routes
+    app.include_router(shared_sessions_router)
     app.include_router(messages_router)
     app.include_router(feedback_router)
     app.include_router(admin_router)
-    app.include_router(shared_sessions_router)
     app.include_router(query_router)
 
     # Register auth router (if enabled)

rem/api/mcp_router/resources.py

@@ -512,9 +512,10 @@ async def load_resource(uri: str) -> dict | str:
     Raises:
         ValueError: If URI is invalid or resource not found
     """
-    # Create temporary MCP instance with resources
+    import inspect
     from fastmcp import FastMCP
 
+    # Create temporary MCP instance with resources
     mcp = FastMCP(name="temp")
 
     # Register all resources
@@ -523,14 +524,18 @@ async def load_resource(uri: str) -> dict | str:
     register_file_resources(mcp)
     register_status_resources(mcp)
 
-    # Get resource handlers from MCP internal registry
-    # FastMCP stores resources in a dict by URI
-    if hasattr(mcp, "_resources"):
-        if uri in mcp._resources:
-            handler = mcp._resources[uri]
-            if callable(handler):
-                result = handler()
-                return result if result else {"error": "Resource returned None"}
+    # Get resources using FastMCP's async get_resources() method
+    resources = await mcp.get_resources()
+
+    if uri in resources:
+        resource = resources[uri]
+        # Call the underlying function
+        result = resource.fn()
+        # Handle async functions
+        if inspect.iscoroutine(result):
+            result = await result
+        return result if result else {"error": "Resource returned None"}
 
     # If not found, raise error
-    raise ValueError(f"Resource not found: {uri}. Available resources: {list(mcp._resources.keys()) if hasattr(mcp, '_resources') else 'unknown'}")
+    available = list(resources.keys())
+    raise ValueError(f"Resource not found: {uri}. Available resources: {available}")

rem/api/mcp_router/server.py

@@ -127,10 +127,12 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
             "AVAILABLE TOOLS\n"
             "═══════════════════════════════════════════════════════════════════════════\n"
             "\n"
-            "• rem_query - Execute REM queries (LOOKUP, FUZZY, SEARCH, SQL, TRAVERSE)\n"
-            "• ask_rem - Natural language to REM query conversion\n"
+            "• search_rem - Execute REM queries (LOOKUP, FUZZY, SEARCH, SQL, TRAVERSE)\n"
+            "• ask_rem_agent - Natural language to REM query conversion\n"
             "  - plan_mode=True: Hints agent to use TRAVERSE with depth=0 for edge analysis\n"
-            "• parse_and_ingest_file - Ingest files from local paths (local server only), s3://, or https://\n"
+            "• ingest_into_rem - Ingest files from local paths (local server only), s3://, or https://\n"
+            "• list_schema - List all database schemas (tables) with row counts\n"
+            "• get_schema - Get detailed schema for a specific table (columns, types, indexes)\n"
             "\n"
             "═══════════════════════════════════════════════════════════════════════════\n"
             "AVAILABLE RESOURCES (Read-Only)\n"
@@ -175,9 +177,12 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     # Register REM tools
     from .tools import (
         ask_rem_agent,
+        get_schema,
         ingest_into_rem,
+        list_schema,
         read_resource,
         register_metadata,
+        save_agent,
         search_rem,
     )
 
@@ -185,6 +190,9 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     mcp.tool()(ask_rem_agent)
     mcp.tool()(read_resource)
     mcp.tool()(register_metadata)
+    mcp.tool()(list_schema)
+    mcp.tool()(get_schema)
+    mcp.tool()(save_agent)
 
     # File ingestion tool (with local path support for local servers)
     # Wrap to inject is_local parameter