remdb 0.3.127__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/agentic/providers/pydantic_ai.py

@@ -96,6 +96,35 @@ TODO:
 
     Priority: HIGH (blocks production scaling beyond 50 req/sec)
 
+    4. Response Format Control (structured_output enhancement):
+       - Current: structured_output is bool (True=strict schema, False=free-form text)
+       - Missing: OpenAI JSON mode (valid JSON without strict schema enforcement)
+       - Missing: Completions API support (some models only support completions, not chat)
+
+       Proposed schema field values for `structured_output`:
+         - True (default): Strict structured output using provider's native schema support
+         - False: Free-form text response (properties converted to prompt guidance)
+         - "json": JSON mode - ensures valid JSON but no schema enforcement
+                   (OpenAI: response_format={"type": "json_object"})
+         - "text": Explicit free-form text (alias for False)
+
+       Implementation:
+         a) Update AgentSchemaMetadata.structured_output type:
+            structured_output: bool | Literal["json", "text"] = True
+         b) In create_agent(), handle each mode:
+            - True: Use output_type with Pydantic model (current behavior)
+            - False/"text": Convert properties to prompt guidance (current behavior)
+            - "json": Use provider's JSON mode without strict schema
+         c) Provider-specific JSON mode:
+            - OpenAI: model_settings={"response_format": {"type": "json_object"}}
+            - Anthropic: Not supported natively, use prompt guidance
+            - Others: Fallback to prompt guidance with JSON instruction
+
+       Related: Some providers (Cerebras) have completions-only models where
+       structured output isn't available. Consider model capability detection.
+
+       Priority: MEDIUM (enables more flexible output control)
+
 Example Agent Schema:
 {
   "type": "object",
@@ -550,10 +579,18 @@ async def create_agent(
     # Extract schema fields using typed helpers
     from ..schema import get_system_prompt, get_metadata
 
+    # Track whether mcp_servers was explicitly configured (even if empty)
+    mcp_servers_explicitly_set = False
+
     if agent_schema:
         system_prompt = get_system_prompt(agent_schema)
         metadata = get_metadata(agent_schema)
-        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') else []
+        # Check if mcp_servers was explicitly set (could be empty list to disable)
+        if hasattr(metadata, 'mcp_servers') and metadata.mcp_servers is not None:
+            mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers]
+            mcp_servers_explicitly_set = True
+        else:
+            mcp_server_configs = []
         resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
 
         if metadata.system_prompt:
@@ -564,15 +601,49 @@ async def create_agent(
         mcp_server_configs = []
         resource_configs = []
 
+    # Auto-detect local MCP server if not explicitly configured
+    # This makes mcp_servers config optional - agents get tools automatically
+    # But if mcp_servers: [] is explicitly set, respect that (no auto-detection)
+    if not mcp_server_configs and not mcp_servers_explicitly_set:
+        import importlib
+        import os
+        import sys
+
+        # Ensure current working directory is in sys.path for local imports
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+
+        # Try common local MCP server module paths first
+        auto_detect_modules = [
+            "tools.mcp_server",  # Convention: tools/mcp_server.py
+            "mcp_server",        # Alternative: mcp_server.py in root
+        ]
+        for module_path in auto_detect_modules:
+            try:
+                mcp_module = importlib.import_module(module_path)
+                if hasattr(mcp_module, "mcp"):
+                    logger.info(f"Auto-detected local MCP server: {module_path}")
+                    mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
+                    break
+            except ImportError:
+                continue
+
+        # Fall back to REM's default MCP server if no local server found
+        if not mcp_server_configs:
+            logger.debug("No local MCP server found, using REM default")
+            mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
+
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
     if metadata:
         temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
         max_iterations = metadata.override_max_iterations if metadata.override_max_iterations is not None else settings.llm.default_max_iterations
-        use_structured_output = metadata.structured_output
+        # Use schema-level structured_output if set, otherwise fall back to global setting
+        use_structured_output = metadata.structured_output if metadata.structured_output is not None else settings.llm.default_structured_output
     else:
         temperature = settings.llm.default_temperature
         max_iterations = settings.llm.default_max_iterations
-        use_structured_output = True
+        use_structured_output = settings.llm.default_structured_output
 
     # Build list of tools - start with built-in tools
     tools = _get_builtin_tools()
@@ -608,50 +679,97 @@ async def create_agent(
             search_rem_suffix += f"Example: `SEARCH \"your query\" FROM {default_table} LIMIT 10`"
 
     # Add tools from MCP server (in-process, no subprocess)
-    if mcp_server_configs:
-        for server_config in mcp_server_configs:
-            server_type = server_config.get("type")
-            server_id = server_config.get("id", "mcp-server")
-
-            if server_type == "local":
-                # Import MCP server directly (in-process)
-                module_path = server_config.get("module", "rem.mcp_server")
-
-                try:
-                    # Dynamic import of MCP server module
-                    import importlib
-                    mcp_module = importlib.import_module(module_path)
-                    mcp_server = mcp_module.mcp
-
-                    # Extract tools from MCP server (get_tools is async)
-                    from ..mcp.tool_wrapper import create_mcp_tool_wrapper
-
-                    # Await async get_tools() call
-                    mcp_tools_dict = await mcp_server.get_tools()
-
-                    for tool_name, tool_func in mcp_tools_dict.items():
-                        # Add description suffix to search_rem tool if schema specifies a default table
-                        tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
-
-                        wrapped_tool = create_mcp_tool_wrapper(
-                            tool_name,
-                            tool_func,
-                            user_id=context.user_id if context else None,
-                            description_suffix=tool_suffix,
-                        )
-                        tools.append(wrapped_tool)
-                        logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
-
-                    logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
-
-                except Exception as e:
-                    logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
-            else:
-                logger.warning(f"Unsupported MCP server type: {server_type}")
+    # Track loaded MCP servers for resource resolution
+    loaded_mcp_server = None
+
+    for server_config in mcp_server_configs:
+        server_type = server_config.get("type")
+        server_id = server_config.get("id", "mcp-server")
+
+        if server_type == "local":
+            # Import MCP server directly (in-process)
+            module_path = server_config.get("module", "rem.mcp_server")
+
+            try:
+                # Dynamic import of MCP server module
+                import importlib
+                mcp_module = importlib.import_module(module_path)
+                mcp_server = mcp_module.mcp
+
+                # Store the loaded server for resource resolution
+                loaded_mcp_server = mcp_server
+
+                # Extract tools from MCP server (get_tools is async)
+                from ..mcp.tool_wrapper import create_mcp_tool_wrapper
+
+                # Await async get_tools() call
+                mcp_tools_dict = await mcp_server.get_tools()
 
+                for tool_name, tool_func in mcp_tools_dict.items():
+                    # Add description suffix to search_rem tool if schema specifies a default table
+                    tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
+
+                    wrapped_tool = create_mcp_tool_wrapper(
+                        tool_name,
+                        tool_func,
+                        user_id=context.user_id if context else None,
+                        description_suffix=tool_suffix,
+                    )
+                    tools.append(wrapped_tool)
+                    logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
+
+                logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
+
+            except Exception as e:
+                logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
+        else:
+            logger.warning(f"Unsupported MCP server type: {server_type}")
+
+    # Convert resources to tools (MCP convenience syntax)
+    # Resources declared in agent YAML become callable tools - eliminates
+    # the artificial MCP distinction between tools and resources
+    #
+    # Supports both concrete and template URIs:
+    # - Concrete: "rem://schemas" -> no-param tool
+    # - Template: "patient-profile://field/{field_key}" -> tool with field_key param
+    from ..mcp.tool_wrapper import create_resource_tool
+
+    # Collect all resource URIs from both resources section AND tools section
+    resource_uris = []
+
+    # From resources section (legacy format)
     if resource_configs:
-        # TODO: Convert resources to tools (MCP convenience syntax)
-        pass
+        for resource_config in resource_configs:
+            if hasattr(resource_config, 'uri'):
+                uri = resource_config.uri
+                usage = resource_config.description or ""
+            else:
+                uri = resource_config.get("uri", "")
+                usage = resource_config.get("description", "")
+            if uri:
+                resource_uris.append((uri, usage))
+
+    # From tools section - detect URIs (anything with ://)
+    # This allows unified syntax: resources as tools
+    tool_configs = metadata.tools if metadata and hasattr(metadata, 'tools') else []
+    for tool_config in tool_configs:
+        if hasattr(tool_config, 'name'):
+            tool_name = tool_config.name
+            tool_desc = tool_config.description or ""
+        else:
+            tool_name = tool_config.get("name", "")
+            tool_desc = tool_config.get("description", "")
+
+        # Auto-detect resource URIs (anything with :// scheme)
+        if "://" in tool_name:
+            resource_uris.append((tool_name, tool_desc))
+
+    # Create tools from collected resource URIs
+    # Pass the loaded MCP server so resources can be resolved from it
+    for uri, usage in resource_uris:
+        resource_tool = create_resource_tool(uri, usage, mcp_server=loaded_mcp_server)
+        tools.append(resource_tool)
+        logger.debug(f"Loaded resource as tool: {uri}")
 
     # Create dynamic result_type from schema if not provided
     # Note: use_structured_output is set earlier from metadata.structured_output

rem/agentic/schema.py

@@ -154,8 +154,10 @@ class MCPServerConfig(BaseModel):
     )
 
     id: str = Field(
+        default="mcp-server",
         description=(
             "Server identifier for logging and debugging. "
+            "Defaults to 'mcp-server' if not specified. "
             "Example: 'rem-local'"
         )
     )
@@ -213,12 +215,13 @@ class AgentSchemaMetadata(BaseModel):
     )
 
     # Structured output toggle
-    structured_output: bool = Field(
-        default=True,
+    structured_output: bool | None = Field(
+        default=None,
         description=(
             "Whether to enforce structured JSON output. "
             "When False, the agent produces free-form text and schema properties "
-            "are converted to prompt guidance instead. Default: True (JSON output)."
+            "are converted to prompt guidance instead. "
+            "Default: None (uses LLM__DEFAULT_STRUCTURED_OUTPUT setting, which defaults to False)."
         ),
     )
 
@@ -228,7 +231,8 @@ class AgentSchemaMetadata(BaseModel):
         description=(
             "MCP server configurations for dynamic tool loading. "
             "Servers are loaded in-process at agent creation time. "
-            "All tools from configured servers become available to the agent."
+            "All tools from configured servers become available to the agent. "
+            "If not specified, defaults to rem.mcp_server (REM's built-in tools)."
         ),
     )
 

rem/api/deps.py

@@ -147,7 +147,6 @@ def is_admin(user: dict | None) -> bool:
 async def get_user_filter(
     request: Request,
     x_user_id: str | None = None,
-    x_tenant_id: str = "default",
 ) -> dict[str, Any]:
     """
     Get user-scoped filter dict for database queries.
@@ -158,7 +157,6 @@ async def get_user_filter(
     Args:
         request: FastAPI request
         x_user_id: Optional user_id filter (admin only for cross-user)
-        x_tenant_id: Tenant ID for multi-tenancy
 
     Returns:
         Filter dict with appropriate user_id constraint
@@ -169,7 +167,7 @@ async def get_user_filter(
             return await repo.find(filters)
     """
     user = get_current_user(request)
-    filters: dict[str, Any] = {"tenant_id": x_tenant_id}
+    filters: dict[str, Any] = {}
 
     if is_admin(user):
         # Admin can filter by any user or see all
@@ -185,8 +183,8 @@ async def get_user_filter(
                 f"User {user.get('email')} attempted to filter by user_id={x_user_id}"
             )
     else:
-        # Anonymous: could use anonymous tracking ID or restrict access
-        # For now, anonymous can't access user-scoped data
+        # Anonymous: use anonymous tracking ID
+        # Note: user_id should come from JWT, not from parameters
         anon_id = getattr(request.state, "anon_id", None)
         if anon_id:
             filters["user_id"] = f"anon:{anon_id}"

rem/api/main.py

@@ -149,19 +149,38 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
         client_host = request.client.host if request.client else "unknown"
         user_agent = request.headers.get('user-agent', 'unknown')[:100]
 
+        # Extract auth info for logging (first 8 chars of token for debugging)
+        auth_header = request.headers.get('authorization', '')
+        auth_preview = ""
+        if auth_header.startswith('Bearer '):
+            token = auth_header[7:]
+            auth_preview = f"Bearer {token[:8]}..." if len(token) > 8 else f"Bearer {token}"
+
         # Process request
         response = await call_next(request)
 
+        # Extract user info set by auth middleware (after processing)
+        user = getattr(request.state, "user", None)
+        user_id = user.get("id", "none")[:12] if user else "anon"
+        user_email = user.get("email", "") if user else ""
+
         # Determine log level based on path AND response status
         duration_ms = (time.time() - start_time) * 1000
         use_debug = self._should_log_at_debug(path, response.status_code)
         log_fn = logger.debug if use_debug else logger.info
 
-        # Log request and response together
+        # Build user info string
+        user_info = f"user={user_id}"
+        if user_email:
+            user_info += f" ({user_email})"
+        if auth_preview:
+            user_info += f" | auth={auth_preview}"
+
+        # Log request and response together with auth info
         log_fn(
             f"→ REQUEST: {request.method} {path} | "
             f"Client: {client_host} | "
-            f"User-Agent: {user_agent}"
+            f"{user_info}"
         )
         log_fn(
             f"← RESPONSE: {request.method} {path} | "
@@ -304,7 +323,7 @@ def create_app() -> FastAPI:
     app.add_middleware(
         AuthMiddleware,
         protected_paths=["/api/v1"],
-        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth"],
+        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth", "/api/v1/slack"],
         # Allow anonymous when auth is disabled, otherwise use setting
         allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,
         # MCP requires auth only when auth is fully enabled

rem/api/mcp_router/resources.py

@@ -512,9 +512,10 @@ async def load_resource(uri: str) -> dict | str:
     Raises:
         ValueError: If URI is invalid or resource not found
     """
-    # Create temporary MCP instance with resources
+    import inspect
     from fastmcp import FastMCP
 
+    # Create temporary MCP instance with resources
     mcp = FastMCP(name="temp")
 
     # Register all resources
@@ -523,14 +524,18 @@ async def load_resource(uri: str) -> dict | str:
     register_file_resources(mcp)
     register_status_resources(mcp)
 
-    # Get resource handlers from MCP internal registry
-    # FastMCP stores resources in a dict by URI
-    if hasattr(mcp, "_resources"):
-        if uri in mcp._resources:
-            handler = mcp._resources[uri]
-            if callable(handler):
-                result = handler()
-                return result if result else {"error": "Resource returned None"}
+    # Get resources using FastMCP's async get_resources() method
+    resources = await mcp.get_resources()
+
+    if uri in resources:
+        resource = resources[uri]
+        # Call the underlying function
+        result = resource.fn()
+        # Handle async functions
+        if inspect.iscoroutine(result):
+            result = await result
+        return result if result else {"error": "Resource returned None"}
 
     # If not found, raise error
-    raise ValueError(f"Resource not found: {uri}. Available resources: {list(mcp._resources.keys()) if hasattr(mcp, '_resources') else 'unknown'}")
+    available = list(resources.keys())
+    raise ValueError(f"Resource not found: {uri}. Available resources: {available}")

rem/api/mcp_router/server.py

@@ -182,6 +182,7 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
         list_schema,
         read_resource,
         register_metadata,
+        save_agent,
         search_rem,
     )
 
@@ -191,6 +192,7 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     mcp.tool()(register_metadata)
     mcp.tool()(list_schema)
     mcp.tool()(get_schema)
+    mcp.tool()(save_agent)
 
     # File ingestion tool (with local path support for local servers)
     # Wrap to inject is_local parameter

rem/api/mcp_router/tools.py

@@ -116,7 +116,8 @@ def mcp_tool_error_handler(func: Callable) -> Callable:
             # Otherwise wrap in success response
             return {"status": "success", **result}
         except Exception as e:
-            logger.error(f"{func.__name__} failed: {e}", exc_info=True)
+            # Use %s format to avoid issues with curly braces in error messages
+            logger.opt(exception=True).error("{} failed: {}", func.__name__, str(e))
             return {
                 "status": "error",
                 "error": str(e),
@@ -380,9 +381,10 @@ async def ask_rem_agent(
     from ...utils.schema_loader import load_agent_schema
 
     # Create agent context
+    # Note: tenant_id defaults to "default" if user_id is None
     context = AgentContext(
         user_id=user_id,
-        tenant_id=user_id,  # Set tenant_id to user_id for backward compat
+        tenant_id=user_id or "default",  # Use default tenant for anonymous users
         default_model=settings.llm.default_model,
     )
 
@@ -1040,3 +1042,93 @@ async def get_schema(
     logger.info(f"Retrieved schema for table '{table_name}' with {len(column_defs)} columns")
 
     return result
+
+
+@mcp_tool_error_handler
+async def save_agent(
+    name: str,
+    description: str,
+    properties: dict[str, Any] | None = None,
+    required: list[str] | None = None,
+    tools: list[str] | None = None,
+    tags: list[str] | None = None,
+    version: str = "1.0.0",
+    user_id: str | None = None,
+) -> dict[str, Any]:
+    """
+    Save an agent schema to REM, making it available for use.
+
+    This tool creates or updates an agent definition in the user's schema space.
+    The agent becomes immediately available for conversations.
+
+    **Default Tools**: All agents automatically get `search_rem` and `register_metadata`
+    tools unless explicitly overridden.
+
+    Args:
+        name: Agent name in kebab-case (e.g., "code-reviewer", "sales-assistant").
+            Must be unique within the user's schema space.
+        description: The agent's system prompt. This is the full instruction set
+            that defines the agent's behavior, personality, and capabilities.
+            Use markdown formatting for structure.
+        properties: Output schema properties as a dict. Each property should have:
+            - type: "string", "number", "boolean", "array", "object"
+            - description: What this field captures
+            Example: {"answer": {"type": "string", "description": "Response to user"}}
+            If not provided, defaults to a simple {"answer": {"type": "string"}} schema.
+        required: List of required property names. Defaults to ["answer"] if not provided.
+        tools: List of tool names the agent can use. Defaults to ["search_rem", "register_metadata"].
+        tags: Optional tags for categorizing the agent.
+        version: Semantic version string (default: "1.0.0").
+        user_id: User identifier for scoping. Uses authenticated user if not provided.
+
+    Returns:
+        Dict with:
+        - status: "success" or "error"
+        - agent_name: Name of the saved agent
+        - version: Version saved
+        - message: Human-readable status
+
+    Examples:
+        # Create a simple agent
+        save_agent(
+            name="greeting-bot",
+            description="You are a friendly greeter. Say hello warmly.",
+            properties={"answer": {"type": "string", "description": "Greeting message"}},
+            required=["answer"]
+        )
+
+        # Create agent with structured output
+        save_agent(
+            name="sentiment-analyzer",
+            description="Analyze sentiment of text provided by the user.",
+            properties={
+                "answer": {"type": "string", "description": "Analysis explanation"},
+                "sentiment": {"type": "string", "enum": ["positive", "negative", "neutral"]},
+                "confidence": {"type": "number", "minimum": 0, "maximum": 1}
+            },
+            required=["answer", "sentiment"],
+            tags=["analysis", "nlp"]
+        )
+    """
+    from ...agentic.agents.agent_manager import save_agent as _save_agent
+
+    # Get user_id from context if not provided
+    user_id = AgentContext.get_user_id_or_default(user_id, source="save_agent")
+
+    # Delegate to agent_manager
+    result = await _save_agent(
+        name=name,
+        description=description,
+        user_id=user_id,
+        properties=properties,
+        required=required,
+        tools=tools,
+        tags=tags,
+        version=version,
+    )
+
+    # Add helpful message for Slack users
+    if result.get("status") == "success":
+        result["message"] = f"Agent '{name}' saved. Use `/custom-agent {name}` to chat with it."
+
+    return result

rem/api/middleware/tracking.py

@@ -102,14 +102,14 @@ class AnonymousTrackingMiddleware(BaseHTTPMiddleware):
             # Tenant ID from header or default
             tenant_id = request.headers.get("X-Tenant-Id", "default")
 
-        # 4. Rate Limiting
-        if settings.postgres.enabled:
+        # 4. Rate Limiting (skip if disabled via settings)
+        if settings.postgres.enabled and settings.api.rate_limit_enabled:
             is_allowed, current, limit = await self.rate_limiter.check_rate_limit(
                 tenant_id=tenant_id,
                 identifier=identifier,
                 tier=tier
             )
-            
+
             if not is_allowed:
                 return JSONResponse(
                     status_code=429,
@@ -141,8 +141,8 @@ class AnonymousTrackingMiddleware(BaseHTTPMiddleware):
                 secure=settings.environment == "production"
             )
             
-        # Add Rate Limit headers
-        if settings.postgres.enabled and 'limit' in locals():
+        # Add Rate Limit headers (only if rate limiting is enabled)
+        if settings.postgres.enabled and settings.api.rate_limit_enabled and 'limit' in locals():
             response.headers["X-RateLimit-Limit"] = str(limit)
             response.headers["X-RateLimit-Remaining"] = str(max(0, limit - current))