remdb 0.3.14__py3-none-any.whl → 0.3.157__py3-none-any.whl

rem/agentic/providers/pydantic_ai.py

@@ -175,6 +175,23 @@ class AgentRuntime:
         return self.agent.iter(*args, **kwargs)
 
 
+def _get_builtin_tools() -> list:
+    """
+    Get built-in tools that are always available to agents.
+
+    Currently returns empty list - all tools come from MCP servers.
+    The register_metadata tool is available via the REM MCP server and
+    agents can opt-in by configuring mcp_servers in their schema.
+
+    Returns:
+        List of Pydantic AI tool functions (currently empty)
+    """
+    # NOTE: register_metadata is now an MCP tool, not a built-in.
+    # Agents that want it should configure mcp_servers to load from rem.mcp_server.
+    # This allows agents to choose which tools they need.
+    return []
+
+
 def _create_model_from_schema(agent_schema: dict[str, Any]) -> type[BaseModel]:
     """
     Create Pydantic model dynamically from JSON Schema.
@@ -303,6 +320,68 @@ def _prepare_schema_for_qwen(schema: dict[str, Any]) -> dict[str, Any]:
     return schema_copy
 
 
+def _convert_properties_to_prompt(properties: dict[str, Any]) -> str:
+    """
+    Convert schema properties to prompt guidance text.
+
+    When structured_output is disabled, this converts the properties
+    definition into natural language guidance that informs the agent
+    about the expected response structure without forcing JSON output.
+
+    Args:
+        properties: JSON Schema properties dict
+
+    Returns:
+        Prompt text describing the expected response elements
+
+    Example:
+        properties = {
+            "answer": {"type": "string", "description": "The answer"},
+            "confidence": {"type": "number", "description": "Confidence 0-1"}
+        }
+        # Returns:
+        # "## Response Structure\n\nYour response should include:\n- **answer**: The answer\n..."
+    """
+    if not properties:
+        return ""
+
+    lines = ["## Response Guidelines", "", "Your response should address the following elements:"]
+
+    for field_name, field_def in properties.items():
+        field_type = field_def.get("type", "any")
+        description = field_def.get("description", "")
+
+        # Format based on type
+        if field_type == "array":
+            type_hint = "list"
+        elif field_type == "number":
+            type_hint = "number"
+            # Include min/max if specified
+            if "minimum" in field_def or "maximum" in field_def:
+                min_val = field_def.get("minimum", "")
+                max_val = field_def.get("maximum", "")
+                if min_val != "" and max_val != "":
+                    type_hint = f"number ({min_val}-{max_val})"
+        elif field_type == "boolean":
+            type_hint = "yes/no"
+        else:
+            type_hint = field_type
+
+        # Build field description
+        field_line = f"- **{field_name}**"
+        if type_hint and type_hint != "string":
+            field_line += f" ({type_hint})"
+        if description:
+            field_line += f": {description}"
+
+        lines.append(field_line)
+
+    lines.append("")
+    lines.append("Respond naturally in prose, addressing these elements where relevant.")
+
+    return "\n".join(lines)
+
+
 def _create_schema_wrapper(
     result_type: type[BaseModel], strip_description: bool = True
 ) -> type[BaseModel]:
@@ -462,23 +541,80 @@ async def create_agent(
         # agent_schema = load_agent_schema(context.agent_schema_uri)
         pass
 
-    # Determine model: override > context.default_model > settings
-    model = (
-        model_override or (context.default_model if context else settings.llm.default_model)
-    )
+    # Determine model: validate override against allowed list, fallback to context or settings
+    from rem.agentic.llm_provider_models import get_valid_model_or_default
 
-    # Extract schema fields
-    system_prompt = agent_schema.get("description", "") if agent_schema else ""
-    metadata = agent_schema.get("json_schema_extra", {}) if agent_schema else {}
-    mcp_server_configs = metadata.get("mcp_servers", [])
-    resource_configs = metadata.get("resources", [])
+    default_model = context.default_model if context else settings.llm.default_model
+    model = get_valid_model_or_default(model_override, default_model)
+
+    # Extract schema fields using typed helpers
+    from ..schema import get_system_prompt, get_metadata
+
+    if agent_schema:
+        system_prompt = get_system_prompt(agent_schema)
+        metadata = get_metadata(agent_schema)
+        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') and metadata.mcp_servers else []
+        resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
+
+        if metadata.system_prompt:
+            logger.debug("Using custom system_prompt from json_schema_extra")
+    else:
+        system_prompt = ""
+        metadata = None
+        mcp_server_configs = []
+        resource_configs = []
+
+    # Auto-detect local MCP server if not explicitly configured
+    # This makes mcp_servers config optional - agents get tools automatically
+    if not mcp_server_configs:
+        import importlib
+        import os
+        import sys
+
+        # Ensure current working directory is in sys.path for local imports
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+
+        # Try common local MCP server module paths first
+        auto_detect_modules = [
+            "tools.mcp_server",  # Convention: tools/mcp_server.py
+            "mcp_server",        # Alternative: mcp_server.py in root
+        ]
+        for module_path in auto_detect_modules:
+            try:
+                mcp_module = importlib.import_module(module_path)
+                if hasattr(mcp_module, "mcp"):
+                    logger.info(f"Auto-detected local MCP server: {module_path}")
+                    mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
+                    break
+            except ImportError:
+                continue
+
+        # Fall back to REM's default MCP server if no local server found
+        if not mcp_server_configs:
+            logger.debug("No local MCP server found, using REM default")
+            mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
 
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
-    temperature = metadata.get("override_temperature", settings.llm.default_temperature)
-    max_iterations = metadata.get("override_max_iterations", settings.llm.default_max_iterations)
+    if metadata:
+        temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
+        max_iterations = metadata.override_max_iterations if metadata.override_max_iterations is not None else settings.llm.default_max_iterations
+        use_structured_output = metadata.structured_output
+    else:
+        temperature = settings.llm.default_temperature
+        max_iterations = settings.llm.default_max_iterations
+        use_structured_output = True
+
+    # Build list of tools - start with built-in tools
+    tools = _get_builtin_tools()
+
+    # Get agent name from metadata for logging
+    agent_name = metadata.name if metadata and hasattr(metadata, 'name') else "unknown"
 
     logger.info(
-        f"Creating agent: model={model}, mcp_servers={len(mcp_server_configs)}, resources={len(resource_configs)}"
+        f"Creating agent '{agent_name}': model={model}, mcp_servers={len(mcp_server_configs)}, "
+        f"resources={len(resource_configs)}, builtin_tools={len(tools)}"
     )
 
     # Set agent resource attributes for OTEL (before creating agent)
@@ -487,53 +623,131 @@ async def create_agent(
 
         set_agent_resource_attributes(agent_schema=agent_schema)
 
-    # Build list of tools from MCP server (in-process, no subprocess)
-    tools = []
-    if mcp_server_configs:
-        for server_config in mcp_server_configs:
-            server_type = server_config.get("type")
-            server_id = server_config.get("id", "mcp-server")
-
-            if server_type == "local":
-                # Import MCP server directly (in-process)
-                module_path = server_config.get("module", "rem.mcp_server")
-
-                try:
-                    # Dynamic import of MCP server module
-                    import importlib
-                    mcp_module = importlib.import_module(module_path)
-                    mcp_server = mcp_module.mcp
-
-                    # Extract tools from MCP server (get_tools is async)
-                    from ..mcp.tool_wrapper import create_mcp_tool_wrapper
-
-                    # Await async get_tools() call
-                    mcp_tools_dict = await mcp_server.get_tools()
-
-                    for tool_name, tool_func in mcp_tools_dict.items():
-                        wrapped_tool = create_mcp_tool_wrapper(tool_name, tool_func, user_id=context.user_id if context else None)
-                        tools.append(wrapped_tool)
-                        logger.debug(f"Loaded MCP tool: {tool_name}")
-
-                    logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
-
-                except Exception as e:
-                    logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
-            else:
-                logger.warning(f"Unsupported MCP server type: {server_type}")
-
+    # Extract schema metadata for search_rem tool description suffix
+    # This allows entity schemas to add context-specific notes to the search_rem tool
+    search_rem_suffix = None
+    if metadata:
+        # Check for default_search_table in metadata (set by entity schemas)
+        extra = agent_schema.get("json_schema_extra", {}) if agent_schema else {}
+        default_table = extra.get("default_search_table")
+        has_embeddings = extra.get("has_embeddings", False)
+
+        if default_table:
+            # Build description suffix for search_rem
+            search_rem_suffix = f"\n\nFor this schema, use `search_rem` to query `{default_table}`. "
+            if has_embeddings:
+                search_rem_suffix += f"SEARCH works well on {default_table} (has embeddings). "
+            search_rem_suffix += f"Example: `SEARCH \"your query\" FROM {default_table} LIMIT 10`"
+
+    # Add tools from MCP server (in-process, no subprocess)
+    # Track loaded MCP servers for resource resolution
+    loaded_mcp_server = None
+
+    for server_config in mcp_server_configs:
+        server_type = server_config.get("type")
+        server_id = server_config.get("id", "mcp-server")
+
+        if server_type == "local":
+            # Import MCP server directly (in-process)
+            module_path = server_config.get("module", "rem.mcp_server")
+
+            try:
+                # Dynamic import of MCP server module
+                import importlib
+                mcp_module = importlib.import_module(module_path)
+                mcp_server = mcp_module.mcp
+
+                # Store the loaded server for resource resolution
+                loaded_mcp_server = mcp_server
+
+                # Extract tools from MCP server (get_tools is async)
+                from ..mcp.tool_wrapper import create_mcp_tool_wrapper
+
+                # Await async get_tools() call
+                mcp_tools_dict = await mcp_server.get_tools()
+
+                for tool_name, tool_func in mcp_tools_dict.items():
+                    # Add description suffix to search_rem tool if schema specifies a default table
+                    tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
+
+                    wrapped_tool = create_mcp_tool_wrapper(
+                        tool_name,
+                        tool_func,
+                        user_id=context.user_id if context else None,
+                        description_suffix=tool_suffix,
+                    )
+                    tools.append(wrapped_tool)
+                    logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
+
+                logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
+
+            except Exception as e:
+                logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
+        else:
+            logger.warning(f"Unsupported MCP server type: {server_type}")
+
+    # Convert resources to tools (MCP convenience syntax)
+    # Resources declared in agent YAML become callable tools - eliminates
+    # the artificial MCP distinction between tools and resources
+    #
+    # Supports both concrete and template URIs:
+    # - Concrete: "rem://schemas" -> no-param tool
+    # - Template: "patient-profile://field/{field_key}" -> tool with field_key param
+    from ..mcp.tool_wrapper import create_resource_tool
+
+    # Collect all resource URIs from both resources section AND tools section
+    resource_uris = []
+
+    # From resources section (legacy format)
     if resource_configs:
-        # TODO: Convert resources to tools (MCP convenience syntax)
-        pass
+        for resource_config in resource_configs:
+            if hasattr(resource_config, 'uri'):
+                uri = resource_config.uri
+                usage = resource_config.description or ""
+            else:
+                uri = resource_config.get("uri", "")
+                usage = resource_config.get("description", "")
+            if uri:
+                resource_uris.append((uri, usage))
+
+    # From tools section - detect URIs (anything with ://)
+    # This allows unified syntax: resources as tools
+    tool_configs = metadata.tools if metadata and hasattr(metadata, 'tools') else []
+    for tool_config in tool_configs:
+        if hasattr(tool_config, 'name'):
+            tool_name = tool_config.name
+            tool_desc = tool_config.description or ""
+        else:
+            tool_name = tool_config.get("name", "")
+            tool_desc = tool_config.get("description", "")
+
+        # Auto-detect resource URIs (anything with :// scheme)
+        if "://" in tool_name:
+            resource_uris.append((tool_name, tool_desc))
+
+    # Create tools from collected resource URIs
+    # Pass the loaded MCP server so resources can be resolved from it
+    for uri, usage in resource_uris:
+        resource_tool = create_resource_tool(uri, usage, mcp_server=loaded_mcp_server)
+        tools.append(resource_tool)
+        logger.debug(f"Loaded resource as tool: {uri}")
 
     # Create dynamic result_type from schema if not provided
+    # Note: use_structured_output is set earlier from metadata.structured_output
     if result_type is None and agent_schema and "properties" in agent_schema:
-        # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
-        # This ensures the generated Pydantic model doesn't have incompatible constraints
-        sanitized_schema = _prepare_schema_for_qwen(agent_schema)
-        result_type = _create_model_from_schema(sanitized_schema)
-        logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
-        logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
+        if use_structured_output:
+            # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
+            # This ensures the generated Pydantic model doesn't have incompatible constraints
+            sanitized_schema = _prepare_schema_for_qwen(agent_schema)
+            result_type = _create_model_from_schema(sanitized_schema)
+            logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
+        else:
+            # Convert properties to prompt guidance instead of structured output
+            # This informs the agent about expected response structure without forcing it
+            properties_prompt = _convert_properties_to_prompt(agent_schema.get("properties", {}))
+            if properties_prompt:
+                system_prompt = system_prompt + "\n\n" + properties_prompt
+            logger.debug("Structured output disabled - properties converted to prompt guidance")
 
     # Create agent with optional output_type for structured output and tools
     if result_type:
@@ -541,21 +755,30 @@ async def create_agent(
         wrapped_result_type = _create_schema_wrapper(
             result_type, strip_description=strip_model_description
         )
+        # Use InstrumentationSettings with version=3 to include agent name in span names
+        from pydantic_ai.models.instrumented import InstrumentationSettings
+        instrumentation = InstrumentationSettings(version=3) if settings.otel.enabled else False
+
         agent = Agent(
             model=model,
+            name=agent_name,  # Used for OTEL span names (version 3: "invoke_agent {name}")
             system_prompt=system_prompt,
             output_type=wrapped_result_type,
             tools=tools,
-            instrument=settings.otel.enabled,  # Conditional OTEL instrumentation
+            instrument=instrumentation,
             model_settings={"temperature": temperature},
             retries=settings.llm.max_retries,
         )
     else:
+        from pydantic_ai.models.instrumented import InstrumentationSettings
+        instrumentation = InstrumentationSettings(version=3) if settings.otel.enabled else False
+
         agent = Agent(
             model=model,
+            name=agent_name,  # Used for OTEL span names (version 3: "invoke_agent {name}")
             system_prompt=system_prompt,
             tools=tools,
-            instrument=settings.otel.enabled,
+            instrument=instrumentation,
             model_settings={"temperature": temperature},
             retries=settings.llm.max_retries,
         )