remdb 0.3.0__py3-none-any.whl → 0.3.114__py3-none-any.whl

rem/agentic/providers/pydantic_ai.py

@@ -175,6 +175,23 @@ class AgentRuntime:
         return self.agent.iter(*args, **kwargs)
 
 
+def _get_builtin_tools() -> list:
+    """
+    Get built-in tools that are always available to agents.
+
+    Currently returns empty list - all tools come from MCP servers.
+    The register_metadata tool is available via the REM MCP server and
+    agents can opt-in by configuring mcp_servers in their schema.
+
+    Returns:
+        List of Pydantic AI tool functions (currently empty)
+    """
+    # NOTE: register_metadata is now an MCP tool, not a built-in.
+    # Agents that want it should configure mcp_servers to load from rem.mcp_server.
+    # This allows agents to choose which tools they need.
+    return []
+
+
 def _create_model_from_schema(agent_schema: dict[str, Any]) -> type[BaseModel]:
     """
     Create Pydantic model dynamically from JSON Schema.
@@ -303,6 +320,68 @@ def _prepare_schema_for_qwen(schema: dict[str, Any]) -> dict[str, Any]:
     return schema_copy
 
 
+def _convert_properties_to_prompt(properties: dict[str, Any]) -> str:
+    """
+    Convert schema properties to prompt guidance text.
+
+    When structured_output is disabled, this converts the properties
+    definition into natural language guidance that informs the agent
+    about the expected response structure without forcing JSON output.
+
+    Args:
+        properties: JSON Schema properties dict
+
+    Returns:
+        Prompt text describing the expected response elements
+
+    Example:
+        properties = {
+            "answer": {"type": "string", "description": "The answer"},
+            "confidence": {"type": "number", "description": "Confidence 0-1"}
+        }
+        # Returns:
+        # "## Response Structure\n\nYour response should include:\n- **answer**: The answer\n..."
+    """
+    if not properties:
+        return ""
+
+    lines = ["## Response Guidelines", "", "Your response should address the following elements:"]
+
+    for field_name, field_def in properties.items():
+        field_type = field_def.get("type", "any")
+        description = field_def.get("description", "")
+
+        # Format based on type
+        if field_type == "array":
+            type_hint = "list"
+        elif field_type == "number":
+            type_hint = "number"
+            # Include min/max if specified
+            if "minimum" in field_def or "maximum" in field_def:
+                min_val = field_def.get("minimum", "")
+                max_val = field_def.get("maximum", "")
+                if min_val != "" and max_val != "":
+                    type_hint = f"number ({min_val}-{max_val})"
+        elif field_type == "boolean":
+            type_hint = "yes/no"
+        else:
+            type_hint = field_type
+
+        # Build field description
+        field_line = f"- **{field_name}**"
+        if type_hint and type_hint != "string":
+            field_line += f" ({type_hint})"
+        if description:
+            field_line += f": {description}"
+
+        lines.append(field_line)
+
+    lines.append("")
+    lines.append("Respond naturally in prose, addressing these elements where relevant.")
+
+    return "\n".join(lines)
+
+
 def _create_schema_wrapper(
     result_type: type[BaseModel], strip_description: bool = True
 ) -> type[BaseModel]:
@@ -462,23 +541,48 @@ async def create_agent(
         # agent_schema = load_agent_schema(context.agent_schema_uri)
         pass
 
-    # Determine model: override > context.default_model > settings
-    model = (
-        model_override or (context.default_model if context else settings.llm.default_model)
-    )
+    # Determine model: validate override against allowed list, fallback to context or settings
+    from rem.agentic.llm_provider_models import get_valid_model_or_default
+
+    default_model = context.default_model if context else settings.llm.default_model
+    model = get_valid_model_or_default(model_override, default_model)
+
+    # Extract schema fields using typed helpers
+    from ..schema import get_system_prompt, get_metadata
 
-    # Extract schema fields
-    system_prompt = agent_schema.get("description", "") if agent_schema else ""
-    metadata = agent_schema.get("json_schema_extra", {}) if agent_schema else {}
-    mcp_server_configs = metadata.get("mcp_servers", [])
-    resource_configs = metadata.get("resources", [])
+    if agent_schema:
+        system_prompt = get_system_prompt(agent_schema)
+        metadata = get_metadata(agent_schema)
+        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') else []
+        resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
+
+        if metadata.system_prompt:
+            logger.debug("Using custom system_prompt from json_schema_extra")
+    else:
+        system_prompt = ""
+        metadata = None
+        mcp_server_configs = []
+        resource_configs = []
 
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
-    temperature = metadata.get("override_temperature", settings.llm.default_temperature)
-    max_iterations = metadata.get("override_max_iterations", settings.llm.default_max_iterations)
+    if metadata:
+        temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
+        max_iterations = metadata.override_max_iterations if metadata.override_max_iterations is not None else settings.llm.default_max_iterations
+        use_structured_output = metadata.structured_output
+    else:
+        temperature = settings.llm.default_temperature
+        max_iterations = settings.llm.default_max_iterations
+        use_structured_output = True
+
+    # Build list of tools - start with built-in tools
+    tools = _get_builtin_tools()
+
+    # Get agent name from metadata for logging
+    agent_name = metadata.name if metadata and hasattr(metadata, 'name') else "unknown"
 
     logger.info(
-        f"Creating agent: model={model}, mcp_servers={len(mcp_server_configs)}, resources={len(resource_configs)}"
+        f"Creating agent '{agent_name}': model={model}, mcp_servers={len(mcp_server_configs)}, "
+        f"resources={len(resource_configs)}, builtin_tools={len(tools)}"
     )
 
     # Set agent resource attributes for OTEL (before creating agent)
@@ -487,8 +591,7 @@ async def create_agent(
 
         set_agent_resource_attributes(agent_schema=agent_schema)
 
-    # Build list of tools from MCP server (in-process, no subprocess)
-    tools = []
+    # Add tools from MCP server (in-process, no subprocess)
     if mcp_server_configs:
         for server_config in mcp_server_configs:
             server_type = server_config.get("type")
@@ -527,13 +630,21 @@ async def create_agent(
         pass
 
     # Create dynamic result_type from schema if not provided
+    # Note: use_structured_output is set earlier from metadata.structured_output
     if result_type is None and agent_schema and "properties" in agent_schema:
-        # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
-        # This ensures the generated Pydantic model doesn't have incompatible constraints
-        sanitized_schema = _prepare_schema_for_qwen(agent_schema)
-        result_type = _create_model_from_schema(sanitized_schema)
-        logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
-        logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
+        if use_structured_output:
+            # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
+            # This ensures the generated Pydantic model doesn't have incompatible constraints
+            sanitized_schema = _prepare_schema_for_qwen(agent_schema)
+            result_type = _create_model_from_schema(sanitized_schema)
+            logger.debug(f"Created dynamic Pydantic model: {result_type.__name__}")
+        else:
+            # Convert properties to prompt guidance instead of structured output
+            # This informs the agent about expected response structure without forcing it
+            properties_prompt = _convert_properties_to_prompt(agent_schema.get("properties", {}))
+            if properties_prompt:
+                system_prompt = system_prompt + "\n\n" + properties_prompt
+            logger.debug("Structured output disabled - properties converted to prompt guidance")
 
     # Create agent with optional output_type for structured output and tools
     if result_type:
@@ -541,21 +652,30 @@ async def create_agent(
         wrapped_result_type = _create_schema_wrapper(
             result_type, strip_description=strip_model_description
         )
+        # Use InstrumentationSettings with version=3 to include agent name in span names
+        from pydantic_ai.models.instrumented import InstrumentationSettings
+        instrumentation = InstrumentationSettings(version=3) if settings.otel.enabled else False
+
         agent = Agent(
             model=model,
+            name=agent_name,  # Used for OTEL span names (version 3: "invoke_agent {name}")
             system_prompt=system_prompt,
             output_type=wrapped_result_type,
             tools=tools,
-            instrument=settings.otel.enabled,  # Conditional OTEL instrumentation
+            instrument=instrumentation,
             model_settings={"temperature": temperature},
             retries=settings.llm.max_retries,
         )
     else:
+        from pydantic_ai.models.instrumented import InstrumentationSettings
+        instrumentation = InstrumentationSettings(version=3) if settings.otel.enabled else False
+
         agent = Agent(
             model=model,
+            name=agent_name,  # Used for OTEL span names (version 3: "invoke_agent {name}")
             system_prompt=system_prompt,
             tools=tools,
-            instrument=settings.otel.enabled,
+            instrument=instrumentation,
             model_settings={"temperature": temperature},
             retries=settings.llm.max_retries,
         )

rem/agentic/schema.py

@@ -13,7 +13,7 @@ The schema protocol serves as:
 """
 
 from typing import Any, Literal
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 
 
 class MCPToolReference(BaseModel):
@@ -23,11 +23,21 @@ class MCPToolReference(BaseModel):
     Tools are functions that agents can call during execution to
     interact with external systems, retrieve data, or perform actions.
 
-    Example:
+    Two usage patterns:
+    1. With mcp_servers config: Just declare name + description, tools loaded from MCP servers
+    2. Explicit MCP server: Specify mcp_server to load tool from specific server
+
+    Example (declarative with mcp_servers):
+        {
+            "name": "search_rem",
+            "description": "Execute REM queries for entity lookup and search"
+        }
+
+    Example (explicit server):
         {
             "name": "lookup_entity",
             "mcp_server": "rem",
-            "description": "Lookup entities by exact key with O(1) performance"
+            "description": "Lookup entities by exact key"
         }
     """
 
@@ -38,20 +48,20 @@ class MCPToolReference(BaseModel):
         )
     )
 
-    mcp_server: str = Field(
+    mcp_server: str | None = Field(
+        default=None,
         description=(
-            "MCP server identifier. Resolved via environment variable: "
-            "MCP_SERVER_{NAME} or MCP__{NAME}__URL. "
-            "Common values: 'rem' (REM knowledge graph), 'filesystem', 'web'."
+            "MCP server identifier (optional when using mcp_servers config). "
+            "If not specified, tool is expected from configured mcp_servers. "
+            "Resolved via environment variable: MCP_SERVER_{NAME} or MCP__{NAME}__URL."
         )
     )
 
     description: str | None = Field(
         default=None,
         description=(
-            "Optional description override. If provided, replaces the tool's "
-            "description from the MCP server in the agent's context. "
-            "Use this to provide agent-specific guidance on tool usage."
+            "Tool description for the agent. Explains what the tool does "
+            "and when to use it. This is visible to the LLM."
         ),
     )
 
@@ -63,29 +73,90 @@ class MCPResourceReference(BaseModel):
     Resources are data sources that can be read by agents, such as
     knowledge graph entities, files, or API endpoints.
 
-    Example:
+    Two formats supported:
+    1. uri: Exact URI or URI with query params
+    2. uri_pattern: Regex pattern for flexible matching
+
+    Example (exact URI):
+        {
+            "uri": "rem://schemas",
+            "name": "Agent Schemas",
+            "description": "List all available agent schemas"
+        }
+
+    Example (pattern):
         {
             "uri_pattern": "rem://resources/.*",
             "mcp_server": "rem"
         }
     """
 
-    uri_pattern: str = Field(
+    # Support both exact URI and pattern
+    uri: str | None = Field(
+        default=None,
+        description=(
+            "Exact resource URI or URI with query parameters. "
+            "Examples: 'rem://schemas', 'rem://resources?category=drug.*'"
+        )
+    )
+
+    uri_pattern: str | None = Field(
+        default=None,
         description=(
             "Regex pattern matching resource URIs. "
-            "Examples: "
-            "'rem://resources/.*' (all resources), "
-            "'rem://moments/.*' (all moments), "
-            "'file:///data/.*' (local files). "
-            "Supports full regex syntax for flexible matching."
+            "Examples: 'rem://resources/.*' (all resources). "
+            "Use uri for exact URIs, uri_pattern for regex matching."
+        )
+    )
+
+    name: str | None = Field(
+        default=None,
+        description="Human-readable name for the resource."
+    )
+
+    description: str | None = Field(
+        default=None,
+        description="Description of what the resource provides."
+    )
+
+    mcp_server: str | None = Field(
+        default=None,
+        description=(
+            "MCP server identifier (optional when using mcp_servers config). "
+            "Resolved via environment variable MCP_SERVER_{NAME}."
+        )
+    )
+
+
+class MCPServerConfig(BaseModel):
+    """
+    MCP server configuration for in-process tool loading.
+
+    Example:
+        {
+            "type": "local",
+            "module": "rem.mcp_server",
+            "id": "rem-local"
+        }
+    """
+
+    type: Literal["local"] = Field(
+        default="local",
+        description="Server type. Currently only 'local' (in-process) is supported.",
+    )
+
+    module: str = Field(
+        description=(
+            "Python module path containing the MCP server. "
+            "The module must export an 'mcp' object that supports get_tools(). "
+            "Example: 'rem.mcp_server'"
         )
     )
 
-    mcp_server: str = Field(
+    id: str = Field(
         description=(
-            "MCP server identifier that provides these resources. "
-            "Resolved via environment variable MCP_SERVER_{NAME}. "
-            "The server must expose resources matching the uri_pattern."
+            "Server identifier for logging and debugging. "
+            "Example: 'rem-local'"
         )
     )
 
@@ -130,6 +201,37 @@ class AgentSchemaMetadata(BaseModel):
         ),
     )
 
+    # System prompt override (takes precedence over description when present)
+    system_prompt: str | None = Field(
+        default=None,
+        description=(
+            "Custom system prompt that overrides or extends the schema description. "
+            "When present, this is combined with the main schema.description field "
+            "to form the complete system prompt. Use this for detailed instructions "
+            "that you don't want in the public schema description."
+        ),
+    )
+
+    # Structured output toggle
+    structured_output: bool = Field(
+        default=True,
+        description=(
+            "Whether to enforce structured JSON output. "
+            "When False, the agent produces free-form text and schema properties "
+            "are converted to prompt guidance instead. Default: True (JSON output)."
+        ),
+    )
+
+    # MCP server configurations (for dynamic tool loading)
+    mcp_servers: list[MCPServerConfig] = Field(
+        default_factory=list,
+        description=(
+            "MCP server configurations for dynamic tool loading. "
+            "Servers are loaded in-process at agent creation time. "
+            "All tools from configured servers become available to the agent."
+        ),
+    )
+
     tools: list[MCPToolReference] = Field(
         default_factory=list,
         description=(
@@ -394,3 +496,238 @@ def create_agent_schema(
         json_schema_extra=metadata.model_dump(),
         **kwargs,
     )
+
+
+# =============================================================================
+# YAML and Database Serialization
+# =============================================================================
+
+
+def schema_to_dict(schema: AgentSchema, exclude_none: bool = True) -> dict[str, Any]:
+    """
+    Serialize AgentSchema to a dictionary suitable for YAML or database storage.
+
+    This produces the canonical format used in:
+    - YAML files (schemas/agents/*.yaml)
+    - Database spec column (schemas table)
+    - API responses
+
+    Args:
+        schema: AgentSchema instance to serialize
+        exclude_none: If True, omit None values from output
+
+    Returns:
+        Dictionary representation of the schema
+
+    Example:
+        >>> schema = AgentSchema(
+        ...     description="System prompt...",
+        ...     properties={"answer": {"type": "string"}},
+        ...     json_schema_extra={"name": "my-agent", "structured_output": False}
+        ... )
+        >>> d = schema_to_dict(schema)
+        >>> d["json_schema_extra"]["name"]
+        "my-agent"
+    """
+    return schema.model_dump(exclude_none=exclude_none)
+
+
+def schema_from_dict(data: dict[str, Any]) -> AgentSchema:
+    """
+    Deserialize a dictionary to AgentSchema.
+
+    This handles:
+    - YAML files loaded with yaml.safe_load()
+    - Database spec column (JSON)
+    - API request bodies
+
+    Args:
+        data: Dictionary containing schema data
+
+    Returns:
+        Validated AgentSchema instance
+
+    Raises:
+        ValidationError: If data doesn't match schema structure
+
+    Example:
+        >>> data = {"type": "object", "description": "...", "properties": {}, "json_schema_extra": {"name": "test"}}
+        >>> schema = schema_from_dict(data)
+        >>> schema.json_schema_extra["name"]
+        "test"
+    """
+    return AgentSchema.model_validate(data)
+
+
+def schema_to_yaml(schema: AgentSchema) -> str:
+    """
+    Serialize AgentSchema to YAML string.
+
+    The output format matches the canonical schema file format:
+    ```yaml
+    type: object
+    description: |
+      System prompt here...
+    properties:
+      answer:
+        type: string
+    json_schema_extra:
+      name: my-agent
+      system_prompt: |
+        Extended prompt here...
+    ```
+
+    Args:
+        schema: AgentSchema instance to serialize
+
+    Returns:
+        YAML string representation
+
+    Example:
+        >>> schema = create_agent_schema(
+        ...     description="You are a test agent",
+        ...     properties={"answer": {"type": "string"}},
+        ...     required=["answer"],
+        ...     name="test-agent"
+        ... )
+        >>> yaml_str = schema_to_yaml(schema)
+        >>> "test-agent" in yaml_str
+        True
+    """
+    import yaml
+
+    return yaml.dump(
+        schema_to_dict(schema),
+        default_flow_style=False,
+        allow_unicode=True,
+        sort_keys=False,
+    )
+
+
+def schema_from_yaml(yaml_content: str) -> AgentSchema:
+    """
+    Deserialize YAML string to AgentSchema.
+
+    Args:
+        yaml_content: YAML string containing schema definition
+
+    Returns:
+        Validated AgentSchema instance
+
+    Raises:
+        yaml.YAMLError: If YAML parsing fails
+        ValidationError: If schema structure is invalid
+
+    Example:
+        >>> yaml_str = '''
+        ... type: object
+        ... description: Test agent
+        ... properties:
+        ...   answer:
+        ...     type: string
+        ... json_schema_extra:
+        ...   name: test
+        ... '''
+        >>> schema = schema_from_yaml(yaml_str)
+        >>> schema.json_schema_extra["name"]
+        "test"
+    """
+    import yaml
+
+    data = yaml.safe_load(yaml_content)
+    return schema_from_dict(data)
+
+
+def schema_from_yaml_file(file_path: str) -> AgentSchema:
+    """
+    Load AgentSchema from a YAML file.
+
+    Args:
+        file_path: Path to YAML file
+
+    Returns:
+        Validated AgentSchema instance
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        yaml.YAMLError: If YAML parsing fails
+        ValidationError: If schema structure is invalid
+
+    Example:
+        >>> schema = schema_from_yaml_file("schemas/agents/rem.yaml")
+        >>> schema.json_schema_extra["name"]
+        "rem"
+    """
+    with open(file_path, "r") as f:
+        return schema_from_yaml(f.read())
+
+
+def get_system_prompt(schema: AgentSchema | dict[str, Any]) -> str:
+    """
+    Extract the complete system prompt from a schema.
+
+    Combines:
+    1. schema.description (base system prompt / public description)
+    2. json_schema_extra.system_prompt (extended instructions if present)
+
+    Args:
+        schema: AgentSchema instance or raw dict
+
+    Returns:
+        Complete system prompt string
+
+    Example:
+        >>> schema = AgentSchema(
+        ...     description="Base description",
+        ...     properties={},
+        ...     json_schema_extra={"name": "test", "system_prompt": "Extended instructions"}
+        ... )
+        >>> prompt = get_system_prompt(schema)
+        >>> "Base description" in prompt and "Extended instructions" in prompt
+        True
+    """
+    if isinstance(schema, dict):
+        base = schema.get("description", "")
+        extra = schema.get("json_schema_extra", {})
+        custom = extra.get("system_prompt") if isinstance(extra, dict) else None
+    else:
+        base = schema.description
+        extra = schema.json_schema_extra
+        if isinstance(extra, dict):
+            custom = extra.get("system_prompt")
+        elif isinstance(extra, AgentSchemaMetadata):
+            custom = extra.system_prompt
+        else:
+            custom = None
+
+    if custom:
+        return f"{base}\n\n{custom}" if base else custom
+    return base
+
+
+def get_metadata(schema: AgentSchema | dict[str, Any]) -> AgentSchemaMetadata:
+    """
+    Extract and validate metadata from a schema.
+
+    Args:
+        schema: AgentSchema instance or raw dict
+
+    Returns:
+        Validated AgentSchemaMetadata instance
+
+    Example:
+        >>> schema = {"json_schema_extra": {"name": "test", "system_prompt": "hello"}}
+        >>> meta = get_metadata(schema)
+        >>> meta.name
+        "test"
+        >>> meta.system_prompt
+        "hello"
+    """
+    if isinstance(schema, dict):
+        extra = schema.get("json_schema_extra", {})
+    else:
+        extra = schema.json_schema_extra
+
+    if isinstance(extra, AgentSchemaMetadata):
+        return extra
+    return AgentSchemaMetadata.model_validate(extra)

rem/agentic/tools/rem_tools.py

@@ -162,10 +162,10 @@ async def search_rem_tool(
             return {"status": "error", "error": f"Unknown query_type: {query_type}"}
 
         # Execute query
-        logger.info(f"Executing REM query: {query_type} for user {user_id}")
+        logger.debug(f"Executing REM query: {query_type} for user {user_id}")
         result = await rem_service.execute_query(query)
 
-        logger.info(f"Query completed: {query_type}")
+        logger.debug(f"Query completed: {query_type}")
         return {
             "status": "success",
             "query_type": query_type,
@@ -212,7 +212,7 @@ async def ingest_file_tool(
             is_local_server=is_local_server,
         )
 
-        logger.info(
+        logger.debug(
             f"File ingestion complete: {result['file_name']} "
             f"(status: {result['processing_status']}, "
             f"resources: {result['resources_created']})"