remdb 0.3.163__py3-none-any.whl → 0.3.200__py3-none-any.whl

rem/agentic/agents/agent_manager.py

@@ -128,8 +128,9 @@ async def save_agent(
     )
 
     # Create Schema entity (user-scoped)
+    # Note: tenant_id defaults to "default" for anonymous users
     schema_entity = Schema(
-        tenant_id=user_id,
+        tenant_id=user_id or "default",
         user_id=user_id,
         name=name,
         spec=spec,

rem/agentic/context.py

@@ -22,14 +22,81 @@ Key Design Pattern:
 - Enables session tracking across API, CLI, and test execution
 - Supports header-based configuration override (model, schema URI)
 - Clean separation: context (who/what) vs agent (how)
+
+Multi-Agent Context Propagation:
+- ContextVar (_current_agent_context) threads context through nested agent calls
+- Parent context is automatically available to child agents via get_current_context()
+- Use agent_context_scope() context manager for scoped context setting
+- Child agents inherit user_id, tenant_id, session_id, is_eval from parent
 """
 
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Generator
+
 from loguru import logger
 from pydantic import BaseModel, Field
 
 from ..settings import settings
 
 
+# Thread-local context for current agent execution
+# This enables context propagation through nested agent calls (multi-agent)
+_current_agent_context: ContextVar["AgentContext | None"] = ContextVar(
+    "current_agent_context", default=None
+)
+
+
+def get_current_context() -> "AgentContext | None":
+    """
+    Get the current agent context from context var.
+
+    Used by MCP tools (like ask_agent) to inherit context from parent agent.
+    Returns None if no context is set (e.g., direct CLI invocation without context).
+
+    Example:
+        # In an MCP tool
+        parent_context = get_current_context()
+        if parent_context:
+            # Inherit user_id, session_id, etc. from parent
+            child_context = parent_context.child_context(agent_schema_uri="child-agent")
+    """
+    return _current_agent_context.get()
+
+
+def set_current_context(ctx: "AgentContext | None") -> None:
+    """
+    Set the current agent context.
+
+    Called by streaming layer before agent execution.
+    Should be cleared (set to None) after execution completes.
+    """
+    _current_agent_context.set(ctx)
+
+
+@contextmanager
+def agent_context_scope(ctx: "AgentContext") -> Generator["AgentContext", None, None]:
+    """
+    Context manager for scoped context setting.
+
+    Automatically restores previous context when exiting scope.
+    Safe for nested agent calls - each level preserves its parent's context.
+
+    Example:
+        context = AgentContext(user_id="user-123")
+        with agent_context_scope(context):
+            # Context is available via get_current_context()
+            result = await agent.run(...)
+        # Previous context (or None) is restored
+    """
+    previous = _current_agent_context.get()
+    _current_agent_context.set(ctx)
+    try:
+        yield ctx
+    finally:
+        _current_agent_context.set(previous)
+
+
 class AgentContext(BaseModel):
     """
     Session and configuration context for agent execution.
@@ -85,6 +152,40 @@ class AgentContext(BaseModel):
 
     model_config = {"populate_by_name": True}
 
+    def child_context(
+        self,
+        agent_schema_uri: str | None = None,
+        model_override: str | None = None,
+    ) -> "AgentContext":
+        """
+        Create a child context for nested agent calls.
+
+        Inherits user_id, tenant_id, session_id, is_eval from parent.
+        Allows overriding agent_schema_uri and default_model for the child.
+
+        Args:
+            agent_schema_uri: Agent schema for the child agent (required for lineage)
+            model_override: Optional model override for child agent
+
+        Returns:
+            New AgentContext for the child agent
+
+        Example:
+            parent_context = get_current_context()
+            child_context = parent_context.child_context(
+                agent_schema_uri="sentiment-analyzer"
+            )
+            agent = await create_agent(context=child_context)
+        """
+        return AgentContext(
+            user_id=self.user_id,
+            tenant_id=self.tenant_id,
+            session_id=self.session_id,
+            default_model=model_override or self.default_model,
+            agent_schema_uri=agent_schema_uri or self.agent_schema_uri,
+            is_eval=self.is_eval,
+        )
+
     @staticmethod
     def get_user_id_or_default(
         user_id: str | None,

rem/agentic/context_builder.py

@@ -12,7 +12,7 @@ User Context (on-demand by default):
 - System message includes REM LOOKUP hint for user profile
 - Agent decides whether to load profile based on query
 - More efficient for queries that don't need personalization
-- Example: "User ID: sarah@example.com. To load user profile: Use REM LOOKUP users/sarah@example.com"
+- Example: "User: sarah@example.com. To load user profile: Use REM LOOKUP \"sarah@example.com\""
 
 User Context (auto-inject when enabled):
 - Set CHAT__AUTO_INJECT_USER_CONTEXT=true
@@ -40,7 +40,7 @@ Usage (on-demand, default):
 
     # Messages list structure (on-demand):
     # [
-    #   {"role": "system", "content": "Today's date: 2025-11-22\nUser ID: sarah@example.com\nTo load user profile: Use REM LOOKUP users/sarah@example.com\nSession ID: sess-123\nTo load session history: Use REM LOOKUP messages?session_id=sess-123"},
+    #   {"role": "system", "content": "Today's date: 2025-11-22\nUser: sarah@example.com\nTo load user profile: Use REM LOOKUP \"sarah@example.com\"\nSession ID: sess-123\nTo load session history: Use REM LOOKUP messages?session_id=sess-123"},
     #   {"role": "user", "content": "What's next for the API migration?"}
     # ]
 
@@ -115,7 +115,7 @@ class ContextBuilder:
         - Agent can retrieve full content on-demand using REM LOOKUP
 
         User Context (on-demand by default):
-        - System message includes REM LOOKUP hint: "User ID: {user_id}. To load user profile: Use REM LOOKUP users/{user_id}"
+        - System message includes REM LOOKUP hint: "User: {email}. To load user profile: Use REM LOOKUP \"{email}\""
         - Agent decides whether to load profile based on query
 
         User Context (auto-inject when enabled):
@@ -137,7 +137,7 @@ class ContextBuilder:
 
             # messages structure:
             # [
-            #   {"role": "system", "content": "Today's date: 2025-11-22\nUser ID: sarah@example.com\nTo load user profile: Use REM LOOKUP users/sarah@example.com"},
+            #   {"role": "system", "content": "Today's date: 2025-11-22\nUser: sarah@example.com\nTo load user profile: Use REM LOOKUP \"sarah@example.com\""},
             #   {"role": "user", "content": "Previous message"},
             #   {"role": "assistant", "content": "Start of long response... [REM LOOKUP session-123-msg-1] ...end"},
             #   {"role": "user", "content": "New message"}
@@ -191,8 +191,16 @@ class ContextBuilder:
                     context_hint += "\n\nNo user context available (anonymous or new user)."
             elif context.user_id:
                 # On-demand: Provide hint to use REM LOOKUP
-                context_hint += f"\n\nUser ID: {context.user_id}"
-                context_hint += f"\nTo load user profile: Use REM LOOKUP users/{context.user_id}"
+                # user_id is UUID5 hash of email - load user to get email for display and LOOKUP
+                user_repo = Repository(User, "users", db=db)
+                user = await user_repo.get_by_id(context.user_id, context.tenant_id)
+                if user and user.email:
+                    # Show email (more useful than UUID) and LOOKUP hint
+                    context_hint += f"\n\nUser: {user.email}"
+                    context_hint += f"\nTo load user profile: Use REM LOOKUP \"{user.email}\""
+                else:
+                    context_hint += f"\n\nUser ID: {context.user_id}"
+                    context_hint += "\nUser profile not available."
 
             # Add system context hint
             messages.append(ContextMessage(role="system", content=context_hint))
@@ -209,11 +217,21 @@ class ContextBuilder:
                 )
 
                 # Convert to ContextMessage format
+                # For tool messages, wrap content with clear markers so the agent
+                # can see previous tool results when the prompt is concatenated
                 for msg_dict in session_history:
+                    role = msg_dict["role"]
+                    content = msg_dict["content"]
+
+                    if role == "tool":
+                        # Wrap tool results with clear markers for visibility
+                        tool_name = msg_dict.get("tool_name", "unknown")
+                        content = f"[TOOL RESULT: {tool_name}]\n{content}\n[/TOOL RESULT]"
+
                     messages.append(
                         ContextMessage(
-                            role=msg_dict["role"],
-                            content=msg_dict["content"],
+                            role=role,
+                            content=content,
                         )
                     )
 
@@ -239,6 +257,9 @@ class ContextBuilder:
         """
         Load user profile from database and format as context.
 
+        user_id is always a UUID5 hash of email (bijection).
+        Looks up user by their id field in the database.
+
         Returns formatted string with:
         - User summary (generated by dreaming worker)
         - Current projects
@@ -252,6 +273,7 @@ class ContextBuilder:
 
         try:
             user_repo = Repository(User, "users", db=db)
+            # user_id is UUID5 hash of email - look up by database id
             user = await user_repo.get_by_id(user_id, tenant_id)
 
             if not user:

rem/agentic/mcp/tool_wrapper.py

@@ -149,12 +149,23 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
     parts = re.sub(r'_+', '_', parts).strip('_')  # Clean up multiple underscores
     func_name = f"get_{parts}"
 
+    # For parameterized URIs, append _by_{params} to avoid naming conflicts
+    # e.g., rem://agents/{name} -> get_rem_agents_by_name (distinct from get_rem_agents)
+    if template_vars:
+        param_suffix = "_by_" + "_".join(template_vars)
+        func_name = f"{func_name}{param_suffix}"
+
     # Build description including parameter info
     description = usage or f"Fetch {uri} resource"
     if template_vars:
         param_desc = ", ".join(template_vars)
         description = f"{description}\n\nParameters: {param_desc}"
 
+    # Capture mcp_server reference at tool creation time (for closure)
+    # This ensures the correct server is used even if called later
+    _captured_mcp_server = mcp_server
+    _captured_uri = uri  # Also capture URI for consistent logging
+
     if template_vars:
         # Template URI -> create parameterized tool
         async def wrapper(**kwargs: Any) -> str:
@@ -162,13 +173,17 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
             import asyncio
             import inspect
 
+            logger.debug(f"Resource tool invoked: uri={_captured_uri}, kwargs={kwargs}, mcp_server={'set' if _captured_mcp_server else 'None'}")
+
             # Try to resolve from MCP server's resource templates first
-            if mcp_server is not None:
+            if _captured_mcp_server is not None:
                 try:
                     # Get resource templates from MCP server
-                    templates = await mcp_server.get_resource_templates()
-                    if uri in templates:
-                        template = templates[uri]
+                    templates = await _captured_mcp_server.get_resource_templates()
+                    logger.debug(f"MCP server templates: {list(templates.keys())}")
+                    if _captured_uri in templates:
+                        template = templates[_captured_uri]
+                        logger.debug(f"Found template for {_captured_uri}, calling fn with kwargs={kwargs}")
                         # Call the template's underlying function directly
                         # The fn expects the template variables as kwargs
                         fn_result = template.fn(**kwargs)
@@ -178,17 +193,22 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
                         if isinstance(fn_result, str):
                             return fn_result
                         return json.dumps(fn_result, indent=2)
+                    else:
+                        logger.warning(f"Template {_captured_uri} not found in MCP server templates: {list(templates.keys())}")
                 except Exception as e:
-                    logger.warning(f"Failed to resolve resource {uri} from MCP server: {e}")
+                    logger.warning(f"Failed to resolve resource {_captured_uri} from MCP server: {e}", exc_info=True)
+            else:
+                logger.warning(f"No MCP server provided for resource tool {_captured_uri}, using fallback")
 
             # Fallback: substitute template variables and use load_resource
-            resolved_uri = uri
+            resolved_uri = _captured_uri
             for var in template_vars:
                 if var in kwargs:
                     resolved_uri = resolved_uri.replace(f"{{{var}}}", str(kwargs[var]))
                 else:
                     return json.dumps({"error": f"Missing required parameter: {var}"})
 
+            logger.debug(f"Using fallback load_resource for resolved URI: {resolved_uri}")
             from rem.api.mcp_router.resources import load_resource
             result = await load_resource(resolved_uri)
             if isinstance(result, str):
@@ -202,7 +222,7 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
         wrapper.__annotations__ = {var: str for var in template_vars}
         wrapper.__annotations__['return'] = str
 
-        logger.info(f"Built parameterized resource tool: {func_name} (uri: {uri}, params: {template_vars})")
+        logger.info(f"Built parameterized resource tool: {func_name} (uri: {uri}, params: {template_vars}, mcp_server={'provided' if mcp_server else 'None'})")
     else:
         # Concrete URI -> no-param tool
         async def wrapper(**kwargs: Any) -> str:
@@ -213,12 +233,16 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
             if kwargs:
                 logger.warning(f"Resource tool {func_name} called with unexpected kwargs: {list(kwargs.keys())}")
 
+            logger.debug(f"Concrete resource tool invoked: uri={_captured_uri}, mcp_server={'set' if _captured_mcp_server else 'None'}")
+
             # Try to resolve from MCP server's resources first
-            if mcp_server is not None:
+            if _captured_mcp_server is not None:
                 try:
-                    resources = await mcp_server.get_resources()
-                    if uri in resources:
-                        resource = resources[uri]
+                    resources = await _captured_mcp_server.get_resources()
+                    logger.debug(f"MCP server resources: {list(resources.keys())}")
+                    if _captured_uri in resources:
+                        resource = resources[_captured_uri]
+                        logger.debug(f"Found resource for {_captured_uri}")
                         # Call the resource's underlying function
                         fn_result = resource.fn()
                         if inspect.iscoroutine(fn_result):
@@ -226,12 +250,17 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
                         if isinstance(fn_result, str):
                             return fn_result
                         return json.dumps(fn_result, indent=2)
+                    else:
+                        logger.warning(f"Resource {_captured_uri} not found in MCP server resources: {list(resources.keys())}")
                 except Exception as e:
-                    logger.warning(f"Failed to resolve resource {uri} from MCP server: {e}")
+                    logger.warning(f"Failed to resolve resource {_captured_uri} from MCP server: {e}", exc_info=True)
+            else:
+                logger.warning(f"No MCP server provided for resource tool {_captured_uri}, using fallback")
 
             # Fallback to load_resource
+            logger.debug(f"Using fallback load_resource for URI: {_captured_uri}")
             from rem.api.mcp_router.resources import load_resource
-            result = await load_resource(uri)
+            result = await load_resource(_captured_uri)
             if isinstance(result, str):
                 return result
             return json.dumps(result, indent=2)
@@ -239,6 +268,6 @@ def create_resource_tool(uri: str, usage: str = "", mcp_server: Any = None) -> T
         wrapper.__name__ = func_name
         wrapper.__doc__ = description
 
-        logger.info(f"Built resource tool: {func_name} (uri: {uri})")
+        logger.info(f"Built resource tool: {func_name} (uri: {uri}, mcp_server={'provided' if mcp_server else 'None'})")
 
     return Tool(wrapper)

rem/agentic/providers/pydantic_ai.py

@@ -96,6 +96,35 @@ TODO:
 
     Priority: HIGH (blocks production scaling beyond 50 req/sec)
 
+    4. Response Format Control (structured_output enhancement):
+       - Current: structured_output is bool (True=strict schema, False=free-form text)
+       - Missing: OpenAI JSON mode (valid JSON without strict schema enforcement)
+       - Missing: Completions API support (some models only support completions, not chat)
+
+       Proposed schema field values for `structured_output`:
+         - True (default): Strict structured output using provider's native schema support
+         - False: Free-form text response (properties converted to prompt guidance)
+         - "json": JSON mode - ensures valid JSON but no schema enforcement
+                   (OpenAI: response_format={"type": "json_object"})
+         - "text": Explicit free-form text (alias for False)
+
+       Implementation:
+         a) Update AgentSchemaMetadata.structured_output type:
+            structured_output: bool | Literal["json", "text"] = True
+         b) In create_agent(), handle each mode:
+            - True: Use output_type with Pydantic model (current behavior)
+            - False/"text": Convert properties to prompt guidance (current behavior)
+            - "json": Use provider's JSON mode without strict schema
+         c) Provider-specific JSON mode:
+            - OpenAI: model_settings={"response_format": {"type": "json_object"}}
+            - Anthropic: Not supported natively, use prompt guidance
+            - Others: Fallback to prompt guidance with JSON instruction
+
+       Related: Some providers (Cerebras) have completions-only models where
+       structured output isn't available. Consider model capability detection.
+
+       Priority: MEDIUM (enables more flexible output control)
+
 Example Agent Schema:
 {
   "type": "object",
@@ -553,58 +582,70 @@ async def create_agent(
     if agent_schema:
         system_prompt = get_system_prompt(agent_schema)
         metadata = get_metadata(agent_schema)
-        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') and metadata.mcp_servers else []
         resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
 
+        # DEPRECATED: mcp_servers in agent schemas is ignored
+        # MCP servers are now always auto-detected at the application level
+        if hasattr(metadata, 'mcp_servers') and metadata.mcp_servers:
+            logger.warning(
+                "DEPRECATED: mcp_servers in agent schema is ignored. "
+                "MCP servers are auto-detected from tools.mcp_server module. "
+                "Remove mcp_servers from your agent schema."
+            )
+
         if metadata.system_prompt:
             logger.debug("Using custom system_prompt from json_schema_extra")
     else:
         system_prompt = ""
         metadata = None
-        mcp_server_configs = []
         resource_configs = []
 
-    # Auto-detect local MCP server if not explicitly configured
-    # This makes mcp_servers config optional - agents get tools automatically
+    # Auto-detect MCP server at application level
+    # Convention: tools/mcp_server.py exports `mcp` FastMCP instance
+    # Falls back to REM's built-in MCP server if no local server found
+    import importlib
+    import os
+    import sys
+
+    # Ensure current working directory is in sys.path for local imports
+    cwd = os.getcwd()
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+
+    mcp_server_configs = []
+    auto_detect_modules = [
+        "tools.mcp_server",  # Convention: tools/mcp_server.py
+        "mcp_server",        # Alternative: mcp_server.py in root
+    ]
+    for module_path in auto_detect_modules:
+        try:
+            mcp_module = importlib.import_module(module_path)
+            if hasattr(mcp_module, "mcp"):
+                logger.info(f"Auto-detected local MCP server: {module_path}")
+                mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
+                break
+        except ImportError as e:
+            logger.debug(f"MCP server auto-detect: {module_path} not found ({e})")
+            continue
+        except Exception as e:
+            logger.warning(f"MCP server auto-detect: {module_path} failed to load: {e}")
+            continue
+
+    # Fall back to REM's default MCP server if no local server found
     if not mcp_server_configs:
-        import importlib
-        import os
-        import sys
-
-        # Ensure current working directory is in sys.path for local imports
-        cwd = os.getcwd()
-        if cwd not in sys.path:
-            sys.path.insert(0, cwd)
-
-        # Try common local MCP server module paths first
-        auto_detect_modules = [
-            "tools.mcp_server",  # Convention: tools/mcp_server.py
-            "mcp_server",        # Alternative: mcp_server.py in root
-        ]
-        for module_path in auto_detect_modules:
-            try:
-                mcp_module = importlib.import_module(module_path)
-                if hasattr(mcp_module, "mcp"):
-                    logger.info(f"Auto-detected local MCP server: {module_path}")
-                    mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
-                    break
-            except ImportError:
-                continue
-
-        # Fall back to REM's default MCP server if no local server found
-        if not mcp_server_configs:
-            logger.debug("No local MCP server found, using REM default")
-            mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
+        logger.info("No local MCP server found, using REM default (rem.mcp_server)")
+        mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
 
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
     if metadata:
         temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
         max_iterations = metadata.override_max_iterations if metadata.override_max_iterations is not None else settings.llm.default_max_iterations
-        use_structured_output = metadata.structured_output
+        # Use schema-level structured_output if set, otherwise fall back to global setting
+        use_structured_output = metadata.structured_output if metadata.structured_output is not None else settings.llm.default_structured_output
     else:
         temperature = settings.llm.default_temperature
         max_iterations = settings.llm.default_max_iterations
-        use_structured_output = True
+        use_structured_output = settings.llm.default_structured_output
 
     # Build list of tools - start with built-in tools
     tools = _get_builtin_tools()
@@ -727,6 +768,7 @@ async def create_agent(
 
     # Create tools from collected resource URIs
     # Pass the loaded MCP server so resources can be resolved from it
+    logger.info(f"Creating {len(resource_uris)} resource tools with mcp_server={'set' if loaded_mcp_server else 'None'}")
     for uri, usage in resource_uris:
         resource_tool = create_resource_tool(uri, usage, mcp_server=loaded_mcp_server)
         tools.append(resource_tool)

rem/agentic/schema.py

@@ -215,12 +215,13 @@ class AgentSchemaMetadata(BaseModel):
     )
 
     # Structured output toggle
-    structured_output: bool = Field(
-        default=True,
+    structured_output: bool | None = Field(
+        default=None,
         description=(
             "Whether to enforce structured JSON output. "
             "When False, the agent produces free-form text and schema properties "
-            "are converted to prompt guidance instead. Default: True (JSON output)."
+            "are converted to prompt guidance instead. "
+            "Default: None (uses LLM__DEFAULT_STRUCTURED_OUTPUT setting, which defaults to False)."
         ),
     )
 

rem/agentic/tools/rem_tools.py

@@ -3,6 +3,17 @@ REM tools for agent execution (CLI and API compatible).
 
 These tools work in both CLI and API contexts by initializing services on-demand.
 They wrap the service layer directly, not MCP tools.
+
+Core tables (always available):
+- resources: Documents, content chunks, artifacts
+- moments: Temporal narratives extracted from resources (usually user-specific)
+- ontologies: Domain entities with semantic links for further lookups (like a wiki)
+
+Other tables (may vary by deployment):
+- users, sessions, messages, files, schemas, feedbacks
+
+Note: Not all tables are populated in all systems. Use FUZZY or SEARCH
+to discover what data exists before assuming specific tables have content.
 """
 
 from typing import Any, Literal, cast

rem/api/main.py

@@ -322,7 +322,7 @@ def create_app() -> FastAPI:
 
     app.add_middleware(
         AuthMiddleware,
-        protected_paths=["/api/v1"],
+        protected_paths=["/api/v1", "/api/admin"],
         excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth", "/api/v1/slack"],
         # Allow anonymous when auth is disabled, otherwise use setting
         allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,