remdb 0.3.103__py3-none-any.whl → 0.3.118__py3-none-any.whl

rem/agentic/context.py

@@ -73,43 +73,47 @@ class AgentContext(BaseModel):
         user_id: str | None,
         source: str = "context",
         default: str | None = None,
-    ) -> str:
+    ) -> str | None:
         """
-        Get user_id or fallback to default with logging.
+        Get user_id or return None for anonymous access.
 
-        Centralized helper for consistent user_id fallback behavior across
-        API endpoints, MCP tools, CLI commands, and services.
+        User ID convention:
+        - user_id is a deterministic UUID5 hash of the user's email address
+        - Use rem.utils.user_id.email_to_user_id(email) to generate
+        - The JWT's `sub` claim is NOT directly used as user_id
+        - Authentication middleware extracts email from JWT and hashes it
+
+        When user_id is None, queries return data with user_id IS NULL
+        (shared/public data). This is intentional - no fake user IDs.
 
         Args:
-            user_id: User identifier (may be None)
+            user_id: User identifier (UUID5 hash of email, may be None for anonymous)
             source: Source of the call (for logging clarity)
-            default: Default value to use (default: settings.test.effective_user_id)
+            default: Explicit default (only for testing, not auto-generated)
 
         Returns:
-            user_id if provided, otherwise default from settings
+            user_id if provided, explicit default if provided, otherwise None
 
         Example:
-            # In MCP tool
-            user_id = AgentContext.get_user_id_or_default(
-                user_id, source="ask_rem_agent"
-            )
+            # Generate user_id from email (done by auth middleware)
+            from rem.utils.user_id import email_to_user_id
+            user_id = email_to_user_id("alice@example.com")
+            # -> "2c5ea4c0-4067-5fef-942d-0a20124e06d8"
 
-            # In API endpoint
+            # In MCP tool - anonymous user sees shared data
             user_id = AgentContext.get_user_id_or_default(
-                temp_context.user_id, source="chat_completions"
-            )
-
-            # In CLI command
-            user_id = AgentContext.get_user_id_or_default(
-                args.user_id, source="rem ask"
+                user_id, source="ask_rem_agent"
             )
+            # Returns None if not authenticated -> queries WHERE user_id IS NULL
         """
-        if user_id is None:
-            from rem.settings import settings
-            effective_default = default or settings.test.effective_user_id
-            logger.debug(f"No user_id provided from {source}, using '{effective_default}'")
-            return effective_default
-        return user_id
+        if user_id is not None:
+            return user_id
+        if default is not None:
+            logger.debug(f"Using explicit default user_id '{default}' from {source}")
+            return default
+        # No fake user IDs - return None for anonymous/unauthenticated
+        logger.debug(f"No user_id from {source}, using None (anonymous/shared data)")
+        return None
 
     @classmethod
     def from_headers(cls, headers: dict[str, str]) -> "AgentContext":

rem/agentic/mcp/tool_wrapper.py

@@ -28,7 +28,12 @@ def create_pydantic_tool(func: Callable[..., Any]) -> Tool:
     return Tool(func)
 
 
-def create_mcp_tool_wrapper(tool_name: str, mcp_tool: Any, user_id: str | None = None) -> Tool:
+def create_mcp_tool_wrapper(
+    tool_name: str,
+    mcp_tool: Any,
+    user_id: str | None = None,
+    description_suffix: str | None = None,
+) -> Tool:
     """
     Create a Pydantic AI Tool from a FastMCP FunctionTool.
 
@@ -40,6 +45,8 @@ def create_mcp_tool_wrapper(tool_name: str, mcp_tool: Any, user_id: str | None =
         tool_name: Name of the MCP tool
         mcp_tool: The FastMCP FunctionTool object
         user_id: Optional user_id to inject into tool calls
+        description_suffix: Optional text to append to the tool's docstring.
+            Used to add schema-specific context (e.g., default table for search_rem).
 
     Returns:
         A Pydantic AI Tool instance
@@ -52,7 +59,11 @@ def create_mcp_tool_wrapper(tool_name: str, mcp_tool: Any, user_id: str | None =
     sig = inspect.signature(tool_func)
     has_user_id = "user_id" in sig.parameters
 
-    # If we need to inject user_id, create a wrapper
+    # Build the docstring with optional suffix
+    base_doc = tool_func.__doc__ or ""
+    final_doc = base_doc + description_suffix if description_suffix else base_doc
+
+    # If we need to inject user_id or modify docstring, create a wrapper
     # Otherwise, use the function directly for better signature preservation
     if user_id and has_user_id:
         async def wrapped_tool(**kwargs) -> Any:
@@ -69,12 +80,27 @@ def create_mcp_tool_wrapper(tool_name: str, mcp_tool: Any, user_id: str | None =
 
         # Copy signature from original function for Pydantic AI inspection
         wrapped_tool.__name__ = tool_name
-        wrapped_tool.__doc__ = tool_func.__doc__
+        wrapped_tool.__doc__ = final_doc
         wrapped_tool.__annotations__ = tool_func.__annotations__
         wrapped_tool.__signature__ = sig  # Important: preserve full signature
 
         logger.debug(f"Creating MCP tool wrapper with user_id injection: {tool_name}")
         return Tool(wrapped_tool)
+    elif description_suffix:
+        # Need to wrap just for docstring modification
+        async def wrapped_tool(**kwargs) -> Any:
+            """Wrapper for docstring modification."""
+            valid_params = set(sig.parameters.keys())
+            filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params}
+            return await tool_func(**filtered_kwargs)
+
+        wrapped_tool.__name__ = tool_name
+        wrapped_tool.__doc__ = final_doc
+        wrapped_tool.__annotations__ = tool_func.__annotations__
+        wrapped_tool.__signature__ = sig
+
+        logger.debug(f"Creating MCP tool wrapper with description suffix: {tool_name}")
+        return Tool(wrapped_tool)
     else:
         # No injection needed - use original function directly
         logger.debug(f"Creating MCP tool wrapper (no injection): {tool_name}")

rem/agentic/otel/setup.py

@@ -14,6 +14,7 @@ from loguru import logger
 
 from ...settings import settings
 
+
 # Global flag to track if instrumentation is initialized
 _instrumentation_initialized = False
 
@@ -52,12 +53,94 @@ def setup_instrumentation() -> None:
 
     try:
         from opentelemetry import trace
-        from opentelemetry.sdk.trace import TracerProvider
-        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.sdk.trace import TracerProvider, ReadableSpan
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
         from opentelemetry.sdk.resources import Resource, SERVICE_NAME, DEPLOYMENT_ENVIRONMENT
         from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HTTPExporter
         from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as GRPCExporter
 
+        class SanitizingSpanExporter(SpanExporter):
+            """
+            Wrapper exporter that sanitizes span attributes before export.
+
+            Removes None values that cause OTLP encoding failures like:
+            - llm.input_messages.3.message.content: None
+            """
+
+            def __init__(self, wrapped_exporter: SpanExporter):
+                self._wrapped = wrapped_exporter
+
+            def _sanitize_value(self, value):
+                """Recursively sanitize a value, replacing None with empty string."""
+                if value is None:
+                    return ""  # Replace None with empty string
+                if isinstance(value, dict):
+                    return {k: self._sanitize_value(v) for k, v in value.items()}
+                if isinstance(value, (list, tuple)):
+                    return [self._sanitize_value(v) for v in value]
+                return value
+
+            def export(self, spans: tuple[ReadableSpan, ...]) -> SpanExportResult:
+                # Create sanitized copies of spans
+                sanitized_spans = []
+                for span in spans:
+                    if span.attributes:
+                        # Sanitize all attribute values - replace None with empty string
+                        sanitized_attrs = {}
+                        for k, v in span.attributes.items():
+                            sanitized_attrs[k] = self._sanitize_value(v)
+                        sanitized_spans.append(_SanitizedSpan(span, sanitized_attrs))
+                    else:
+                        sanitized_spans.append(span)
+
+                return self._wrapped.export(tuple(sanitized_spans))
+
+            def shutdown(self) -> None:
+                self._wrapped.shutdown()
+
+            def force_flush(self, timeout_millis: int = 30000) -> bool:
+                return self._wrapped.force_flush(timeout_millis)
+
+        class _SanitizedSpan(ReadableSpan):
+            """ReadableSpan wrapper with sanitized attributes."""
+
+            def __init__(self, original: ReadableSpan, sanitized_attributes: dict):
+                self._original = original
+                self._sanitized_attributes = sanitized_attributes
+
+            @property
+            def name(self): return self._original.name
+            @property
+            def context(self): return self._original.context
+            @property
+            def parent(self): return self._original.parent
+            @property
+            def resource(self): return self._original.resource
+            @property
+            def instrumentation_scope(self): return self._original.instrumentation_scope
+            @property
+            def status(self): return self._original.status
+            @property
+            def start_time(self): return self._original.start_time
+            @property
+            def end_time(self): return self._original.end_time
+            @property
+            def links(self): return self._original.links
+            @property
+            def events(self): return self._original.events
+            @property
+            def kind(self): return self._original.kind
+            @property
+            def attributes(self): return self._sanitized_attributes
+            @property
+            def dropped_attributes(self): return self._original.dropped_attributes
+            @property
+            def dropped_events(self): return self._original.dropped_events
+            @property
+            def dropped_links(self): return self._original.dropped_links
+
+            def get_span_context(self): return self._original.get_span_context()
+
         # Create resource with service metadata
         resource = Resource(
             attributes={
@@ -72,16 +155,19 @@ def setup_instrumentation() -> None:
 
         # Configure OTLP exporter based on protocol
         if settings.otel.protocol == "grpc":
-            exporter = GRPCExporter(
+            base_exporter = GRPCExporter(
                 endpoint=settings.otel.collector_endpoint,
                 timeout=settings.otel.export_timeout,
             )
         else:  # http
-            exporter = HTTPExporter(
+            base_exporter = HTTPExporter(
                 endpoint=f"{settings.otel.collector_endpoint}/v1/traces",
                 timeout=settings.otel.export_timeout,
             )
 
+        # Wrap with sanitizing exporter to handle None values
+        exporter = SanitizingSpanExporter(base_exporter)
+
         # Add span processor
         tracer_provider.add_span_processor(BatchSpanProcessor(exporter))
 
@@ -95,6 +181,8 @@ def setup_instrumentation() -> None:
         # Add OpenInference span processor for Pydantic AI
         # This adds rich attributes (openinference.span.kind, input/output, etc.) to ALL traces
         # Phoenix receives these traces via the OTLP collector - no separate "Phoenix integration" needed
+        # Note: The OTEL exporter may log warnings about None values in tool call messages,
+        # but this is a known limitation in openinference-instrumentation-pydantic-ai
         try:
             from openinference.instrumentation.pydantic_ai import OpenInferenceSpanProcessor as PydanticAISpanProcessor
 

rem/agentic/providers/pydantic_ai.py

@@ -175,6 +175,23 @@ class AgentRuntime:
         return self.agent.iter(*args, **kwargs)
 
 
+def _get_builtin_tools() -> list:
+    """
+    Get built-in tools that are always available to agents.
+
+    Currently returns empty list - all tools come from MCP servers.
+    The register_metadata tool is available via the REM MCP server and
+    agents can opt-in by configuring mcp_servers in their schema.
+
+    Returns:
+        List of Pydantic AI tool functions (currently empty)
+    """
+    # NOTE: register_metadata is now an MCP tool, not a built-in.
+    # Agents that want it should configure mcp_servers to load from rem.mcp_server.
+    # This allows agents to choose which tools they need.
+    return []
+
+
 def _create_model_from_schema(agent_schema: dict[str, Any]) -> type[BaseModel]:
     """
     Create Pydantic model dynamically from JSON Schema.
@@ -530,18 +547,42 @@ async def create_agent(
     default_model = context.default_model if context else settings.llm.default_model
     model = get_valid_model_or_default(model_override, default_model)
 
-    # Extract schema fields
-    system_prompt = agent_schema.get("description", "") if agent_schema else ""
-    metadata = agent_schema.get("json_schema_extra", {}) if agent_schema else {}
-    mcp_server_configs = metadata.get("mcp_servers", [])
-    resource_configs = metadata.get("resources", [])
+    # Extract schema fields using typed helpers
+    from ..schema import get_system_prompt, get_metadata
+
+    if agent_schema:
+        system_prompt = get_system_prompt(agent_schema)
+        metadata = get_metadata(agent_schema)
+        mcp_server_configs = [s.model_dump() for s in metadata.mcp_servers] if hasattr(metadata, 'mcp_servers') else []
+        resource_configs = metadata.resources if hasattr(metadata, 'resources') else []
+
+        if metadata.system_prompt:
+            logger.debug("Using custom system_prompt from json_schema_extra")
+    else:
+        system_prompt = ""
+        metadata = None
+        mcp_server_configs = []
+        resource_configs = []
 
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
-    temperature = metadata.get("override_temperature", settings.llm.default_temperature)
-    max_iterations = metadata.get("override_max_iterations", settings.llm.default_max_iterations)
+    if metadata:
+        temperature = metadata.override_temperature if metadata.override_temperature is not None else settings.llm.default_temperature
+        max_iterations = metadata.override_max_iterations if metadata.override_max_iterations is not None else settings.llm.default_max_iterations
+        use_structured_output = metadata.structured_output
+    else:
+        temperature = settings.llm.default_temperature
+        max_iterations = settings.llm.default_max_iterations
+        use_structured_output = True
+
+    # Build list of tools - start with built-in tools
+    tools = _get_builtin_tools()
+
+    # Get agent name from metadata for logging
+    agent_name = metadata.name if metadata and hasattr(metadata, 'name') else "unknown"
 
     logger.info(
-        f"Creating agent: model={model}, mcp_servers={len(mcp_server_configs)}, resources={len(resource_configs)}"
+        f"Creating agent '{agent_name}': model={model}, mcp_servers={len(mcp_server_configs)}, "
+        f"resources={len(resource_configs)}, builtin_tools={len(tools)}"
     )
 
     # Set agent resource attributes for OTEL (before creating agent)
@@ -550,8 +591,23 @@ async def create_agent(
 
         set_agent_resource_attributes(agent_schema=agent_schema)
 
-    # Build list of tools from MCP server (in-process, no subprocess)
-    tools = []
+    # Extract schema metadata for search_rem tool description suffix
+    # This allows entity schemas to add context-specific notes to the search_rem tool
+    search_rem_suffix = None
+    if metadata:
+        # Check for default_search_table in metadata (set by entity schemas)
+        extra = agent_schema.get("json_schema_extra", {}) if agent_schema else {}
+        default_table = extra.get("default_search_table")
+        has_embeddings = extra.get("has_embeddings", False)
+
+        if default_table:
+            # Build description suffix for search_rem
+            search_rem_suffix = f"\n\nFor this schema, use `search_rem` to query `{default_table}`. "
+            if has_embeddings:
+                search_rem_suffix += f"SEARCH works well on {default_table} (has embeddings). "
+            search_rem_suffix += f"Example: `SEARCH \"your query\" FROM {default_table} LIMIT 10`"
+
+    # Add tools from MCP server (in-process, no subprocess)
     if mcp_server_configs:
         for server_config in mcp_server_configs:
             server_type = server_config.get("type")
@@ -574,9 +630,17 @@ async def create_agent(
                     mcp_tools_dict = await mcp_server.get_tools()
 
                     for tool_name, tool_func in mcp_tools_dict.items():
-                        wrapped_tool = create_mcp_tool_wrapper(tool_name, tool_func, user_id=context.user_id if context else None)
+                        # Add description suffix to search_rem tool if schema specifies a default table
+                        tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
+
+                        wrapped_tool = create_mcp_tool_wrapper(
+                            tool_name,
+                            tool_func,
+                            user_id=context.user_id if context else None,
+                            description_suffix=tool_suffix,
+                        )
                         tools.append(wrapped_tool)
-                        logger.debug(f"Loaded MCP tool: {tool_name}")
+                        logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
 
                     logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
 
@@ -589,11 +653,8 @@ async def create_agent(
         # TODO: Convert resources to tools (MCP convenience syntax)
         pass
 
-    # Check if structured output is disabled for this schema
-    # When structured_output: false, properties become part of prompt instead of output_type
-    use_structured_output = metadata.get("structured_output", True)
-
     # Create dynamic result_type from schema if not provided
+    # Note: use_structured_output is set earlier from metadata.structured_output
     if result_type is None and agent_schema and "properties" in agent_schema:
         if use_structured_output:
             # Pre-process schema for Qwen compatibility (strips min/max, sets additionalProperties=False)
@@ -615,21 +676,30 @@ async def create_agent(
         wrapped_result_type = _create_schema_wrapper(
             result_type, strip_description=strip_model_description
         )
+        # Use InstrumentationSettings with version=3 to include agent name in span names
+        from pydantic_ai.models.instrumented import InstrumentationSettings
+        instrumentation = InstrumentationSettings(version=3) if settings.otel.enabled else False
+
         agent = Agent(
             model=model,
+            name=agent_name,  # Used for OTEL span names (version 3: "invoke_agent {name}")
             system_prompt=system_prompt,
             output_type=wrapped_result_type,
             tools=tools,
-            instrument=settings.otel.enabled,  # Conditional OTEL instrumentation
+            instrument=instrumentation,
             model_settings={"temperature": temperature},
             retries=settings.llm.max_retries,
         )
     else:
+        from pydantic_ai.models.instrumented import InstrumentationSettings
+        instrumentation = InstrumentationSettings(version=3) if settings.otel.enabled else False
+
         agent = Agent(
             model=model,
+            name=agent_name,  # Used for OTEL span names (version 3: "invoke_agent {name}")
             system_prompt=system_prompt,
             tools=tools,
-            instrument=settings.otel.enabled,
+            instrument=instrumentation,
             model_settings={"temperature": temperature},
             retries=settings.llm.max_retries,
         )