remdb 0.3.133__py3-none-any.whl → 0.3.157__py3-none-any.whl

rem/agentic/providers/pydantic_ai.py

@@ -564,9 +564,37 @@ async def create_agent(
         mcp_server_configs = []
         resource_configs = []
 
-    # Default to rem.mcp_server if no MCP servers configured
+    # Auto-detect local MCP server if not explicitly configured
+    # This makes mcp_servers config optional - agents get tools automatically
     if not mcp_server_configs:
-        mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
+        import importlib
+        import os
+        import sys
+
+        # Ensure current working directory is in sys.path for local imports
+        cwd = os.getcwd()
+        if cwd not in sys.path:
+            sys.path.insert(0, cwd)
+
+        # Try common local MCP server module paths first
+        auto_detect_modules = [
+            "tools.mcp_server",  # Convention: tools/mcp_server.py
+            "mcp_server",        # Alternative: mcp_server.py in root
+        ]
+        for module_path in auto_detect_modules:
+            try:
+                mcp_module = importlib.import_module(module_path)
+                if hasattr(mcp_module, "mcp"):
+                    logger.info(f"Auto-detected local MCP server: {module_path}")
+                    mcp_server_configs = [{"type": "local", "module": module_path, "id": "auto-detected"}]
+                    break
+            except ImportError:
+                continue
+
+        # Fall back to REM's default MCP server if no local server found
+        if not mcp_server_configs:
+            logger.debug("No local MCP server found, using REM default")
+            mcp_server_configs = [{"type": "local", "module": "rem.mcp_server", "id": "rem"}]
 
     # Extract temperature and max_iterations from schema metadata (with fallback to settings defaults)
     if metadata:
@@ -612,46 +640,51 @@ async def create_agent(
             search_rem_suffix += f"Example: `SEARCH \"your query\" FROM {default_table} LIMIT 10`"
 
     # Add tools from MCP server (in-process, no subprocess)
-    if mcp_server_configs:
-        for server_config in mcp_server_configs:
-            server_type = server_config.get("type")
-            server_id = server_config.get("id", "mcp-server")
-
-            if server_type == "local":
-                # Import MCP server directly (in-process)
-                module_path = server_config.get("module", "rem.mcp_server")
-
-                try:
-                    # Dynamic import of MCP server module
-                    import importlib
-                    mcp_module = importlib.import_module(module_path)
-                    mcp_server = mcp_module.mcp
-
-                    # Extract tools from MCP server (get_tools is async)
-                    from ..mcp.tool_wrapper import create_mcp_tool_wrapper
-
-                    # Await async get_tools() call
-                    mcp_tools_dict = await mcp_server.get_tools()
-
-                    for tool_name, tool_func in mcp_tools_dict.items():
-                        # Add description suffix to search_rem tool if schema specifies a default table
-                        tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
-
-                        wrapped_tool = create_mcp_tool_wrapper(
-                            tool_name,
-                            tool_func,
-                            user_id=context.user_id if context else None,
-                            description_suffix=tool_suffix,
-                        )
-                        tools.append(wrapped_tool)
-                        logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
-
-                    logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
-
-                except Exception as e:
-                    logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
-            else:
-                logger.warning(f"Unsupported MCP server type: {server_type}")
+    # Track loaded MCP servers for resource resolution
+    loaded_mcp_server = None
+
+    for server_config in mcp_server_configs:
+        server_type = server_config.get("type")
+        server_id = server_config.get("id", "mcp-server")
+
+        if server_type == "local":
+            # Import MCP server directly (in-process)
+            module_path = server_config.get("module", "rem.mcp_server")
+
+            try:
+                # Dynamic import of MCP server module
+                import importlib
+                mcp_module = importlib.import_module(module_path)
+                mcp_server = mcp_module.mcp
+
+                # Store the loaded server for resource resolution
+                loaded_mcp_server = mcp_server
+
+                # Extract tools from MCP server (get_tools is async)
+                from ..mcp.tool_wrapper import create_mcp_tool_wrapper
+
+                # Await async get_tools() call
+                mcp_tools_dict = await mcp_server.get_tools()
+
+                for tool_name, tool_func in mcp_tools_dict.items():
+                    # Add description suffix to search_rem tool if schema specifies a default table
+                    tool_suffix = search_rem_suffix if tool_name == "search_rem" else None
+
+                    wrapped_tool = create_mcp_tool_wrapper(
+                        tool_name,
+                        tool_func,
+                        user_id=context.user_id if context else None,
+                        description_suffix=tool_suffix,
+                    )
+                    tools.append(wrapped_tool)
+                    logger.debug(f"Loaded MCP tool: {tool_name}" + (" (with schema suffix)" if tool_suffix else ""))
+
+                logger.info(f"Loaded {len(mcp_tools_dict)} tools from MCP server: {server_id} (in-process)")
+
+            except Exception as e:
+                logger.error(f"Failed to load MCP server {server_id}: {e}", exc_info=True)
+        else:
+            logger.warning(f"Unsupported MCP server type: {server_type}")
 
     # Convert resources to tools (MCP convenience syntax)
     # Resources declared in agent YAML become callable tools - eliminates
@@ -693,8 +726,9 @@ async def create_agent(
             resource_uris.append((tool_name, tool_desc))
 
     # Create tools from collected resource URIs
+    # Pass the loaded MCP server so resources can be resolved from it
     for uri, usage in resource_uris:
-        resource_tool = create_resource_tool(uri, usage)
+        resource_tool = create_resource_tool(uri, usage, mcp_server=loaded_mcp_server)
         tools.append(resource_tool)
         logger.debug(f"Loaded resource as tool: {uri}")
 

rem/api/deps.py

@@ -185,8 +185,8 @@ async def get_user_filter(
                 f"User {user.get('email')} attempted to filter by user_id={x_user_id}"
             )
     else:
-        # Anonymous: could use anonymous tracking ID or restrict access
-        # For now, anonymous can't access user-scoped data
+        # Anonymous: use anonymous tracking ID
+        # Note: user_id should come from JWT, not from parameters
         anon_id = getattr(request.state, "anon_id", None)
         if anon_id:
             filters["user_id"] = f"anon:{anon_id}"

rem/api/main.py

@@ -304,7 +304,7 @@ def create_app() -> FastAPI:
     app.add_middleware(
         AuthMiddleware,
         protected_paths=["/api/v1"],
-        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth"],
+        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth", "/api/v1/slack"],
         # Allow anonymous when auth is disabled, otherwise use setting
         allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,
         # MCP requires auth only when auth is fully enabled

rem/api/mcp_router/server.py

@@ -182,6 +182,7 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
         list_schema,
         read_resource,
         register_metadata,
+        save_agent,
         search_rem,
     )
 
@@ -191,6 +192,7 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     mcp.tool()(register_metadata)
     mcp.tool()(list_schema)
     mcp.tool()(get_schema)
+    mcp.tool()(save_agent)
 
     # File ingestion tool (with local path support for local servers)
     # Wrap to inject is_local parameter

rem/api/mcp_router/tools.py

@@ -1040,3 +1040,93 @@ async def get_schema(
     logger.info(f"Retrieved schema for table '{table_name}' with {len(column_defs)} columns")
 
     return result
+
+
+@mcp_tool_error_handler
+async def save_agent(
+    name: str,
+    description: str,
+    properties: dict[str, Any] | None = None,
+    required: list[str] | None = None,
+    tools: list[str] | None = None,
+    tags: list[str] | None = None,
+    version: str = "1.0.0",
+    user_id: str | None = None,
+) -> dict[str, Any]:
+    """
+    Save an agent schema to REM, making it available for use.
+
+    This tool creates or updates an agent definition in the user's schema space.
+    The agent becomes immediately available for conversations.
+
+    **Default Tools**: All agents automatically get `search_rem` and `register_metadata`
+    tools unless explicitly overridden.
+
+    Args:
+        name: Agent name in kebab-case (e.g., "code-reviewer", "sales-assistant").
+            Must be unique within the user's schema space.
+        description: The agent's system prompt. This is the full instruction set
+            that defines the agent's behavior, personality, and capabilities.
+            Use markdown formatting for structure.
+        properties: Output schema properties as a dict. Each property should have:
+            - type: "string", "number", "boolean", "array", "object"
+            - description: What this field captures
+            Example: {"answer": {"type": "string", "description": "Response to user"}}
+            If not provided, defaults to a simple {"answer": {"type": "string"}} schema.
+        required: List of required property names. Defaults to ["answer"] if not provided.
+        tools: List of tool names the agent can use. Defaults to ["search_rem", "register_metadata"].
+        tags: Optional tags for categorizing the agent.
+        version: Semantic version string (default: "1.0.0").
+        user_id: User identifier for scoping. Uses authenticated user if not provided.
+
+    Returns:
+        Dict with:
+        - status: "success" or "error"
+        - agent_name: Name of the saved agent
+        - version: Version saved
+        - message: Human-readable status
+
+    Examples:
+        # Create a simple agent
+        save_agent(
+            name="greeting-bot",
+            description="You are a friendly greeter. Say hello warmly.",
+            properties={"answer": {"type": "string", "description": "Greeting message"}},
+            required=["answer"]
+        )
+
+        # Create agent with structured output
+        save_agent(
+            name="sentiment-analyzer",
+            description="Analyze sentiment of text provided by the user.",
+            properties={
+                "answer": {"type": "string", "description": "Analysis explanation"},
+                "sentiment": {"type": "string", "enum": ["positive", "negative", "neutral"]},
+                "confidence": {"type": "number", "minimum": 0, "maximum": 1}
+            },
+            required=["answer", "sentiment"],
+            tags=["analysis", "nlp"]
+        )
+    """
+    from ...agentic.agents.agent_manager import save_agent as _save_agent
+
+    # Get user_id from context if not provided
+    user_id = AgentContext.get_user_id_or_default(user_id, source="save_agent")
+
+    # Delegate to agent_manager
+    result = await _save_agent(
+        name=name,
+        description=description,
+        user_id=user_id,
+        properties=properties,
+        required=required,
+        tools=tools,
+        tags=tags,
+        version=version,
+    )
+
+    # Add helpful message for Slack users
+    if result.get("status") == "success":
+        result["message"] = f"Agent '{name}' saved. Use `/custom-agent {name}` to chat with it."
+
+    return result

rem/api/routers/auth.py

@@ -1,20 +1,68 @@
 """
-OAuth 2.1 Authentication Router.
+Authentication Router.
 
-Leverages Authlib for standards-compliant OAuth/OIDC implementation.
-Minimal custom code - Authlib handles PKCE, token validation, JWKS.
+Supports multiple authentication methods:
+1. Email (passwordless): POST /api/auth/email/send-code, POST /api/auth/email/verify
+2. OAuth (Google, Microsoft): GET /api/auth/{provider}/login, GET /api/auth/{provider}/callback
 
 Endpoints:
+- POST /api/auth/email/send-code     - Send login code to email
+- POST /api/auth/email/verify        - Verify code and create session
 - GET  /api/auth/{provider}/login    - Initiate OAuth flow
 - GET  /api/auth/{provider}/callback - OAuth callback
 - POST /api/auth/logout              - Clear session
 - GET  /api/auth/me                  - Current user info
 
 Supported providers:
+- email: Passwordless email login
 - google: Google OAuth 2.0 / OIDC
 - microsoft: Microsoft Entra ID OIDC
 
-Design Pattern (OAuth 2.1 + PKCE):
+=============================================================================
+Email Authentication Access Control
+=============================================================================
+
+The email auth provider implements a tiered access control system:
+
+Access Control Flow (send-code):
+    User requests login code
+    ├── User exists in database?
+    │   ├── Yes → Check user.tier
+    │   │   ├── tier == BLOCKED → Reject "Account is blocked"
+    │   │   └── tier != BLOCKED → Allow (send code, existing users grandfathered)
+    │   └── No (new user) → Check EMAIL__TRUSTED_EMAIL_DOMAINS
+    │       ├── Setting configured → domain in trusted list?
+    │       │   ├── Yes → Create user & send code
+    │       │   └── No → Reject "Email domain not allowed for signup"
+    │       └── Not configured (empty) → Create user & send code (no restrictions)
+
+Key Behaviors:
+- Existing users: Always allowed to login (unless tier=BLOCKED)
+- New users: Must have email from trusted domain (if EMAIL__TRUSTED_EMAIL_DOMAINS is set)
+- No restrictions: Leave EMAIL__TRUSTED_EMAIL_DOMAINS empty to allow all domains
+
+User Tiers (models.entities.UserTier):
+- BLOCKED: Cannot login (rejected at send-code)
+- ANONYMOUS: Rate-limited anonymous access
+- FREE: Standard free tier
+- BASIC/PRO: Paid tiers with additional features
+
+Configuration:
+    # Allow only specific domains for new signups
+    EMAIL__TRUSTED_EMAIL_DOMAINS=siggymd.ai,example.com
+
+    # Allow all domains (no restrictions)
+    EMAIL__TRUSTED_EMAIL_DOMAINS=
+
+Example blocking a user:
+    user = await user_repo.get_by_id(user_id, tenant_id="default")
+    user.tier = UserTier.BLOCKED
+    await user_repo.upsert(user)
+
+=============================================================================
+OAuth Design Pattern (OAuth 2.1 + PKCE)
+=============================================================================
+
 1. User clicks "Login with Google"
 2. /login generates state + PKCE code_verifier
 3. Store code_verifier in session
@@ -37,6 +85,7 @@ Environment variables:
     AUTH__MICROSOFT__CLIENT_ID=<microsoft-client-id>
     AUTH__MICROSOFT__CLIENT_SECRET=<microsoft-client-secret>
     AUTH__MICROSOFT__TENANT=common
+    EMAIL__TRUSTED_EMAIL_DOMAINS=example.com  # Optional: restrict new signups
 
 References:
 - Authlib: https://docs.authlib.org/en/latest/
@@ -46,11 +95,13 @@ References:
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import RedirectResponse
 from authlib.integrations.starlette_client import OAuth
+from pydantic import BaseModel, EmailStr
 from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.service import PostgresService
 from ...services.user_service import UserService
+from ...auth.providers.email import EmailAuthProvider
 
 router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -87,6 +138,159 @@ if settings.auth.microsoft.client_id:
     logger.info(f"Microsoft OAuth provider registered (tenant: {tenant})")
 
 
+# =============================================================================
+# Email Authentication Endpoints
+# =============================================================================
+
+
+class EmailSendCodeRequest(BaseModel):
+    """Request to send login code."""
+    email: EmailStr
+
+
+class EmailVerifyRequest(BaseModel):
+    """Request to verify login code."""
+    email: EmailStr
+    code: str
+
+
+@router.post("/email/send-code")
+async def send_email_code(request: Request, body: EmailSendCodeRequest):
+    """
+    Send a login code to an email address.
+
+    Creates user if not exists (using deterministic UUID from email).
+    Stores code in user metadata with expiry.
+
+    Args:
+        request: FastAPI request
+        body: EmailSendCodeRequest with email
+
+    Returns:
+        Success status and message
+    """
+    if not settings.email.is_configured:
+        raise HTTPException(
+            status_code=501,
+            detail="Email authentication is not configured"
+        )
+
+    # Get database connection
+    if not settings.postgres.enabled:
+        raise HTTPException(
+            status_code=501,
+            detail="Database is required for email authentication"
+        )
+
+    db = PostgresService()
+    try:
+        await db.connect()
+
+        # Initialize email auth provider
+        email_auth = EmailAuthProvider()
+
+        # Send code
+        result = await email_auth.send_code(
+            email=body.email,
+            db=db,
+        )
+
+        if result.success:
+            return {
+                "success": True,
+                "message": result.message,
+                "email": result.email,
+            }
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=result.message or result.error
+            )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error sending login code: {e}")
+        raise HTTPException(status_code=500, detail="Failed to send login code")
+    finally:
+        await db.disconnect()
+
+
+@router.post("/email/verify")
+async def verify_email_code(request: Request, body: EmailVerifyRequest):
+    """
+    Verify login code and create session.
+
+    Args:
+        request: FastAPI request
+        body: EmailVerifyRequest with email and code
+
+    Returns:
+        Success status with user info
+    """
+    if not settings.email.is_configured:
+        raise HTTPException(
+            status_code=501,
+            detail="Email authentication is not configured"
+        )
+
+    if not settings.postgres.enabled:
+        raise HTTPException(
+            status_code=501,
+            detail="Database is required for email authentication"
+        )
+
+    db = PostgresService()
+    try:
+        await db.connect()
+
+        # Initialize email auth provider
+        email_auth = EmailAuthProvider()
+
+        # Verify code
+        result = await email_auth.verify_code(
+            email=body.email,
+            code=body.code,
+            db=db,
+        )
+
+        if not result.success:
+            raise HTTPException(
+                status_code=400,
+                detail=result.message or result.error
+            )
+
+        # Create session - compatible with OAuth session format
+        user_dict = email_auth.get_user_dict(
+            email=result.email,
+            user_id=result.user_id,
+        )
+
+        # Store user in session
+        request.session["user"] = user_dict
+
+        logger.info(f"User authenticated via email: {result.email}")
+
+        return {
+            "success": True,
+            "message": result.message,
+            "user": user_dict,
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error verifying login code: {e}")
+        raise HTTPException(status_code=500, detail="Failed to verify login code")
+    finally:
+        await db.disconnect()
+
+
+# =============================================================================
+# OAuth Authentication Endpoints
+# =============================================================================
+
+
 @router.get("/{provider}/login")
 async def login(provider: str, request: Request):
     """

rem/api/routers/chat/streaming.py

@@ -76,6 +76,9 @@ async def stream_openai_response(
     agent_schema: str | None = None,
     # Mutable container to capture trace context (deterministic, not AI-dependent)
     trace_context_out: dict | None = None,
+    # Mutable container to capture tool calls for persistence
+    # Format: list of {"tool_name": str, "tool_id": str, "arguments": dict, "result": any}
+    tool_calls_out: list | None = None,
 ) -> AsyncGenerator[str, None]:
     """
     Stream Pydantic AI agent responses with rich SSE events.
@@ -146,6 +149,9 @@ async def stream_openai_response(
     pending_tool_completions: list[tuple[str, str]] = []
     # Track if metadata was registered via register_metadata tool
     metadata_registered = False
+    # Track pending tool calls with full data for persistence
+    # Maps tool_id -> {"tool_name": str, "tool_id": str, "arguments": dict}
+    pending_tool_data: dict[str, dict] = {}
 
     try:
         # Emit initial progress event
@@ -299,6 +305,13 @@ async def stream_openai_response(
                                     arguments=args_dict
                                 ))
 
+                                # Track tool call data for persistence (especially register_metadata)
+                                pending_tool_data[tool_id] = {
+                                    "tool_name": tool_name,
+                                    "tool_id": tool_id,
+                                    "arguments": args_dict,
+                                }
+
                                 # Update progress
                                 current_step = 2
                                 total_steps = 4  # Added tool execution step
@@ -421,6 +434,15 @@ async def stream_openai_response(
                                         hidden=False,
                                     ))
 
+                                # Capture tool call with result for persistence
+                                # Special handling for register_metadata - always capture full data
+                                if tool_calls_out is not None and tool_id in pending_tool_data:
+                                    tool_data = pending_tool_data[tool_id]
+                                    tool_data["result"] = result_content
+                                    tool_data["is_metadata"] = is_metadata_event
+                                    tool_calls_out.append(tool_data)
+                                    del pending_tool_data[tool_id]
+
                                 if not is_metadata_event:
                                     # Normal tool completion - emit ToolCallEvent
                                     result_str = str(result_content)
@@ -728,6 +750,9 @@ async def stream_openai_response_with_save(
     # Accumulate content during streaming
     accumulated_content = []
 
+    # Capture tool calls for persistence (especially register_metadata)
+    tool_calls: list = []
+
     async for chunk in stream_openai_response(
         agent=agent,
         prompt=prompt,
@@ -737,6 +762,7 @@ async def stream_openai_response_with_save(
         session_id=session_id,
         message_id=message_id,
         trace_context_out=trace_context,  # Pass container to capture trace IDs
+        tool_calls_out=tool_calls,  # Capture tool calls for persistence
     ):
         yield chunk
 
@@ -755,28 +781,57 @@ async def stream_openai_response_with_save(
             except (json.JSONDecodeError, KeyError, IndexError):
                 pass  # Skip non-JSON or malformed chunks
 
-    # After streaming completes, save the assistant response
-    if settings.postgres.enabled and session_id and accumulated_content:
-        full_content = "".join(accumulated_content)
+    # After streaming completes, save tool calls and assistant response
+    # Note: All messages stored UNCOMPRESSED. Compression happens on reload.
+    if settings.postgres.enabled and session_id:
         # Get captured trace context from container (deterministically captured inside agent execution)
         captured_trace_id = trace_context.get("trace_id")
         captured_span_id = trace_context.get("span_id")
-        assistant_message = {
-            "id": message_id,  # Use pre-generated ID for consistency with metadata event
-            "role": "assistant",
-            "content": full_content,
-            "timestamp": to_iso(utc_now()),
-            "trace_id": captured_trace_id,
-            "span_id": captured_span_id,
-        }
-        try:
-            store = SessionMessageStore(user_id=user_id or settings.test.effective_user_id)
-            await store.store_session_messages(
-                session_id=session_id,
-                messages=[assistant_message],
-                user_id=user_id,
-                compress=True,  # Compress long assistant responses
-            )
-            logger.debug(f"Saved assistant response {message_id} to session {session_id} ({len(full_content)} chars)")
-        except Exception as e:
-            logger.error(f"Failed to save assistant response: {e}", exc_info=True)
+        timestamp = to_iso(utc_now())
+
+        messages_to_store = []
+
+        # First, store tool call messages (message_type: "tool")
+        for tool_call in tool_calls:
+            tool_message = {
+                "role": "tool",
+                "content": json.dumps(tool_call.get("result", {}), default=str),
+                "timestamp": timestamp,
+                "trace_id": captured_trace_id,
+                "span_id": captured_span_id,
+                # Store tool call details in a way that can be reconstructed
+                "tool_call_id": tool_call.get("tool_id"),
+                "tool_name": tool_call.get("tool_name"),
+                "tool_arguments": tool_call.get("arguments"),
+            }
+            messages_to_store.append(tool_message)
+
+        # Then store assistant text response (if any)
+        if accumulated_content:
+            full_content = "".join(accumulated_content)
+            assistant_message = {
+                "id": message_id,  # Use pre-generated ID for consistency with metadata event
+                "role": "assistant",
+                "content": full_content,
+                "timestamp": timestamp,
+                "trace_id": captured_trace_id,
+                "span_id": captured_span_id,
+            }
+            messages_to_store.append(assistant_message)
+
+        if messages_to_store:
+            try:
+                store = SessionMessageStore(user_id=user_id or settings.test.effective_user_id)
+                await store.store_session_messages(
+                    session_id=session_id,
+                    messages=messages_to_store,
+                    user_id=user_id,
+                    compress=False,  # Store uncompressed; compression happens on reload
+                )
+                logger.debug(
+                    f"Saved {len(tool_calls)} tool calls and "
+                    f"{'assistant response' if accumulated_content else 'no text'} "
+                    f"to session {session_id}"
+                )
+            except Exception as e:
+                logger.error(f"Failed to save session messages: {e}", exc_info=True)