remdb 0.3.127__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/api/routers/auth.py

@@ -1,20 +1,68 @@
 """
-OAuth 2.1 Authentication Router.
+Authentication Router.
 
-Leverages Authlib for standards-compliant OAuth/OIDC implementation.
-Minimal custom code - Authlib handles PKCE, token validation, JWKS.
+Supports multiple authentication methods:
+1. Email (passwordless): POST /api/auth/email/send-code, POST /api/auth/email/verify
+2. OAuth (Google, Microsoft): GET /api/auth/{provider}/login, GET /api/auth/{provider}/callback
 
 Endpoints:
+- POST /api/auth/email/send-code     - Send login code to email
+- POST /api/auth/email/verify        - Verify code and create session
 - GET  /api/auth/{provider}/login    - Initiate OAuth flow
 - GET  /api/auth/{provider}/callback - OAuth callback
 - POST /api/auth/logout              - Clear session
 - GET  /api/auth/me                  - Current user info
 
 Supported providers:
+- email: Passwordless email login
 - google: Google OAuth 2.0 / OIDC
 - microsoft: Microsoft Entra ID OIDC
 
-Design Pattern (OAuth 2.1 + PKCE):
+=============================================================================
+Email Authentication Access Control
+=============================================================================
+
+The email auth provider implements a tiered access control system:
+
+Access Control Flow (send-code):
+    User requests login code
+    ├── User exists in database?
+    │   ├── Yes → Check user.tier
+    │   │   ├── tier == BLOCKED → Reject "Account is blocked"
+    │   │   └── tier != BLOCKED → Allow (send code, existing users grandfathered)
+    │   └── No (new user) → Check EMAIL__TRUSTED_EMAIL_DOMAINS
+    │       ├── Setting configured → domain in trusted list?
+    │       │   ├── Yes → Create user & send code
+    │       │   └── No → Reject "Email domain not allowed for signup"
+    │       └── Not configured (empty) → Create user & send code (no restrictions)
+
+Key Behaviors:
+- Existing users: Always allowed to login (unless tier=BLOCKED)
+- New users: Must have email from trusted domain (if EMAIL__TRUSTED_EMAIL_DOMAINS is set)
+- No restrictions: Leave EMAIL__TRUSTED_EMAIL_DOMAINS empty to allow all domains
+
+User Tiers (models.entities.UserTier):
+- BLOCKED: Cannot login (rejected at send-code)
+- ANONYMOUS: Rate-limited anonymous access
+- FREE: Standard free tier
+- BASIC/PRO: Paid tiers with additional features
+
+Configuration:
+    # Allow only specific domains for new signups
+    EMAIL__TRUSTED_EMAIL_DOMAINS=siggymd.ai,example.com
+
+    # Allow all domains (no restrictions)
+    EMAIL__TRUSTED_EMAIL_DOMAINS=
+
+Example blocking a user:
+    user = await user_repo.get_by_id(user_id, tenant_id="default")
+    user.tier = UserTier.BLOCKED
+    await user_repo.upsert(user)
+
+=============================================================================
+OAuth Design Pattern (OAuth 2.1 + PKCE)
+=============================================================================
+
 1. User clicks "Login with Google"
 2. /login generates state + PKCE code_verifier
 3. Store code_verifier in session
@@ -37,6 +85,7 @@ Environment variables:
     AUTH__MICROSOFT__CLIENT_ID=<microsoft-client-id>
     AUTH__MICROSOFT__CLIENT_SECRET=<microsoft-client-secret>
     AUTH__MICROSOFT__TENANT=common
+    EMAIL__TRUSTED_EMAIL_DOMAINS=example.com  # Optional: restrict new signups
 
 References:
 - Authlib: https://docs.authlib.org/en/latest/
@@ -46,11 +95,15 @@ References:
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import RedirectResponse
 from authlib.integrations.starlette_client import OAuth
+from pydantic import BaseModel, EmailStr
 from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.service import PostgresService
 from ...services.user_service import UserService
+from ...auth.providers.email import EmailAuthProvider
+from ...auth.jwt import JWTService, get_jwt_service
+from ...utils.user_id import email_to_user_id
 
 router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -87,6 +140,182 @@ if settings.auth.microsoft.client_id:
     logger.info(f"Microsoft OAuth provider registered (tenant: {tenant})")
 
 
+# =============================================================================
+# Email Authentication Endpoints
+# =============================================================================
+
+
+class EmailSendCodeRequest(BaseModel):
+    """Request to send login code."""
+    email: EmailStr
+
+
+class EmailVerifyRequest(BaseModel):
+    """Request to verify login code."""
+    email: EmailStr
+    code: str
+
+
+@router.post("/email/send-code")
+async def send_email_code(request: Request, body: EmailSendCodeRequest):
+    """
+    Send a login code to an email address.
+
+    Creates user if not exists (using deterministic UUID from email).
+    Stores code in user metadata with expiry.
+
+    Args:
+        request: FastAPI request
+        body: EmailSendCodeRequest with email
+
+    Returns:
+        Success status and message
+    """
+    if not settings.email.is_configured:
+        raise HTTPException(
+            status_code=501,
+            detail="Email authentication is not configured"
+        )
+
+    # Get database connection
+    if not settings.postgres.enabled:
+        raise HTTPException(
+            status_code=501,
+            detail="Database is required for email authentication"
+        )
+
+    db = PostgresService()
+    try:
+        await db.connect()
+
+        # Initialize email auth provider
+        email_auth = EmailAuthProvider()
+
+        # Send code
+        result = await email_auth.send_code(
+            email=body.email,
+            db=db,
+        )
+
+        if result.success:
+            return {
+                "success": True,
+                "message": result.message,
+                "email": result.email,
+            }
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=result.message or result.error
+            )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error sending login code: {e}")
+        raise HTTPException(status_code=500, detail="Failed to send login code")
+    finally:
+        await db.disconnect()
+
+
+@router.post("/email/verify")
+async def verify_email_code(request: Request, body: EmailVerifyRequest):
+    """
+    Verify login code and create session with JWT tokens.
+
+    Args:
+        request: FastAPI request
+        body: EmailVerifyRequest with email and code
+
+    Returns:
+        Success status with user info and JWT tokens
+    """
+    if not settings.email.is_configured:
+        raise HTTPException(
+            status_code=501,
+            detail="Email authentication is not configured"
+        )
+
+    if not settings.postgres.enabled:
+        raise HTTPException(
+            status_code=501,
+            detail="Database is required for email authentication"
+        )
+
+    db = PostgresService()
+    try:
+        await db.connect()
+
+        # Initialize email auth provider
+        email_auth = EmailAuthProvider()
+
+        # Verify code
+        result = await email_auth.verify_code(
+            email=body.email,
+            code=body.code,
+            db=db,
+        )
+
+        if not result.success:
+            raise HTTPException(
+                status_code=400,
+                detail=result.message or result.error
+            )
+
+        # Create session - compatible with OAuth session format
+        user_dict = email_auth.get_user_dict(
+            email=result.email,
+            user_id=result.user_id,
+        )
+
+        # Fetch actual user data from database to get role/tier
+        user_service = UserService(db)
+        try:
+            user_entity = await user_service.get_user_by_id(result.user_id)
+            if user_entity:
+                # Override defaults with actual database values
+                user_dict["role"] = user_entity.role or "user"
+                user_dict["roles"] = [user_entity.role] if user_entity.role else ["user"]
+                user_dict["tier"] = user_entity.tier.value if user_entity.tier else "free"
+                user_dict["name"] = user_entity.name or user_dict["name"]
+        except Exception as e:
+            logger.warning(f"Could not fetch user details: {e}")
+            # Continue with defaults from get_user_dict
+
+        # Generate JWT tokens
+        jwt_service = get_jwt_service()
+        tokens = jwt_service.create_tokens(user_dict)
+
+        # Store user in session (for backward compatibility)
+        request.session["user"] = user_dict
+
+        logger.info(f"User authenticated via email: {result.email}")
+
+        return {
+            "success": True,
+            "message": result.message,
+            "user": user_dict,
+            # JWT tokens for stateless auth
+            "access_token": tokens["access_token"],
+            "refresh_token": tokens["refresh_token"],
+            "token_type": tokens["token_type"],
+            "expires_in": tokens["expires_in"],
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error verifying login code: {e}")
+        raise HTTPException(status_code=500, detail="Failed to verify login code")
+    finally:
+        await db.disconnect()
+
+
+# =============================================================================
+# OAuth Authentication Endpoints
+# =============================================================================
+
+
 @router.get("/{provider}/login")
 async def login(provider: str, request: Request):
     """
@@ -201,8 +430,9 @@ async def callback(provider: str, request: Request):
                     await user_service.link_anonymous_session(user_entity, anon_id)
                     
                 # Enrich session user with DB info
+                # user_id = UUID5 hash of email (deterministic, bijection)
                 db_info = {
-                    "id": str(user_entity.id),
+                    "id": email_to_user_id(user_info.get("email")),
                     "tenant_id": user_entity.tenant_id,
                     "tier": user_entity.tier.value if user_entity.tier else "free",
                     "roles": [user_entity.role] if user_entity.role else [],
@@ -268,7 +498,7 @@ async def logout(request: Request):
 @router.get("/me")
 async def me(request: Request):
     """
-    Get current user information from session.
+    Get current user information from session or JWT.
 
     Args:
         request: FastAPI request
@@ -276,6 +506,16 @@ async def me(request: Request):
     Returns:
         User information or 401 if not authenticated
     """
+    # First check for JWT in Authorization header
+    auth_header = request.headers.get("Authorization")
+    if auth_header and auth_header.startswith("Bearer "):
+        token = auth_header[7:]
+        jwt_service = get_jwt_service()
+        user = jwt_service.verify_token(token)
+        if user:
+            return user
+
+    # Fall back to session
     user = request.session.get("user")
     if not user:
         raise HTTPException(status_code=401, detail="Not authenticated")
@@ -283,6 +523,69 @@ async def me(request: Request):
     return user
 
 
+# =============================================================================
+# JWT Token Endpoints
+# =============================================================================
+
+
+class TokenRefreshRequest(BaseModel):
+    """Request to refresh access token."""
+    refresh_token: str
+
+
+@router.post("/token/refresh")
+async def refresh_token(body: TokenRefreshRequest):
+    """
+    Refresh access token using refresh token.
+
+    Args:
+        body: TokenRefreshRequest with refresh_token
+
+    Returns:
+        New access token or 401 if refresh token is invalid
+    """
+    jwt_service = get_jwt_service()
+    result = jwt_service.refresh_access_token(body.refresh_token)
+
+    if not result:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid or expired refresh token"
+        )
+
+    return result
+
+
+@router.post("/token/verify")
+async def verify_token(request: Request):
+    """
+    Verify an access token is valid.
+
+    Pass the token in the Authorization header: Bearer <token>
+
+    Returns:
+        User info if valid, 401 if invalid
+    """
+    auth_header = request.headers.get("Authorization")
+    if not auth_header or not auth_header.startswith("Bearer "):
+        raise HTTPException(
+            status_code=401,
+            detail="Missing Authorization header"
+        )
+
+    token = auth_header[7:]
+    jwt_service = get_jwt_service()
+    user = jwt_service.verify_token(token)
+
+    if not user:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid or expired token"
+        )
+
+    return {"valid": True, "user": user}
+
+
 # =============================================================================
 # Development Token Endpoints (non-production only)
 # =============================================================================
@@ -351,3 +654,43 @@ async def get_dev_token(request: Request):
         "usage": f'curl -H "Authorization: Bearer {token}" http://localhost:8000/api/v1/...',
         "warning": "This token is for development/testing only and will not work in production.",
     }
+
+
+@router.get("/dev/mock-code/{email}")
+async def get_mock_code(email: str, request: Request):
+    """
+    Get the mock login code for testing (non-production only).
+
+    This endpoint retrieves the code that was "sent" via email in mock mode.
+    Use this for automated testing without real email delivery.
+
+    Usage:
+        1. POST /api/auth/email/send-code with email
+        2. GET /api/auth/dev/mock-code/{email} to retrieve the code
+        3. POST /api/auth/email/verify with email and code
+
+    Returns:
+        401 if in production environment
+        404 if no code found for the email
+        The code and email otherwise
+    """
+    if settings.environment == "production":
+        raise HTTPException(
+            status_code=401,
+            detail="Mock codes are not available in production"
+        )
+
+    from ...services.email import EmailService
+
+    code = EmailService.get_mock_code(email)
+    if not code:
+        raise HTTPException(
+            status_code=404,
+            detail=f"No mock code found for {email}. Send a code first."
+        )
+
+    return {
+        "email": email,
+        "code": code,
+        "warning": "This endpoint is for testing only and will not work in production.",
+    }

rem/api/routers/chat/completions.py

@@ -97,7 +97,7 @@ Context Building Flow:
    - Long messages include REM LOOKUP hints: "... [REM LOOKUP session-{id}-msg-{index}] ..."
    - Agent can retrieve full content on-demand using REM LOOKUP
 3. User profile provided as REM LOOKUP hint (on-demand by default)
-   - Agent receives: "User ID: {user_id}. To load user profile: Use REM LOOKUP users/{user_id}"
+   - Agent receives: "User: {email}. To load user profile: Use REM LOOKUP \"{email}\""
    - Agent decides whether to load profile based on query
 4. If CHAT__AUTO_INJECT_USER_CONTEXT=true: User profile auto-loaded and injected
 5. Combines: system context + compressed session history + new messages
@@ -330,8 +330,8 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
     - Useful for A/B testing, model comparison, and feedback collection
     """
     # Load agent schema: use header value from context or default
-    # Extract AgentContext first to get schema name
-    temp_context = AgentContext.from_headers(dict(request.headers))
+    # Extract AgentContext from request (gets user_id from JWT token)
+    temp_context = AgentContext.from_request(request)
     schema_name = temp_context.agent_schema_uri or DEFAULT_AGENT_SCHEMA
 
     # Resolve model: use body.model if provided, otherwise settings default
@@ -350,6 +350,7 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
         context, messages = await ContextBuilder.build_from_headers(
             headers=dict(request.headers),
             new_messages=new_messages,
+            user_id=temp_context.user_id,  # From JWT token (source of truth)
         )
 
         # Ensure session exists with metadata and eval mode if applicable
@@ -509,6 +510,7 @@ async def chat_completions(body: ChatCompletionRequest, request: Request):
     context, messages = await ContextBuilder.build_from_headers(
         headers=dict(request.headers),
         new_messages=new_messages,
+        user_id=temp_context.user_id,  # From JWT token (source of truth)
     )
 
     logger.info(f"Built context with {len(messages)} total messages (includes history + user context)")

rem/api/routers/chat/streaming.py

@@ -76,6 +76,9 @@ async def stream_openai_response(
     agent_schema: str | None = None,
     # Mutable container to capture trace context (deterministic, not AI-dependent)
     trace_context_out: dict | None = None,
+    # Mutable container to capture tool calls for persistence
+    # Format: list of {"tool_name": str, "tool_id": str, "arguments": dict, "result": any}
+    tool_calls_out: list | None = None,
 ) -> AsyncGenerator[str, None]:
     """
     Stream Pydantic AI agent responses with rich SSE events.
@@ -146,6 +149,9 @@ async def stream_openai_response(
     pending_tool_completions: list[tuple[str, str]] = []
     # Track if metadata was registered via register_metadata tool
     metadata_registered = False
+    # Track pending tool calls with full data for persistence
+    # Maps tool_id -> {"tool_name": str, "tool_id": str, "arguments": dict}
+    pending_tool_data: dict[str, dict] = {}
 
     try:
         # Emit initial progress event
@@ -299,6 +305,13 @@ async def stream_openai_response(
                                     arguments=args_dict
                                 ))
 
+                                # Track tool call data for persistence (especially register_metadata)
+                                pending_tool_data[tool_id] = {
+                                    "tool_name": tool_name,
+                                    "tool_id": tool_id,
+                                    "arguments": args_dict,
+                                }
+
                                 # Update progress
                                 current_step = 2
                                 total_steps = 4  # Added tool execution step
@@ -421,6 +434,15 @@ async def stream_openai_response(
                                         hidden=False,
                                     ))
 
+                                # Capture tool call with result for persistence
+                                # Special handling for register_metadata - always capture full data
+                                if tool_calls_out is not None and tool_id in pending_tool_data:
+                                    tool_data = pending_tool_data[tool_id]
+                                    tool_data["result"] = result_content
+                                    tool_data["is_metadata"] = is_metadata_event
+                                    tool_calls_out.append(tool_data)
+                                    del pending_tool_data[tool_id]
+
                                 if not is_metadata_event:
                                     # Normal tool completion - emit ToolCallEvent
                                     result_str = str(result_content)
@@ -728,6 +750,9 @@ async def stream_openai_response_with_save(
     # Accumulate content during streaming
     accumulated_content = []
 
+    # Capture tool calls for persistence (especially register_metadata)
+    tool_calls: list = []
+
     async for chunk in stream_openai_response(
         agent=agent,
         prompt=prompt,
@@ -737,6 +762,7 @@ async def stream_openai_response_with_save(
         session_id=session_id,
         message_id=message_id,
         trace_context_out=trace_context,  # Pass container to capture trace IDs
+        tool_calls_out=tool_calls,  # Capture tool calls for persistence
     ):
         yield chunk
 
@@ -755,28 +781,75 @@ async def stream_openai_response_with_save(
             except (json.JSONDecodeError, KeyError, IndexError):
                 pass  # Skip non-JSON or malformed chunks
 
-    # After streaming completes, save the assistant response
-    if settings.postgres.enabled and session_id and accumulated_content:
-        full_content = "".join(accumulated_content)
+    # After streaming completes, save tool calls and assistant response
+    # Note: All messages stored UNCOMPRESSED. Compression happens on reload.
+    if settings.postgres.enabled and session_id:
         # Get captured trace context from container (deterministically captured inside agent execution)
         captured_trace_id = trace_context.get("trace_id")
         captured_span_id = trace_context.get("span_id")
-        assistant_message = {
-            "id": message_id,  # Use pre-generated ID for consistency with metadata event
-            "role": "assistant",
-            "content": full_content,
-            "timestamp": to_iso(utc_now()),
-            "trace_id": captured_trace_id,
-            "span_id": captured_span_id,
-        }
-        try:
-            store = SessionMessageStore(user_id=user_id or settings.test.effective_user_id)
-            await store.store_session_messages(
-                session_id=session_id,
-                messages=[assistant_message],
-                user_id=user_id,
-                compress=True,  # Compress long assistant responses
-            )
-            logger.debug(f"Saved assistant response {message_id} to session {session_id} ({len(full_content)} chars)")
-        except Exception as e:
-            logger.error(f"Failed to save assistant response: {e}", exc_info=True)
+        timestamp = to_iso(utc_now())
+
+        messages_to_store = []
+
+        # First, store tool call messages (message_type: "tool")
+        for tool_call in tool_calls:
+            tool_message = {
+                "role": "tool",
+                "content": json.dumps(tool_call.get("result", {}), default=str),
+                "timestamp": timestamp,
+                "trace_id": captured_trace_id,
+                "span_id": captured_span_id,
+                # Store tool call details in a way that can be reconstructed
+                "tool_call_id": tool_call.get("tool_id"),
+                "tool_name": tool_call.get("tool_name"),
+                "tool_arguments": tool_call.get("arguments"),
+            }
+            messages_to_store.append(tool_message)
+
+        # Then store assistant text response (if any)
+        if accumulated_content:
+            full_content = "".join(accumulated_content)
+            assistant_message = {
+                "id": message_id,  # Use pre-generated ID for consistency with metadata event
+                "role": "assistant",
+                "content": full_content,
+                "timestamp": timestamp,
+                "trace_id": captured_trace_id,
+                "span_id": captured_span_id,
+            }
+            messages_to_store.append(assistant_message)
+
+        if messages_to_store:
+            try:
+                store = SessionMessageStore(user_id=user_id or settings.test.effective_user_id)
+                await store.store_session_messages(
+                    session_id=session_id,
+                    messages=messages_to_store,
+                    user_id=user_id,
+                    compress=False,  # Store uncompressed; compression happens on reload
+                )
+                logger.debug(
+                    f"Saved {len(tool_calls)} tool calls and "
+                    f"{'assistant response' if accumulated_content else 'no text'} "
+                    f"to session {session_id}"
+                )
+            except Exception as e:
+                logger.error(f"Failed to save session messages: {e}", exc_info=True)
+
+        # Update session description with session_name (non-blocking, after all yields)
+        for tool_call in tool_calls:
+            if tool_call.get("tool_name") == "register_metadata" and tool_call.get("is_metadata"):
+                session_name = tool_call.get("arguments", {}).get("session_name")
+                if session_name:
+                    try:
+                        from ....models.entities import Session
+                        from ....services.postgres import Repository
+                        repo = Repository(Session, table_name="sessions")
+                        session = await repo.get_by_id(session_id)
+                        if session and session.description != session_name:
+                            session.description = session_name
+                            await repo.update(session)
+                            logger.debug(f"Updated session {session_id} description to '{session_name}'")
+                    except Exception as e:
+                        logger.warning(f"Failed to update session description: {e}")
+                    break