remdb 0.3.141__py3-none-any.whl → 0.3.163__py3-none-any.whl

rem/api/routers/chat/streaming.py

@@ -76,6 +76,9 @@ async def stream_openai_response(
     agent_schema: str | None = None,
     # Mutable container to capture trace context (deterministic, not AI-dependent)
     trace_context_out: dict | None = None,
+    # Mutable container to capture tool calls for persistence
+    # Format: list of {"tool_name": str, "tool_id": str, "arguments": dict, "result": any}
+    tool_calls_out: list | None = None,
 ) -> AsyncGenerator[str, None]:
     """
     Stream Pydantic AI agent responses with rich SSE events.
@@ -146,6 +149,9 @@ async def stream_openai_response(
     pending_tool_completions: list[tuple[str, str]] = []
     # Track if metadata was registered via register_metadata tool
     metadata_registered = False
+    # Track pending tool calls with full data for persistence
+    # Maps tool_id -> {"tool_name": str, "tool_id": str, "arguments": dict}
+    pending_tool_data: dict[str, dict] = {}
 
     try:
         # Emit initial progress event
@@ -299,6 +305,13 @@ async def stream_openai_response(
                                     arguments=args_dict
                                 ))
 
+                                # Track tool call data for persistence (especially register_metadata)
+                                pending_tool_data[tool_id] = {
+                                    "tool_name": tool_name,
+                                    "tool_id": tool_id,
+                                    "arguments": args_dict,
+                                }
+
                                 # Update progress
                                 current_step = 2
                                 total_steps = 4  # Added tool execution step
@@ -421,6 +434,15 @@ async def stream_openai_response(
                                         hidden=False,
                                     ))
 
+                                # Capture tool call with result for persistence
+                                # Special handling for register_metadata - always capture full data
+                                if tool_calls_out is not None and tool_id in pending_tool_data:
+                                    tool_data = pending_tool_data[tool_id]
+                                    tool_data["result"] = result_content
+                                    tool_data["is_metadata"] = is_metadata_event
+                                    tool_calls_out.append(tool_data)
+                                    del pending_tool_data[tool_id]
+
                                 if not is_metadata_event:
                                     # Normal tool completion - emit ToolCallEvent
                                     result_str = str(result_content)
@@ -728,6 +750,9 @@ async def stream_openai_response_with_save(
     # Accumulate content during streaming
     accumulated_content = []
 
+    # Capture tool calls for persistence (especially register_metadata)
+    tool_calls: list = []
+
     async for chunk in stream_openai_response(
         agent=agent,
         prompt=prompt,
@@ -737,6 +762,7 @@ async def stream_openai_response_with_save(
         session_id=session_id,
         message_id=message_id,
         trace_context_out=trace_context,  # Pass container to capture trace IDs
+        tool_calls_out=tool_calls,  # Capture tool calls for persistence
     ):
         yield chunk
 
@@ -755,28 +781,57 @@ async def stream_openai_response_with_save(
             except (json.JSONDecodeError, KeyError, IndexError):
                 pass  # Skip non-JSON or malformed chunks
 
-    # After streaming completes, save the assistant response
-    if settings.postgres.enabled and session_id and accumulated_content:
-        full_content = "".join(accumulated_content)
+    # After streaming completes, save tool calls and assistant response
+    # Note: All messages stored UNCOMPRESSED. Compression happens on reload.
+    if settings.postgres.enabled and session_id:
         # Get captured trace context from container (deterministically captured inside agent execution)
         captured_trace_id = trace_context.get("trace_id")
         captured_span_id = trace_context.get("span_id")
-        assistant_message = {
-            "id": message_id,  # Use pre-generated ID for consistency with metadata event
-            "role": "assistant",
-            "content": full_content,
-            "timestamp": to_iso(utc_now()),
-            "trace_id": captured_trace_id,
-            "span_id": captured_span_id,
-        }
-        try:
-            store = SessionMessageStore(user_id=user_id or settings.test.effective_user_id)
-            await store.store_session_messages(
-                session_id=session_id,
-                messages=[assistant_message],
-                user_id=user_id,
-                compress=True,  # Compress long assistant responses
-            )
-            logger.debug(f"Saved assistant response {message_id} to session {session_id} ({len(full_content)} chars)")
-        except Exception as e:
-            logger.error(f"Failed to save assistant response: {e}", exc_info=True)
+        timestamp = to_iso(utc_now())
+
+        messages_to_store = []
+
+        # First, store tool call messages (message_type: "tool")
+        for tool_call in tool_calls:
+            tool_message = {
+                "role": "tool",
+                "content": json.dumps(tool_call.get("result", {}), default=str),
+                "timestamp": timestamp,
+                "trace_id": captured_trace_id,
+                "span_id": captured_span_id,
+                # Store tool call details in a way that can be reconstructed
+                "tool_call_id": tool_call.get("tool_id"),
+                "tool_name": tool_call.get("tool_name"),
+                "tool_arguments": tool_call.get("arguments"),
+            }
+            messages_to_store.append(tool_message)
+
+        # Then store assistant text response (if any)
+        if accumulated_content:
+            full_content = "".join(accumulated_content)
+            assistant_message = {
+                "id": message_id,  # Use pre-generated ID for consistency with metadata event
+                "role": "assistant",
+                "content": full_content,
+                "timestamp": timestamp,
+                "trace_id": captured_trace_id,
+                "span_id": captured_span_id,
+            }
+            messages_to_store.append(assistant_message)
+
+        if messages_to_store:
+            try:
+                store = SessionMessageStore(user_id=user_id or settings.test.effective_user_id)
+                await store.store_session_messages(
+                    session_id=session_id,
+                    messages=messages_to_store,
+                    user_id=user_id,
+                    compress=False,  # Store uncompressed; compression happens on reload
+                )
+                logger.debug(
+                    f"Saved {len(tool_calls)} tool calls and "
+                    f"{'assistant response' if accumulated_content else 'no text'} "
+                    f"to session {session_id}"
+                )
+            except Exception as e:
+                logger.error(f"Failed to save session messages: {e}", exc_info=True)

rem/api/routers/messages.py

@@ -134,7 +134,6 @@ async def list_messages(
     ),
     limit: int = Query(default=50, ge=1, le=100, description="Max results to return"),
     offset: int = Query(default=0, ge=0, description="Offset for pagination"),
-    x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
 ) -> MessageListResponse:
     """
     List messages with optional filters.
@@ -158,15 +157,18 @@ async def list_messages(
 
     repo = Repository(Message, table_name="messages")
 
+    # Get current user for logging
+    current_user = get_current_user(request)
+    jwt_user_id = current_user.get("id") if current_user else None
+
     # If mine=true, force filter to current user's ID from JWT
     effective_user_id = user_id
     if mine:
-        current_user = get_current_user(request)
         if current_user:
             effective_user_id = current_user.get("id")
 
     # Build user-scoped filters (admin can see all, regular users see only their own)
-    filters = await get_user_filter(request, x_user_id=effective_user_id, x_tenant_id=x_tenant_id)
+    filters = await get_user_filter(request, x_user_id=effective_user_id)
 
     # Apply optional filters
     if session_id:
@@ -174,6 +176,13 @@ async def list_messages(
     if message_type:
         filters["message_type"] = message_type
 
+    # Log the query parameters for debugging
+    logger.debug(
+        f"[messages] Query: session_id={session_id} | "
+        f"jwt_user_id={jwt_user_id} | "
+        f"filters={filters}"
+    )
+
     # For date filtering, we need custom SQL (not supported by basic Repository)
     # For now, fetch all matching base filters and filter in Python
     # TODO: Extend Repository to support date range filters
@@ -206,6 +215,12 @@ async def list_messages(
     # Get total count for pagination info
     total = await repo.count(filters)
 
+    # Log result count
+    logger.debug(
+        f"[messages] Result: returned={len(messages)} | total={total} | "
+        f"session_id={session_id}"
+    )
+
     return MessageListResponse(data=messages, total=total, has_more=has_more)
 
 
@@ -213,7 +228,6 @@ async def list_messages(
 async def get_message(
     request: Request,
     message_id: str,
-    x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
 ) -> Message:
     """
     Get a specific message by ID.
@@ -236,7 +250,7 @@ async def get_message(
         raise HTTPException(status_code=503, detail="Database not enabled")
 
     repo = Repository(Message, table_name="messages")
-    message = await repo.get_by_id(message_id, x_tenant_id)
+    message = await repo.get_by_id(message_id)
 
     if not message:
         raise HTTPException(status_code=404, detail=f"Message '{message_id}' not found")
@@ -263,7 +277,6 @@ async def list_sessions(
     mode: SessionMode | None = Query(default=None, description="Filter by session mode"),
     page: int = Query(default=1, ge=1, description="Page number (1-indexed)"),
     page_size: int = Query(default=50, ge=1, le=100, description="Number of results per page"),
-    x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
 ) -> SessionsQueryResponse:
     """
     List sessions with optional filters and page-based pagination.
@@ -288,7 +301,7 @@ async def list_sessions(
     repo = Repository(Session, table_name="sessions")
 
     # Build user-scoped filters (admin can see all, regular users see only their own)
-    filters = await get_user_filter(request, x_user_id=user_id, x_tenant_id=x_tenant_id)
+    filters = await get_user_filter(request, x_user_id=user_id)
     if mode:
         filters["mode"] = mode.value
 
@@ -319,7 +332,6 @@ async def create_session(
     request_body: SessionCreateRequest,
     user: dict = Depends(require_admin),
     x_user_id: str = Header(alias="X-User-Id", default="default"),
-    x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
 ) -> Session:
     """
     Create a new session.
@@ -334,7 +346,6 @@ async def create_session(
 
     Headers:
     - X-User-Id: User identifier (owner of the session)
-    - X-Tenant-Id: Tenant identifier
 
     Returns:
         Created session object
@@ -354,7 +365,7 @@ async def create_session(
         prompt=request_body.prompt,
         agent_schema_uri=request_body.agent_schema_uri,
         user_id=effective_user_id,
-        tenant_id=x_tenant_id,
+        tenant_id="default",  # tenant_id not used for filtering, set to default
     )
 
     repo = Repository(Session, table_name="sessions")
@@ -372,7 +383,6 @@ async def create_session(
 async def get_session(
     request: Request,
     session_id: str,
-    x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
 ) -> Session:
     """
     Get a specific session by ID.
@@ -395,11 +405,11 @@ async def get_session(
         raise HTTPException(status_code=503, detail="Database not enabled")
 
     repo = Repository(Session, table_name="sessions")
-    session = await repo.get_by_id(session_id, x_tenant_id)
+    session = await repo.get_by_id(session_id)
 
     if not session:
         # Try finding by name
-        sessions = await repo.find({"name": session_id, "tenant_id": x_tenant_id}, limit=1)
+        sessions = await repo.find({"name": session_id}, limit=1)
         if sessions:
             session = sessions[0]
         else:
@@ -420,7 +430,6 @@ async def update_session(
     request: Request,
     session_id: str,
     request_body: SessionUpdateRequest,
-    x_tenant_id: str = Header(alias="X-Tenant-Id", default="default"),
 ) -> Session:
     """
     Update an existing session.
@@ -450,7 +459,7 @@ async def update_session(
         raise HTTPException(status_code=503, detail="Database not enabled")
 
     repo = Repository(Session, table_name="sessions")
-    session = await repo.get_by_id(session_id, x_tenant_id)
+    session = await repo.get_by_id(session_id)
 
     if not session:
         raise HTTPException(status_code=404, detail=f"Session '{session_id}' not found")

rem/auth/__init__.py

@@ -1,26 +1,36 @@
 """
 REM Authentication Module.
 
-OAuth 2.1 compliant authentication with support for:
+Authentication with support for:
+- Email passwordless login (verification codes)
 - Google OAuth
 - Microsoft Entra ID (Azure AD) OIDC
 - Custom OIDC providers
 
 Design Pattern:
 - Provider-agnostic base classes
-- PKCE (Proof Key for Code Exchange) for all flows
+- PKCE (Proof Key for Code Exchange) for OAuth flows
 - State parameter for CSRF protection
 - Nonce for ID token replay protection
 - Token validation with JWKS
-- Clean separation: providers/ for OAuth logic, middleware.py for FastAPI integration
+- Clean separation: providers/ for auth logic, middleware.py for FastAPI integration
+
+Email Auth Flow:
+1. POST /api/auth/email/send-code with {email}
+2. User receives code via email
+3. POST /api/auth/email/verify with {email, code}
+4. Session created, user authenticated
 """
 
 from .providers.base import OAuthProvider
+from .providers.email import EmailAuthProvider, EmailAuthResult
 from .providers.google import GoogleOAuthProvider
 from .providers.microsoft import MicrosoftOAuthProvider
 
 __all__ = [
     "OAuthProvider",
+    "EmailAuthProvider",
+    "EmailAuthResult",
     "GoogleOAuthProvider",
     "MicrosoftOAuthProvider",
 ]

rem/auth/jwt.py

@@ -0,0 +1,352 @@
+"""
+JWT Token Service for REM Authentication.
+
+Provides JWT token generation and validation for stateless authentication.
+Uses HS256 algorithm with the session secret for signing.
+
+Token Types:
+- Access Token: Short-lived (default 1 hour), used for API authentication
+- Refresh Token: Long-lived (default 7 days), used to obtain new access tokens
+
+Token Claims:
+- sub: User ID (UUID string)
+- email: User email
+- name: User display name
+- role: User role (user, admin)
+- tier: User subscription tier
+- roles: List of roles for authorization
+- provider: Auth provider (email, google, microsoft)
+- tenant_id: Tenant identifier for multi-tenancy
+- exp: Expiration timestamp
+- iat: Issued at timestamp
+- type: Token type (access, refresh)
+
+Usage:
+    from rem.auth.jwt import JWTService
+
+    jwt_service = JWTService()
+
+    # Generate tokens after successful authentication
+    tokens = jwt_service.create_tokens(user_dict)
+    # Returns: {"access_token": "...", "refresh_token": "...", "token_type": "bearer", "expires_in": 3600}
+
+    # Validate token from Authorization header
+    user = jwt_service.verify_token(token)
+    # Returns user dict or None if invalid
+
+    # Refresh access token
+    new_tokens = jwt_service.refresh_access_token(refresh_token)
+"""
+
+import time
+import hmac
+import hashlib
+import base64
+import json
+from datetime import datetime, timezone
+from typing import Optional
+
+from loguru import logger
+
+
+class JWTService:
+    """
+    JWT token service for authentication.
+
+    Uses HMAC-SHA256 for signing - simple and secure for single-service deployment.
+    For multi-service deployments, consider switching to RS256 with public/private keys.
+    """
+
+    def __init__(
+        self,
+        secret: str | None = None,
+        access_token_expiry_seconds: int = 3600,  # 1 hour
+        refresh_token_expiry_seconds: int = 604800,  # 7 days
+        issuer: str = "rem",
+    ):
+        """
+        Initialize JWT service.
+
+        Args:
+            secret: Secret key for signing (uses settings.auth.session_secret if not provided)
+            access_token_expiry_seconds: Access token lifetime in seconds
+            refresh_token_expiry_seconds: Refresh token lifetime in seconds
+            issuer: Token issuer identifier
+        """
+        if secret:
+            self._secret = secret
+        else:
+            from ..settings import settings
+            self._secret = settings.auth.session_secret
+
+        self._access_expiry = access_token_expiry_seconds
+        self._refresh_expiry = refresh_token_expiry_seconds
+        self._issuer = issuer
+
+    def _base64url_encode(self, data: bytes) -> str:
+        """Base64url encode without padding."""
+        return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8")
+
+    def _base64url_decode(self, data: str) -> bytes:
+        """Base64url decode with padding restoration."""
+        padding = 4 - len(data) % 4
+        if padding != 4:
+            data += "=" * padding
+        return base64.urlsafe_b64decode(data)
+
+    def _sign(self, message: str) -> str:
+        """Create HMAC-SHA256 signature."""
+        signature = hmac.new(
+            self._secret.encode("utf-8"),
+            message.encode("utf-8"),
+            hashlib.sha256
+        ).digest()
+        return self._base64url_encode(signature)
+
+    def _create_token(self, payload: dict) -> str:
+        """
+        Create a JWT token.
+
+        Args:
+            payload: Token claims
+
+        Returns:
+            Encoded JWT string
+        """
+        header = {"alg": "HS256", "typ": "JWT"}
+
+        header_encoded = self._base64url_encode(json.dumps(header, separators=(",", ":")).encode())
+        payload_encoded = self._base64url_encode(json.dumps(payload, separators=(",", ":")).encode())
+
+        message = f"{header_encoded}.{payload_encoded}"
+        signature = self._sign(message)
+
+        return f"{message}.{signature}"
+
+    def _verify_signature(self, token: str) -> dict | None:
+        """
+        Verify token signature and decode payload.
+
+        Args:
+            token: JWT token string
+
+        Returns:
+            Decoded payload dict or None if invalid
+        """
+        try:
+            parts = token.split(".")
+            if len(parts) != 3:
+                return None
+
+            header_encoded, payload_encoded, signature = parts
+
+            # Verify signature
+            message = f"{header_encoded}.{payload_encoded}"
+            expected_signature = self._sign(message)
+
+            if not hmac.compare_digest(signature, expected_signature):
+                logger.debug("JWT signature verification failed")
+                return None
+
+            # Decode payload
+            payload = json.loads(self._base64url_decode(payload_encoded))
+            return payload
+
+        except Exception as e:
+            logger.debug(f"JWT decode error: {e}")
+            return None
+
+    def create_tokens(
+        self,
+        user: dict,
+        access_expiry: int | None = None,
+        refresh_expiry: int | None = None,
+    ) -> dict:
+        """
+        Create access and refresh tokens for a user.
+
+        Args:
+            user: User dict with id, email, name, role, tier, roles, provider, tenant_id
+            access_expiry: Override access token expiry (seconds)
+            refresh_expiry: Override refresh token expiry (seconds)
+
+        Returns:
+            Dict with access_token, refresh_token, token_type, expires_in
+        """
+        now = int(time.time())
+        access_exp = access_expiry or self._access_expiry
+        refresh_exp = refresh_expiry or self._refresh_expiry
+
+        # Common claims
+        base_claims = {
+            "sub": user.get("id", ""),
+            "email": user.get("email", ""),
+            "name": user.get("name", ""),
+            "role": user.get("role"),
+            "tier": user.get("tier", "free"),
+            "roles": user.get("roles", ["user"]),
+            "provider": user.get("provider", "email"),
+            "tenant_id": user.get("tenant_id", "default"),
+            "iss": self._issuer,
+            "iat": now,
+        }
+
+        # Access token
+        access_payload = {
+            **base_claims,
+            "type": "access",
+            "exp": now + access_exp,
+        }
+        access_token = self._create_token(access_payload)
+
+        # Refresh token (minimal claims for security)
+        refresh_payload = {
+            "sub": user.get("id", ""),
+            "email": user.get("email", ""),
+            "type": "refresh",
+            "iss": self._issuer,
+            "iat": now,
+            "exp": now + refresh_exp,
+        }
+        refresh_token = self._create_token(refresh_payload)
+
+        return {
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "token_type": "bearer",
+            "expires_in": access_exp,
+        }
+
+    def verify_token(self, token: str, token_type: str = "access") -> dict | None:
+        """
+        Verify a token and return user claims.
+
+        Args:
+            token: JWT token string
+            token_type: Expected token type ("access" or "refresh")
+
+        Returns:
+            User dict with claims or None if invalid/expired
+        """
+        payload = self._verify_signature(token)
+        if not payload:
+            return None
+
+        # Check token type
+        if payload.get("type") != token_type:
+            logger.debug(f"Token type mismatch: expected {token_type}, got {payload.get('type')}")
+            return None
+
+        # Check expiration
+        exp = payload.get("exp", 0)
+        if exp < time.time():
+            logger.debug("Token expired")
+            return None
+
+        # Check issuer
+        if payload.get("iss") != self._issuer:
+            logger.debug(f"Token issuer mismatch: expected {self._issuer}, got {payload.get('iss')}")
+            return None
+
+        # Return user dict (compatible with session user format)
+        return {
+            "id": payload.get("sub"),
+            "email": payload.get("email"),
+            "name": payload.get("name"),
+            "role": payload.get("role"),
+            "tier": payload.get("tier", "free"),
+            "roles": payload.get("roles", ["user"]),
+            "provider": payload.get("provider", "email"),
+            "tenant_id": payload.get("tenant_id", "default"),
+        }
+
+    def refresh_access_token(self, refresh_token: str) -> dict | None:
+        """
+        Create new access token using refresh token.
+
+        Args:
+            refresh_token: Valid refresh token
+
+        Returns:
+            New token dict or None if refresh token is invalid
+        """
+        # Verify refresh token
+        payload = self._verify_signature(refresh_token)
+        if not payload:
+            return None
+
+        if payload.get("type") != "refresh":
+            logger.debug("Not a refresh token")
+            return None
+
+        # Check expiration
+        exp = payload.get("exp", 0)
+        if exp < time.time():
+            logger.debug("Refresh token expired")
+            return None
+
+        # Create new access token with minimal info from refresh token
+        # In production, you'd look up the full user from database
+        user = {
+            "id": payload.get("sub"),
+            "email": payload.get("email"),
+            "name": payload.get("email", "").split("@")[0],
+            "provider": "email",
+            "tenant_id": "default",
+            "tier": "free",
+            "roles": ["user"],
+        }
+
+        # Only return new access token, keep same refresh token
+        now = int(time.time())
+        access_payload = {
+            "sub": user["id"],
+            "email": user["email"],
+            "name": user["name"],
+            "role": user.get("role"),
+            "tier": user["tier"],
+            "roles": user["roles"],
+            "provider": user["provider"],
+            "tenant_id": user["tenant_id"],
+            "type": "access",
+            "iss": self._issuer,
+            "iat": now,
+            "exp": now + self._access_expiry,
+        }
+
+        return {
+            "access_token": self._create_token(access_payload),
+            "token_type": "bearer",
+            "expires_in": self._access_expiry,
+        }
+
+    def decode_without_verification(self, token: str) -> dict | None:
+        """
+        Decode token without verification (for debugging only).
+
+        Args:
+            token: JWT token string
+
+        Returns:
+            Decoded payload or None
+        """
+        try:
+            parts = token.split(".")
+            if len(parts) != 3:
+                return None
+            payload = json.loads(self._base64url_decode(parts[1]))
+            return payload
+        except Exception:
+            return None
+
+
+# Singleton instance for convenience
+_jwt_service: Optional[JWTService] = None
+
+
+def get_jwt_service() -> JWTService:
+    """Get or create the JWT service singleton."""
+    global _jwt_service
+    if _jwt_service is None:
+        _jwt_service = JWTService()
+    return _jwt_service