remdb 0.2.6__py3-none-any.whl → 0.3.103__py3-none-any.whl

rem/api/mcp_router/tools.py

@@ -422,6 +422,7 @@ async def ingest_into_rem(
     tags: list[str] | None = None,
     is_local_server: bool = False,
     user_id: str | None = None,
+    resource_type: str | None = None,
 ) -> dict[str, Any]:
     """
     Ingest file into REM, creating searchable resources and embeddings.
@@ -448,6 +449,11 @@ async def ingest_into_rem(
         tags: Optional tags for file
         is_local_server: True if running as local/stdio MCP server
         user_id: Optional user identifier (defaults to authenticated user or "default")
+        resource_type: Optional resource type for storing chunks (case-insensitive).
+            Supports flexible naming:
+            - "resource", "resources", "Resource" → Resource (default)
+            - "domain-resource", "domain_resource", "DomainResource",
+              "domain-resources" → DomainResource (curated internal knowledge)
 
     Returns:
         Dict with:
@@ -478,6 +484,13 @@ async def ingest_into_rem(
             file_uri="https://example.com/whitepaper.pdf",
             tags=["research", "whitepaper"]
         )
+
+        # Ingest as curated domain knowledge
+        ingest_into_rem(
+            file_uri="s3://bucket/internal/procedures.pdf",
+            resource_type="domain-resource",
+            category="procedures"
+        )
     """
     from ...services.content import ContentService
 
@@ -493,6 +506,7 @@ async def ingest_into_rem(
         category=category,
         tags=tags,
         is_local_server=is_local_server,
+        resource_type=resource_type,
     )
 
     logger.info(
@@ -582,3 +596,69 @@ async def read_resource(uri: str) -> dict[str, Any]:
             "uri": uri,
             "data": {"content": result},
         }
+
+
+async def register_metadata(
+    confidence: float | None = None,
+    references: list[str] | None = None,
+    sources: list[str] | None = None,
+    flags: list[str] | None = None,
+) -> dict[str, Any]:
+    """
+    Register response metadata to be emitted as an SSE MetadataEvent.
+
+    Call this tool BEFORE generating your final response to provide structured
+    metadata that will be sent to the client alongside your natural language output.
+    This allows you to stream conversational responses while still providing
+    machine-readable confidence scores, references, and other metadata.
+
+    **Design Pattern**: Agents can call this once before their final response to
+    register metadata that the streaming layer will emit as a MetadataEvent.
+    This decouples structured metadata from the response format.
+
+    Args:
+        confidence: Confidence score (0.0-1.0) for the response quality.
+            - 0.9-1.0: High confidence, answer is well-supported
+            - 0.7-0.9: Medium confidence, some uncertainty
+            - 0.5-0.7: Low confidence, significant gaps
+            - <0.5: Very uncertain, may need clarification
+        references: List of reference identifiers (file paths, document IDs,
+            entity labels) that support the response.
+        sources: List of source descriptions (e.g., "REM database",
+            "search results", "user context").
+        flags: Optional flags for the response (e.g., "needs_review",
+            "uncertain", "incomplete").
+
+    Returns:
+        Dict with:
+        - status: "success"
+        - _metadata_event: True (marker for streaming layer)
+        - confidence, references, sources, flags: The registered values
+
+    Examples:
+        # High confidence answer with references
+        register_metadata(
+            confidence=0.95,
+            references=["sarah-chen", "q3-report-2024"],
+            sources=["REM database lookup"]
+        )
+
+        # Lower confidence with flags
+        register_metadata(
+            confidence=0.65,
+            flags=["needs_review", "incomplete_data"]
+        )
+    """
+    logger.info(
+        f"📊 Registering metadata: confidence={confidence}, "
+        f"refs={len(references or [])}, sources={len(sources or [])}"
+    )
+
+    return {
+        "status": "success",
+        "_metadata_event": True,  # Marker for streaming layer
+        "confidence": confidence,
+        "references": references,
+        "sources": sources,
+        "flags": flags,
+    }

rem/api/middleware/tracking.py

@@ -0,0 +1,172 @@
+"""
+Anonymous User Tracking & Rate Limiting Middleware.
+
+Handles:
+1. Anonymous Identity: Generates/Validates 'rem_anon_id' cookie.
+2. Context Injection: Sets request.state.anon_id.
+3. Rate Limiting: Enforces tenant-aware tiered limits via RateLimitService.
+"""
+
+import hmac
+import hashlib
+import uuid
+import secrets
+from typing import Optional
+
+from fastapi import Request, Response
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.types import ASGIApp
+
+from ...services.postgres.service import PostgresService
+from ...services.rate_limit import RateLimitService
+from ...models.entities.user import UserTier
+from ...settings import settings
+
+
+class AnonymousTrackingMiddleware(BaseHTTPMiddleware):
+    """
+    Middleware for anonymous user tracking and rate limiting.
+    
+    Design Pattern:
+    - Uses a secure, signed cookie for anonymous ID.
+    - Enforces rate limits before request processing.
+    - Injects anon_id into request state.
+    """
+    
+    def __init__(self, app: ASGIApp):
+        super().__init__(app)
+        # Secret for signing cookies (should be in settings, fallback for safety)
+        self.secret_key = settings.auth.session_secret or "fallback-secret-change-me"
+        self.cookie_name = "rem_anon_id"
+        
+        # Dedicated DB service for this middleware (one pool per app instance)
+        self.db = PostgresService()
+        self.rate_limiter = RateLimitService(self.db)
+        
+        # Excluded paths (health checks, static assets, auth callbacks)
+        self.excluded_paths = {
+            "/health", 
+            "/docs", 
+            "/openapi.json", 
+            "/favicon.ico",
+            "/api/auth", # Don't rate limit auth flow heavily
+        }
+
+    async def dispatch(self, request: Request, call_next):
+        # 0. Skip excluded paths
+        if any(request.url.path.startswith(p) for p in self.excluded_paths):
+            return await call_next(request)
+
+        # 1. Lazy DB Connection
+        if not self.db.pool:
+            # Note: simple lazy init. In high concurrency startup, might trigger multiple connects
+            # followed by disconnects, but asyncpg pool handles this gracefully usually.
+            # Ideally hook into lifespan, but middleware is separate.
+            if settings.postgres.enabled:
+                await self.db.connect()
+
+        # 2. Identification (Cookie Strategy)
+        anon_id = request.cookies.get(self.cookie_name)
+        is_new_anon = False
+        
+        if not anon_id or not self._validate_signature(anon_id):
+            anon_id = self._generate_signed_id()
+            is_new_anon = True
+            
+        # Strip signature for internal use
+        raw_anon_id = anon_id.split(".")[0]
+        request.state.anon_id = raw_anon_id
+        
+        # 3. Determine User Tier & ID for Rate Limiting
+        # Check if user is authenticated (set by AuthMiddleware usually, but that runs AFTER?)
+        # Actually middleware runs in reverse order of addition. 
+        # If AuthMiddleware adds user to request.session, we might need to access session directly.
+        # request.user is standard.
+        
+        user = getattr(request.state, "user", None)
+        if user:
+            # Authenticated User
+            identifier = user.get("id") # Assuming user dict or object
+            # Determine tier from user object
+            tier_str = user.get("tier", UserTier.FREE.value)
+            try:
+                tier = UserTier(tier_str)
+            except ValueError:
+                tier = UserTier.FREE
+            tenant_id = user.get("tenant_id", "default")
+        else:
+            # Anonymous User
+            identifier = raw_anon_id
+            tier = UserTier.ANONYMOUS
+            # Tenant ID from header or default
+            tenant_id = request.headers.get("X-Tenant-Id", "default")
+
+        # 4. Rate Limiting
+        if settings.postgres.enabled:
+            is_allowed, current, limit = await self.rate_limiter.check_rate_limit(
+                tenant_id=tenant_id,
+                identifier=identifier,
+                tier=tier
+            )
+            
+            if not is_allowed:
+                return JSONResponse(
+                    status_code=429,
+                    content={
+                        "error": {
+                            "code": "rate_limit_exceeded",
+                            "message": "You have exceeded your rate limit. Please sign in or upgrade to continue.",
+                            "details": {
+                                "limit": limit,
+                                "tier": tier.value,
+                                "retry_after": 60
+                            }
+                        }
+                    },
+                    headers={"Retry-After": "60"}
+                )
+        
+        # 5. Process Request
+        response = await call_next(request)
+        
+        # 6. Set Cookie if new
+        if is_new_anon:
+            response.set_cookie(
+                key=self.cookie_name,
+                value=anon_id,
+                max_age=31536000, # 1 year
+                httponly=True,
+                samesite="lax",
+                secure=settings.environment == "production"
+            )
+            
+        # Add Rate Limit headers
+        if settings.postgres.enabled and 'limit' in locals():
+            response.headers["X-RateLimit-Limit"] = str(limit)
+            response.headers["X-RateLimit-Remaining"] = str(max(0, limit - current))
+            
+        return response
+
+    def _generate_signed_id(self) -> str:
+        """Generate a UUID4 signed with HMAC."""
+        val = str(uuid.uuid4())
+        sig = hmac.new(
+            self.secret_key.encode(), 
+            val.encode(), 
+            hashlib.sha256
+        ).hexdigest()[:12] # Short signature
+        return f"{val}.{sig}"
+
+    def _validate_signature(self, signed_val: str) -> bool:
+        """Validate the HMAC signature."""
+        try:
+            val, sig = signed_val.split(".")
+            expected_sig = hmac.new(
+                self.secret_key.encode(), 
+                val.encode(), 
+                hashlib.sha256
+            ).hexdigest()[:12]
+            return secrets.compare_digest(sig, expected_sig)
+        except ValueError:
+            return False

rem/api/routers/admin.py

@@ -0,0 +1,277 @@
+"""
+Admin API Router.
+
+Protected endpoints requiring admin role for system management tasks.
+
+Endpoints:
+    GET  /api/admin/users          - List all users (admin only)
+    GET  /api/admin/sessions       - List all sessions across users (admin only)
+    GET  /api/admin/messages       - List all messages across users (admin only)
+    GET  /api/admin/stats          - System statistics (admin only)
+
+All endpoints require:
+1. Authentication (valid session)
+2. Admin role in user's roles list
+
+Design Pattern:
+- Uses require_admin dependency for role enforcement
+- Cross-tenant queries (no user_id filtering)
+- Audit logging for admin actions
+"""
+
+from typing import Literal
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from loguru import logger
+from pydantic import BaseModel
+
+from ..deps import require_admin
+from ...models.entities import Message, Session, SessionMode
+from ...services.postgres import Repository
+from ...settings import settings
+
+router = APIRouter(prefix="/api/admin", tags=["admin"])
+
+
+# =============================================================================
+# Response Models
+# =============================================================================
+
+
+class UserSummary(BaseModel):
+    """User summary for admin listing."""
+
+    id: str
+    email: str | None
+    name: str | None
+    tier: str
+    role: str | None
+    created_at: str | None
+
+
+class UserListResponse(BaseModel):
+    """Response for user list endpoint."""
+
+    object: Literal["list"] = "list"
+    data: list[UserSummary]
+    total: int
+    has_more: bool
+
+
+class SessionListResponse(BaseModel):
+    """Response for session list endpoint."""
+
+    object: Literal["list"] = "list"
+    data: list[Session]
+    total: int
+    has_more: bool
+
+
+class MessageListResponse(BaseModel):
+    """Response for message list endpoint."""
+
+    object: Literal["list"] = "list"
+    data: list[Message]
+    total: int
+    has_more: bool
+
+
+class SystemStats(BaseModel):
+    """System statistics for admin dashboard."""
+
+    total_users: int
+    total_sessions: int
+    total_messages: int
+    active_sessions_24h: int
+    messages_24h: int
+
+
+# =============================================================================
+# Admin Endpoints
+# =============================================================================
+
+
+@router.get("/users", response_model=UserListResponse)
+async def list_all_users(
+    user: dict = Depends(require_admin),
+    limit: int = Query(default=50, ge=1, le=100),
+    offset: int = Query(default=0, ge=0),
+) -> UserListResponse:
+    """
+    List all users in the system.
+
+    Admin-only endpoint for user management.
+    Returns users across all tenants.
+    """
+    if not settings.postgres.enabled:
+        raise HTTPException(status_code=503, detail="Database not enabled")
+
+    logger.info(f"Admin {user.get('email')} listing all users")
+
+    # Import User model dynamically to avoid circular imports
+    from ...models.entities import User
+
+    repo = Repository(User, table_name="users")
+
+    # No tenant filter - admin sees all
+    users = await repo.find(
+        filters={},
+        order_by="created_at DESC",
+        limit=limit + 1,
+        offset=offset,
+    )
+
+    has_more = len(users) > limit
+    if has_more:
+        users = users[:limit]
+
+    total = await repo.count({})
+
+    # Convert to summary format
+    summaries = [
+        UserSummary(
+            id=str(u.id),
+            email=u.email,
+            name=u.name,
+            tier=u.tier.value if u.tier else "free",
+            role=u.role,
+            created_at=u.created_at.isoformat() if u.created_at else None,
+        )
+        for u in users
+    ]
+
+    return UserListResponse(data=summaries, total=total, has_more=has_more)
+
+
+@router.get("/sessions", response_model=SessionListResponse)
+async def list_all_sessions(
+    user: dict = Depends(require_admin),
+    user_id: str | None = Query(default=None, description="Filter by user ID"),
+    mode: SessionMode | None = Query(default=None, description="Filter by mode"),
+    limit: int = Query(default=50, ge=1, le=100),
+    offset: int = Query(default=0, ge=0),
+) -> SessionListResponse:
+    """
+    List all sessions across all users.
+
+    Admin-only endpoint for session monitoring.
+    Can optionally filter by user_id or mode.
+    """
+    if not settings.postgres.enabled:
+        raise HTTPException(status_code=503, detail="Database not enabled")
+
+    logger.info(
+        f"Admin {user.get('email')} listing sessions "
+        f"(user_id={user_id}, mode={mode})"
+    )
+
+    repo = Repository(Session, table_name="sessions")
+
+    # Build optional filters
+    filters: dict = {}
+    if user_id:
+        filters["user_id"] = user_id
+    if mode:
+        filters["mode"] = mode.value
+
+    sessions = await repo.find(
+        filters=filters,
+        order_by="created_at DESC",
+        limit=limit + 1,
+        offset=offset,
+    )
+
+    has_more = len(sessions) > limit
+    if has_more:
+        sessions = sessions[:limit]
+
+    total = await repo.count(filters)
+
+    return SessionListResponse(data=sessions, total=total, has_more=has_more)
+
+
+@router.get("/messages", response_model=MessageListResponse)
+async def list_all_messages(
+    user: dict = Depends(require_admin),
+    user_id: str | None = Query(default=None, description="Filter by user ID"),
+    session_id: str | None = Query(default=None, description="Filter by session ID"),
+    message_type: str | None = Query(default=None, description="Filter by type"),
+    limit: int = Query(default=50, ge=1, le=100),
+    offset: int = Query(default=0, ge=0),
+) -> MessageListResponse:
+    """
+    List all messages across all users.
+
+    Admin-only endpoint for message auditing.
+    Can filter by user_id, session_id, or message_type.
+    """
+    if not settings.postgres.enabled:
+        raise HTTPException(status_code=503, detail="Database not enabled")
+
+    logger.info(
+        f"Admin {user.get('email')} listing messages "
+        f"(user_id={user_id}, session_id={session_id})"
+    )
+
+    repo = Repository(Message, table_name="messages")
+
+    # Build optional filters
+    filters: dict = {}
+    if user_id:
+        filters["user_id"] = user_id
+    if session_id:
+        filters["session_id"] = session_id
+    if message_type:
+        filters["message_type"] = message_type
+
+    messages = await repo.find(
+        filters=filters,
+        order_by="created_at DESC",
+        limit=limit + 1,
+        offset=offset,
+    )
+
+    has_more = len(messages) > limit
+    if has_more:
+        messages = messages[:limit]
+
+    total = await repo.count(filters)
+
+    return MessageListResponse(data=messages, total=total, has_more=has_more)
+
+
+@router.get("/stats", response_model=SystemStats)
+async def get_system_stats(
+    user: dict = Depends(require_admin),
+) -> SystemStats:
+    """
+    Get system-wide statistics.
+
+    Admin-only endpoint for monitoring dashboard.
+    """
+    if not settings.postgres.enabled:
+        raise HTTPException(status_code=503, detail="Database not enabled")
+
+    logger.info(f"Admin {user.get('email')} fetching system stats")
+
+    from ...models.entities import User
+    from ...utils.date_utils import days_ago
+
+    user_repo = Repository(User, table_name="users")
+    session_repo = Repository(Session, table_name="sessions")
+    message_repo = Repository(Message, table_name="messages")
+
+    # Get totals
+    total_users = await user_repo.count({})
+    total_sessions = await session_repo.count({})
+    total_messages = await message_repo.count({})
+
+    # For 24h stats, we'd need date filtering in Repository
+    # For now, return totals (TODO: add date range support)
+    return SystemStats(
+        total_users=total_users,
+        total_sessions=total_sessions,
+        total_messages=total_messages,
+        active_sessions_24h=0,  # TODO: implement
+        messages_24h=0,  # TODO: implement
+    )

rem/api/routers/auth.py

@@ -49,6 +49,8 @@ from authlib.integrations.starlette_client import OAuth
 from loguru import logger
 
 from ...settings import settings
+from ...services.postgres.service import PostgresService
+from ...services.user_service import UserService
 
 router = APIRouter(prefix="/api/auth", tags=["auth"])
 
@@ -168,6 +170,53 @@ async def callback(provider: str, request: Request):
         if not user_info:
             # Fetch from userinfo endpoint if not in ID token
             user_info = await client.userinfo(token=token)
+            
+        # --- REM Integration Start ---
+        if settings.postgres.enabled:
+            # Connect to DB
+            db = PostgresService()
+            try:
+                await db.connect()
+                user_service = UserService(db)
+                
+                # Get/Create User
+                user_entity = await user_service.get_or_create_user(
+                    email=user_info.get("email"),
+                    name=user_info.get("name", "New User"),
+                    avatar_url=user_info.get("picture"),
+                    tenant_id="default", # Single tenant for now
+                )
+                
+                # Link Anonymous Session
+                # TrackingMiddleware sets request.state.anon_id
+                anon_id = getattr(request.state, "anon_id", None)
+                # Fallback to cookie if middleware didn't run or state missing
+                if not anon_id:
+                    # Attempt to parse cookie manually if needed, but middleware 
+                    # usually handles the signature logic.
+                    # Just check raw cookie for simple case (not recommended if signed)
+                    pass 
+                
+                if anon_id:
+                    await user_service.link_anonymous_session(user_entity, anon_id)
+                    
+                # Enrich session user with DB info
+                db_info = {
+                    "id": str(user_entity.id),
+                    "tenant_id": user_entity.tenant_id,
+                    "tier": user_entity.tier.value if user_entity.tier else "free",
+                    "roles": [user_entity.role] if user_entity.role else [],
+                }
+                
+            except Exception as db_e:
+                logger.error(f"Database error during auth callback: {db_e}")
+                # Continue login even if DB fails, but warn
+                db_info = {"id": "db_error", "tier": "free"}
+            finally:
+                await db.disconnect()
+        else:
+            db_info = {"id": "no_db", "tier": "free"}
+        # --- REM Integration End ---
 
         # Store user info in session
         request.session["user"] = {
@@ -176,6 +225,11 @@ async def callback(provider: str, request: Request):
             "email": user_info.get("email"),
             "name": user_info.get("name"),
             "picture": user_info.get("picture"),
+            # Add DB info
+            "id": db_info.get("id"),
+            "tenant_id": db_info.get("tenant_id", "default"),
+            "tier": db_info.get("tier"),
+            "roles": db_info.get("roles", []),
         }
 
         # Store tokens in session for API access
@@ -227,3 +281,73 @@ async def me(request: Request):
         raise HTTPException(status_code=401, detail="Not authenticated")
 
     return user
+
+
+# =============================================================================
+# Development Token Endpoints (non-production only)
+# =============================================================================
+
+
+def generate_dev_token() -> str:
+    """
+    Generate a dev token for testing.
+
+    Token format: dev_<hmac_signature>
+    The signature is based on the session secret to ensure only valid tokens work.
+    """
+    import hashlib
+    import hmac
+
+    # Use session secret as key
+    secret = settings.auth.session_secret or "dev-secret"
+    message = "test-user:dev-token"
+
+    signature = hmac.new(
+        secret.encode(),
+        message.encode(),
+        hashlib.sha256
+    ).hexdigest()[:32]
+
+    return f"dev_{signature}"
+
+
+def verify_dev_token(token: str) -> bool:
+    """Verify a dev token is valid."""
+    expected = generate_dev_token()
+    return token == expected
+
+
+@router.get("/dev/token")
+async def get_dev_token(request: Request):
+    """
+    Get a development token for testing (non-production only).
+
+    This token can be used as a Bearer token to authenticate as the
+    test user (test-user / test@rem.local) without going through OAuth.
+
+    Usage:
+        curl -H "Authorization: Bearer <token>" http://localhost:8000/api/v1/...
+
+    Returns:
+        401 if in production environment
+        Token and usage instructions otherwise
+    """
+    if settings.environment == "production":
+        raise HTTPException(
+            status_code=401,
+            detail="Dev tokens are not available in production"
+        )
+
+    token = generate_dev_token()
+
+    return {
+        "token": token,
+        "type": "Bearer",
+        "user": {
+            "id": "test-user",
+            "email": "test@rem.local",
+            "name": "Test User",
+        },
+        "usage": f'curl -H "Authorization: Bearer {token}" http://localhost:8000/api/v1/...',
+        "warning": "This token is for development/testing only and will not work in production.",
+    }