remdb 0.3.0__py3-none-any.whl → 0.3.114__py3-none-any.whl

rem/api/main.py

@@ -26,10 +26,10 @@ Endpoints:
 - /health                    : Health check
 - /api/v1/mcp                : MCP endpoint (HTTP transport)
 - /api/v1/chat/completions   : OpenAI-compatible chat completions (streaming & non-streaming)
-- /api/v1/query              : REM query execution (TODO)
+- /api/v1/query              : REM query execution (rem-dialect or natural-language)
 - /api/v1/resources          : Resource CRUD (TODO)
 - /api/v1/moments            : Moment CRUD (TODO)
-- /api/auth/*                : OAuth/OIDC authentication (TODO)
+- /api/auth/*                : OAuth/OIDC authentication
 - /docs                      : OpenAPI documentation
 
 Headers → AgentContext Mapping:
@@ -59,8 +59,16 @@ Running:
     hypercorn rem.api.main:app --bind 0.0.0.0:8000
 """
 
+import importlib.metadata
 import secrets
+import sys
 import time
+
+# Get package version for API responses
+try:
+    __version__ = importlib.metadata.version("remdb")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "0.0.0-dev"
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, Request
@@ -73,6 +81,23 @@ from starlette.middleware.sessions import SessionMiddleware
 from .mcp_router.server import create_mcp_server
 from ..settings import settings
 
+# Configure loguru based on settings
+# Remove default handler and add one with configured level
+logger.remove()
+
+# Configure level icons - only warnings and errors get visual indicators
+logger.level("DEBUG", icon=" ")
+logger.level("INFO", icon=" ")
+logger.level("WARNING", icon="🟠")
+logger.level("ERROR", icon="🔴")
+logger.level("CRITICAL", icon="🔴")
+
+logger.add(
+    sys.stderr,
+    level=settings.api.log_level.upper(),
+    format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | {level.icon} <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+)
+
 
 class RequestLoggingMiddleware(BaseHTTPMiddleware):
     """
@@ -82,26 +107,64 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
     - Logs request method, path, client, user-agent
     - Logs response status, content-type, duration
     - Essential for debugging OAuth flow and MCP sessions
+    - Health checks and 404s logged at DEBUG level to reduce noise
+    - Scanner/exploit attempts (common vulnerability probes) logged at DEBUG
     """
 
+    # Paths to log at DEBUG level (health checks, probes)
+    DEBUG_PATHS = {"/health", "/healthz", "/ready", "/readyz", "/livez"}
+
+    # Path patterns that indicate vulnerability scanners (log at DEBUG)
+    SCANNER_PATTERNS = (
+        "/vendor/",      # PHP composer exploits
+        "/.git/",        # Git config exposure
+        "/.env",         # Environment file exposure
+        "/wp-",          # WordPress exploits
+        "/phpunit/",     # PHPUnit RCE
+        "/eval-stdin",   # PHP eval exploits
+        "/console/",     # Console exposure
+        "/actuator/",    # Spring Boot actuator
+        "/debug/",       # Debug endpoints
+        "/admin/",       # Admin panel probes (when we don't have one)
+    )
+
+    def _should_log_at_debug(self, path: str, status_code: int) -> bool:
+        """Determine if request should be logged at DEBUG level."""
+        # Health checks
+        if path in self.DEBUG_PATHS:
+            return True
+        # 404 responses (not found - includes scanner probes)
+        if status_code == 404:
+            return True
+        # Known scanner patterns
+        if any(pattern in path for pattern in self.SCANNER_PATTERNS):
+            return True
+        return False
+
     async def dispatch(self, request: Request, call_next):
         start_time = time.time()
+        path = request.url.path
 
-        # Log incoming request
+        # Log incoming request (preliminary - may adjust after response)
         client_host = request.client.host if request.client else "unknown"
-        logger.info(
-            f"→ REQUEST: {request.method} {request.url.path} | "
-            f"Client: {client_host} | "
-            f"User-Agent: {request.headers.get('user-agent', 'unknown')[:100]}"
-        )
+        user_agent = request.headers.get('user-agent', 'unknown')[:100]
 
         # Process request
         response = await call_next(request)
 
-        # Log response
+        # Determine log level based on path AND response status
         duration_ms = (time.time() - start_time) * 1000
-        logger.info(
-            f"← RESPONSE: {request.method} {request.url.path} | "
+        use_debug = self._should_log_at_debug(path, response.status_code)
+        log_fn = logger.debug if use_debug else logger.info
+
+        # Log request and response together
+        log_fn(
+            f"→ REQUEST: {request.method} {path} | "
+            f"Client: {client_host} | "
+            f"User-Agent: {user_agent}"
+        )
+        log_fn(
+            f"← RESPONSE: {request.method} {path} | "
             f"Status: {response.status_code} | "
             f"Duration: {duration_ms:.2f}ms"
         )
@@ -154,7 +217,8 @@ async def lifespan(app: FastAPI):
             "and history lookups are unavailable. Enable database with POSTGRES__ENABLED=true"
         )
     else:
-        logger.info(f"Database enabled: {settings.postgres.connection_string}")
+        # Log database host only - never log credentials
+        logger.info(f"Database enabled: {settings.postgres.host}:{settings.postgres.port}/{settings.postgres.database}")
 
     yield
 
@@ -163,7 +227,22 @@ async def lifespan(app: FastAPI):
 
 def create_app() -> FastAPI:
     """
-    Create and configure the FastAPI application.
+    Create and configure the FastAPI application with MCP server.
+
+    The returned app exposes `app.mcp_server` (FastMCP instance) for adding
+    custom tools, resources, and prompts:
+
+        app = create_app()
+
+        @app.mcp_server.tool()
+        async def my_tool(query: str) -> dict:
+            '''Custom MCP tool.'''
+            return {"result": query}
+
+        @app.mcp_server.resource("custom://data")
+        async def my_resource() -> str:
+            '''Custom resource.'''
+            return '{"data": "value"}'
 
     Design Pattern:
     1. Create MCP server
@@ -174,9 +253,10 @@ def create_app() -> FastAPI:
     6. Define health endpoints
     7. Register API routers
     8. Mount MCP app
+    9. Expose mcp_server on app for extension
 
     Returns:
-        Configured FastAPI application
+        Configured FastAPI application with .mcp_server attribute
     """
     # Create MCP server and get HTTP app
     # path="/" creates routes at root, then mount at /api/v1/mcp
@@ -198,15 +278,42 @@ def create_app() -> FastAPI:
                 yield
 
     app = FastAPI(
-        title="REM API",
-        description="Resources Entities Moments system for agentic AI",
-        version="0.1.0",
+        title=f"{settings.app_name} API",
+        description=f"{settings.app_name} - Resources Entities Moments system for agentic AI",
+        version=__version__,
         lifespan=combined_lifespan,
         root_path=settings.root_path if settings.root_path else "",
         redirect_slashes=False,  # Don't redirect /mcp/ -> /mcp
     )
 
+    # Add request logging middleware
+    app.add_middleware(RequestLoggingMiddleware)
+
+    # Add SSE buffering middleware (for MCP SSE transport)
+    app.add_middleware(SSEBufferingMiddleware)
+
+    # Add Anonymous Tracking & Rate Limiting (Runs AFTER Auth if Auth is enabled)
+    # Must be added BEFORE AuthMiddleware in code to be INNER in the stack
+    from .middleware.tracking import AnonymousTrackingMiddleware
+    app.add_middleware(AnonymousTrackingMiddleware)
+
+    # Add authentication middleware
+    # Always load middleware for dev token support, but allow anonymous when auth disabled
+    from ..auth.middleware import AuthMiddleware
+
+    app.add_middleware(
+        AuthMiddleware,
+        protected_paths=["/api/v1"],
+        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth"],
+        # Allow anonymous when auth is disabled, otherwise use setting
+        allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,
+        # MCP requires auth only when auth is fully enabled
+        mcp_requires_auth=settings.auth.enabled and settings.auth.mcp_requires_auth,
+    )
+
     # Add session middleware for OAuth state management
+    # Must be added AFTER AuthMiddleware in code so it runs BEFORE (middleware runs in reverse)
+    # AuthMiddleware needs request.session to be available
     session_secret = settings.auth.session_secret or secrets.token_hex(32)
     if not settings.auth.session_secret:
         logger.warning(
@@ -223,27 +330,12 @@ def create_app() -> FastAPI:
         https_only=settings.environment == "production",
     )
 
-    # Add request logging middleware
-    app.add_middleware(RequestLoggingMiddleware)
-
-    # Add SSE buffering middleware (for MCP SSE transport)
-    app.add_middleware(SSEBufferingMiddleware)
-
-    # Add authentication middleware (if enabled)
-    if settings.auth.enabled:
-        from ..auth.middleware import AuthMiddleware
-
-        app.add_middleware(
-            AuthMiddleware,
-            protected_paths=["/api/v1"],
-            excluded_paths=["/api/auth", "/api/v1/mcp/auth"],
-        )
-
     # Add CORS middleware LAST (runs first in middleware chain)
     # Must expose mcp-session-id header for MCP session management
     CORS_ORIGIN_WHITELIST = [
-        "http://localhost:5173",  # Local development (Vite)
         "http://localhost:3000",  # Local development (React)
+        "http://localhost:5000",  # Local development (Flask/other)
+        "http://localhost:5173",  # Local development (Vite)
     ]
 
     app.add_middleware(
@@ -261,8 +353,8 @@ def create_app() -> FastAPI:
         """API information endpoint."""
         # TODO: If auth enabled and no user, return 401 with WWW-Authenticate
         return {
-            "name": "REM API",
-            "version": "0.1.0",
+            "name": f"{settings.app_name} API",
+            "version": __version__,
             "mcp_endpoint": "/api/v1/mcp",
             "docs": "/docs",
         }
@@ -271,12 +363,24 @@ def create_app() -> FastAPI:
     @app.get("/health")
     async def health():
         """Health check endpoint."""
-        return {"status": "healthy", "version": "0.1.0"}
+        return {"status": "healthy", "version": __version__}
 
     # Register API routers
     from .routers.chat import router as chat_router
+    from .routers.models import router as models_router
+    from .routers.messages import router as messages_router
+    from .routers.feedback import router as feedback_router
+    from .routers.admin import router as admin_router
+    from .routers.shared_sessions import router as shared_sessions_router
+    from .routers.query import router as query_router
 
     app.include_router(chat_router)
+    app.include_router(models_router)
+    app.include_router(messages_router)
+    app.include_router(feedback_router)
+    app.include_router(admin_router)
+    app.include_router(shared_sessions_router)
+    app.include_router(query_router)
 
     # Register auth router (if enabled)
     if settings.auth.enabled:
@@ -284,6 +388,12 @@ def create_app() -> FastAPI:
 
         app.include_router(auth_router)
 
+    # Register dev router (non-production only)
+    if settings.environment != "production":
+        from .routers.dev import router as dev_router
+
+        app.include_router(dev_router)
+
     # TODO: Register additional routers
     # from .routers.query import router as query_router
     # from .routers.resources import router as resources_router
@@ -305,6 +415,10 @@ def create_app() -> FastAPI:
     # Mount MCP app at /api/v1/mcp
     app.mount("/api/v1/mcp", mcp_app)
 
+    # Expose MCP server on app for extension
+    # Users can add tools/resources/prompts via app.mcp_server
+    app.mcp_server = mcp_server  # type: ignore[attr-defined]
+
     return app
 
 

rem/api/mcp_router/resources.py

@@ -181,7 +181,7 @@ Parameters:
 - table_name (required): Table to search (resources, moments, etc.)
 - field_name (optional): Field to search (defaults to "content")
 - provider (optional): Embedding provider (default: from LLM__EMBEDDING_PROVIDER setting)
-- min_similarity (optional): Minimum similarity 0.0-1.0 (default: 0.7)
+- min_similarity (optional): Minimum similarity 0.0-1.0 (default: 0.3)
 - limit (optional): Max results (default: 10)
 - user_id (optional): User scoping
 

rem/api/mcp_router/server.py

@@ -19,10 +19,18 @@ FastMCP Features:
 - Built-in auth that can be disabled for testing
 """
 
+import importlib.metadata
+
 from fastmcp import FastMCP
 
 from ...settings import settings
 
+# Get package version
+try:
+    __version__ = importlib.metadata.version("remdb")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "0.0.0-dev"
+
 
 def create_mcp_server(is_local: bool = False) -> FastMCP:
     """
@@ -52,7 +60,7 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     """
     mcp = FastMCP(
         name=f"REM MCP Server ({settings.team}/{settings.environment})",
-        version="0.1.0",
+        version=__version__,
         instructions=(
             "REM (Resource-Entity-Moment) MCP Server - Unified memory infrastructure for agentic systems.\n\n"
             "═══════════════════════════════════════════════════════════════════════════\n"
@@ -165,11 +173,18 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     )
 
     # Register REM tools
-    from .tools import ask_rem_agent, ingest_into_rem, read_resource, search_rem
+    from .tools import (
+        ask_rem_agent,
+        ingest_into_rem,
+        read_resource,
+        register_metadata,
+        search_rem,
+    )
 
     mcp.tool()(search_rem)
     mcp.tool()(ask_rem_agent)
     mcp.tool()(read_resource)
+    mcp.tool()(register_metadata)
 
     # File ingestion tool (with local path support for local servers)
     # Wrap to inject is_local parameter

rem/api/mcp_router/tools.py

@@ -53,7 +53,7 @@ def init_services(postgres_service: PostgresService, rem_service: RemService):
     """
     _service_cache["postgres"] = postgres_service
     _service_cache["rem"] = rem_service
-    logger.info("MCP tools initialized with service instances")
+    logger.debug("MCP tools initialized with service instances")
 
 
 async def get_rem_service() -> RemService:
@@ -79,7 +79,7 @@ async def get_rem_service() -> RemService:
     _service_cache["postgres"] = postgres_service
     _service_cache["rem"] = rem_service
 
-    logger.info("MCP tools: lazy initialized services")
+    logger.debug("MCP tools: lazy initialized services")
     return rem_service
 
 
@@ -399,14 +399,14 @@ async def ask_rem_agent(
     )
 
     # Run agent (errors handled by decorator)
-    logger.info(f"Running ask_rem agent for query: {query[:100]}...")
+    logger.debug(f"Running ask_rem agent for query: {query[:100]}...")
     result = await agent_runtime.run(query)
 
     # Extract output
     from rem.agentic.serialization import serialize_agent_result
     query_output = serialize_agent_result(result.output)
 
-    logger.info("Agent execution completed successfully")
+    logger.debug("Agent execution completed successfully")
 
     return {
         "response": str(result.output),
@@ -422,6 +422,7 @@ async def ingest_into_rem(
     tags: list[str] | None = None,
     is_local_server: bool = False,
     user_id: str | None = None,
+    resource_type: str | None = None,
 ) -> dict[str, Any]:
     """
     Ingest file into REM, creating searchable resources and embeddings.
@@ -448,6 +449,11 @@ async def ingest_into_rem(
         tags: Optional tags for file
         is_local_server: True if running as local/stdio MCP server
         user_id: Optional user identifier (defaults to authenticated user or "default")
+        resource_type: Optional resource type for storing chunks (case-insensitive).
+            Supports flexible naming:
+            - "resource", "resources", "Resource" → Resource (default)
+            - "domain-resource", "domain_resource", "DomainResource",
+              "domain-resources" → DomainResource (curated internal knowledge)
 
     Returns:
         Dict with:
@@ -478,6 +484,13 @@ async def ingest_into_rem(
             file_uri="https://example.com/whitepaper.pdf",
             tags=["research", "whitepaper"]
         )
+
+        # Ingest as curated domain knowledge
+        ingest_into_rem(
+            file_uri="s3://bucket/internal/procedures.pdf",
+            resource_type="domain-resource",
+            category="procedures"
+        )
     """
     from ...services.content import ContentService
 
@@ -493,9 +506,10 @@ async def ingest_into_rem(
         category=category,
         tags=tags,
         is_local_server=is_local_server,
+        resource_type=resource_type,
     )
 
-    logger.info(
+    logger.debug(
         f"MCP ingestion complete: {result['file_name']} "
         f"(status: {result['processing_status']}, "
         f"resources: {result['resources_created']})"
@@ -550,7 +564,7 @@ async def read_resource(uri: str) -> dict[str, Any]:
         # Check system status
         read_resource(uri="rem://status")
     """
-    logger.info(f"📖 Reading resource: {uri}")
+    logger.debug(f"Reading resource: {uri}")
 
     # Import here to avoid circular dependency
     from .resources import load_resource
@@ -558,7 +572,7 @@ async def read_resource(uri: str) -> dict[str, Any]:
     # Load resource using the existing resource handler (errors handled by decorator)
     result = await load_resource(uri)
 
-    logger.info(f"✓ Resource loaded successfully: {uri}")
+    logger.debug(f"Resource loaded successfully: {uri}")
 
     # If result is already a dict, return it
     if isinstance(result, dict):
@@ -582,3 +596,125 @@ async def read_resource(uri: str) -> dict[str, Any]:
             "uri": uri,
             "data": {"content": result},
         }
+
+
+async def register_metadata(
+    confidence: float | None = None,
+    references: list[str] | None = None,
+    sources: list[str] | None = None,
+    flags: list[str] | None = None,
+    # Risk assessment fields (used by mental health agents like Siggy)
+    risk_level: str | None = None,
+    risk_score: int | None = None,
+    risk_reasoning: str | None = None,
+    recommended_action: str | None = None,
+    # Generic extension - any additional key-value pairs
+    extra: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """
+    Register response metadata to be emitted as an SSE MetadataEvent.
+
+    Call this tool BEFORE generating your final response to provide structured
+    metadata that will be sent to the client alongside your natural language output.
+    This allows you to stream conversational responses while still providing
+    machine-readable confidence scores, references, and other metadata.
+
+    **Design Pattern**: Agents can call this once before their final response to
+    register metadata that the streaming layer will emit as a MetadataEvent.
+    This decouples structured metadata from the response format.
+
+    Args:
+        confidence: Confidence score (0.0-1.0) for the response quality.
+            - 0.9-1.0: High confidence, answer is well-supported
+            - 0.7-0.9: Medium confidence, some uncertainty
+            - 0.5-0.7: Low confidence, significant gaps
+            - <0.5: Very uncertain, may need clarification
+        references: List of reference identifiers (file paths, document IDs,
+            entity labels) that support the response.
+        sources: List of source descriptions (e.g., "REM database",
+            "search results", "user context").
+        flags: Optional flags for the response (e.g., "needs_review",
+            "uncertain", "incomplete", "crisis_alert").
+
+        risk_level: Risk level indicator (e.g., "green", "orange", "red").
+            Used by mental health agents for C-SSRS style assessment.
+        risk_score: Numeric risk score (e.g., 0-6 for C-SSRS).
+        risk_reasoning: Brief explanation of risk assessment.
+        recommended_action: Suggested next steps based on assessment.
+
+        extra: Dict of arbitrary additional metadata. Use this for any
+            domain-specific fields not covered by the standard parameters.
+            Example: {"topics_detected": ["anxiety", "sleep"], "session_count": 5}
+
+    Returns:
+        Dict with:
+        - status: "success"
+        - _metadata_event: True (marker for streaming layer)
+        - All provided fields merged into response
+
+    Examples:
+        # High confidence answer with references
+        register_metadata(
+            confidence=0.95,
+            references=["sarah-chen", "q3-report-2024"],
+            sources=["REM database lookup"]
+        )
+
+        # Mental health risk assessment (Siggy-style)
+        register_metadata(
+            confidence=0.9,
+            risk_level="green",
+            risk_score=0,
+            risk_reasoning="No risk indicators detected in message",
+            sources=["mental_health_resources"]
+        )
+
+        # Orange risk with recommended action
+        register_metadata(
+            risk_level="orange",
+            risk_score=2,
+            risk_reasoning="Passive ideation detected - 'feeling hopeless'",
+            recommended_action="Schedule care team check-in within 24-48 hours",
+            flags=["care_team_alert"]
+        )
+
+        # Custom domain-specific metadata
+        register_metadata(
+            confidence=0.8,
+            extra={
+                "topics_detected": ["medication", "side_effects"],
+                "drug_mentioned": "sertraline",
+                "sentiment": "concerned"
+            }
+        )
+    """
+    logger.debug(
+        f"Registering metadata: confidence={confidence}, "
+        f"risk_level={risk_level}, refs={len(references or [])}, "
+        f"sources={len(sources or [])}"
+    )
+
+    result = {
+        "status": "success",
+        "_metadata_event": True,  # Marker for streaming layer
+        "confidence": confidence,
+        "references": references,
+        "sources": sources,
+        "flags": flags,
+    }
+
+    # Add risk assessment fields if provided
+    if risk_level is not None:
+        result["risk_level"] = risk_level
+    if risk_score is not None:
+        result["risk_score"] = risk_score
+    if risk_reasoning is not None:
+        result["risk_reasoning"] = risk_reasoning
+    if recommended_action is not None:
+        result["recommended_action"] = recommended_action
+
+    # Merge any extra fields
+    if extra:
+        result["extra"] = extra
+
+    return result