remdb 0.3.7__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/api/main.py

@@ -26,10 +26,10 @@ Endpoints:
 - /health                    : Health check
 - /api/v1/mcp                : MCP endpoint (HTTP transport)
 - /api/v1/chat/completions   : OpenAI-compatible chat completions (streaming & non-streaming)
-- /api/v1/query              : REM query execution (TODO)
+- /api/v1/query              : REM query execution (rem-dialect or natural-language)
 - /api/v1/resources          : Resource CRUD (TODO)
 - /api/v1/moments            : Moment CRUD (TODO)
-- /api/auth/*                : OAuth/OIDC authentication (TODO)
+- /api/auth/*                : OAuth/OIDC authentication
 - /docs                      : OpenAPI documentation
 
 Headers → AgentContext Mapping:
@@ -59,8 +59,16 @@ Running:
     hypercorn rem.api.main:app --bind 0.0.0.0:8000
 """
 
+import importlib.metadata
 import secrets
+import sys
 import time
+
+# Get package version for API responses
+try:
+    __version__ = importlib.metadata.version("remdb")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "0.0.0-dev"
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, Request
@@ -73,6 +81,23 @@ from starlette.middleware.sessions import SessionMiddleware
 from .mcp_router.server import create_mcp_server
 from ..settings import settings
 
+# Configure loguru based on settings
+# Remove default handler and add one with configured level
+logger.remove()
+
+# Configure level icons - only warnings and errors get visual indicators
+logger.level("DEBUG", icon=" ")
+logger.level("INFO", icon=" ")
+logger.level("WARNING", icon="🟠")
+logger.level("ERROR", icon="🔴")
+logger.level("CRITICAL", icon="🔴")
+
+logger.add(
+    sys.stderr,
+    level=settings.api.log_level.upper(),
+    format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | {level.icon} <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+)
+
 
 class RequestLoggingMiddleware(BaseHTTPMiddleware):
     """
@@ -82,26 +107,64 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
     - Logs request method, path, client, user-agent
     - Logs response status, content-type, duration
     - Essential for debugging OAuth flow and MCP sessions
+    - Health checks and 404s logged at DEBUG level to reduce noise
+    - Scanner/exploit attempts (common vulnerability probes) logged at DEBUG
     """
 
+    # Paths to log at DEBUG level (health checks, probes)
+    DEBUG_PATHS = {"/health", "/healthz", "/ready", "/readyz", "/livez"}
+
+    # Path patterns that indicate vulnerability scanners (log at DEBUG)
+    SCANNER_PATTERNS = (
+        "/vendor/",      # PHP composer exploits
+        "/.git/",        # Git config exposure
+        "/.env",         # Environment file exposure
+        "/wp-",          # WordPress exploits
+        "/phpunit/",     # PHPUnit RCE
+        "/eval-stdin",   # PHP eval exploits
+        "/console/",     # Console exposure
+        "/actuator/",    # Spring Boot actuator
+        "/debug/",       # Debug endpoints
+        "/admin/",       # Admin panel probes (when we don't have one)
+    )
+
+    def _should_log_at_debug(self, path: str, status_code: int) -> bool:
+        """Determine if request should be logged at DEBUG level."""
+        # Health checks
+        if path in self.DEBUG_PATHS:
+            return True
+        # 404 responses (not found - includes scanner probes)
+        if status_code == 404:
+            return True
+        # Known scanner patterns
+        if any(pattern in path for pattern in self.SCANNER_PATTERNS):
+            return True
+        return False
+
     async def dispatch(self, request: Request, call_next):
         start_time = time.time()
+        path = request.url.path
 
-        # Log incoming request
+        # Log incoming request (preliminary - may adjust after response)
         client_host = request.client.host if request.client else "unknown"
-        logger.info(
-            f"→ REQUEST: {request.method} {request.url.path} | "
-            f"Client: {client_host} | "
-            f"User-Agent: {request.headers.get('user-agent', 'unknown')[:100]}"
-        )
+        user_agent = request.headers.get('user-agent', 'unknown')[:100]
 
         # Process request
         response = await call_next(request)
 
-        # Log response
+        # Determine log level based on path AND response status
         duration_ms = (time.time() - start_time) * 1000
-        logger.info(
-            f"← RESPONSE: {request.method} {request.url.path} | "
+        use_debug = self._should_log_at_debug(path, response.status_code)
+        log_fn = logger.debug if use_debug else logger.info
+
+        # Log request and response together
+        log_fn(
+            f"→ REQUEST: {request.method} {path} | "
+            f"Client: {client_host} | "
+            f"User-Agent: {user_agent}"
+        )
+        log_fn(
+            f"← RESPONSE: {request.method} {path} | "
             f"Status: {response.status_code} | "
             f"Duration: {duration_ms:.2f}ms"
         )
@@ -154,7 +217,8 @@ async def lifespan(app: FastAPI):
             "and history lookups are unavailable. Enable database with POSTGRES__ENABLED=true"
         )
     else:
-        logger.info(f"Database enabled: {settings.postgres.connection_string}")
+        # Log database host only - never log credentials
+        logger.info(f"Database enabled: {settings.postgres.host}:{settings.postgres.port}/{settings.postgres.database}")
 
     yield
 
@@ -163,7 +227,22 @@ async def lifespan(app: FastAPI):
 
 def create_app() -> FastAPI:
     """
-    Create and configure the FastAPI application.
+    Create and configure the FastAPI application with MCP server.
+
+    The returned app exposes `app.mcp_server` (FastMCP instance) for adding
+    custom tools, resources, and prompts:
+
+        app = create_app()
+
+        @app.mcp_server.tool()
+        async def my_tool(query: str) -> dict:
+            '''Custom MCP tool.'''
+            return {"result": query}
+
+        @app.mcp_server.resource("custom://data")
+        async def my_resource() -> str:
+            '''Custom resource.'''
+            return '{"data": "value"}'
 
     Design Pattern:
     1. Create MCP server
@@ -174,9 +253,10 @@ def create_app() -> FastAPI:
     6. Define health endpoints
     7. Register API routers
     8. Mount MCP app
+    9. Expose mcp_server on app for extension
 
     Returns:
-        Configured FastAPI application
+        Configured FastAPI application with .mcp_server attribute
     """
     # Create MCP server and get HTTP app
     # path="/" creates routes at root, then mount at /api/v1/mcp
@@ -198,15 +278,42 @@ def create_app() -> FastAPI:
                 yield
 
     app = FastAPI(
-        title="REM API",
-        description="Resources Entities Moments system for agentic AI",
-        version="0.1.0",
+        title=f"{settings.app_name} API",
+        description=f"{settings.app_name} - Resources Entities Moments system for agentic AI",
+        version=__version__,
         lifespan=combined_lifespan,
         root_path=settings.root_path if settings.root_path else "",
         redirect_slashes=False,  # Don't redirect /mcp/ -> /mcp
     )
 
+    # Add request logging middleware
+    app.add_middleware(RequestLoggingMiddleware)
+
+    # Add SSE buffering middleware (for MCP SSE transport)
+    app.add_middleware(SSEBufferingMiddleware)
+
+    # Add Anonymous Tracking & Rate Limiting (Runs AFTER Auth if Auth is enabled)
+    # Must be added BEFORE AuthMiddleware in code to be INNER in the stack
+    from .middleware.tracking import AnonymousTrackingMiddleware
+    app.add_middleware(AnonymousTrackingMiddleware)
+
+    # Add authentication middleware
+    # Always load middleware for dev token support, but allow anonymous when auth disabled
+    from ..auth.middleware import AuthMiddleware
+
+    app.add_middleware(
+        AuthMiddleware,
+        protected_paths=["/api/v1"],
+        excluded_paths=["/api/auth", "/api/dev", "/api/v1/mcp/auth"],
+        # Allow anonymous when auth is disabled, otherwise use setting
+        allow_anonymous=(not settings.auth.enabled) or settings.auth.allow_anonymous,
+        # MCP requires auth only when auth is fully enabled
+        mcp_requires_auth=settings.auth.enabled and settings.auth.mcp_requires_auth,
+    )
+
     # Add session middleware for OAuth state management
+    # Must be added AFTER AuthMiddleware in code so it runs BEFORE (middleware runs in reverse)
+    # AuthMiddleware needs request.session to be available
     session_secret = settings.auth.session_secret or secrets.token_hex(32)
     if not settings.auth.session_secret:
         logger.warning(
@@ -223,27 +330,12 @@ def create_app() -> FastAPI:
         https_only=settings.environment == "production",
     )
 
-    # Add request logging middleware
-    app.add_middleware(RequestLoggingMiddleware)
-
-    # Add SSE buffering middleware (for MCP SSE transport)
-    app.add_middleware(SSEBufferingMiddleware)
-
-    # Add authentication middleware (if enabled)
-    if settings.auth.enabled:
-        from ..auth.middleware import AuthMiddleware
-
-        app.add_middleware(
-            AuthMiddleware,
-            protected_paths=["/api/v1"],
-            excluded_paths=["/api/auth", "/api/v1/mcp/auth"],
-        )
-
     # Add CORS middleware LAST (runs first in middleware chain)
     # Must expose mcp-session-id header for MCP session management
     CORS_ORIGIN_WHITELIST = [
-        "http://localhost:5173",  # Local development (Vite)
         "http://localhost:3000",  # Local development (React)
+        "http://localhost:5000",  # Local development (Flask/other)
+        "http://localhost:5173",  # Local development (Vite)
     ]
 
     app.add_middleware(
@@ -261,8 +353,8 @@ def create_app() -> FastAPI:
         """API information endpoint."""
         # TODO: If auth enabled and no user, return 401 with WWW-Authenticate
         return {
-            "name": "REM API",
-            "version": "0.1.0",
+            "name": f"{settings.app_name} API",
+            "version": __version__,
             "mcp_endpoint": "/api/v1/mcp",
             "docs": "/docs",
         }
@@ -271,12 +363,27 @@ def create_app() -> FastAPI:
     @app.get("/health")
     async def health():
         """Health check endpoint."""
-        return {"status": "healthy", "version": "0.1.0"}
+        return {"status": "healthy", "version": __version__}
 
     # Register API routers
     from .routers.chat import router as chat_router
+    from .routers.models import router as models_router
+    from .routers.messages import router as messages_router
+    from .routers.feedback import router as feedback_router
+    from .routers.admin import router as admin_router
+    from .routers.shared_sessions import router as shared_sessions_router
+    from .routers.query import router as query_router
 
     app.include_router(chat_router)
+    app.include_router(models_router)
+    # shared_sessions_router MUST be before messages_router
+    # because messages_router has /sessions/{session_id} which would match
+    # before the more specific /sessions/shared-with-me routes
+    app.include_router(shared_sessions_router)
+    app.include_router(messages_router)
+    app.include_router(feedback_router)
+    app.include_router(admin_router)
+    app.include_router(query_router)
 
     # Register auth router (if enabled)
     if settings.auth.enabled:
@@ -284,6 +391,12 @@ def create_app() -> FastAPI:
 
         app.include_router(auth_router)
 
+    # Register dev router (non-production only)
+    if settings.environment != "production":
+        from .routers.dev import router as dev_router
+
+        app.include_router(dev_router)
+
     # TODO: Register additional routers
     # from .routers.query import router as query_router
     # from .routers.resources import router as resources_router
@@ -305,6 +418,10 @@ def create_app() -> FastAPI:
     # Mount MCP app at /api/v1/mcp
     app.mount("/api/v1/mcp", mcp_app)
 
+    # Expose MCP server on app for extension
+    # Users can add tools/resources/prompts via app.mcp_server
+    app.mcp_server = mcp_server  # type: ignore[attr-defined]
+
     return app
 
 

rem/api/mcp_router/resources.py

@@ -181,7 +181,7 @@ Parameters:
 - table_name (required): Table to search (resources, moments, etc.)
 - field_name (optional): Field to search (defaults to "content")
 - provider (optional): Embedding provider (default: from LLM__EMBEDDING_PROVIDER setting)
-- min_similarity (optional): Minimum similarity 0.0-1.0 (default: 0.7)
+- min_similarity (optional): Minimum similarity 0.0-1.0 (default: 0.3)
 - limit (optional): Max results (default: 10)
 - user_id (optional): User scoping
 

rem/api/mcp_router/server.py

@@ -19,10 +19,18 @@ FastMCP Features:
 - Built-in auth that can be disabled for testing
 """
 
+import importlib.metadata
+
 from fastmcp import FastMCP
 
 from ...settings import settings
 
+# Get package version
+try:
+    __version__ = importlib.metadata.version("remdb")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "0.0.0-dev"
+
 
 def create_mcp_server(is_local: bool = False) -> FastMCP:
     """
@@ -52,7 +60,7 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     """
     mcp = FastMCP(
         name=f"REM MCP Server ({settings.team}/{settings.environment})",
-        version="0.1.0",
+        version=__version__,
         instructions=(
             "REM (Resource-Entity-Moment) MCP Server - Unified memory infrastructure for agentic systems.\n\n"
             "═══════════════════════════════════════════════════════════════════════════\n"
@@ -119,10 +127,12 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
             "AVAILABLE TOOLS\n"
             "═══════════════════════════════════════════════════════════════════════════\n"
             "\n"
-            "• rem_query - Execute REM queries (LOOKUP, FUZZY, SEARCH, SQL, TRAVERSE)\n"
-            "• ask_rem - Natural language to REM query conversion\n"
+            "• search_rem - Execute REM queries (LOOKUP, FUZZY, SEARCH, SQL, TRAVERSE)\n"
+            "• ask_rem_agent - Natural language to REM query conversion\n"
             "  - plan_mode=True: Hints agent to use TRAVERSE with depth=0 for edge analysis\n"
-            "• parse_and_ingest_file - Ingest files from local paths (local server only), s3://, or https://\n"
+            "• ingest_into_rem - Ingest files from local paths (local server only), s3://, or https://\n"
+            "• list_schema - List all database schemas (tables) with row counts\n"
+            "• get_schema - Get detailed schema for a specific table (columns, types, indexes)\n"
             "\n"
             "═══════════════════════════════════════════════════════════════════════════\n"
             "AVAILABLE RESOURCES (Read-Only)\n"
@@ -165,11 +175,22 @@ def create_mcp_server(is_local: bool = False) -> FastMCP:
     )
 
     # Register REM tools
-    from .tools import ask_rem_agent, ingest_into_rem, read_resource, search_rem
+    from .tools import (
+        ask_rem_agent,
+        get_schema,
+        ingest_into_rem,
+        list_schema,
+        read_resource,
+        register_metadata,
+        search_rem,
+    )
 
     mcp.tool()(search_rem)
     mcp.tool()(ask_rem_agent)
     mcp.tool()(read_resource)
+    mcp.tool()(register_metadata)
+    mcp.tool()(list_schema)
+    mcp.tool()(get_schema)
 
     # File ingestion tool (with local path support for local servers)
     # Wrap to inject is_local parameter