isa-model 0.4.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

isa_model/serving/api/fastapi_server.py

@@ -15,34 +15,29 @@ import logging
 import os
 from typing import Dict, Any, Optional
 
-from .routes import health, unified, deployments, logs, analytics, settings, evaluations
+from .routes import health, unified, deployments, logs, analytics, settings, inference_monitoring, webhooks, tenants  # config, training, annotation, and evaluations temporarily disabled
 from .middleware.request_logger import RequestLoggerMiddleware
 from .middleware.security import setup_security_middleware, check_redis_health
+from .middleware.tenant_context import TenantContextMiddleware
 from .startup import run_startup_initialization
+from ...core.logging import api_logger, setup_logger
 
-logger = logging.getLogger(__name__)
+logger = api_logger  # Use Loki-configured logger instead of standard logging
 
 def configure_logging():
-    """Configure logging based on environment variables"""
+    """Configure logging based on environment variables
+
+    Note: Loki integration is handled automatically by isa_model.core.logging.setup_logger
+    This function only sets log levels for existing loggers.
+    """
     log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
     verbose_logging = os.getenv('VERBOSE_LOGGING', 'false').lower() == 'true'
-    
+
     # Set log level
     level = getattr(logging, log_level, logging.INFO)
-    
-    # Configure format
-    if verbose_logging:
-        log_format = '%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s'
-    else:
-        log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-    
-    # Configure root logger
-    logging.basicConfig(
-        level=level,
-        format=log_format,
-        datefmt='%Y-%m-%d %H:%M:%S',
-        force=True  # Override existing configuration
-    )
+
+    # Note: Don't call logging.basicConfig() here as it conflicts with Loki handlers
+    # The Loki logger (api_logger) is already configured with proper handlers
     
     # Set uvicorn logger level to match
     uvicorn_logger = logging.getLogger("uvicorn")
@@ -52,6 +47,69 @@ def configure_logging():
     app_logger = logging.getLogger("isa_model")
     app_logger.setLevel(level)
     
+    # Suppress verbose third-party library logs
+    # HTTP libraries - only show WARNING and above
+    logging.getLogger("httpcore").setLevel(logging.WARNING)
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("httpcore.http11").setLevel(logging.WARNING)
+    logging.getLogger("httpcore.connection").setLevel(logging.WARNING)
+    
+    # Database and ORM libraries
+    logging.getLogger("supabase").setLevel(logging.WARNING)
+    logging.getLogger("postgrest").setLevel(logging.WARNING)
+    
+    # AI/ML libraries
+    logging.getLogger("openai").setLevel(logging.WARNING)
+    logging.getLogger("anthropic").setLevel(logging.WARNING)
+    logging.getLogger("google").setLevel(logging.WARNING)
+    logging.getLogger("google.cloud").setLevel(logging.WARNING)
+    logging.getLogger("google.generativeai").setLevel(logging.WARNING)
+    
+    # Other verbose libraries
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("aiohttp").setLevel(logging.WARNING)
+    
+    # Reduce startup debug logs
+    if not verbose_logging:
+        # Reduce startup initialization debug logs
+        startup_logger = logging.getLogger("isa_model.serving.api.startup")
+        startup_logger.setLevel(logging.WARNING)
+        
+        # Reduce model registry debug logs
+        model_logger = logging.getLogger("isa_model.core.models.model_repo")
+        model_logger.setLevel(logging.WARNING)
+        
+        # Reduce intelligent selector debug logs
+        selector_logger = logging.getLogger("isa_model.core.services.intelligent_model_selector")
+        selector_logger.setLevel(logging.WARNING)
+        
+        # Training module removed - logger configuration no longer needed
+        
+        # Reduce knowledge base logs
+        kb_logger = logging.getLogger("isa_model.core.knowledge_base")
+        kb_logger.setLevel(logging.WARNING)
+        
+        # Reduce database migration logs
+        migration_logger = logging.getLogger("isa_model.core.database.migrations")
+        migration_logger.setLevel(logging.WARNING)
+        
+        # Reduce AI factory logs
+        ai_factory_logger = logging.getLogger("isa_model.inference.ai_factory")
+        ai_factory_logger.setLevel(logging.WARNING)
+        
+        # Reduce embedding service logs
+        embed_logger = logging.getLogger("isa_model.inference.services.embedding")
+        embed_logger.setLevel(logging.WARNING)
+        
+        # Reduce config manager logs
+        config_logger = logging.getLogger("isa_model.core.config")
+        config_logger.setLevel(logging.WARNING)
+        
+        # Reduce core integration logs
+        core_logger = logging.getLogger("isa_model.core")
+        core_logger.setLevel(logging.WARNING)
+    
     logger.info(f"Logging configured - Level: {log_level}, Verbose: {verbose_logging}")
 
 def create_app(config: Dict[str, Any] = None) -> FastAPI:
@@ -79,6 +137,9 @@ def create_app(config: Dict[str, Any] = None) -> FastAPI:
     # This includes CORS, rate limiting, security headers, request validation
     setup_security_middleware(app)
     
+    # Add tenant context middleware (before request logger)
+    app.add_middleware(TenantContextMiddleware)
+    
     # Add custom middleware
     app.add_middleware(RequestLoggerMiddleware)
     
@@ -112,8 +173,26 @@ def create_app(config: Dict[str, Any] = None) -> FastAPI:
     # SETTINGS API - Configuration and API key management
     app.include_router(settings.router, prefix="/api/v1/settings", tags=["settings"])
     
-    # EVALUATIONS API - Model evaluation and benchmarking
-    app.include_router(evaluations.router, prefix="/api/v1/evaluations", tags=["evaluations"])
+    # EVALUATIONS API - Temporarily disabled for staging optimization
+    # app.include_router(evaluations.router, prefix="/api/v1/evaluations", tags=["evaluations"])
+    
+    # INFERENCE MONITORING API - InfluxDB-based inference monitoring and analytics
+    app.include_router(inference_monitoring.router, prefix="/api/v1/monitoring", tags=["monitoring"])
+    
+    # TRAINING API - Disabled for staging optimization
+    # app.include_router(training.router, prefix="/api/v1/training", tags=["training"])
+    
+    # WEBHOOKS API - Webhook management and notifications
+    app.include_router(webhooks.router, prefix="/api/v1/webhooks", tags=["webhooks"])
+    
+    # TENANTS API - Multi-tenancy and organization management
+    app.include_router(tenants.router, prefix="/api/v1/tenants", tags=["tenants"])
+    
+    # ANNOTATION API - Temporarily disabled for staging optimization
+    # app.include_router(annotation.router, prefix="/api/v1/annotations", tags=["annotations"])
+    
+    # CONFIG API - Configuration management
+    # app.include_router(config.router, prefix="/api/v1/config", tags=["config"])  # Temporarily disabled
     
     # Mount static files
     static_path = os.path.join(os.path.dirname(__file__), "../static")
@@ -149,7 +228,45 @@ def create_app(config: Dict[str, Any] = None) -> FastAPI:
             logger.info("✅ Application startup completed successfully")
         except Exception as e:
             logger.error(f"❌ Application startup failed: {e}")
-            # Don't raise - let the app start anyway
+            logger.error("⚠️ Server will continue but may have reduced functionality")
+            # Store startup failure state for health checks
+            app.state.startup_failed = True
+            app.state.startup_error = str(e)
+            # Continue running to allow debugging and partial functionality
+    
+    # Add shutdown event handler
+    @app.on_event("shutdown")
+    async def shutdown_event():
+        logger.info("🧹 Starting application shutdown cleanup...")
+        try:
+            # Close database connections
+            try:
+                from .dependencies.database import close_database_pool
+                await close_database_pool()
+                logger.info("✅ Database connections closed")
+            except Exception as e:
+                logger.error(f"❌ Error closing database connections: {e}")
+            
+            # Clean up AI factory and services
+            try:
+                from ...inference.ai_factory import AIFactory
+                factory = AIFactory.get_instance()
+                await factory.cleanup()
+                logger.info("✅ AI Factory cleaned up")
+            except Exception as e:
+                logger.error(f"❌ Error cleaning up AI Factory: {e}")
+            
+            # Clean up startup initializer resources
+            try:
+                from .startup import startup_initializer
+                await startup_initializer.cleanup()
+                logger.info("✅ Startup resources cleaned up")
+            except Exception as e:
+                logger.error(f"❌ Error cleaning up startup resources: {e}")
+            
+            logger.info("✅ Application shutdown completed successfully")
+        except Exception as e:
+            logger.error(f"❌ Error during application shutdown: {e}")
     
     return app
 
@@ -158,4 +275,37 @@ app = create_app()
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    import os
+    import signal
+    
+    port = int(os.getenv("PORT", 8082))
+    
+    # Configure uvicorn for graceful shutdown
+    config = uvicorn.Config(
+        app,
+        host="0.0.0.0",
+        port=port,
+        log_level=os.getenv("LOG_LEVEL", "info").lower(),
+        access_log=True,
+        loop="asyncio",
+        # Graceful shutdown configuration
+        timeout_keep_alive=30,  # Keep connections alive for 30 seconds
+        timeout_graceful_shutdown=30,  # 30 second graceful shutdown timeout
+    )
+    
+    server = uvicorn.Server(config)
+    
+    # Setup signal handlers for graceful shutdown
+    def signal_handler(signum, frame):
+        logger.info(f"Received signal {signum}, initiating graceful shutdown...")
+        server.should_exit = True
+    
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+    
+    try:
+        server.run()
+    except KeyboardInterrupt:
+        logger.info("Keyboard interrupt received, shutting down...")
+    finally:
+        logger.info("Server shutdown complete")

isa_model/serving/api/middleware/auth.py

@@ -35,10 +35,16 @@ class APIKeyManager:
         # Load API keys first to check if auth should be enabled
         self.load_api_keys()
         
-        # Determine auth state: enabled if explicitly set OR if API keys exist
+        # Determine auth state: check explicit setting first, then auto-detect from keys
         explicit_auth = AUTH_ENABLED
         has_keys = len(self.api_keys) > 0
-        self.auth_enabled = explicit_auth or has_keys
+        
+        # If explicitly disabled (REQUIRE_API_KEYS=false), respect that setting
+        if os.getenv("REQUIRE_API_KEYS", "").lower() == "false":
+            self.auth_enabled = False
+        else:
+            # Otherwise, enable if explicitly set OR if API keys exist
+            self.auth_enabled = explicit_auth or has_keys
         
         if self.auth_enabled:
             logger.info(f"API Key authentication is ENABLED ({'explicit' if explicit_auth else 'auto-detected from keys'})")

isa_model/serving/api/middleware/security.py

@@ -13,7 +13,6 @@ import time
 import logging
 import os
 import redis
-import structlog
 from typing import Dict, Any, Optional, Callable
 from fastapi import FastAPI, Request, Response, HTTPException, status
 from fastapi.middleware.cors import CORSMiddleware
@@ -25,12 +24,16 @@ from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.responses import JSONResponse
 import html
 
-# Configure structured logging
-logger = structlog.get_logger(__name__)
+from ....core.config.config_manager import ConfigManager
+
+# Configure logging
+logger = logging.getLogger(__name__)
 
 # Configuration from environment variables
+config_manager = ConfigManager()
 MAX_REQUEST_SIZE = int(os.getenv("MAX_REQUEST_SIZE_MB", "50")) * 1024 * 1024  # 50MB default
-REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379")
+# Use Consul discovery for Redis URL with fallback
+REDIS_URL = os.getenv("REDIS_URL", config_manager.get_redis_url())
 ENABLE_RATE_LIMITING = os.getenv("ENABLE_RATE_LIMITING", "true").lower() == "true"
 RATE_LIMIT_PER_MINUTE = os.getenv("RATE_LIMIT_PER_MINUTE", "100")
 RATE_LIMIT_PER_HOUR = os.getenv("RATE_LIMIT_PER_HOUR", "1000")
@@ -41,7 +44,7 @@ SECURITY_HEADERS = {
     "X-Frame-Options": "DENY",
     "X-XSS-Protection": "1; mode=block",
     "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
-    "Content-Security-Policy": "default-src 'self'; script-src 'self' 'unsafe-inline' https://unpkg.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; connect-src 'self'",
+    "Content-Security-Policy": "default-src 'self'; script-src 'self' 'unsafe-inline' https://unpkg.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://cdn.jsdelivr.net; font-src 'self' https://fonts.gstatic.com; img-src 'self' https://fastapi.tiangolo.com data:; connect-src 'self'",
     "Referrer-Policy": "strict-origin-when-cross-origin",
     "Permissions-Policy": "geolocation=(), microphone=(), camera=()"
 }
@@ -100,7 +103,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
             return response
             
         except Exception as e:
-            logger.error("Error in security headers middleware", error=str(e))
+            logger.error(f"Error in security headers middleware: {e}")
             return JSONResponse(
                 status_code=500,
                 content={"error": "Internal server error"},
@@ -119,10 +122,8 @@ class RequestValidationMiddleware(BaseHTTPMiddleware):
             content_length = request.headers.get("content-length")
             if content_length and int(content_length) > MAX_REQUEST_SIZE:
                 logger.warning(
-                    "Request too large",
-                    content_length=content_length,
-                    max_size=MAX_REQUEST_SIZE,
-                    client_ip=get_remote_address_with_proxy(request)
+                    f"Request too large: {content_length} bytes > {MAX_REQUEST_SIZE} bytes "
+                    f"from client {get_remote_address_with_proxy(request)}"
                 )
                 raise HTTPException(
                     status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
@@ -134,19 +135,15 @@ class RequestValidationMiddleware(BaseHTTPMiddleware):
                 sanitized_query = html.escape(request.url.query)
                 if sanitized_query != request.url.query:
                     logger.warning(
-                        "Potentially malicious query parameters detected",
-                        original=request.url.query,
-                        sanitized=sanitized_query,
-                        client_ip=get_remote_address_with_proxy(request)
+                        f"Potentially malicious query parameters detected from client {get_remote_address_with_proxy(request)}: "
+                        f"'{request.url.query}' -> '{sanitized_query}'"
                     )
             
             # Log request details for monitoring
             logger.info(
-                "Request received",
-                method=request.method,
-                path=request.url.path,
-                client_ip=get_remote_address_with_proxy(request),
-                user_agent=request.headers.get("user-agent", "unknown")
+                f"Request received: {request.method} {request.url.path} "
+                f"from {get_remote_address_with_proxy(request)} "
+                f"(UA: {request.headers.get('user-agent', 'unknown')})"
             )
             
             response = await call_next(request)
@@ -154,12 +151,8 @@ class RequestValidationMiddleware(BaseHTTPMiddleware):
             # Log response details
             process_time = time.time() - request.state.start_time
             logger.info(
-                "Request completed",
-                method=request.method,
-                path=request.url.path,
-                status_code=response.status_code,
-                process_time=process_time,
-                client_ip=get_remote_address_with_proxy(request)
+                f"Request completed: {request.method} {request.url.path} -> {response.status_code} "
+                f"in {process_time:.3f}s from {get_remote_address_with_proxy(request)}"
             )
             
             return response
@@ -168,11 +161,8 @@ class RequestValidationMiddleware(BaseHTTPMiddleware):
             raise
         except Exception as e:
             logger.error(
-                "Error in request validation middleware",
-                error=str(e),
-                path=request.url.path,
-                method=request.method,
-                client_ip=get_remote_address_with_proxy(request)
+                f"Error in request validation middleware: {e} "
+                f"({request.method} {request.url.path} from {get_remote_address_with_proxy(request)})"
             )
             raise HTTPException(
                 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -186,13 +176,13 @@ def setup_security_middleware(app: FastAPI):
     if ENABLE_RATE_LIMITING:
         app.state.limiter = limiter
         app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
-        logger.info("Rate limiting enabled", redis_backend=redis_client is not None)
+        logger.info(f"Rate limiting enabled (Redis backend: {redis_client is not None})")
     
     # Trusted hosts (production should specify allowed hosts)
     allowed_hosts = os.getenv("ALLOWED_HOSTS", "*").split(",")
     if allowed_hosts != ["*"]:
         app.add_middleware(TrustedHostMiddleware, allowed_hosts=allowed_hosts)
-        logger.info("Trusted hosts middleware enabled", allowed_hosts=allowed_hosts)
+        logger.info(f"Trusted hosts middleware enabled: {allowed_hosts}")
     
     # CORS configuration
     cors_origins = os.getenv("CORS_ORIGINS", "*").split(",")
@@ -204,7 +194,7 @@ def setup_security_middleware(app: FastAPI):
         allow_headers=["*"],
         expose_headers=["X-Process-Time"]
     )
-    logger.info("CORS middleware enabled", origins=cors_origins)
+    logger.info(f"CORS middleware enabled for origins: {cors_origins}")
     
     # Custom security middleware
     app.add_middleware(SecurityHeadersMiddleware)