mcli-framework 7.0.0__py3-none-any.whl

mcli/lib/watcher/watcher.py

@@ -0,0 +1,185 @@
+# import click
+# import os
+# from watchdog.observers import Observer
+# from watchdog.events import FileSystemEventHandler
+# # from .config import PACKAGES_TO_SYNC, PATH_TO_PACKAGE_REPO, ENDPOINT
+# # # from mcli.types.file.file import NO_CHANGE_TO_FILE
+# # import re
+
+
+# # # TODO: To test
+# # class Watcher:
+# #     def __init__(self, directories):
+# #         self.observer = Observer()
+# #         self.directories = directories
+
+# #     def run(self):
+# #         event_handler = Handler()
+# #         for directory in self.directories:
+# #             self.observer.schedule(event_handler, directory, recursive=True)
+# #         self.observer.start()
+# #         try:
+# #             while True:
+# #                 pass
+# #         except KeyboardInterrupt:
+# #             self.observer.stop()
+# #         self.observer.join()
+
+# # class Handler(FileSystemEventHandler):
+# #     @staticmethod
+# #     def process(event):
+# #         if (event.is_directory):
+# #             logger.info("No rwx")
+# #             # # or re.search(r'\.\w+$', event.src_path)
+# #             # or event.src_path.endswith('~')):
+# #             # logger.info("Badly formatted file string")
+# #             # logger.info(event)
+# #             # logger.info(event.src_path)
+# #             # logger.info(event.is_synthetic)
+# #             return
+# #         elif event.event_type == 'created' or event.event_type == 'modified':
+# #             logger.info("writing content")
+# #             # logger.info(event)
+# #             # logger.info(event.src_path)
+# #             write_content(event.src_path)
+# #         elif event.event_type == 'deleted':
+# #             logger.info("delete content")
+# #             logger.info(event.src_path)
+# #             delete_content(event.src_path)
+
+# #     def on_created(self, event):
+# #         self.process(event)
+
+# #     def on_modified(self, event):
+# #         self.process(event)
+
+# #     def on_deleted(self, event):
+# #         self.process(event)
+
+
+# # def get_metadata_path(path):
+# #     return path[len(PATH_TO_PACKAGE_REPO):]
+
+# # def get_pkg_id():
+# #     global pkg_id
+# #     if pkg_id:
+# #         return pkg_id
+
+# #     def handle_response(body):
+# #         global pkg_id
+# #         pkg_id = body
+
+# #     make_post_request('Pkg', 'inst', ['Pkg'], handle_response)
+# #     return pkg_id
+
+# # def write_content(path):
+# #     logger.info("write_content")
+# #     # if re.search(r'\.\w+$', path) or path.endswith('~'):
+# #     #     logger.info("Badly formatted file")
+# #     #     pass
+# #     # pkg_id = get_pkg_id()
+# #     metadata_path = get_metadata_path(path)
+# #     content = None
+# #     with open(path, 'rb') as file:
+# #         logger.info(file)
+# #         content = file
+# #     logger.info(metadata_path)
+# #     logger.info(content)
+# #     if content == NO_CHANGE_TO_FILE:
+# #         return
+# #     # return make_post_request('Pkg', 'writeContent', [pkg_id, metadata_path, {
+# #     #     'type': 'ContentValue',
+# #     #     'content': content
+# #     # }])
+
+# # def delete_content(path):
+# #     # pkg_id = get_pkg_id()
+# #     metadata_path = get_metadata_path(path)
+# #     logger.info(metadata_path)
+# #     # return make_post_request('Pkg', 'deleteContent', [pkg_id, metadata_path, True])
+
+
+# # @click.command()
+# # def watch():
+# #     """watcher utility - use this to watch changes to your packages"""
+# #     watch_dirs = [os.path.join(PATH_TO_PACKAGE_REPO, pkg) for pkg in PACKAGES_TO_SYNC]
+# #     watcher = Watcher(watch_dirs)
+# #     click.echo(f"Listening to file updates at: {', '.join(watch_dirs)}")
+# #     watcher.run()
+
+
+# #  # @pkg.command()
+# # # @click.argument('path')
+# # # def deploy(path):
+# # #     """Deploy a package at a given directory"""
+# # #     return _deploy(path)
+
+# # # def _deploy(path):
+# # #     def _find_repos(path):
+# # #         repos = []
+# # #         logger.info(f"PATH: {path}")
+# # #         for root, dirs, files in os.walk(path):
+# # #             logger.info(f"\troot: {root}")
+# # #             # logger.info(f"\tdirs: {dirs}")
+# # #             logger.info(f"\tfiles: {files}")
+# # #             # logger.info(f"_find_repos\npath: {path}\n")
+# # #             # for file in files:
+# # #                 # logger.info(file)
+# # #         return repos
+
+
+# # #         if len(repos) == 0:
+# # #             raise FileNotFoundError('Could not find a `repository.json` file. Are you provisioning from the right location?')
+# # #         if len(repos) > 1:
+# # #             raise NotImplementedError(f'Too many repositories ({len(repos)}) found! This is not yet implemented')
+
+
+# # #     def _generate_zip_content(repos):
+# # #         tmpZipFile = _generate_zip(repos)
+# # #         return _zip_file_to_content(tmpZipFile)
+
+
+# # #     def _generate_zip(repos):
+# # #         repo = repos[0]
+# # #         repoDir = OS_SEP.join(repo.split(OS_SEP)[:-1])
+# # #         zipDir = f'{LOCAL_ZIP_DIR}{OS_SEP}provtmp-{round(time.time())}'
+# # #         shutil.make_archive(zipDir, 'zip', repoDir)
+# # #         return zipDir
+
+
+# # #     def _zip_file_to_content(zipFile):
+# # #         with open(zipFile + '.zip', 'rb') as f:
+# # #             bytes = f.read()
+# # #             zipContent = base64.b64encode(bytes).decode()
+# # #             shutil.rmtree(LOCAL_ZIP_DIR)
+# # #             return zip
+
+
+# # #     return _find_repos(path)
+
+# # # # @pkg.command()
+# # # # @click.argument('path')
+# # # # def write(path):
+# # # #     """Write/Edit content from mcli server."""
+# # # #     click.echo(f"TODO: Not tested")
+# # #     return
+# # #     write_content(path)
+# # #     click.echo(f"Content written for path: {path}")
+
+# # # @pkg.command()
+# # # @click.argument('path')
+# # # def delete(path):p
+# # #     """Delete content from mcli server."""
+# # #     click.echo(f"TODO: Not tested")
+# # #     return
+# # #     delete_content(path)
+# # #     click.echo(f"Content deleted for path: {path}")
+
+
+# # if __name__ == "__main__":
+# #      watch()
+
+
+def watch(*args, **kwargs):
+    """Dummy watch function for CLI test pass."""
+    pass

mcli/ml/api/app.py

@@ -0,0 +1,215 @@
+"""FastAPI application factory and configuration"""
+
+from contextlib import asynccontextmanager
+from typing import Dict, Any
+
+from fastapi import FastAPI, Request, Response
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.middleware.trustedhost import TrustedHostMiddleware
+from fastapi.responses import JSONResponse
+from starlette.middleware.sessions import SessionMiddleware
+import uvicorn
+
+from mcli.ml.config import settings
+from mcli.ml.database.session import init_db
+from mcli.ml.cache import init_cache
+from mcli.ml.logging import setup_logging, get_logger
+from .routers import (
+    auth_router,
+    model_router,
+    prediction_router,
+    portfolio_router,
+    data_router,
+    trade_router,
+    backtest_router,
+    monitoring_router,
+    admin_router,
+    websocket_router,
+)
+from .middleware import (
+    RequestLoggingMiddleware,
+    RateLimitMiddleware,
+    ErrorHandlingMiddleware,
+)
+
+logger = get_logger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan manager"""
+    # Startup
+    logger.info("Starting ML API server...")
+
+    # Initialize database
+    init_db()
+    logger.info("Database initialized")
+
+    # Initialize cache
+    await init_cache()
+    logger.info("Cache initialized")
+
+    # Initialize ML models
+    from mcli.ml.models import load_production_models
+    await load_production_models()
+    logger.info("ML models loaded")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down ML API server...")
+
+    # Cleanup cache connections
+    from mcli.ml.cache import close_cache
+    await close_cache()
+
+    # Cleanup database connections
+    from mcli.ml.database.session import async_engine
+    await async_engine.dispose()
+
+
+def create_app() -> FastAPI:
+    """Create FastAPI application with all configurations"""
+
+    # Setup logging
+    setup_logging()
+
+    # Create FastAPI app
+    app = FastAPI(
+        title="MCLI ML System API",
+        description="ML system for politician trading analysis and stock recommendations",
+        version="1.0.0",
+        docs_url="/docs" if settings.debug else None,
+        redoc_url="/redoc" if settings.debug else None,
+        openapi_url="/openapi.json" if settings.debug else None,
+        lifespan=lifespan,
+    )
+
+    # Add middlewares
+    app.add_middleware(ErrorHandlingMiddleware)
+    app.add_middleware(RequestLoggingMiddleware)
+    app.add_middleware(RateLimitMiddleware, requests_per_minute=settings.api.rate_limit)
+    app.add_middleware(
+        SessionMiddleware,
+        secret_key=settings.api.secret_key,
+        session_cookie="ml_session",
+        max_age=3600,
+    )
+    app.add_middleware(GZipMiddleware, minimum_size=1000)
+
+    # CORS configuration
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.security.cors_origins,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Trusted host middleware
+    if settings.is_production:
+        app.add_middleware(
+            TrustedHostMiddleware,
+            allowed_hosts=["*.mcli-ml.com", "mcli-ml.com"]
+        )
+
+    # Include routers
+    app.include_router(auth_router.router, prefix="/api/v1/auth", tags=["Authentication"])
+    app.include_router(model_router.router, prefix="/api/v1/models", tags=["Models"])
+    app.include_router(prediction_router.router, prefix="/api/v1/predictions", tags=["Predictions"])
+    app.include_router(portfolio_router.router, prefix="/api/v1/portfolios", tags=["Portfolios"])
+    app.include_router(data_router.router, prefix="/api/v1/data", tags=["Data"])
+    app.include_router(trade_router.router, prefix="/api/v1/trades", tags=["Trades"])
+    app.include_router(backtest_router.router, prefix="/api/v1/backtests", tags=["Backtesting"])
+    app.include_router(monitoring_router.router, prefix="/api/v1/monitoring", tags=["Monitoring"])
+    app.include_router(admin_router.router, prefix="/api/v1/admin", tags=["Admin"])
+    app.include_router(websocket_router.router, prefix="/ws", tags=["WebSocket"])
+
+    # Health check endpoint
+    @app.get("/health", tags=["Health"])
+    async def health_check():
+        """Health check endpoint"""
+        return {
+            "status": "healthy",
+            "environment": settings.environment,
+            "version": "1.0.0"
+        }
+
+    # Ready check endpoint
+    @app.get("/ready", tags=["Health"])
+    async def ready_check():
+        """Readiness check endpoint"""
+        from mcli.ml.database.session import check_database_health
+        from mcli.ml.cache import check_cache_health
+
+        db_healthy = await check_database_health()
+        cache_healthy = await check_cache_health()
+
+        if db_healthy and cache_healthy:
+            return {"status": "ready", "database": "healthy", "cache": "healthy"}
+        else:
+            return JSONResponse(
+                status_code=503,
+                content={
+                    "status": "not ready",
+                    "database": "healthy" if db_healthy else "unhealthy",
+                    "cache": "healthy" if cache_healthy else "unhealthy"
+                }
+            )
+
+    # Metrics endpoint (Prometheus format)
+    @app.get("/metrics", tags=["Monitoring"])
+    async def metrics():
+        """Prometheus metrics endpoint"""
+        from mcli.ml.monitoring.metrics import get_metrics
+        return Response(content=get_metrics(), media_type="text/plain")
+
+    # Root endpoint
+    @app.get("/")
+    async def root():
+        """Root endpoint"""
+        return {
+            "message": "MCLI ML System API",
+            "version": "1.0.0",
+            "docs": "/docs" if settings.debug else None
+        }
+
+    # Exception handlers
+    @app.exception_handler(404)
+    async def not_found_handler(request: Request, exc):
+        return JSONResponse(
+            status_code=404,
+            content={"detail": "Resource not found"}
+        )
+
+    @app.exception_handler(500)
+    async def internal_server_error_handler(request: Request, exc):
+        logger.error(f"Internal server error: {exc}")
+        return JSONResponse(
+            status_code=500,
+            content={"detail": "Internal server error"}
+        )
+
+    return app
+
+
+def get_application() -> FastAPI:
+    """Get configured FastAPI application"""
+    return create_app()
+
+
+# Create app instance
+app = get_application()
+
+
+if __name__ == "__main__":
+    """Run the application directly"""
+    uvicorn.run(
+        "mcli.ml.api.app:app",
+        host=settings.api.host,
+        port=settings.api.port,
+        workers=settings.api.workers,
+        reload=settings.debug,
+        log_level="debug" if settings.debug else "info",
+    )

mcli/ml/api/middleware.py

@@ -0,0 +1,224 @@
+"""Custom middleware for API"""
+
+import time
+import uuid
+from typing import Callable
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+from fastapi import Request, Response, HTTPException
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.types import ASGIApp
+
+from mcli.ml.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class RequestLoggingMiddleware(BaseHTTPMiddleware):
+    """Log all incoming requests and responses"""
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        # Generate request ID
+        request_id = str(uuid.uuid4())
+        request.state.request_id = request_id
+
+        # Log request
+        start_time = time.time()
+        logger.info(f"Request {request_id}: {request.method} {request.url.path}")
+
+        # Process request
+        response = await call_next(request)
+
+        # Log response
+        process_time = time.time() - start_time
+        logger.info(
+            f"Response {request_id}: status={response.status_code} "
+            f"duration={process_time:.3f}s"
+        )
+
+        # Add headers
+        response.headers["X-Request-ID"] = request_id
+        response.headers["X-Process-Time"] = str(process_time)
+
+        return response
+
+
+class RateLimitMiddleware(BaseHTTPMiddleware):
+    """Rate limiting middleware"""
+
+    def __init__(self, app: ASGIApp, requests_per_minute: int = 60):
+        super().__init__(app)
+        self.requests_per_minute = requests_per_minute
+        self.clients = defaultdict(list)
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        # Skip rate limiting for health checks
+        if request.url.path in ["/health", "/ready", "/metrics"]:
+            return await call_next(request)
+
+        # Get client IP
+        client_ip = request.client.host
+
+        # Check rate limit
+        now = datetime.utcnow()
+        minute_ago = now - timedelta(minutes=1)
+
+        # Clean old requests
+        self.clients[client_ip] = [
+            req_time for req_time in self.clients[client_ip]
+            if req_time > minute_ago
+        ]
+
+        # Check if limit exceeded
+        if len(self.clients[client_ip]) >= self.requests_per_minute:
+            logger.warning(f"Rate limit exceeded for {client_ip}")
+            return JSONResponse(
+                status_code=429,
+                content={"detail": "Rate limit exceeded. Please try again later."},
+                headers={"Retry-After": "60"}
+            )
+
+        # Record request
+        self.clients[client_ip].append(now)
+
+        # Process request
+        return await call_next(request)
+
+
+class ErrorHandlingMiddleware(BaseHTTPMiddleware):
+    """Global error handling middleware"""
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        try:
+            response = await call_next(request)
+            return response
+
+        except HTTPException as e:
+            # Let FastAPI handle HTTP exceptions
+            raise e
+
+        except Exception as e:
+            # Log unexpected errors
+            request_id = getattr(request.state, "request_id", "unknown")
+            logger.error(
+                f"Unhandled exception in request {request_id}: {str(e)}",
+                exc_info=True
+            )
+
+            # Return generic error response
+            return JSONResponse(
+                status_code=500,
+                content={
+                    "detail": "An internal error occurred",
+                    "request_id": request_id
+                }
+            )
+
+
+class CompressionMiddleware(BaseHTTPMiddleware):
+    """Response compression middleware"""
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        response = await call_next(request)
+
+        # Check if client accepts gzip
+        accept_encoding = request.headers.get("accept-encoding", "")
+        if "gzip" in accept_encoding.lower():
+            # Response will be compressed by GZipMiddleware
+            response.headers["Vary"] = "Accept-Encoding"
+
+        return response
+
+
+class CacheControlMiddleware(BaseHTTPMiddleware):
+    """Add cache control headers"""
+
+    def __init__(self, app: ASGIApp, max_age: int = 0):
+        super().__init__(app)
+        self.max_age = max_age
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        response = await call_next(request)
+
+        # Add cache control headers based on endpoint
+        if request.url.path.startswith("/api/v1/data"):
+            # Cache data endpoints
+            response.headers["Cache-Control"] = f"public, max-age=300"
+        elif request.url.path.startswith("/api/v1/predictions"):
+            # Don't cache predictions
+            response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
+        else:
+            # Default cache control
+            response.headers["Cache-Control"] = f"public, max-age={self.max_age}"
+
+        return response
+
+
+class SecurityHeadersMiddleware(BaseHTTPMiddleware):
+    """Add security headers to responses"""
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        response = await call_next(request)
+
+        # Add security headers
+        response.headers["X-Content-Type-Options"] = "nosniff"
+        response.headers["X-Frame-Options"] = "DENY"
+        response.headers["X-XSS-Protection"] = "1; mode=block"
+        response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
+        response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+
+        # Content Security Policy
+        response.headers["Content-Security-Policy"] = (
+            "default-src 'self'; "
+            "script-src 'self' 'unsafe-inline' 'unsafe-eval'; "
+            "style-src 'self' 'unsafe-inline'; "
+            "img-src 'self' data: https:; "
+            "font-src 'self' data:; "
+            "connect-src 'self' wss: https:;"
+        )
+
+        return response
+
+
+class MetricsMiddleware(BaseHTTPMiddleware):
+    """Collect metrics for monitoring"""
+
+    def __init__(self, app: ASGIApp):
+        super().__init__(app)
+        self.request_count = defaultdict(int)
+        self.request_duration = defaultdict(list)
+
+    async def dispatch(self, request: Request, call_next: Callable) -> Response:
+        # Record metrics
+        endpoint = f"{request.method} {request.url.path}"
+        start_time = time.time()
+
+        # Process request
+        response = await call_next(request)
+
+        # Update metrics
+        duration = time.time() - start_time
+        self.request_count[endpoint] += 1
+        self.request_duration[endpoint].append(duration)
+
+        # Limit history size
+        if len(self.request_duration[endpoint]) > 1000:
+            self.request_duration[endpoint] = self.request_duration[endpoint][-1000:]
+
+        return response
+
+    def get_metrics(self) -> dict:
+        """Get collected metrics"""
+        metrics = {}
+        for endpoint, count in self.request_count.items():
+            durations = self.request_duration[endpoint]
+            if durations:
+                metrics[endpoint] = {
+                    "count": count,
+                    "avg_duration": sum(durations) / len(durations),
+                    "min_duration": min(durations),
+                    "max_duration": max(durations),
+                }
+        return metrics

mcli/ml/api/routers/admin_router.py

@@ -0,0 +1,12 @@
+"""Admin API routes"""
+
+from fastapi import APIRouter, Depends
+from mcli.ml.auth import get_current_active_user, require_role
+from mcli.ml.database.models import User, UserRole
+
+router = APIRouter()
+
+@router.get("/users")
+async def list_users(current_user: User = Depends(require_role(UserRole.ADMIN))):
+    """List all users"""
+    return {"users": []}