cortex-engine 0.1.0__py3-none-any.whl

cortex_engine/__init__.py

@@ -0,0 +1,12 @@
+"""
+Cortex-Engine — AI Operating System for Coding.
+
+Routes queries across 30–50+ language models, manages GPU scheduling,
+LRU caching, output evaluation, and continuous feedback-driven routing.
+"""
+
+__version__ = "0.1.0"
+__author__  = "Cortex-Engine Contributors"
+__license__ = "MIT"
+
+from cortex_engine.main import app  # noqa: F401 — expose ASGI app at top level

cortex_engine/cli.py

@@ -0,0 +1,51 @@
+"""
+Cortex-Engine — CLI entry point.
+
+Usage:
+    cortex-engine              # start the API server (default)
+    cortex-engine serve        # same as above
+    cortex-engine --help
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="cortex-engine",
+        description="Cortex-Engine — AI Operating System for Coding.",
+    )
+    parser.add_argument(
+        "command",
+        nargs="?",
+        default="serve",
+        choices=["serve"],
+        help="Command to run (default: serve)",
+    )
+    parser.add_argument("--host", default="0.0.0.0", help="Bind host (default: 0.0.0.0)")
+    parser.add_argument("--port", type=int, default=8000, help="Bind port (default: 8000)")
+    parser.add_argument("--workers", type=int, default=1, help="Worker count (default: 1)")
+    parser.add_argument("--reload", action="store_true", help="Enable auto-reload (dev mode)")
+    args = parser.parse_args()
+
+    if args.command == "serve":
+        try:
+            import uvicorn
+        except ImportError:
+            print("ERROR: uvicorn is required. Run: uv add uvicorn", file=sys.stderr)
+            sys.exit(1)
+
+        uvicorn.run(
+            "cortex_engine.main:app",
+            host=args.host,
+            port=args.port,
+            workers=args.workers,
+            reload=args.reload,
+        )
+
+
+if __name__ == "__main__":
+    main()

cortex_engine/config.py

@@ -0,0 +1,35 @@
+"""
+Cortex-Engine — Configuration
+Reads from environment variables with sensible defaults.
+"""
+
+from __future__ import annotations
+from typing import List
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
+
+    # Redis
+    redis_url:             str  = "redis://localhost:6379/0"
+    redis_max_connections: int  = 50
+
+    # Server
+    host:    str = "0.0.0.0"
+    port:    int = 8000
+    workers: int = 1
+    reload:  bool = False
+
+    # CORS
+    allowed_origins: List[str] = ["*"]
+
+    # Cache
+    cache_ttl_seconds: int = 3600
+
+    # Feature flags
+    enable_evaluation: bool = True
+    enable_feedback:   bool = True
+
+
+settings = Settings()

cortex_engine/dependencies.py

@@ -0,0 +1,41 @@
+"""
+Cortex-Engine — Dependency Injection
+All FastAPI Depends() providers live here.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import AsyncGenerator
+
+import redis.asyncio as redis
+from fastapi import Request
+
+from cortex_engine.services.cache_manager import CacheManager
+from cortex_engine.services.evaluator     import EvaluationEngine
+from cortex_engine.services.feedback      import FeedbackSystem
+from cortex_engine.services.orchestrator  import ModelOrchestrator
+from cortex_engine.services.registry      import ModelRegistry
+from cortex_engine.services.router        import RouterEngine
+from cortex_engine.services.scheduler     import Scheduler
+
+
+# ── Low-level services (pulled from app.state) ────────────────────────────────
+
+def get_redis(request: Request) -> redis.Redis:
+    return request.app.state.redis
+
+def get_registry(request: Request) -> ModelRegistry:
+    return request.app.state.registry
+
+def get_cache(request: Request) -> CacheManager:
+    return request.app.state.cache
+
+def get_scheduler(request: Request) -> Scheduler:
+    return request.app.state.scheduler
+
+def get_feedback(request: Request) -> FeedbackSystem:
+    return request.app.state.feedback
+
+def get_orchestrator(request: Request) -> ModelOrchestrator:
+    return request.app.state.orchestrator

cortex_engine/main.py

@@ -0,0 +1,180 @@
+"""
+Cortex-Engine — Main Application
+FastAPI entry point. Bootstraps all services on startup.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from contextlib import asynccontextmanager
+
+import redis.asyncio as redis_asyncio
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+
+from cortex_engine.config import settings
+from cortex_engine.routers.inference import router as inference_router
+from cortex_engine.routers.api import registry_router, admin_router
+from cortex_engine.services.cache_manager import CacheManager
+from cortex_engine.services.evaluator     import EvaluationEngine
+from cortex_engine.services.feedback      import FeedbackSystem
+from cortex_engine.services.orchestrator  import ModelOrchestrator
+from cortex_engine.services.registry      import ModelRegistry
+from cortex_engine.services.router        import RouterEngine
+from cortex_engine.services.scheduler     import Scheduler
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)-8s | %(name)s — %(message)s",
+)
+log = logging.getLogger("cortex_engine")
+
+_BOOT_TIME = time.time()
+
+
+# ── Lifespan (startup / shutdown) ─────────────────────────────────────────────
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    log.info("🚀 Cortex-Engine booting …")
+
+    # Redis
+    pool = redis_asyncio.ConnectionPool.from_url(
+        settings.redis_url,
+        max_connections=settings.redis_max_connections,
+        decode_responses=True,
+    )
+    r = redis_asyncio.Redis(connection_pool=pool)
+    await r.ping()
+    log.info("✅ Redis connected: %s", settings.redis_url)
+
+    # Services
+    registry  = ModelRegistry(r)
+    cache     = CacheManager(r)
+    scheduler = Scheduler(r, registry)
+    router    = RouterEngine(registry=registry)
+    evaluator = EvaluationEngine()
+    feedback  = FeedbackSystem(r)
+
+    await registry.seed()
+
+    orchestrator = ModelOrchestrator(
+        registry=registry,
+        router=router,
+        scheduler=scheduler,
+        cache=cache,
+        evaluator=evaluator,
+        feedback=feedback,
+    )
+
+    # Attach to app state
+    app.state.redis        = r
+    app.state.registry     = registry
+    app.state.cache        = cache
+    app.state.scheduler    = scheduler
+    app.state.router       = router
+    app.state.evaluator    = evaluator
+    app.state.feedback     = feedback
+    app.state.orchestrator = orchestrator
+
+    log.info("✅ All services initialised.")
+    yield
+
+    # Shutdown
+    log.info("🛑 Cortex-Engine shutting down …")
+    await r.aclose()
+    await pool.aclose()
+
+
+# ── App ───────────────────────────────────────────────────────────────────────
+
+app = FastAPI(
+    title="Cortex-Engine",
+    description=(
+        "Distributed AI system: routes queries to 30–50+ models, "
+        "manages GPU scheduling, caching, and continuous feedback."
+    ),
+    version="0.1.0",
+    lifespan=lifespan,
+    docs_url="/docs",
+    redoc_url="/redoc",
+)
+
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.allowed_origins,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# ── Latency middleware ─────────────────────────────────────────────────────────
+
+import time as _time
+from fastapi import Request, Response
+
+@app.middleware("http")
+async def latency_header(request: Request, call_next):
+    t0 = _time.perf_counter()
+    response: Response = await call_next(request)
+    response.headers["X-Process-Time-Ms"] = str(
+        round((_time.perf_counter() - t0) * 1000, 2)
+    )
+    return response
+
+
+# ── Routers ───────────────────────────────────────────────────────────────────
+
+app.include_router(inference_router)
+app.include_router(registry_router)
+app.include_router(admin_router)
+
+
+# ── Health endpoints ───────────────────────────────────────────────────────────
+
+@app.get("/", tags=["Health"])
+async def root():
+    return {"service": "Cortex-Engine", "version": "0.1.0", "status": "ok"}
+
+
+@app.get("/health", tags=["Health"])
+async def health(request: Request):
+    r: redis_asyncio.Redis = request.app.state.redis
+    try:
+        await r.ping()
+        redis_ok = True
+    except Exception:
+        redis_ok = False
+
+    reg_stats  = await request.app.state.registry.stats()
+    cache_info = await request.app.state.cache.stats()
+
+    return {
+        "status":          "ok" if redis_ok else "degraded",
+        "uptime_seconds":  round(time.time() - _BOOT_TIME, 1),
+        "redis":           "up" if redis_ok else "down",
+        "models":          reg_stats,
+        "cache":           cache_info,
+    }
+
+
+@app.get("/metrics", tags=["Health"])
+async def metrics(request: Request):
+    fb_stats    = await request.app.state.feedback.accuracy_stats()
+    cache_stats = await request.app.state.cache.stats()
+    reg_stats   = await request.app.state.registry.stats()
+    queue_depth = await request.app.state.scheduler.queue_depth()
+    gpu_loads   = await request.app.state.scheduler.gpu_loads()
+
+    return {
+        "uptime_seconds":    round(time.time() - _BOOT_TIME, 1),
+        "routing_accuracy":  fb_stats["accuracy"],
+        "cache_hit_rate":    cache_stats["hit_rate"],
+        "models":            reg_stats,
+        "queue_depth":       queue_depth,
+        "gpu_loads":         gpu_loads,
+        "feedback":          fb_stats,
+    }

cortex_engine/models/__init__.py

@@ -0,0 +1 @@
+# models sub-package

cortex_engine/models/schemas.py

@@ -0,0 +1,158 @@
+"""
+Cortex-Engine — Pydantic Schemas
+Defines all request/response models used across the system.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# ─────────────────────────────────────────────
+# Enums
+# ─────────────────────────────────────────────
+
+class ModelType(str, Enum):
+    CODING      = "coding"
+    DEBUGGING   = "debugging"
+    EXPLANATION = "explanation"
+    TESTING     = "testing"
+    REFACTORING = "refactoring"
+    DOCS        = "docs"
+    GENERAL     = "general"
+
+
+class ModelStatus(str, Enum):
+    AVAILABLE = "available"
+    LOADING   = "loading"
+    BUSY      = "busy"
+    OFFLINE   = "offline"
+
+
+class EvalMethod(str, Enum):
+    UNIT_TEST       = "unit_test"
+    STATIC_ANALYSIS = "static_analysis"
+    LLM_GRADING     = "llm_grading"
+    EXECUTION       = "execution"
+
+
+# ─────────────────────────────────────────────
+# Model Registry Schemas
+# ─────────────────────────────────────────────
+
+class ModelMetadata(BaseModel):
+    model_name:   str
+    type:         ModelType
+    cluster:      str                      # e.g. "python", "typescript", "rust"
+    size:         str                      # e.g. "7B", "14B", "70B"
+    latency_ms:   int   = Field(ge=0)      # expected p50 latency in ms
+    cost_per_tok: float = Field(ge=0.0)    # USD per 1K tokens
+    gpu_location: str                      # e.g. "gpu-0", "gpu-3"
+    status:       ModelStatus = ModelStatus.AVAILABLE
+    priority:     int = 5                  # 1 (lowest) – 10 (highest)
+    tags:         List[str] = []
+
+    model_config = ConfigDict(use_enum_values=True)
+
+
+class ModelRegistryResponse(BaseModel):
+    models:      List[ModelMetadata]
+    total:       int
+    available:   int
+
+
+# ─────────────────────────────────────────────
+# Inference Schemas
+# ─────────────────────────────────────────────
+
+class InferenceRequest(BaseModel):
+    query:           str  = Field(..., min_length=1, max_length=32_000)
+    preferred_type:  Optional[ModelType]  = None
+    preferred_model: Optional[str]        = None  # override routing
+    max_tokens:      int  = Field(default=2048, ge=64, le=16384)
+    temperature:     float = Field(default=0.2, ge=0.0, le=2.0)
+    evaluate:        bool = True
+    user_id:         Optional[str] = None
+    session_id:      Optional[str] = None
+    metadata:        Dict[str, Any] = {}
+
+
+class RouteDecision(BaseModel):
+    cluster:          str
+    selected_model:   str
+    confidence:       float = Field(ge=0.0, le=1.0)
+    fallback_models:  List[str] = []
+    routing_latency_ms: float = 0.0
+
+
+class EvaluationResult(BaseModel):
+    success:  bool
+    score:    float = Field(ge=0.0, le=1.0)
+    method:   EvalMethod
+    details:  str = ""
+
+
+class InferenceResponse(BaseModel):
+    request_id:       str
+    output:           str
+    model_used:       str
+    route:            RouteDecision
+    evaluation:       Optional[EvaluationResult] = None
+    total_latency_ms: float
+    cached:           bool = False
+    tokens_used:      int  = 0
+
+
+# ─────────────────────────────────────────────
+# Scheduler Schemas
+# ─────────────────────────────────────────────
+
+class ScheduleRequest(BaseModel):
+    model_name: str
+    gpu_hint:   Optional[str] = None   # prefer specific GPU
+    priority:   int = 5
+
+class ScheduleResult(BaseModel):
+    model_name:     str
+    gpu_location:   str
+    queue_position: int
+    estimated_wait_ms: float
+
+
+# ─────────────────────────────────────────────
+# Feedback Schemas
+# ─────────────────────────────────────────────
+
+class FeedbackEntry(BaseModel):
+    request_id:      str
+    query:           str
+    predicted_route: str
+    actual_route:    Optional[str] = None
+    success:         bool
+    score:           float
+    user_correction: Optional[str] = None   # what the correct model should have been
+
+
+# ─────────────────────────────────────────────
+# Health / Metrics
+# ─────────────────────────────────────────────
+
+class GPUStats(BaseModel):
+    gpu_id:       str
+    utilization:  float   # 0.0 – 1.0
+    memory_used:  float   # GB
+    memory_total: float   # GB
+    models_loaded: List[str] = []
+
+
+class KernelMetrics(BaseModel):
+    uptime_seconds:      float
+    total_requests:      int
+    requests_per_second: float
+    cache_hit_rate:      float
+    avg_latency_ms:      float
+    routing_accuracy:    float
+    gpu_stats:           List[GPUStats] = []
+    models_available:    int

cortex_engine/py.typed

@@ -0,0 +1 @@
+# This file marks cortex_engine as a typed package (PEP 561).

cortex_engine/routers/__init__.py

@@ -0,0 +1 @@
+# routers sub-package

cortex_engine/routers/api.py

@@ -0,0 +1,86 @@
+"""
+Cortex-Engine — /registry and /admin routes
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from cortex_engine.models.schemas import ModelMetadata, ModelRegistryResponse, ModelStatus
+from cortex_engine.services.registry import ModelRegistry
+from cortex_engine.services.scheduler import Scheduler
+from cortex_engine.services.cache_manager import CacheManager
+from cortex_engine.services.feedback import FeedbackSystem
+from cortex_engine.dependencies import get_registry, get_scheduler, get_cache, get_feedback
+
+registry_router = APIRouter(prefix="/registry", tags=["Registry"])
+admin_router    = APIRouter(prefix="/admin",    tags=["Admin"])
+
+
+# ── Registry ──────────────────────────────────────────────────────────────────
+
+@registry_router.get("/", response_model=ModelRegistryResponse)
+async def list_models(reg: ModelRegistry = Depends(get_registry)):
+    models = await reg.list_all()
+    stats  = await reg.stats()
+    return ModelRegistryResponse(models=models, total=stats["total"], available=stats["available"])
+
+
+@registry_router.post("/", status_code=status.HTTP_201_CREATED)
+async def register_model(model: ModelMetadata, reg: ModelRegistry = Depends(get_registry)):
+    await reg.register(model)
+    return {"registered": model.model_name}
+
+
+@registry_router.get("/{model_name}")
+async def get_model(model_name: str, reg: ModelRegistry = Depends(get_registry)):
+    model = await reg.get(model_name)
+    if not model:
+        raise HTTPException(404, f"Model '{model_name}' not found")
+    return model
+
+
+@registry_router.patch("/{model_name}/status")
+async def set_model_status(
+    model_name: str, status: ModelStatus,
+    reg: ModelRegistry = Depends(get_registry),
+):
+    ok = await reg.update_status(model_name, status)
+    if not ok:
+        raise HTTPException(404, f"Model '{model_name}' not found")
+    return {"updated": model_name, "status": status}
+
+
+@registry_router.delete("/{model_name}", status_code=204)
+async def deregister_model(model_name: str, reg: ModelRegistry = Depends(get_registry)):
+    ok = await reg.deregister(model_name)
+    if not ok:
+        raise HTTPException(404, f"Model '{model_name}' not found")
+
+
+# ── Admin ─────────────────────────────────────────────────────────────────────
+
+@admin_router.get("/queue")
+async def queue_status(sched: Scheduler = Depends(get_scheduler)):
+    return {
+        "queue_depth":   await sched.queue_depth(),
+        "running_jobs":  await sched.running_jobs(),
+        "gpu_loads":     await sched.gpu_loads(),
+    }
+
+
+@admin_router.get("/cache")
+async def cache_status(cache: CacheManager = Depends(get_cache)):
+    return {
+        "stats":       await cache.stats(),
+        "warm_pool":   await cache.warm_pool(),
+        "eviction_log": await cache.eviction_log(20),
+    }
+
+
+@admin_router.get("/feedback")
+async def feedback_status(
+    limit: int = 50,
+    fb: FeedbackSystem = Depends(get_feedback),
+):
+    return {
+        "accuracy_stats": await fb.accuracy_stats(),
+        "recent":         await fb.recent(limit),
+    }

cortex_engine/routers/inference.py

@@ -0,0 +1,38 @@
+"""
+Cortex-Engine — /inference routes
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from cortex_engine.models.schemas import InferenceRequest, InferenceResponse
+from cortex_engine.services.orchestrator import ModelOrchestrator
+from cortex_engine.dependencies import get_orchestrator
+
+router = APIRouter(prefix="/inference", tags=["Inference"])
+
+
+@router.post(
+    "/",
+    response_model=InferenceResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Submit a query for AI inference",
+)
+async def infer(
+    req: InferenceRequest,
+    orchestrator: ModelOrchestrator = Depends(get_orchestrator),
+) -> InferenceResponse:
+    try:
+        return await orchestrator.infer(req)
+    except RuntimeError as exc:
+        raise HTTPException(status_code=503, detail=str(exc))
+
+
+@router.get(
+    "/route-preview",
+    summary="Preview which model would be selected without running inference",
+)
+async def route_preview(
+    query: str,
+    orchestrator: ModelOrchestrator = Depends(get_orchestrator),
+):
+    route = await orchestrator._router.route(query=query)
+    return route

cortex_engine/services/__init__.py

@@ -0,0 +1 @@
+# services sub-package