llm-bench-studio 0.1.0__tar.gz

llm_bench_studio-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: llm-bench-studio
+Version: 0.1.0
+Summary: Production-grade benchmarking platform for OpenAI-compatible LLM endpoints
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: uvicorn[standard]>=0.30.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: sqlalchemy[asyncio]>=2.0.0
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: openpyxl>=3.1.0
+Requires-Dist: reportlab>=4.0.0
+Requires-Dist: psutil>=6.0.0
+Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: python-dotenv>=1.0.0

llm_bench_studio-0.1.0/backend/__init__.py

@@ -0,0 +1 @@
+# Backend package

llm_bench_studio-0.1.0/backend/api/__init__.py

@@ -0,0 +1,17 @@
+"""API package."""
+
+from backend.api.connections import router as connections_router
+from backend.api.system import router as system_router
+from backend.api.prompts import router as prompts_router
+from backend.api.benchmarks import router as benchmarks_router
+from backend.api.websocket import router as websocket_router
+
+__all__ = [
+    "connections_router",
+    "system_router",
+    "prompts_router",
+    "benchmarks_router",
+    "websocket_router",
+]
+
+

llm_bench_studio-0.1.0/backend/api/benchmarks.py

@@ -0,0 +1,438 @@
+"""API router for starting, stopping, and inspecting benchmark runs."""
+
+from datetime import datetime
+import json
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel, Field
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.benchmark.engine import benchmark_engine
+from backend.database.engine import get_db_session
+from backend.services.connection_service import ConnectionService
+from backend.models.connection import Connection
+from backend.models.benchmark_run import BenchmarkRun
+from backend.models.benchmark_request import BenchmarkRequest
+from backend.utils.errors import ValidationError
+from backend.api.websocket import broadcast_progress_callback
+
+router = APIRouter(prefix="/api/benchmarks", tags=["benchmarks"])
+
+
+class StartBenchmarkRequest(BaseModel):
+    """Payload schema to initiate a new benchmark."""
+
+    connection_id: str = Field(..., description="ID of the target connection endpoint")
+    prompt_ids: list[str] | None = Field(None, description="Optional list of prompt IDs to run")
+    
+    input_tokens: int = Field(128, ge=1, le=128000)
+    output_tokens: int = Field(128, ge=1, le=128000)
+    
+    concurrency: int = Field(10, ge=1, le=500)
+    total_requests: int = Field(100, ge=1, le=10000)
+    warmup_requests: int = Field(0, ge=0, le=1000)
+    ignore_eos: bool = Field(False, description="Ignore EOS token to force max tokens generation")
+
+
+class StartBenchmarkResponse(BaseModel):
+    """Response returned on successful start."""
+
+    run_id: str
+    message: str
+
+
+class StopBenchmarkResponse(BaseModel):
+    """Response returned on stop command."""
+
+    message: str
+
+
+class BenchmarkStatusResponse(BaseModel):
+    """Response carrying current run status information."""
+
+    run_id: str | None
+    status: str
+    completed_requests: int
+    total_requests: int
+    failed_requests: int
+    requests_per_second: float
+    tokens_per_second: float
+    avg_latency_ms: float
+    avg_ttft_ms: float
+    avg_itl_ms: float
+
+
+@router.post("/start", response_model=StartBenchmarkResponse, status_code=202)
+async def start_benchmark(
+    data: StartBenchmarkRequest,
+    session: AsyncSession = Depends(get_db_session),
+) -> StartBenchmarkResponse:
+    """Initiates a concurrent LLM endpoint benchmark run."""
+    # Validate connection exists
+    conn_service = ConnectionService(session)
+    await conn_service.get_by_id(data.connection_id)
+
+    # Format configuration values
+    config = {
+        "concurrency": data.concurrency,
+        "total_requests": data.total_requests,
+        "warmup_requests": data.warmup_requests,
+        "ignore_eos": data.ignore_eos,
+        "input_tokens": data.input_tokens,
+        "output_tokens": data.output_tokens,
+    }
+
+    # Start the benchmark engine
+    run_id = await benchmark_engine.start(
+        connection_id=data.connection_id,
+        config=config,
+        prompt_ids=data.prompt_ids or [],
+        progress_callback=broadcast_progress_callback,
+    )
+
+    return StartBenchmarkResponse(
+        run_id=run_id,
+        message="Benchmark run started successfully",
+    )
+
+
+@router.post("/stop", response_model=StopBenchmarkResponse)
+async def stop_benchmark() -> StopBenchmarkResponse:
+    """Stops the currently running benchmark run."""
+    if not benchmark_engine.is_running():
+        raise ValidationError("No benchmark is currently running")
+
+    await benchmark_engine.stop()
+    return StopBenchmarkResponse(message="Stop signal sent to benchmark engine")
+
+
+@router.get("/status", response_model=BenchmarkStatusResponse)
+async def get_status() -> BenchmarkStatusResponse:
+    """Retrieves the real-time status of the benchmark engine."""
+    return BenchmarkStatusResponse(
+        is_running=benchmark_engine.is_running(),
+        active_run_id=benchmark_engine.active_run_id,
+        completed_requests=benchmark_engine.completed_requests,
+        failed_requests=benchmark_engine.failed_requests,
+        current_concurrency=benchmark_engine.active_controller.concurrency
+        if benchmark_engine.active_controller
+        else 0,
+        avg_latency_ms=0.0,
+    )
+
+
+# --- History and Comparison schemas ---
+
+class BenchmarkRunSummary(BaseModel):
+    id: str
+    connection_id: str
+    connection_name: str | None = None
+    model_name: str | None = None
+    status: str
+    is_favorite: bool
+    config: dict
+    total_requests: int
+    successful_requests: int
+    failed_requests: int
+    avg_latency_ms: float
+    avg_ttft_ms: float
+    avg_itl_ms: float
+    throughput_rps: float
+    throughput_tps: float
+    success_rate: float
+    started_at: datetime
+    completed_at: datetime | None = None
+    created_at: datetime
+
+
+class BenchmarkRequestDetail(BaseModel):
+    id: str
+    request_index: int
+    prompt: str
+    response: str
+    status: str
+    status_code: int
+    error_message: str | None = None
+    latency_ms: float
+    ttft_ms: float | None = None
+    input_tokens: int
+    output_tokens: int
+    started_at: datetime
+    completed_at: datetime
+
+
+class BenchmarkRunDetail(BaseModel):
+    id: str
+    connection_id: str
+    connection_name: str | None = None
+    model_name: str | None = None
+    status: str
+    is_favorite: bool
+    config: dict
+    total_requests: int
+    successful_requests: int
+    failed_requests: int
+    avg_latency_ms: float
+    avg_ttft_ms: float
+    avg_itl_ms: float
+    throughput_rps: float
+    throughput_tps: float
+    success_rate: float
+    metrics_summary: dict | None = None
+    started_at: datetime
+    completed_at: datetime | None = None
+    created_at: datetime
+    requests: list[BenchmarkRequestDetail] = []
+
+
+# --- History and Comparison endpoints ---
+
+@router.get("/history", response_model=list[BenchmarkRunSummary])
+async def get_history(
+    offset: int = 0,
+    limit: int = 50,
+    session: AsyncSession = Depends(get_db_session),
+) -> list[BenchmarkRunSummary]:
+    """Retrieves all past benchmark executions with joined connection details."""
+    query = (
+        select(BenchmarkRun, Connection.name, Connection.model_name)
+        .outerjoin(Connection, BenchmarkRun.connection_id == Connection.id)
+        .order_by(BenchmarkRun.created_at.desc())
+        .offset(offset)
+        .limit(limit)
+    )
+    result = await session.execute(query)
+    
+    summaries = []
+    for row in result.all():
+        db_run, conn_name, model_name = row
+        try:
+            cfg = json.loads(db_run.config)
+        except Exception:
+            cfg = {}
+        summaries.append(
+            BenchmarkRunSummary(
+                id=db_run.id,
+                connection_id=db_run.connection_id,
+                connection_name=conn_name,
+                model_name=model_name,
+                status=db_run.status,
+                is_favorite=db_run.is_favorite,
+                config=cfg,
+                total_requests=db_run.total_requests,
+                successful_requests=db_run.successful_requests,
+                failed_requests=db_run.failed_requests,
+                avg_latency_ms=db_run.avg_latency_ms,
+                avg_ttft_ms=db_run.avg_ttft_ms,
+                avg_itl_ms=db_run.avg_itl_ms,
+                throughput_rps=db_run.throughput_rps,
+                throughput_tps=db_run.throughput_tps,
+                success_rate=db_run.success_rate,
+                started_at=db_run.started_at,
+                completed_at=db_run.completed_at,
+                created_at=db_run.created_at,
+            )
+        )
+    return summaries
+
+
+@router.get("/history/{run_id}", response_model=BenchmarkRunDetail)
+async def get_run_detail(
+    run_id: str,
+    session: AsyncSession = Depends(get_db_session),
+) -> BenchmarkRunDetail:
+    """Fetches details of a completed benchmark run including statistics and individual request trials."""
+    query = (
+        select(BenchmarkRun, Connection.name, Connection.model_name)
+        .outerjoin(Connection, BenchmarkRun.connection_id == Connection.id)
+        .where(BenchmarkRun.id == run_id)
+    )
+    result = await session.execute(query)
+    row = result.one_or_none()
+    if not row:
+        raise ValidationError("Benchmark run not found")
+        
+    db_run, conn_name, model_name = row
+    
+    # Load requests
+    req_query = (
+        select(BenchmarkRequest)
+        .where(BenchmarkRequest.benchmark_run_id == run_id)
+        .order_by(BenchmarkRequest.request_index.asc())
+    )
+    req_result = await session.execute(req_query)
+    db_requests = req_result.scalars().all()
+    
+    requests_detail = [
+        BenchmarkRequestDetail(
+            id=r.id,
+            request_index=r.request_index,
+            prompt=r.prompt,
+            response=r.response,
+            status=r.status,
+            status_code=r.status_code,
+            error_message=r.error_message,
+            latency_ms=r.latency_ms,
+            ttft_ms=r.ttft_ms,
+            input_tokens=r.input_tokens,
+            output_tokens=r.output_tokens,
+            started_at=r.started_at,
+            completed_at=r.completed_at,
+        )
+        for r in db_requests
+    ]
+    
+    try:
+        cfg = json.loads(db_run.config)
+    except Exception:
+        cfg = {}
+        
+    try:
+        metrics = json.loads(db_run.metrics_summary) if db_run.metrics_summary else {}
+    except Exception:
+        metrics = {}
+
+    return BenchmarkRunDetail(
+        id=db_run.id,
+        connection_id=db_run.connection_id,
+        connection_name=conn_name,
+        model_name=model_name,
+        status=db_run.status,
+        is_favorite=db_run.is_favorite,
+        config=cfg,
+        total_requests=db_run.total_requests,
+        successful_requests=db_run.successful_requests,
+        failed_requests=db_run.failed_requests,
+        avg_latency_ms=db_run.avg_latency_ms,
+        avg_ttft_ms=db_run.avg_ttft_ms,
+        avg_itl_ms=db_run.avg_itl_ms,
+        throughput_rps=db_run.throughput_rps,
+        throughput_tps=db_run.throughput_tps,
+        success_rate=db_run.success_rate,
+        metrics_summary=metrics,
+        started_at=db_run.started_at,
+        completed_at=db_run.completed_at,
+        created_at=db_run.created_at,
+        requests=requests_detail,
+    )
+
+
+@router.post("/history/{run_id}/favorite")
+async def toggle_favorite(
+    run_id: str,
+    session: AsyncSession = Depends(get_db_session),
+):
+    """Toggles the favorite state of a historical benchmark run."""
+    stmt = select(BenchmarkRun).where(BenchmarkRun.id == run_id)
+    result = await session.execute(stmt)
+    db_run = result.scalar_one_or_none()
+    if not db_run:
+        raise ValidationError("Benchmark run not found")
+        
+    db_run.is_favorite = not db_run.is_favorite
+    await session.commit()
+    return {"id": run_id, "is_favorite": db_run.is_favorite}
+
+
+@router.delete("/history/{run_id}")
+async def delete_run(
+    run_id: str,
+    session: AsyncSession = Depends(get_db_session),
+):
+    """Deletes a benchmark run from history database."""
+    stmt = select(BenchmarkRun).where(BenchmarkRun.id == run_id)
+    result = await session.execute(stmt)
+    db_run = result.scalar_one_or_none()
+    if not db_run:
+        raise ValidationError("Benchmark run not found")
+        
+    await session.delete(db_run)
+    await session.commit()
+    return {"message": "Run deleted successfully", "id": run_id}
+
+
+@router.get("/compare", response_model=list[BenchmarkRunDetail])
+async def compare_runs(
+    run_ids: list[str] = Query(..., description="List of run IDs to compare side by side"),
+    session: AsyncSession = Depends(get_db_session),
+) -> list[BenchmarkRunDetail]:
+    """Fetches details for multiple runs side by side for comparative analysis."""
+    if not run_ids:
+        raise ValidationError("At least one run ID is required for comparison")
+        
+    query = (
+        select(BenchmarkRun, Connection.name, Connection.model_name)
+        .outerjoin(Connection, BenchmarkRun.connection_id == Connection.id)
+        .where(BenchmarkRun.id.in_(run_ids))
+    )
+    result = await session.execute(query)
+    rows = result.all()
+    
+    comparisons = []
+    for row in rows:
+        db_run, conn_name, model_name = row
+        
+        # Load requests for each compared run
+        req_query = (
+            select(BenchmarkRequest)
+            .where(BenchmarkRequest.benchmark_run_id == db_run.id)
+            .order_by(BenchmarkRequest.request_index.asc())
+        )
+        req_result = await session.execute(req_query)
+        db_requests = req_result.scalars().all()
+        
+        requests_detail = [
+            BenchmarkRequestDetail(
+                id=r.id,
+                request_index=r.request_index,
+                prompt=r.prompt,
+                response=r.response,
+                status=r.status,
+                status_code=r.status_code,
+                error_message=r.error_message,
+                latency_ms=r.latency_ms,
+                ttft_ms=r.ttft_ms,
+                input_tokens=r.input_tokens,
+                output_tokens=r.output_tokens,
+                started_at=r.started_at,
+                completed_at=r.completed_at,
+            )
+            for r in db_requests
+        ]
+        
+        try:
+            cfg = json.loads(db_run.config)
+        except Exception:
+            cfg = {}
+            
+        try:
+            metrics = json.loads(db_run.metrics_summary) if db_run.metrics_summary else {}
+        except Exception:
+            metrics = {}
+
+        comparisons.append(
+            BenchmarkRunDetail(
+                id=db_run.id,
+                connection_id=db_run.connection_id,
+                connection_name=conn_name,
+                model_name=model_name,
+                status=db_run.status,
+                is_favorite=db_run.is_favorite,
+                config=cfg,
+                total_requests=db_run.total_requests,
+                successful_requests=db_run.successful_requests,
+                failed_requests=db_run.failed_requests,
+                avg_latency_ms=db_run.avg_latency_ms,
+                avg_ttft_ms=db_run.avg_ttft_ms,
+                avg_itl_ms=db_run.avg_itl_ms,
+                throughput_rps=db_run.throughput_rps,
+                throughput_tps=db_run.throughput_tps,
+                success_rate=db_run.success_rate,
+                metrics_summary=metrics,
+                started_at=db_run.started_at,
+                completed_at=db_run.completed_at,
+                created_at=db_run.created_at,
+                requests=requests_detail,
+            )
+        )
+    return comparisons

llm_bench_studio-0.1.0/backend/api/connections.py

@@ -0,0 +1,87 @@
+"""Connection management API endpoints."""
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.database.engine import get_db_session
+from backend.services.connection_service import (
+    ConnectionCreate,
+    ConnectionResponse,
+    ConnectionService,
+    ConnectionTestResult,
+    ConnectionUpdate,
+)
+
+router = APIRouter(prefix="/api/connections", tags=["connections"])
+
+
+def _get_service(session: AsyncSession = Depends(get_db_session)) -> ConnectionService:
+    return ConnectionService(session)
+
+
+@router.post("", response_model=ConnectionResponse, status_code=201)
+async def create_connection(
+    data: ConnectionCreate,
+    service: ConnectionService = Depends(_get_service),
+) -> ConnectionResponse:
+    """Create a new LLM endpoint connection."""
+    return await service.create(data)
+
+
+@router.get("", response_model=list[ConnectionResponse])
+async def list_connections(
+    offset: int = Query(0, ge=0),
+    limit: int = Query(100, ge=1, le=500),
+    service: ConnectionService = Depends(_get_service),
+) -> list[ConnectionResponse]:
+    """List all saved connections with pagination."""
+    return await service.get_all(offset=offset, limit=limit)
+
+
+@router.get("/{connection_id}", response_model=ConnectionResponse)
+async def get_connection(
+    connection_id: str,
+    service: ConnectionService = Depends(_get_service),
+) -> ConnectionResponse:
+    """Get a specific connection by ID."""
+    return await service.get_by_id(connection_id)
+
+
+@router.put("/{connection_id}", response_model=ConnectionResponse)
+async def update_connection(
+    connection_id: str,
+    data: ConnectionUpdate,
+    service: ConnectionService = Depends(_get_service),
+) -> ConnectionResponse:
+    """Update an existing connection."""
+    return await service.update(connection_id, data)
+
+
+@router.delete("/{connection_id}", status_code=204)
+async def delete_connection(
+    connection_id: str,
+    service: ConnectionService = Depends(_get_service),
+) -> None:
+    """Delete a connection."""
+    await service.delete(connection_id)
+
+
+@router.post(
+    "/{connection_id}/test",
+    response_model=ConnectionTestResult,
+)
+async def test_connection(
+    connection_id: str,
+    service: ConnectionService = Depends(_get_service),
+) -> ConnectionTestResult:
+    """Test a saved connection by sending a minimal request."""
+    return await service.test_connection(connection_id)
+
+
+@router.post("/test", response_model=ConnectionTestResult)
+async def test_connection_config(
+    data: ConnectionCreate,
+    service: ConnectionService = Depends(_get_service),
+) -> ConnectionTestResult:
+    """Test a connection from raw config without saving."""
+    return await service.test_connection_config(data)