valcore 0.0.1__py3-none-any.whl

valcore/__init__.py

@@ -0,0 +1 @@
+"""valcore: a local tool for developing, improving, and running agentic evaluations."""

valcore/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI application layer for valcore. Never imported by the valcore library."""

valcore/api/deps.py

@@ -0,0 +1,18 @@
+"""FastAPI dependencies shared across resource routers."""
+
+import functools
+
+from valcore.settings import get_settings
+from valcore.store import Store, create_engine, init_db
+
+
+@functools.lru_cache
+def get_store() -> Store:
+    """Return the process-wide Store, creating the engine and tables on first use.
+
+    Cached so a single engine is shared across requests. Tests override this via
+    ``app.dependency_overrides[get_store]``.
+    """
+    engine = create_engine(get_settings().db_path)
+    init_db(engine)
+    return Store(engine)

valcore/api/dtos.py

@@ -0,0 +1,21 @@
+"""Pydantic request/response schemas for the API surface.
+
+These are kept separate from the SQLModel entities so responses never leak raw SQL
+objects or secrets. Only the shared error envelope lives here; resource routers define
+their own request/response models.
+"""
+
+from pydantic import BaseModel
+
+
+class ErrorBody(BaseModel):
+    """The inner error payload: the exception class name and its message."""
+
+    type: str
+    message: str
+
+
+class ErrorResponse(BaseModel):
+    """Uniform error envelope returned by every exception handler."""
+
+    error: ErrorBody

valcore/api/events.py

@@ -0,0 +1,91 @@
+"""In-process pub/sub so the runner can push run progress to SSE subscribers."""
+
+import asyncio
+from collections.abc import AsyncIterator
+
+_QUEUE_MAXSIZE = 256
+_CLOSE_SENTINEL: dict = {}
+
+
+def _evict_oldest_progress(queue: "asyncio.Queue[dict]") -> bool:
+    """Drop the oldest ``progress`` event from a full queue, preserving order and others.
+
+    Returns True if a progress event was dropped, making room for a new event.
+    """
+    buffered: list[dict] = []
+    while True:
+        try:
+            buffered.append(queue.get_nowait())
+        except asyncio.QueueEmpty:
+            break
+    dropped = False
+    for event in buffered:
+        if not dropped and event.get("type") == "progress":
+            dropped = True
+            continue
+        queue.put_nowait(event)
+    return dropped
+
+
+class EventBus:
+    """Fan-out of per-run events to each subscriber's own bounded queue.
+
+    Slow subscribers never block publishers: when a queue is full the oldest
+    ``progress`` event is dropped to make room. ``row``, ``started``, and ``finished``
+    events are never dropped.
+    """
+
+    def __init__(self) -> None:
+        self._subscribers: dict[str, list[asyncio.Queue[dict]]] = {}
+
+    def publish(self, run_id: str, event: dict) -> None:
+        """Deliver an event to every subscriber of a run without blocking."""
+        for queue in list(self._subscribers.get(run_id, ())):
+            self._offer(queue, event)
+
+    def _offer(self, queue: "asyncio.Queue[dict]", event: dict) -> None:
+        """Enqueue an event, evicting old progress under pressure rather than blocking."""
+        try:
+            queue.put_nowait(event)
+            return
+        except asyncio.QueueFull:
+            pass
+        if event.get("type") == "progress":
+            return
+        if _evict_oldest_progress(queue):
+            try:
+                queue.put_nowait(event)
+            except asyncio.QueueFull:
+                pass
+
+    async def subscribe(self, run_id: str) -> AsyncIterator[dict]:
+        """Yield events published to ``run_id`` until the run is closed."""
+        queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=_QUEUE_MAXSIZE)
+        self._subscribers.setdefault(run_id, []).append(queue)
+        try:
+            while True:
+                event = await queue.get()
+                if event is _CLOSE_SENTINEL:
+                    return
+                yield event
+        finally:
+            subscribers = self._subscribers.get(run_id)
+            if subscribers is not None and queue in subscribers:
+                subscribers.remove(queue)
+            if subscribers is not None and not subscribers:
+                self._subscribers.pop(run_id, None)
+
+    def close(self, run_id: str) -> None:
+        """Signal every subscriber of a run to stop iterating."""
+        for queue in list(self._subscribers.get(run_id, ())):
+            try:
+                queue.put_nowait(_CLOSE_SENTINEL)
+            except asyncio.QueueFull:
+                _evict_oldest_progress(queue)
+                try:
+                    queue.put_nowait(_CLOSE_SENTINEL)
+                except asyncio.QueueFull:
+                    pass
+
+
+bus = EventBus()

valcore/api/main.py

@@ -0,0 +1,113 @@
+"""FastAPI application factory: CORS, exception handlers, health/config, routers, static SPA."""
+
+import importlib
+from importlib.resources import files as _package_files
+from pathlib import Path
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from valcore.api.dtos import ErrorBody, ErrorResponse
+from valcore.errors import (
+    ConfigError,
+    ContractError,
+    FrozenVersionError,
+    NotFoundError,
+    ValcoreError,
+)
+from valcore.models import VALID_CAPABILITIES
+from valcore.settings import MODEL_CATALOG
+from valcore.tools import tool_names
+
+_STATUS_BY_ERROR: tuple[tuple[type[ValcoreError], int], ...] = (
+    (NotFoundError, 404),
+    (ContractError, 422),
+    (ConfigError, 422),
+    (FrozenVersionError, 409),
+    (ValcoreError, 400),
+)
+
+
+def _resolve_dist_dir() -> Path | None:
+    """Locate the built SPA: packaged assets first, then a repo checkout, else nothing."""
+    try:
+        packaged = _package_files("valcore") / "web_dist"
+        if packaged.is_dir():
+            return Path(str(packaged))
+    except (ModuleNotFoundError, FileNotFoundError):
+        pass
+
+    for parent in Path(__file__).resolve().parents:
+        candidate = parent / "web" / "dist"
+        if candidate.is_dir():
+            return candidate
+
+    return None
+
+
+def _error_response(status_code: int, exc: Exception) -> JSONResponse:
+    """Render an exception into the uniform ``{"error": {type, message}}`` envelope."""
+    body = ErrorResponse(error=ErrorBody(type=type(exc).__name__, message=str(exc)))
+    return JSONResponse(status_code=status_code, content=body.model_dump())
+
+
+def _register_exception_handlers(app: FastAPI) -> None:
+    """Map each domain error to its documented HTTP status via the uniform envelope."""
+
+    def make_handler(status_code: int):
+        async def handler(_request: Request, exc: ValcoreError) -> JSONResponse:
+            return _error_response(status_code, exc)
+
+        return handler
+
+    for error_type, status_code in _STATUS_BY_ERROR:
+        app.add_exception_handler(error_type, make_handler(status_code))
+
+
+def _include_routers(app: FastAPI) -> None:
+    """Discover and mount resource routers, tolerating ones that do not exist yet."""
+    for module_name in ("evaluators", "datasets", "runs"):
+        try:
+            module = importlib.import_module(f"valcore.api.routes.{module_name}")
+        except ImportError:
+            continue
+        app.include_router(module.router)
+
+
+def create_app() -> FastAPI:
+    """Build and return the valcore FastAPI application."""
+    app = FastAPI(title="valcore")
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["http://localhost:5173"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    _register_exception_handlers(app)
+
+    @app.get("/api/health")
+    async def health() -> dict[str, str]:
+        """Liveness probe."""
+        return {"status": "ok"}
+
+    @app.get("/api/config")
+    async def config() -> dict[str, list[str]]:
+        """Return the pickers the SPA needs: models, tools, and capabilities."""
+        return {
+            "models": list(MODEL_CATALOG),
+            "tools": tool_names(),
+            "capabilities": sorted(VALID_CAPABILITIES),
+        }
+
+    _include_routers(app)
+
+    dist_dir = _resolve_dist_dir()
+    if dist_dir:
+        app.mount("/", StaticFiles(directory=dist_dir, html=True), name="spa")
+
+    return app

valcore/api/routes/__init__.py

@@ -0,0 +1 @@
+"""Resource routers. Each module exports a module-level ``router: APIRouter``."""

valcore/api/routes/datasets.py

@@ -0,0 +1,346 @@
+"""Dataset CRUD, file upload, generation, and labeling endpoints."""
+
+import csv
+import io
+import json
+from datetime import datetime
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, File, Form, UploadFile
+from pydantic import BaseModel, ConfigDict
+
+from valcore.api.deps import get_store
+from valcore.datagen import generate_rows
+from valcore.errors import ContractError
+from valcore.models import LabelSchema, LabelSource, ScoreKind
+from valcore.store import Store
+
+router = APIRouter(prefix="/api/datasets", tags=["datasets"])
+
+StoreDep = Annotated[Store, Depends(get_store)]
+
+_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
+_MAX_GENERATE_COUNT = 200
+_JSONL_INFER_LIMIT = 50
+
+
+class DatasetCreate(BaseModel):
+    """Request body to create an empty dataset."""
+
+    name: str
+    description: str = ""
+    columns: list[str]
+    label_schema: LabelSchema
+
+
+class DatasetGenerate(BaseModel):
+    """Request body to generate a dataset and its rows."""
+
+    name: str
+    description: str = ""
+    columns: list[str]
+    label_schema: LabelSchema
+    count: int
+
+
+class RowsAppend(BaseModel):
+    """Request body to append plain data rows to a dataset."""
+
+    rows: list[dict]
+
+
+class RowPatch(BaseModel):
+    """Request body to relabel or annotate a single row."""
+
+    label: str | float | None = None
+    note: str | None = None
+    accept_suggestion: bool = False
+    clear_label: bool = False
+
+
+class DatasetOut(BaseModel):
+    """A dataset as returned to the client."""
+
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    created_at: datetime
+    name: str
+    description: str
+    columns: list[str]
+    label_schema: dict
+
+
+class DatasetCreatedOut(BaseModel):
+    """A newly created dataset paired with the number of rows persisted."""
+
+    dataset: DatasetOut
+    row_count: int
+
+
+class RowOut(BaseModel):
+    """A dataset row with its labels as returned to the client."""
+
+    model_config = ConfigDict(from_attributes=True)
+
+    id: str
+    dataset_id: str
+    idx: int
+    data: dict
+    label: dict | None
+    suggested_label: dict | None
+    label_reasoning: str | None
+    label_source: LabelSource | None
+    note: str | None
+
+
+class RowsPage(BaseModel):
+    """A paginated slice of dataset rows."""
+
+    rows: list[RowOut]
+    total: int
+    limit: int
+    offset: int
+
+
+class StatsOut(BaseModel):
+    """Labeling progress for a dataset."""
+
+    total: int
+    labeled: int
+    unlabeled: int
+    label_distribution: dict[str, int]
+
+
+def _label_matches_schema(value: str | float, schema: LabelSchema) -> bool:
+    """Return True if ``value`` is a valid label under ``schema``."""
+    if schema.kind is ScoreKind.CATEGORICAL:
+        return isinstance(value, str) and value in (schema.labels or [])
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return False
+    below = schema.minimum is not None and value < schema.minimum
+    above = schema.maximum is not None and value > schema.maximum
+    return not (below or above)
+
+
+def _parse_csv(text: str, label_column: str | None) -> tuple[list[str], list[dict]]:
+    """Parse CSV text into inferred data columns and prepared row dicts."""
+    reader = csv.DictReader(io.StringIO(text))
+    header = reader.fieldnames or []
+    if label_column is not None and label_column not in header:
+        raise ContractError(f"label_column {label_column!r} is not one of the columns {header}.")
+    columns = [name for name in header if name != label_column]
+    prepared = [_prepare_row(dict(record), columns, label_column) for record in reader]
+    return columns, prepared
+
+
+def _parse_jsonl(text: str, label_column: str | None) -> tuple[list[str], list[dict]]:
+    """Parse JSONL text into inferred data columns and prepared row dicts."""
+    records: list[dict] = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise ContractError(f"Invalid JSON line: {exc}.") from exc
+        if not isinstance(record, dict):
+            raise ContractError("Every JSONL record must be an object.")
+        records.append(record)
+
+    keys: list[str] = []
+    for record in records[:_JSONL_INFER_LIMIT]:
+        for key in record:
+            if key != label_column and key not in keys:
+                keys.append(key)
+    prepared = [_prepare_row(record, keys, label_column) for record in records]
+    return keys, prepared
+
+
+def _prepare_row(record: dict, columns: list[str], label_column: str | None) -> dict:
+    """Split a raw record into a prepared row, moving any label column into a manual label."""
+    fields: dict = {"data": {k: v for k, v in record.items() if k != label_column}}
+    if label_column is not None and record.get(label_column) is not None:
+        fields["label"] = {"value": record[label_column]}
+        fields["label_source"] = LabelSource.MANUAL
+    return fields
+
+
+@router.get("")
+async def list_datasets(store: StoreDep) -> list[DatasetOut]:
+    """List every dataset."""
+    return [DatasetOut.model_validate(ds) for ds in store.list_datasets()]
+
+
+@router.post("")
+async def create_dataset(body: DatasetCreate, store: StoreDep) -> DatasetOut:
+    """Create an empty dataset."""
+    dataset = store.create_dataset(
+        name=body.name,
+        description=body.description,
+        columns=body.columns,
+        label_schema=body.label_schema.model_dump(mode="json"),
+    )
+    return DatasetOut.model_validate(dataset)
+
+
+@router.get("/{id}")
+async def get_dataset(id: str, store: StoreDep) -> DatasetOut:
+    """Return a single dataset."""
+    return DatasetOut.model_validate(store.get_dataset(id))
+
+
+@router.delete("/{id}")
+async def delete_dataset(id: str, store: StoreDep) -> dict[str, str]:
+    """Delete a dataset and all of its rows."""
+    store.delete_dataset(id)
+    return {"status": "deleted"}
+
+
+@router.post("/upload")
+async def upload_dataset(
+    store: StoreDep,
+    file: Annotated[UploadFile, File()],
+    name: Annotated[str, Form()],
+    label_column: Annotated[str | None, Form()] = None,
+    label_schema: Annotated[str | None, Form()] = None,
+) -> DatasetCreatedOut:
+    """Create a dataset from an uploaded CSV or JSONL file."""
+    contents = await file.read()
+    if len(contents) > _MAX_UPLOAD_BYTES:
+        raise ContractError(
+            f"File exceeds the {_MAX_UPLOAD_BYTES // (1024 * 1024)} MB upload limit."
+        )
+
+    text = contents.decode("utf-8-sig")
+    filename = (file.filename or "").lower()
+    if filename.endswith(".csv"):
+        columns, prepared = _parse_csv(text, label_column)
+    elif filename.endswith((".jsonl", ".json")):
+        columns, prepared = _parse_jsonl(text, label_column)
+    else:
+        raise ContractError("Unsupported file type; upload a .csv or .jsonl file.")
+
+    if not prepared:
+        raise ContractError("File contains no data rows.")
+
+    schema_dict = _parse_label_schema(label_schema)
+    dataset = store.create_dataset(
+        name=name, description="", columns=columns, label_schema=schema_dict
+    )
+    rows = store.add_prepared_rows(dataset.id, prepared)
+    return DatasetCreatedOut(dataset=DatasetOut.model_validate(dataset), row_count=len(rows))
+
+
+def _parse_label_schema(raw: str | None) -> dict:
+    """Parse an optional JSON label-schema form field, validating its shape."""
+    if raw is None or not raw.strip():
+        return {}
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ContractError(f"Invalid label_schema JSON: {exc}.") from exc
+    try:
+        return LabelSchema.model_validate(parsed).model_dump(mode="json")
+    except ValueError as exc:
+        raise ContractError(f"Invalid label_schema: {exc}.") from exc
+
+
+@router.post("/{id}/rows")
+async def append_rows(id: str, body: RowsAppend, store: StoreDep) -> dict[str, int]:
+    """Append plain data rows to a dataset."""
+    store.get_dataset(id)
+    rows = store.add_rows(id, body.rows)
+    return {"row_count": len(rows)}
+
+
+@router.post("/generate")
+async def generate_dataset(body: DatasetGenerate, store: StoreDep) -> DatasetCreatedOut:
+    """Generate a dataset and its rows with suggested labels."""
+    if body.count > _MAX_GENERATE_COUNT:
+        raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
+
+    dataset = store.create_dataset(
+        name=body.name,
+        description=body.description,
+        columns=body.columns,
+        label_schema=body.label_schema.model_dump(mode="json"),
+    )
+    generated = await generate_rows(body.description, body.columns, body.label_schema, body.count)
+    prepared = [
+        {
+            "data": row.data,
+            "suggested_label": {"value": row.suggested_label},
+            "label_reasoning": row.reasoning,
+            "label_source": LabelSource.GENERATED,
+        }
+        for row in generated
+    ]
+    rows = store.add_prepared_rows(dataset.id, prepared)
+    return DatasetCreatedOut(dataset=DatasetOut.model_validate(dataset), row_count=len(rows))
+
+
+@router.get("/{id}/rows")
+async def list_dataset_rows(
+    id: str, store: StoreDep, limit: int = 100, offset: int = 0
+) -> RowsPage:
+    """Return a paginated slice of a dataset's rows."""
+    store.get_dataset(id)
+    rows = store.list_rows(id, limit=limit, offset=offset)
+    _, total = store.labeled_count(id)
+    return RowsPage(
+        rows=[RowOut.model_validate(row) for row in rows],
+        total=total,
+        limit=limit,
+        offset=offset,
+    )
+
+
+@router.patch("/rows/{row_id}")
+async def patch_row(row_id: str, body: RowPatch, store: StoreDep) -> RowOut:
+    """Relabel or annotate a single dataset row."""
+    row = store.get_row(row_id)
+    updates: dict = {}
+
+    if body.accept_suggestion:
+        if row.suggested_label is None:
+            raise ContractError("Row has no suggested label to accept.")
+        updates["label"] = row.suggested_label
+        updates["label_source"] = LabelSource.ACCEPTED
+
+    # A null ``label`` is indistinguishable from an omitted one, so clearing needs an
+    # explicit flag rather than relying on ``label=None``.
+    if body.clear_label:
+        updates["label"] = None
+        updates["label_source"] = None
+
+    if body.label is not None:
+        dataset = store.get_dataset(row.dataset_id)
+        schema = LabelSchema.model_validate(dataset.label_schema)
+        if not _label_matches_schema(body.label, schema):
+            raise ContractError(
+                f"Label {body.label!r} is not valid for this dataset's label schema."
+            )
+        updates["label"] = {"value": body.label}
+        updates["label_source"] = LabelSource.MANUAL
+
+    if body.note is not None:
+        updates["note"] = body.note
+
+    if not updates:
+        return RowOut.model_validate(row)
+    return RowOut.model_validate(store.update_row(row_id, **updates))
+
+
+@router.get("/{id}/stats")
+async def dataset_stats(id: str, store: StoreDep) -> StatsOut:
+    """Return labeling progress for a dataset."""
+    store.get_dataset(id)
+    labeled, total = store.labeled_count(id)
+    return StatsOut(
+        total=total,
+        labeled=labeled,
+        unlabeled=total - labeled,
+        label_distribution=store.label_distribution(id),
+    )