scorepilot 0.1.2__py3-none-any.whl

scorepilot/__init__.py

@@ -0,0 +1,3 @@
+"""ScorePilot: a web-based tool for PCA/PLS model analysis in chemometrics."""
+
+__version__ = "0.1.0"

scorepilot/api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI routers: a thin layer translating HTTP to core/db."""

scorepilot/api/datasets.py

@@ -0,0 +1,116 @@
+"""Dataset import and metadata endpoints.
+
+Uploaded datasets live in an in-memory store keyed by a generated id. Import keeps
+every column (identifiers and qualitative columns included) and infers each
+column's data type. Roles like X/Y and exclusions are *not* set here - those are
+modelling choices captured in a preprocessing spec.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, File, HTTPException, Query, UploadFile, status
+
+from scorepilot.api.deps import DatasetStoreDep
+from scorepilot.core import IdentifierRole
+from scorepilot.dataset_store import Dataset, load_table
+from scorepilot.schemas import ColumnMetaModel, ColumnUpdate, DatasetDetail
+
+router = APIRouter(prefix="/datasets", tags=["datasets"])
+
+
+def to_detail(dataset: Dataset) -> DatasetDetail:
+    """Build the API detail model for a dataset."""
+    return DatasetDetail(
+        dataset_id=dataset.id,
+        name=dataset.name,
+        source=dataset.source,
+        sheet=dataset.sheet,
+        sheets=dataset.sheets,
+        n_rows=dataset.n_rows,
+        n_columns=dataset.n_columns,
+        primary_id=dataset.primary_id,
+        columns=[
+            ColumnMetaModel(
+                name=c.name,
+                column_type=c.column_type,
+                identifier_role=c.identifier_role,
+            )
+            for c in dataset.columns
+        ],
+    )
+
+
+@router.post("", response_model=DatasetDetail, status_code=status.HTTP_201_CREATED)
+async def upload_dataset(
+    store: DatasetStoreDep,
+    file: Annotated[UploadFile, File(description="A CSV or Excel file")],
+    sheet: Annotated[str | None, Query(description="Excel sheet name")] = None,
+) -> DatasetDetail:
+    """Import a CSV or Excel file as a dataset."""
+    raw = await file.read()
+    filename = file.filename or "dataset.csv"
+    try:
+        frame, source, sheets, used = load_table(raw, filename, sheet)
+    except Exception as exc:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Could not parse file: {exc}",
+        ) from exc
+
+    if frame.shape[1] == 0:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="No columns found in the uploaded file.",
+        )
+
+    dataset = store.add(filename, frame, source=source, sheets=sheets, sheet=used)
+    return to_detail(dataset)
+
+
+@router.get("", response_model=list[DatasetDetail])
+def list_datasets(store: DatasetStoreDep) -> list[DatasetDetail]:
+    """List all imported datasets."""
+    return [to_detail(d) for d in store.list()]
+
+
+@router.get("/{dataset_id}", response_model=DatasetDetail)
+def get_dataset(dataset_id: str, store: DatasetStoreDep) -> DatasetDetail:
+    """Return one dataset's metadata."""
+    return to_detail(_require(store, dataset_id))
+
+
+@router.patch("/{dataset_id}/columns/{column}", response_model=DatasetDetail)
+def update_column(
+    dataset_id: str, column: str, update: ColumnUpdate, store: DatasetStoreDep
+) -> DatasetDetail:
+    """Update a column's data type or identifier role."""
+    dataset = _require(store, dataset_id)
+    meta = dataset.column(column)
+    if meta is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Unknown column: {column}",
+        )
+
+    if update.column_type is not None:
+        meta.column_type = update.column_type
+    if update.identifier_role is not None:
+        if update.identifier_role is IdentifierRole.PRIMARY:
+            for other in dataset.columns:
+                if other.identifier_role is IdentifierRole.PRIMARY:
+                    other.identifier_role = IdentifierRole.NONE
+        meta.identifier_role = update.identifier_role
+
+    return to_detail(dataset)
+
+
+def _require(store: DatasetStoreDep, dataset_id: str) -> Dataset:
+    dataset = store.get(dataset_id)
+    if dataset is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Unknown dataset_id: {dataset_id}",
+        )
+    return dataset

scorepilot/api/deps.py

@@ -0,0 +1,34 @@
+"""FastAPI dependencies wiring requests to the dataset store and repository."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Annotated
+
+from fastapi import Depends, Request
+from sqlalchemy.orm import Session
+
+from scorepilot.dataset_store import DatasetStore
+from scorepilot.db import SqlModelRepository, session_scope
+
+
+def get_dataset_store(request: Request) -> DatasetStore:
+    """Return the process-wide in-memory dataset store."""
+    return request.app.state.dataset_store
+
+
+def get_session(request: Request) -> Iterator[Session]:
+    """Yield a transactional session, committed when the request succeeds."""
+    with session_scope(request.app.state.session_factory) as session:
+        yield session
+
+
+def get_repository(
+    session: Annotated[Session, Depends(get_session)],
+) -> SqlModelRepository:
+    """Return a repository bound to the request's session."""
+    return SqlModelRepository(session)
+
+
+DatasetStoreDep = Annotated[DatasetStore, Depends(get_dataset_store)]
+RepositoryDep = Annotated[SqlModelRepository, Depends(get_repository)]

scorepilot/api/exploration.py

@@ -0,0 +1,195 @@
+"""Exploration endpoints: data-quality, grid windows, and the variable inspector.
+
+These are read-only views over an immutable dataset. The transform shown in the
+inspector is a non-destructive *preview*; it never changes the stored data.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import pandas as pd
+from fastapi import APIRouter, HTTPException, Query, status
+
+from scorepilot.api.deps import DatasetStoreDep
+from scorepilot.core import (
+    PreprocessingSpec,
+    TransformKind,
+    apply_spec,
+    apply_transform,
+    histogram,
+    quality_report,
+    sequence,
+    suggest_transform,
+    variable_summary,
+)
+from scorepilot.core._pandas import column as get_column
+from scorepilot.dataset_store import Dataset
+from scorepilot.schemas import (
+    ColumnQualityModel,
+    DuplicateIdentifierModel,
+    GridWindow,
+    ObservationQualityModel,
+    QualityReportModel,
+    VariableInspector,
+)
+
+router = APIRouter(prefix="/datasets", tags=["exploration"])
+
+
+def _require(store: DatasetStoreDep, dataset_id: str) -> Dataset:
+    dataset = store.get(dataset_id)
+    if dataset is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Unknown dataset_id: {dataset_id}",
+        )
+    return dataset
+
+
+def _format_cell(value: object) -> str | None:
+    if pd.isna(value):  # type: ignore[arg-type]
+        return None
+    if isinstance(value, float):
+        return f"{value:.6g}"
+    return str(value)
+
+
+@router.get("/{dataset_id}/quality", response_model=QualityReportModel)
+def get_quality(dataset_id: str, store: DatasetStoreDep) -> QualityReportModel:
+    """Return the data-quality report for a dataset."""
+    dataset = _require(store, dataset_id)
+    report = quality_report(dataset.raw, types=dataset.types(), primary_id=dataset.primary_id)
+    return QualityReportModel(
+        n_rows=report.n_rows,
+        n_columns=report.n_columns,
+        n_missing_cells=report.n_missing_cells,
+        pct_missing=report.pct_missing,
+        primary_id_unique=report.primary_id_unique,
+        duplicate_primary_ids=[
+            DuplicateIdentifierModel(value=d.value, rows=d.rows)
+            for d in report.duplicate_primary_ids
+        ],
+        columns=[
+            ColumnQualityModel(
+                name=c.name,
+                n_missing=c.n_missing,
+                pct_missing=c.pct_missing,
+                n_invalid=c.n_invalid,
+                invalid_rows=c.invalid_rows,
+                exceeds_tolerance=c.exceeds_tolerance,
+            )
+            for c in report.columns
+        ],
+        observations_exceeding=[
+            ObservationQualityModel(
+                index=o.index,
+                identifier=o.identifier,
+                n_missing=o.n_missing,
+                pct_missing=o.pct_missing,
+            )
+            for o in report.observations_exceeding
+        ],
+    )
+
+
+@router.get("/{dataset_id}/grid", response_model=GridWindow)
+def get_grid(
+    dataset_id: str,
+    store: DatasetStoreDep,
+    row_offset: Annotated[int, Query(ge=0)] = 0,
+    row_limit: Annotated[int, Query(ge=1, le=1000)] = 100,
+    col_offset: Annotated[int, Query(ge=0)] = 0,
+    col_limit: Annotated[int, Query(ge=1, le=200)] = 50,
+    form: Annotated[str, Query(pattern="^(raw|scaled)$")] = "raw",
+) -> GridWindow:
+    """Return a windowed block of cells for the grid, raw or autoscaled."""
+    dataset = _require(store, dataset_id)
+    display = _display_frame(dataset, form)
+
+    all_columns = [str(c) for c in display.columns]
+    column_names = all_columns[col_offset : col_offset + col_limit]
+    window = display.iloc[row_offset : row_offset + row_limit][column_names]
+
+    row_identifiers = _row_identifiers(dataset, row_offset, len(window))
+    cells = [
+        [_format_cell(window.iat[r, c]) for c in range(window.shape[1])]
+        for r in range(window.shape[0])
+    ]
+    return GridWindow(
+        row_offset=row_offset,
+        column_names=column_names,
+        row_identifiers=row_identifiers,
+        cells=cells,
+    )
+
+
+def _display_frame(dataset: Dataset, form: str) -> pd.DataFrame:
+    if form != "scaled":
+        return dataset.raw
+    quantitative = dataset.quantitative_columns()
+    if not quantitative:
+        return dataset.raw
+    # Scale using full-column statistics so the view is stable across scrolling.
+    scaled = apply_spec(dataset.raw, PreprocessingSpec(x_columns=tuple(quantitative))).X
+    display = dataset.raw.copy()
+    for name in quantitative:
+        display[name] = scaled[name]
+    return display
+
+
+def _row_identifiers(dataset: Dataset, row_offset: int, count: int) -> list[str | None]:
+    if dataset.primary_id is not None:
+        ids = get_column(dataset.raw, dataset.primary_id).iloc[row_offset : row_offset + count]
+        return [_format_cell(v) for v in ids]
+    return [str(row_offset + i) for i in range(count)]
+
+
+@router.get("/{dataset_id}/variables/{column}", response_model=VariableInspector)
+def inspect_variable(
+    dataset_id: str,
+    column: str,
+    store: DatasetStoreDep,
+    transform: Annotated[TransformKind, Query()] = TransformKind.NONE,
+    c1: Annotated[float, Query()] = 0.0,
+    c2: Annotated[float, Query()] = 1.0,
+) -> VariableInspector:
+    """Return summary, histogram, and sequence for a variable (with optional preview)."""
+    dataset = _require(store, dataset_id)
+    meta = dataset.column(column)
+    if meta is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Unknown column: {column}",
+        )
+
+    series = get_column(dataset.raw, column)
+    suggested = suggest_transform(variable_summary(series, column_type=meta.column_type))
+
+    display = series
+    if transform is not TransformKind.NONE:
+        display = apply_transform(series, transform, c1=c1, c2=c2)
+    summary = variable_summary(display, column_type=meta.column_type)
+    counts, edges = histogram(display)
+
+    return VariableInspector(
+        name=meta.name,
+        column_type=meta.column_type,
+        n=summary.n,
+        n_missing=summary.n_missing,
+        pct_missing=summary.pct_missing,
+        n_unique=summary.n_unique,
+        mean=summary.mean,
+        std=summary.std,
+        minimum=summary.minimum,
+        maximum=summary.maximum,
+        median=summary.median,
+        q25=summary.q25,
+        q75=summary.q75,
+        skewness=summary.skewness,
+        min_max_ratio=summary.min_max_ratio,
+        suggested_transform=suggested,
+        histogram_counts=counts,
+        histogram_edges=edges,
+        sequence=sequence(display),
+    )

scorepilot/api/models.py

@@ -0,0 +1,243 @@
+"""Model-fitting endpoints, the Hangar (list), and the Logbook (detail)."""
+
+from __future__ import annotations
+
+import io
+
+import numpy as np
+from fastapi import APIRouter, HTTPException, status
+
+from scorepilot.api.deps import DatasetStoreDep, RepositoryDep
+from scorepilot.core import (
+    ModelDiagnostics,
+    PreprocessingSpec,
+    apply_spec,
+    fit_model,
+)
+from scorepilot.dataset_store import Dataset
+from scorepilot.db import Model
+from scorepilot.schemas import (
+    FitModelRequest,
+    FitPCARequest,
+    LoadingsPayload,
+    ModelDetail,
+    ModelDiagnosticsModel,
+    ModelSummary,
+    PCAFitResponse,
+    ScoresPayload,
+)
+
+router = APIRouter(prefix="/models", tags=["models"])
+
+
+# --- helpers ----------------------------------------------------------------
+
+
+def _pack_params(diag: ModelDiagnostics) -> bytes:
+    buffer = io.BytesIO()
+    np.savez_compressed(
+        buffer,
+        x_loadings=diag.x_loadings.to_numpy(),
+        explained_variance=diag.explained_variance,
+        r2_cumulative=np.asarray(diag.r2_cumulative),
+    )
+    return buffer.getvalue()
+
+
+def _default_spec(dataset: Dataset, spec: PreprocessingSpec | None) -> PreprocessingSpec:
+    if spec is not None:
+        return spec
+    return PreprocessingSpec(x_columns=tuple(dataset.quantitative_columns()))
+
+
+def _run_fit(
+    dataset: Dataset, spec: PreprocessingSpec, kind: str, n_components: int, conf_level: float
+) -> ModelDiagnostics:
+    applied = apply_spec(dataset.raw, spec)
+    return fit_model(applied.X, applied.Y, kind, n_components, conf_level=conf_level)
+
+
+def _summary(model: Model) -> ModelSummary:
+    return ModelSummary(
+        id=model.id,
+        kind=model.kind,
+        name=model.name,
+        n_components=model.n_components,
+        parent_id=model.parent_id,
+        dataset_id=model.dataset_id,
+        created_at=model.created_at,
+    )
+
+
+def _diagnostics_model(diag: ModelDiagnostics) -> ModelDiagnosticsModel:
+    y_loadings = None
+    if diag.y_loadings is not None:
+        y_loadings = LoadingsPayload(
+            component_names=diag.component_names,
+            variable_names=diag.y_variable_names,
+            data=diag.y_loadings.to_numpy().tolist(),
+        )
+    return ModelDiagnosticsModel(
+        kind=diag.kind,
+        n_components=diag.n_components,
+        conf_level=diag.conf_level,
+        component_names=diag.component_names,
+        explained_variance=diag.explained_variance.tolist(),
+        r2_per_component=diag.r2_per_component,
+        r2_cumulative=diag.r2_cumulative,
+        scores=ScoresPayload(
+            component_names=diag.component_names,
+            observation_names=diag.observation_names,
+            data=diag.scores.to_numpy().tolist(),
+        ),
+        x_loadings=LoadingsPayload(
+            component_names=diag.component_names,
+            variable_names=diag.x_variable_names,
+            data=diag.x_loadings.to_numpy().tolist(),
+        ),
+        y_loadings=y_loadings,
+        hotellings_t2=diag.hotellings_t2.to_numpy().tolist(),
+        spe=diag.spe.to_numpy().tolist(),
+        t2_limit=diag.t2_limit,
+        spe_limit=diag.spe_limit,
+        ellipse_x=diag.ellipse_x,
+        ellipse_y=diag.ellipse_y,
+        vip=diag.vip,
+    )
+
+
+def _require_dataset(store: DatasetStoreDep, dataset_id: str) -> Dataset:
+    dataset = store.get(dataset_id)
+    if dataset is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Unknown dataset_id: {dataset_id}",
+        )
+    return dataset
+
+
+# --- endpoints --------------------------------------------------------------
+
+
+@router.post("", response_model=ModelDetail, status_code=status.HTTP_201_CREATED)
+def fit_model_endpoint(
+    request: FitModelRequest, store: DatasetStoreDep, repository: RepositoryDep
+) -> ModelDetail:
+    """Fit a PCA/PLS model variant from a dataset and a spec, and persist it."""
+    dataset = _require_dataset(store, request.dataset_id)
+    if request.parent_id is not None and repository.get(request.parent_id) is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Unknown parent_id: {request.parent_id}",
+        )
+
+    spec = _default_spec(dataset, request.spec.to_core() if request.spec else None)
+    try:
+        diag = _run_fit(dataset, spec, request.kind, request.n_components, request.conf_level)
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)
+        ) from exc
+
+    model = repository.add(
+        Model(
+            kind=request.kind,
+            name=request.name,
+            dataset_id=dataset.id,
+            n_components=diag.n_components,
+            preprocessing=spec.to_dict(),
+            excluded_samples=list(spec.excluded_rows),
+            params=_pack_params(diag),
+            parent_id=request.parent_id,
+        )
+    )
+    return ModelDetail(
+        summary=_summary(model),
+        preprocessing=spec.to_dict(),
+        excluded_samples=list(spec.excluded_rows),
+        lineage=[_summary(m) for m in repository.lineage(model.id)],
+        diagnostics=_diagnostics_model(diag),
+    )
+
+
+@router.get("", response_model=list[ModelSummary])
+def list_models(repository: RepositoryDep) -> list[ModelSummary]:
+    """List all model variants (the Hangar)."""
+    return [_summary(m) for m in repository.list()]
+
+
+@router.get("/{model_id}", response_model=ModelDetail)
+def get_model(model_id: int, store: DatasetStoreDep, repository: RepositoryDep) -> ModelDetail:
+    """Return a model's Logbook: metadata, recipe, lineage, and diagnostics.
+
+    Diagnostics are recomputed from the source dataset and stored spec. If the
+    dataset is no longer in memory, the entry is returned without diagnostics.
+    """
+    model = repository.get(model_id)
+    if model is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=f"Unknown model id: {model_id}"
+        )
+
+    diagnostics = None
+    dataset = store.get(model.dataset_id) if model.dataset_id else None
+    if dataset is not None:
+        spec = PreprocessingSpec.from_dict(model.preprocessing)
+        try:
+            diag = _run_fit(dataset, spec, model.kind, model.n_components, 0.95)
+            diagnostics = _diagnostics_model(diag)
+        except ValueError:
+            diagnostics = None
+
+    return ModelDetail(
+        summary=_summary(model),
+        preprocessing=dict(model.preprocessing),
+        excluded_samples=list(model.excluded_samples),
+        lineage=[_summary(m) for m in repository.lineage(model.id)],
+        diagnostics=diagnostics,
+    )
+
+
+@router.post("/pca", response_model=PCAFitResponse, status_code=status.HTTP_201_CREATED)
+def fit_pca_model(
+    request: FitPCARequest, store: DatasetStoreDep, repository: RepositoryDep
+) -> PCAFitResponse:
+    """Fit a PCA model (legacy endpoint used by the scores playground)."""
+    dataset = _require_dataset(store, request.dataset_id)
+    spec = _default_spec(dataset, request.spec.to_core() if request.spec else None)
+    try:
+        diag = _run_fit(dataset, spec, "PCA", request.n_components, request.conf_level)
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc)
+        ) from exc
+
+    model = repository.add(
+        Model(
+            kind="PCA",
+            name=request.name,
+            dataset_id=dataset.id,
+            n_components=diag.n_components,
+            preprocessing=spec.to_dict(),
+            excluded_samples=list(spec.excluded_rows),
+            params=_pack_params(diag),
+        )
+    )
+    return PCAFitResponse(
+        model_id=model.id,
+        kind="PCA",
+        n_components=diag.n_components,
+        conf_level=diag.conf_level,
+        component_names=diag.component_names,
+        explained_variance=diag.explained_variance.tolist(),
+        r2_cumulative=diag.r2_cumulative,
+        scores=ScoresPayload(
+            component_names=diag.component_names,
+            observation_names=diag.observation_names,
+            data=diag.scores.to_numpy().tolist(),
+        ),
+        hotellings_t2=diag.hotellings_t2.to_numpy().tolist(),
+        spe=diag.spe.to_numpy().tolist(),
+        t2_limit=diag.t2_limit,
+        spe_limit=diag.spe_limit,
+    )

scorepilot/app.py

@@ -0,0 +1,80 @@
+"""FastAPI application factory.
+
+The packaged app is a single process: the API is served under ``/api`` and the
+built static frontend at ``/``. A small ``StaticFiles`` subclass falls back to
+``index.html`` so client-side (SPA) routes resolve.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from starlette.exceptions import HTTPException as StarletteHTTPException
+from starlette.types import Scope
+
+from scorepilot.api import datasets, exploration, models
+from scorepilot.config import Settings, get_settings
+from scorepilot.dataset_store import DatasetStore
+from scorepilot.db import Base, make_engine, make_session_factory
+
+WEB_DIR = Path(__file__).resolve().parent / "web"
+
+
+class SpaStaticFiles(StaticFiles):
+    """Static files that fall back to ``index.html`` on a 404.
+
+    This lets the single-page app own client-side routing while non-existent
+    asset paths still return a real 404.
+    """
+
+    async def get_response(self, path: str, scope: Scope):  # noqa: ANN201
+        try:
+            return await super().get_response(path, scope)
+        except StarletteHTTPException as exc:
+            if exc.status_code == 404:
+                return await super().get_response("index.html", scope)
+            raise
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    """Build and configure the FastAPI application.
+
+    Parameters
+    ----------
+    settings
+        Optional settings override (used by tests). Defaults to the cached
+        environment-derived settings.
+    """
+    settings = settings or get_settings()
+
+    engine = make_engine(settings.database_url)
+    # Convenience for local/dev use: ensure tables exist. Production schema
+    # changes are managed with Alembic.
+    Base.metadata.create_all(engine)
+
+    app = FastAPI(
+        title="ScorePilot",
+        version="0.1.0",
+        docs_url="/api/docs",
+        redoc_url=None,
+        openapi_url="/api/openapi.json",
+    )
+    app.state.settings = settings
+    app.state.engine = engine
+    app.state.session_factory = make_session_factory(engine)
+    app.state.dataset_store = DatasetStore()
+
+    @app.get("/api/health", tags=["meta"])
+    def health() -> dict[str, str]:
+        return {"status": "ok"}
+
+    app.include_router(datasets.router, prefix="/api")
+    app.include_router(exploration.router, prefix="/api")
+    app.include_router(models.router, prefix="/api")
+
+    # Mount the static SPA last so it never shadows the API routes above.
+    app.mount("/", SpaStaticFiles(directory=WEB_DIR, html=True), name="web")
+
+    return app