deriva-dev 0.8.3__tar.gz → 0.8.4__tar.gz

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: deriva-dev
-Version: 0.8.3
+Version: 0.8.4
 Summary: Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks.
 Author: S. T. Grond
 Author-email: S. T. Grond <steven@deriva.dev>
 License-Expression: AGPL-3.0-or-later
 License-File: LICENSE
-Requires-Dist: deriva-studio==0.8.3
+Requires-Dist: deriva-studio==0.8.4
 Requires-Python: >=3.14
 Project-URL: Homepage, https://deriva.dev
 Project-URL: Documentation, https://docs.deriva.dev

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "deriva-dev"
-version = "0.8.3"
+version = "0.8.4"
 description = "Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks."
 readme = "README.md"
 requires-python = ">=3.14"
 license = "AGPL-3.0-or-later"
 license-files = ["LICENSE"]
-dependencies = ["deriva-studio==0.8.3"]
+dependencies = ["deriva-studio==0.8.4"]
 
 [[project.authors]]
 name = "S. T. Grond"

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "deriva-dev"
-version = "0.8.3"
+version = "0.8.4"
 description = "Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -8,7 +8,7 @@ license = "AGPL-3.0-or-later"
 # A copy of the repository LICENSE (a package cannot point outside its folder); tests/test_packaging.py keeps it identical
 license-files = ["LICENSE"]
 authors = [{ name = "S. T. Grond", email = "steven@deriva.dev" }]
-dependencies = ["deriva-studio==0.8.3"]
+dependencies = ["deriva-studio==0.8.4"]
 
 [project.urls]
 Homepage = "https://deriva.dev"

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/src/deriva/bench/benchmarking.py

@@ -1347,7 +1347,7 @@ class BenchmarkOrchestrator:
             return response
 
         # Attach cache keys list to function for retrieval after run
-        query_fn.used_cache_keys = used_cache_keys  # type: ignore[attr-defined]
+        query_fn.used_cache_keys = used_cache_keys  # type: ignore[attr-defined]  # ty: ignore[unresolved-attribute]
         return query_fn
 
     def _record_llm_call(self, llm: Any, step: str | None, prompt: str, schema: dict | None, system_prompt: str | None, response: Any, error: str | None) -> None:
@@ -1884,27 +1884,66 @@ def export_process_logs(benchmarks_dir: str | Path, session_ids: list[str], out_
     return paths
 
 
-def list_benchmark_sessions(engine: Any, limit: int = 10) -> list[dict[str, Any]]:
+def list_benchmark_sessions(
+    engine: Any,
+    limit: int = 10,
+    offset: int = 0,
+    search: str | None = None,
+    repository: str | None = None,
+    model: str | None = None,
+    status: str | None = None,
+) -> list[dict[str, Any]]:
     """
-    List recent benchmark sessions.
+    List benchmark sessions, newest first, with the repositories, models and runs they hold.
 
     Args:
         engine: DuckDB connection
         limit: Maximum number of sessions to return
+        offset: Sessions to skip (a page at a time)
+        search: Text in the session id or description (any case)
+        repository: Only sessions with a run of this repository
+        model: Only sessions with a run of this model config
+        status: Only sessions with this status (completed, failed, running)
 
     Returns:
         List of session dictionaries
     """
+    where: list[str] = []
+    params: list[Any] = []
+    if search:
+        where.append("(s.session_id ILIKE ? OR coalesce(s.description, '') ILIKE ?)")
+        params += [f"%{search}%", f"%{search}%"]
+    if repository:
+        where.append("EXISTS (SELECT 1 FROM benchmark_runs r WHERE r.session_id = s.session_id AND r.repository = ?)")
+        params.append(repository)
+    if model:
+        where.append("EXISTS (SELECT 1 FROM benchmark_runs r WHERE r.session_id = s.session_id AND r.model_name = ?)")
+        params.append(model)
+    if status:
+        where.append("s.status = ?")
+        params.append(status)
+    clause = f"WHERE {' AND '.join(where)}" if where else ""
     rows = engine.execute(
-        """
-        SELECT session_id, description, status, started_at, completed_at
-        FROM benchmark_sessions
-        ORDER BY started_at DESC
-        LIMIT ?
+        f"""
+        SELECT s.session_id, s.description, s.status, s.started_at, s.completed_at, s.config,
+               count(r.run_id), list(DISTINCT r.repository), list(DISTINCT r.model_name)
+        FROM benchmark_sessions s
+        LEFT JOIN benchmark_runs r ON r.session_id = s.session_id
+        {clause}
+        GROUP BY s.session_id, s.description, s.status, s.started_at, s.completed_at, s.config
+        ORDER BY s.started_at DESC, s.session_id DESC
+        LIMIT ? OFFSET ?
         """,
-        [limit],
+        [*params, limit, offset],
     ).fetchall()
 
+    def stages(config: Any) -> list[str]:
+        try:
+            found = json.loads(config) if isinstance(config, str) else (config or {})
+        except ValueError:
+            return []
+        return [str(stage) for stage in found.get("stages") or []] if isinstance(found, dict) else []
+
     return [
         {
             "session_id": row[0],
@@ -1912,11 +1951,59 @@ def list_benchmark_sessions(engine: Any, limit: int = 10) -> list[dict[str, Any]
             "status": row[2],
             "started_at": row[3],
             "completed_at": row[4],
+            "stages": stages(row[5]),
+            "runs": int(row[6]),
+            "repositories": sorted(x for x in row[7] or [] if x is not None),
+            "models": sorted(x for x in row[8] or [] if x is not None),
         }
         for row in rows
     ]
 
 
+def folder_bytes(folder: Path) -> int:
+    """Bytes the files under a folder take (0 when it does not exist)."""
+    return sum(f.stat().st_size for f in folder.rglob("*") if f.is_file()) if folder.is_dir() else 0
+
+
+def delete_benchmark_session(engine: Any, session_id: str, benchmarks_dir: str | Path) -> dict[str, Any]:
+    """Delete a session: its rows (session, runs, metrics) and its folder (event logs, snapshots, inputs).
+
+    Raises:
+        KeyError: no such session
+        ValueError: the session is still running (mark it failed first when its process stopped)
+    """
+    info = get_benchmark_session(engine, session_id)
+    if info is None:
+        raise KeyError(session_id)
+    if info["status"] == "running":
+        raise ValueError(f"Session {session_id} is running; mark it failed first when its process has stopped")
+    (runs,) = engine.execute("SELECT count(*) FROM benchmark_runs WHERE session_id = ?", [session_id]).fetchone()
+    for table in ("benchmark_metrics", "benchmark_runs", "benchmark_sessions"):
+        engine.execute(f"DELETE FROM {table} WHERE session_id = ?", [session_id])
+    folder = Path(benchmarks_dir) / session_id
+    size = folder_bytes(folder)
+    if folder.is_dir():
+        shutil.rmtree(folder)
+    return {"session_id": session_id, "runs": int(runs), "bytes": size}
+
+
+def prune_benchmark_sessions(engine: Any, benchmarks_dir: str | Path, keep: int | None = None, failed: bool = False) -> list[str]:
+    """Delete the failed sessions, or every session past the latest ``keep``; running sessions always stay.
+
+    Returns the deleted session ids, newest first.
+    """
+    rows = engine.execute("SELECT session_id, status FROM benchmark_sessions ORDER BY started_at DESC, session_id DESC").fetchall()
+    if failed:
+        victims = [sid for sid, status in rows if status == "failed"]
+    elif keep is not None:
+        victims = [sid for sid, status in rows[max(keep, 0) :] if status != "running"]
+    else:
+        return []
+    for session_id in victims:
+        delete_benchmark_session(engine, session_id, benchmarks_dir)
+    return victims
+
+
 def get_benchmark_session(engine: Any, session_id: str) -> dict[str, Any] | None:
     """
     Get details for a specific benchmark session.

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/src/deriva/bench/cli.py

@@ -7,7 +7,7 @@ Provides commands for multi-model benchmarking.
 from __future__ import annotations
 
 import json
-from collections.abc import Iterator
+from collections.abc import Generator
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated
@@ -26,7 +26,7 @@ app = typer.Typer(name="benchmark", help="Multi-model benchmarking")
 
 
 @contextmanager
-def _benchmarks() -> Iterator[BenchmarkSession]:
+def _benchmarks() -> Generator[BenchmarkSession]:
     """A connected PipelineSession with the benchmark operations."""
     with PipelineSession() as session:
         yield BenchmarkSession(session)
@@ -328,6 +328,54 @@ def benchmark_list(
         typer.echo("")
 
 
+def _megabytes(size: int) -> str:
+    return f"{size / 1024 / 1024:.1f} MB"
+
+
+@app.command("delete")
+def benchmark_delete(
+    session_ids: Annotated[list[str], typer.Argument(help="Benchmark sessions to delete")],
+) -> None:
+    """Delete benchmark sessions with their runs, metrics and folders (a running session is refused)."""
+    failed = False
+    with _benchmarks() as session:
+        for session_id in session_ids:
+            try:
+                result = session.delete_benchmark(session_id)
+            except KeyError:
+                typer.echo(f"Error: Benchmark session not found: {session_id}", err=True)
+                failed = True
+                continue
+            except ValueError as e:
+                typer.echo(f"Error: {e}", err=True)
+                failed = True
+                continue
+            typer.echo(f"Deleted {session_id} ({result['runs']} runs, {_megabytes(result['bytes'])})")
+    if failed:
+        raise typer.Exit(1)
+
+
+@app.command("prune")
+def benchmark_prune(
+    keep: Annotated[int | None, typer.Option("--keep", min=0, help="Keep the latest N sessions, delete the older ones")] = None,
+    failed: Annotated[bool, typer.Option("--failed", help="Delete the failed sessions")] = False,
+    yes: Annotated[bool, typer.Option("-y", "--yes", help="Do not ask first")] = False,
+) -> None:
+    """Delete the failed sessions, or every session past the latest N, with their runs, metrics and folders; running sessions stay."""
+    if keep is None and not failed:
+        typer.echo("Error: say which sessions to delete: --keep N or --failed", err=True)
+        raise typer.Exit(1)
+    which = "the failed sessions" if failed else f"every session past the latest {keep}"
+    if not yes and not typer.confirm(f"Delete {which}, with their runs, metrics and folders?"):
+        raise typer.Exit(1)
+    with _benchmarks() as session:
+        deleted = session.prune_benchmarks(keep=None if failed else keep, failed=failed)
+    if not deleted:
+        typer.echo("No sessions to delete.")
+        return
+    typer.echo(f"Deleted {len(deleted)} session{'' if len(deleted) == 1 else 's'}: {', '.join(deleted)}")
+
+
 @app.command("conventions")
 def benchmark_conventions(
     session_ids: Annotated[list[str], typer.Argument(help="Benchmark sessions whose run snapshots are checked")],

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/src/deriva/bench/config_deviation.py

@@ -198,7 +198,7 @@ class ConfigDeviationAnalyzer:
             return {"config_id": config_id, "error": "No objects found for config"}
 
         # Build per-object breakdown
-        object_details = []
+        object_details: list[dict[str, Any]] = []
         objects_by_run = {run_id: nodes_by_run.get(run_id, set()) | elements_by_run.get(run_id, set()) for run_id in all_runs}
 
         for obj_id in config_objects:

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/src/deriva/bench/router.py

@@ -53,8 +53,41 @@ def _view(read: Any) -> Any:
 
 
 @router.get("")
-def list_sessions(limit: int = Query(20, ge=1, le=500), session: Any = Depends(get_benchmarks)) -> list[dict[str, Any]]:
-    return session.list_benchmarks(limit)
+def list_sessions(
+    limit: int = Query(20, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+    search: str | None = None,
+    repository: str | None = None,
+    model: str | None = None,
+    status: str | None = None,
+    session: Any = Depends(get_benchmarks),
+) -> list[dict[str, Any]]:
+    """Sessions newest first, a page at a time: by text (id or description), repository, model and status."""
+    return session.list_benchmarks(limit, offset=offset, search=search, repository=repository, model=model, status=status)
+
+
+def _when_idle(request: Request, action: Any) -> Any:
+    """Session changes wait for a run to end: a benchmark in progress writes its session."""
+    try:
+        return request.app.state.runs.while_idle(action)
+    except RunBusy as exc:
+        raise HTTPException(status_code=409, detail="A run is in progress; try again when it has finished") from exc
+
+
+@router.post("/{session_id}/mark-failed")
+def mark_session_failed(session_id: str, request: Request, session: Any = Depends(get_benchmarks)) -> dict[str, Any]:
+    """End a session whose process stopped without finishing (still running) as failed; its files stay."""
+    runs = _view(lambda: _when_idle(request, lambda: session.mark_benchmark_failed(session_id)))
+    return {"session_id": session_id, "runs_marked": runs}
+
+
+@router.delete("/{session_id}")
+def delete_session(session_id: str, request: Request, session: Any = Depends(get_benchmarks)) -> dict[str, Any]:
+    """Delete a finished or failed session with its runs, metrics and folder."""
+    try:
+        return _view(lambda: _when_idle(request, lambda: session.delete_benchmark(session_id)))
+    except KeyError as exc:
+        raise HTTPException(status_code=404, detail=f"Benchmark session not found: {session_id}") from exc
 
 
 @router.get("/models")

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/src/deriva/bench/session.py

@@ -297,16 +297,39 @@ class BenchmarkSession:
         order = _pipeline_order(engine)
         return {repo: [s.to_dict() for s in sorted(steps, key=lambda s: order(s.step))] for repo, steps in stability.items()}
 
-    def list_benchmarks(self, limit: int = 10) -> list[dict]:
+    def list_benchmarks(
+        self,
+        limit: int = 10,
+        offset: int = 0,
+        search: str | None = None,
+        repository: str | None = None,
+        model: str | None = None,
+        status: str | None = None,
+        benchmarks_dir: str | Path | None = None,
+    ) -> list[dict]:
         """
-        List recent benchmark sessions.
+        List benchmark sessions, newest first, each with the bytes its folder takes.
 
         Args:
             limit: Maximum number of sessions to return
+            offset: Sessions to skip (a page at a time)
+            search, repository, model, status: filters (see ``benchmarking.list_benchmark_sessions``)
 
         Returns:
             List of session dictionaries
         """
         engine, _, _ = self.session.managers()
+        rows = benchmarking.list_benchmark_sessions(engine, limit, offset=offset, search=search, repository=repository, model=model, status=status)
+        folder = _folder(benchmarks_dir)
+        return [{**row, "bytes": benchmarking.folder_bytes(folder / str(row["session_id"]))} for row in rows]
 
-        return benchmarking.list_benchmark_sessions(engine, limit)
+    def delete_benchmark(self, session_id: str, benchmarks_dir: str | Path | None = None) -> dict[str, Any]:
+        """Delete a finished or failed session with its runs, metrics and folder."""
+        _check_session_id(session_id)
+        engine, _, _ = self.session.managers()
+        return benchmarking.delete_benchmark_session(engine, session_id, _folder(benchmarks_dir))
+
+    def prune_benchmarks(self, keep: int | None = None, failed: bool = False, benchmarks_dir: str | Path | None = None) -> list[str]:
+        """Delete the failed sessions, or every session past the latest ``keep``; running sessions stay."""
+        engine, _, _ = self.session.managers()
+        return benchmarking.prune_benchmark_sessions(engine, _folder(benchmarks_dir), keep=keep, failed=failed)

{deriva_dev-0.8.3 → deriva_dev-0.8.4}/src/deriva/bench/session_metrics.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 import ast
 import itertools
 import json
-from collections import Counter
+from collections import defaultdict
 from pathlib import Path
 from typing import Any
 
@@ -128,10 +128,11 @@ def _identity_report(views: list[dict[str, Any]], identity: str) -> dict[str, An
         return [{x for x in s if keep(x)} for s in sets]
 
     all_elements = set().union(*elements) if elements else set()
+    present = sorted({str(layer) for x in all_elements if (layer := layers.get(x)) is not None})
     report = {
         "elements": {"cons": _share(elements), "jbar": _jbar(elements), "count": len(all_elements)},
         "relationships": {"cons": _share(rels), "jbar": _jbar(rels), "count": len(set().union(*rels)) if rels else 0},
-        "layers": {layer: _share(part(elements, lambda x, layer=layer: layers.get(x) == layer)) for layer in sorted({layers.get(x) for x in all_elements} - {None})},
+        "layers": {layer: _share(part(elements, lambda x, layer=layer: layers.get(x) == layer)) for layer in present},
         "origin": {},
     }
     for name, wanted in (("llm", True), ("structural", False)):
@@ -174,13 +175,13 @@ def krippendorff_alpha(runs: list[dict[str, Any]]) -> float | None:
     for run in runs:
         for item, value in run.items():
             values.setdefault(item, []).append(repr(value))
-    coincidence: Counter = Counter()
+    coincidence: dict[tuple[str, str], float] = defaultdict(float)
     for found in values.values():
         if len(found) < 2:
             continue
         for a, b in itertools.permutations(range(len(found)), 2):
             coincidence[(found[a], found[b])] += 1 / (len(found) - 1)
-    totals: Counter = Counter()
+    totals: dict[str, float] = defaultdict(float)
     for (a, _), n in coincidence.items():
         totals[a] += n
     n = sum(totals.values())