deriva-dev 0.8.2__tar.gz → 0.8.4__tar.gz

{deriva_dev-0.8.2 → deriva_dev-0.8.4}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: deriva-dev
-Version: 0.8.2
+Version: 0.8.4
 Summary: Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks.
 Author: S. T. Grond
 Author-email: S. T. Grond <steven@deriva.dev>
 License-Expression: AGPL-3.0-or-later
 License-File: LICENSE
-Requires-Dist: deriva-studio==0.8.2
+Requires-Dist: deriva-studio==0.8.4
 Requires-Python: >=3.14
 Project-URL: Homepage, https://deriva.dev
 Project-URL: Documentation, https://docs.deriva.dev

{deriva_dev-0.8.2 → deriva_dev-0.8.4}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "deriva-dev"
-version = "0.8.2"
+version = "0.8.4"
 description = "Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks."
 readme = "README.md"
 requires-python = ">=3.14"
 license = "AGPL-3.0-or-later"
 license-files = ["LICENSE"]
-dependencies = ["deriva-studio==0.8.2"]
+dependencies = ["deriva-studio==0.8.4"]
 
 [[project.authors]]
 name = "S. T. Grond"

{deriva_dev-0.8.2 → deriva_dev-0.8.4}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "deriva-dev"
-version = "0.8.2"
+version = "0.8.4"
 description = "Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -8,7 +8,7 @@ license = "AGPL-3.0-or-later"
 # A copy of the repository LICENSE (a package cannot point outside its folder); tests/test_packaging.py keeps it identical
 license-files = ["LICENSE"]
 authors = [{ name = "S. T. Grond", email = "steven@deriva.dev" }]
-dependencies = ["deriva-studio==0.8.2"]
+dependencies = ["deriva-studio==0.8.4"]
 
 [project.urls]
 Homepage = "https://deriva.dev"

{deriva_dev-0.8.2 → deriva_dev-0.8.4}/src/deriva/bench/benchmarking.py

@@ -297,13 +297,14 @@ class OCELStepContext:
         self._created_relationships.append(relationship_id)
 
     def _elapsed_seconds(self) -> float:
-        return round(time.perf_counter() - self._start_time, 2)
+        """The step's duration to the microsecond: the case log derives the start from it, so steps must not overlap."""
+        return round(time.perf_counter() - self._start_time, 6)
 
     def complete(self, message: str = "") -> None:
         """Mark step as completed and log to OCEL."""
         self._completed = True
         if self.logger.verbose:
-            print(f"    {self.step}: {self._elapsed_seconds()}s, {self.items_created} created")
+            print(f"    {self.step}: {self._elapsed_seconds():.2f}s, {self.items_created} created")
         config_type = "extraction" if self.logger._current_phase == "extraction" else "derivation"
         self.logger.log_config_result(
             config_type=config_type,
@@ -319,7 +320,7 @@ class OCELStepContext:
         """Mark step as errored."""
         self._completed = True
         if self.logger.verbose:
-            print(f"    {self.step}: {self._elapsed_seconds()}s, ERROR {error[:120]}")
+            print(f"    {self.step}: {self._elapsed_seconds():.2f}s, ERROR {error[:120]}")
         config_type = "extraction" if self.logger._current_phase == "extraction" else "derivation"
         self.logger.log_config_result(
             config_type=config_type,
@@ -506,6 +507,8 @@ class BenchmarkOrchestrator:
         self._call_logs: dict[str, LlmCallLog] = {}
         # LLM answers per decision per step, recorded in the session metadata
         self._llm_samples: dict[str, int] = {}
+        # The code commit and dirty flag when the session started, recorded in the session metadata
+        self._code: dict[str, Any] = {"commit": None, "dirty": None}
         self._current_model: str | None = None
         self._current_repo: str | None = None
 
@@ -1344,7 +1347,7 @@ class BenchmarkOrchestrator:
             return response
 
         # Attach cache keys list to function for retrieval after run
-        query_fn.used_cache_keys = used_cache_keys  # type: ignore[attr-defined]
+        query_fn.used_cache_keys = used_cache_keys  # type: ignore[attr-defined]  # ty: ignore[unresolved-attribute]
         return query_fn
 
     def _record_llm_call(self, llm: Any, step: str | None, prompt: str, schema: dict | None, system_prompt: str | None, response: Any, error: str | None) -> None:
@@ -1457,6 +1460,7 @@ class BenchmarkOrchestrator:
                 model_name=model_display_name,
                 properties=leveled.properties,
                 views=leveled.views,
+                relationship_properties=leveled.relationship_properties,
             )
 
             # JSON snapshot for consistency analysis: the XML carries no properties,
@@ -1478,11 +1482,18 @@ class BenchmarkOrchestrator:
                         "name": e.name,
                         "source": (e.properties or {}).get("source"),
                         "level": "architect",
-                        # The name came from an LLM choice among several candidates (origin split)
-                        "name_choice": bool((e.properties or {}).get("name_choice")) or len((e.properties or {}).get("name_candidates") or []) > 1,
+                        # The name came from an LLM choice among several candidates (origin split): as the naming
+                        # step recorded it, else (steps that record none) several candidates mean a choice
+                        "name_choice": bool((e.properties or {})["name_choice"])
+                        if "name_choice" in (e.properties or {})
+                        else len((e.properties or {}).get("name_candidates") or []) > 1,
                         # The structural name (first candidate) next to the chosen one: what LLM naming adds
                         # (no naming call: the name is the structural one)
                         "structure_name": (e.properties or {}).get("structure_name") or next(iter((e.properties or {}).get("name_candidates") or []), None) or e.name,
+                        # A closed-set name's candidates in order (None for a name that is no choice)
+                        "name_candidates": list((e.properties or {}).get("name_candidates") or []) or None,
+                        # The abstraction levels the abstraction_levels step wrote (None: not tagged, enterprise)
+                        "levels": list((e.properties or {})["levels"]) if isinstance((e.properties or {}).get("levels"), list) else None,
                     }
                     for e in elements
                 ],
@@ -1493,6 +1504,9 @@ class BenchmarkOrchestrator:
                         "type": r.relationship_type,
                         "derived_from": (r.properties or {}).get("derived_from"),
                         "confidence": (r.properties or {}).get("confidence"),
+                        # Its levels (those both ends share); a cross-level link (no shared level) counts in the union only
+                        "levels": list((r.properties or {})["levels"]) if isinstance((r.properties or {}).get("levels"), list) else None,
+                        "cross_level": bool((r.properties or {}).get("cross_level")),
                     }
                     for r in relationships
                 ],
@@ -1569,12 +1583,20 @@ class BenchmarkOrchestrator:
                 self.session_start.isoformat(),
             ],
         )
+        self._capture_code()
         try:
             self._write_session_inputs()
         except Exception as exc:
             # A record that cannot be written does not stop the session (as a failed model export does not fail its run)
             logger.warning("Could not record the inputs of session %s: %s", self.session_id, exc)
 
+    def _capture_code(self) -> None:
+        """The commit the session starts on, with a dirty flag (the code may change while the session runs)."""
+        try:
+            self._code = export_bundle.code_commit()
+        except Exception as exc:  # A commit that cannot be read does not stop the session
+            logger.warning("Could not read the code commit of session %s: %s", self.session_id, exc)
+
     def _write_session_inputs(self) -> None:
         """Everything that changes results, as it is when the session starts, into the session folder.
 
@@ -1703,6 +1725,8 @@ class BenchmarkOrchestrator:
         summary = {
             "session_id": self.session_id,
             "config": self.config.to_dict(),
+            # The code the session ran: its commit and whether the working tree had changes at the start
+            "code": self._code,
             "started_at": self.session_start.isoformat() if self.session_start else None,
             "completed_at": datetime.now().isoformat(),
             "total_events": len(self.ocel_log.events),
@@ -1860,27 +1884,66 @@ def export_process_logs(benchmarks_dir: str | Path, session_ids: list[str], out_
     return paths
 
 
-def list_benchmark_sessions(engine: Any, limit: int = 10) -> list[dict[str, Any]]:
+def list_benchmark_sessions(
+    engine: Any,
+    limit: int = 10,
+    offset: int = 0,
+    search: str | None = None,
+    repository: str | None = None,
+    model: str | None = None,
+    status: str | None = None,
+) -> list[dict[str, Any]]:
     """
-    List recent benchmark sessions.
+    List benchmark sessions, newest first, with the repositories, models and runs they hold.
 
     Args:
         engine: DuckDB connection
         limit: Maximum number of sessions to return
+        offset: Sessions to skip (a page at a time)
+        search: Text in the session id or description (any case)
+        repository: Only sessions with a run of this repository
+        model: Only sessions with a run of this model config
+        status: Only sessions with this status (completed, failed, running)
 
     Returns:
         List of session dictionaries
     """
+    where: list[str] = []
+    params: list[Any] = []
+    if search:
+        where.append("(s.session_id ILIKE ? OR coalesce(s.description, '') ILIKE ?)")
+        params += [f"%{search}%", f"%{search}%"]
+    if repository:
+        where.append("EXISTS (SELECT 1 FROM benchmark_runs r WHERE r.session_id = s.session_id AND r.repository = ?)")
+        params.append(repository)
+    if model:
+        where.append("EXISTS (SELECT 1 FROM benchmark_runs r WHERE r.session_id = s.session_id AND r.model_name = ?)")
+        params.append(model)
+    if status:
+        where.append("s.status = ?")
+        params.append(status)
+    clause = f"WHERE {' AND '.join(where)}" if where else ""
     rows = engine.execute(
-        """
-        SELECT session_id, description, status, started_at, completed_at
-        FROM benchmark_sessions
-        ORDER BY started_at DESC
-        LIMIT ?
+        f"""
+        SELECT s.session_id, s.description, s.status, s.started_at, s.completed_at, s.config,
+               count(r.run_id), list(DISTINCT r.repository), list(DISTINCT r.model_name)
+        FROM benchmark_sessions s
+        LEFT JOIN benchmark_runs r ON r.session_id = s.session_id
+        {clause}
+        GROUP BY s.session_id, s.description, s.status, s.started_at, s.completed_at, s.config
+        ORDER BY s.started_at DESC, s.session_id DESC
+        LIMIT ? OFFSET ?
         """,
-        [limit],
+        [*params, limit, offset],
     ).fetchall()
 
+    def stages(config: Any) -> list[str]:
+        try:
+            found = json.loads(config) if isinstance(config, str) else (config or {})
+        except ValueError:
+            return []
+        return [str(stage) for stage in found.get("stages") or []] if isinstance(found, dict) else []
+
     return [
         {
             "session_id": row[0],
@@ -1888,11 +1951,59 @@ def list_benchmark_sessions(engine: Any, limit: int = 10) -> list[dict[str, Any]
             "status": row[2],
             "started_at": row[3],
             "completed_at": row[4],
+            "stages": stages(row[5]),
+            "runs": int(row[6]),
+            "repositories": sorted(x for x in row[7] or [] if x is not None),
+            "models": sorted(x for x in row[8] or [] if x is not None),
         }
         for row in rows
     ]
 
 
+def folder_bytes(folder: Path) -> int:
+    """Bytes the files under a folder take (0 when it does not exist)."""
+    return sum(f.stat().st_size for f in folder.rglob("*") if f.is_file()) if folder.is_dir() else 0
+
+
+def delete_benchmark_session(engine: Any, session_id: str, benchmarks_dir: str | Path) -> dict[str, Any]:
+    """Delete a session: its rows (session, runs, metrics) and its folder (event logs, snapshots, inputs).
+
+    Raises:
+        KeyError: no such session
+        ValueError: the session is still running (mark it failed first when its process stopped)
+    """
+    info = get_benchmark_session(engine, session_id)
+    if info is None:
+        raise KeyError(session_id)
+    if info["status"] == "running":
+        raise ValueError(f"Session {session_id} is running; mark it failed first when its process has stopped")
+    (runs,) = engine.execute("SELECT count(*) FROM benchmark_runs WHERE session_id = ?", [session_id]).fetchone()
+    for table in ("benchmark_metrics", "benchmark_runs", "benchmark_sessions"):
+        engine.execute(f"DELETE FROM {table} WHERE session_id = ?", [session_id])
+    folder = Path(benchmarks_dir) / session_id
+    size = folder_bytes(folder)
+    if folder.is_dir():
+        shutil.rmtree(folder)
+    return {"session_id": session_id, "runs": int(runs), "bytes": size}
+
+
+def prune_benchmark_sessions(engine: Any, benchmarks_dir: str | Path, keep: int | None = None, failed: bool = False) -> list[str]:
+    """Delete the failed sessions, or every session past the latest ``keep``; running sessions always stay.
+
+    Returns the deleted session ids, newest first.
+    """
+    rows = engine.execute("SELECT session_id, status FROM benchmark_sessions ORDER BY started_at DESC, session_id DESC").fetchall()
+    if failed:
+        victims = [sid for sid, status in rows if status == "failed"]
+    elif keep is not None:
+        victims = [sid for sid, status in rows[max(keep, 0) :] if status != "running"]
+    else:
+        return []
+    for session_id in victims:
+        delete_benchmark_session(engine, session_id, benchmarks_dir)
+    return victims
+
+
 def get_benchmark_session(engine: Any, session_id: str) -> dict[str, Any] | None:
     """
     Get details for a specific benchmark session.
@@ -2176,13 +2287,25 @@ def model_quality_for_session(
                 row: dict[str, Any] = {"repository": repo, "model": model, "run": int(run), **compute_model_quality(elements, relationships).to_dict()}
                 snapshot_path = path.with_suffix(".json")
                 try:
-                    row["solve"] = json.loads(snapshot_path.read_text(encoding="utf-8")).get("solve") if snapshot_path.is_file() else None
+                    snapshot = json.loads(snapshot_path.read_text(encoding="utf-8")) if snapshot_path.is_file() else {}
                 except OSError, ValueError:
-                    row["solve"] = None
+                    snapshot = {}
+                row["solve"] = snapshot.get("solve")
                 row["reference"] = None
                 if reference:
-                    # Lineage-free: name + type, the user's manual matches and derivable labels next to the reference
-                    derived = [{"id": e.identifier, "name": e.name, "type": e.element_type} for e in elements]
+                    # Name + type, the user's accepted names and structural refs (sources and lineage from the run's
+                    # snapshot: the XML carries none) and derivable labels: the gate's matching (golden_mapping)
+                    structure = {str(e.get("identifier")): e for e in snapshot.get("elements", []) if isinstance(e, dict)}
+                    derived = [
+                        {
+                            "id": e.identifier,
+                            "name": e.name,
+                            "type": e.element_type,
+                            "source": structure.get(e.identifier, {}).get("source"),
+                            "lineage": structure.get(e.identifier, {}).get("lineage") or [],
+                        }
+                        for e in elements
+                    ]
                     aliases, derivable = session_metrics.manual_inputs(reference_models[repo])
                     row["reference"] = session_metrics.golden_comparison(derived, reference, aliases=aliases, derivable=derivable)
                 rows.append(row)

{deriva_dev-0.8.2 → deriva_dev-0.8.4}/src/deriva/bench/cli.py

@@ -7,7 +7,7 @@ Provides commands for multi-model benchmarking.
 from __future__ import annotations
 
 import json
-from collections.abc import Iterator
+from collections.abc import Generator
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated
@@ -26,7 +26,7 @@ app = typer.Typer(name="benchmark", help="Multi-model benchmarking")
 
 
 @contextmanager
-def _benchmarks() -> Iterator[BenchmarkSession]:
+def _benchmarks() -> Generator[BenchmarkSession]:
     """A connected PipelineSession with the benchmark operations."""
     with PipelineSession() as session:
         yield BenchmarkSession(session)
@@ -238,7 +238,7 @@ def benchmark_run(
                 typer.echo(f"  ... and {len(result.errors) - 5} more")
 
         typer.echo("\nTo analyze results:")
-        typer.echo(f"  deriva benchmark analyze {result.session_id}")
+        typer.echo(f"  deriva-cli benchmark analyze {result.session_id}")
 
         if not result.success:
             raise typer.Exit(1)
@@ -328,6 +328,54 @@ def benchmark_list(
         typer.echo("")
 
 
+def _megabytes(size: int) -> str:
+    return f"{size / 1024 / 1024:.1f} MB"
+
+
+@app.command("delete")
+def benchmark_delete(
+    session_ids: Annotated[list[str], typer.Argument(help="Benchmark sessions to delete")],
+) -> None:
+    """Delete benchmark sessions with their runs, metrics and folders (a running session is refused)."""
+    failed = False
+    with _benchmarks() as session:
+        for session_id in session_ids:
+            try:
+                result = session.delete_benchmark(session_id)
+            except KeyError:
+                typer.echo(f"Error: Benchmark session not found: {session_id}", err=True)
+                failed = True
+                continue
+            except ValueError as e:
+                typer.echo(f"Error: {e}", err=True)
+                failed = True
+                continue
+            typer.echo(f"Deleted {session_id} ({result['runs']} runs, {_megabytes(result['bytes'])})")
+    if failed:
+        raise typer.Exit(1)
+
+
+@app.command("prune")
+def benchmark_prune(
+    keep: Annotated[int | None, typer.Option("--keep", min=0, help="Keep the latest N sessions, delete the older ones")] = None,
+    failed: Annotated[bool, typer.Option("--failed", help="Delete the failed sessions")] = False,
+    yes: Annotated[bool, typer.Option("-y", "--yes", help="Do not ask first")] = False,
+) -> None:
+    """Delete the failed sessions, or every session past the latest N, with their runs, metrics and folders; running sessions stay."""
+    if keep is None and not failed:
+        typer.echo("Error: say which sessions to delete: --keep N or --failed", err=True)
+        raise typer.Exit(1)
+    which = "the failed sessions" if failed else f"every session past the latest {keep}"
+    if not yes and not typer.confirm(f"Delete {which}, with their runs, metrics and folders?"):
+        raise typer.Exit(1)
+    with _benchmarks() as session:
+        deleted = session.prune_benchmarks(keep=None if failed else keep, failed=failed)
+    if not deleted:
+        typer.echo("No sessions to delete.")
+        return
+    typer.echo(f"Deleted {len(deleted)} session{'' if len(deleted) == 1 else 's'}: {', '.join(deleted)}")
+
+
 @app.command("conventions")
 def benchmark_conventions(
     session_ids: Annotated[list[str], typer.Argument(help="Benchmark sessions whose run snapshots are checked")],
@@ -383,10 +431,10 @@ def benchmark_gate(
     reference: Annotated[str | None, typer.Option("--reference", help="Golden model folder (default: workspace/reference)")] = None,
     benchmarks: Annotated[str | None, typer.Option("--benchmarks", help="Benchmark folder (default: workspace/benchmarks)")] = None,
 ) -> None:
-    """Recall, precision and consistency by id per repository, and the verdict of a candidate against a baseline.
+    """Element and relationship recall and precision and consistency by id per repository, and the verdict of a candidate.
 
-    Accepted when recall or precision rises on at least two thirds of the measured repositories, none drops by more
-    than 2 points on either, and element and relationship consistency by id stay at 95% or more everywhere.
+    Accepted when one of the four measures rises by 1 point or more on at least two thirds of the measured repositories,
+    none of them drops by more than 2 points anywhere, and element and relationship consistency by id stay at 95% or more.
     """
     from deriva.bench import gate
 
@@ -399,17 +447,27 @@ def benchmark_gate(
             typer.echo(f"Error: no benchmark session {', '.join(missing)} in {root}", err=True)
             raise typer.Exit(1)
     try:
-        measures = {name: gate.measure([root / s for s in side], goldens_dir, reference_dir, model=model) for name, side in sides.items() if side}
+        measures = {
+            level: {name: gate.measure([root / s for s in side], goldens_dir, reference_dir, model=model, level=level) for name, side in sides.items() if side}
+            for level in gate.GATE_LEVELS
+        }
     except ValueError as exc:
         typer.echo(f"Error: {exc}", err=True)
         raise typer.Exit(1) from exc
     lines = [f"# Gate: {', '.join(sides['baseline'])}" + (f" vs {', '.join(sides['candidate'])}" if sides["candidate"] else ""), ""]
-    lines += gate.table_markdown(measures["baseline"], f"Baseline ({', '.join(sides['baseline'])})")
-    payload: dict = {"sessions": sides, "baseline": measures["baseline"]}
-    if "candidate" in measures:
-        lines += gate.table_markdown(measures["candidate"], f"Candidate ({', '.join(sides['candidate'])})")
-        payload |= {"candidate": measures["candidate"], "verdict": gate.verdict(measures["baseline"], measures["candidate"])}
-        lines += gate.verdict_markdown(payload["verdict"])
+    payload: dict = {"sessions": sides, "levels": {}}
+    for level, by_side in measures.items():
+        lines += [f"## {level.capitalize()} level", ""]
+        lines += gate.table_markdown(by_side["baseline"], f"Baseline ({', '.join(sides['baseline'])})")
+        payload["levels"][level] = {"baseline": by_side["baseline"]}
+        if "candidate" in by_side:
+            lines += gate.table_markdown(by_side["candidate"], f"Candidate ({', '.join(sides['candidate'])})")
+            payload["levels"][level] |= {"candidate": by_side["candidate"], "verdict": gate.verdict(by_side["baseline"], by_side["candidate"])}
+            lines += gate.verdict_markdown(payload["levels"][level]["verdict"])
+    if sides["candidate"]:
+        result, voting = gate.overall_verdict({level: entry["verdict"] for level, entry in payload["levels"].items()})
+        payload["verdict"] = {"verdict": result, "levels": voting}
+        lines += [f"**Overall: {result}** (levels that voted: {', '.join(voting) or 'none'})", ""]
     for line in lines:
         typer.echo(line)
     name = "_".join(sides["baseline"]) + (f"_vs_{'_'.join(sides['candidate'])}" if sides["candidate"] else "")

{deriva_dev-0.8.2 → deriva_dev-0.8.4}/src/deriva/bench/config_deviation.py

@@ -198,7 +198,7 @@ class ConfigDeviationAnalyzer:
             return {"config_id": config_id, "error": "No objects found for config"}
 
         # Build per-object breakdown
-        object_details = []
+        object_details: list[dict[str, Any]] = []
         objects_by_run = {run_id: nodes_by_run.get(run_id, set()) | elements_by_run.get(run_id, set()) for run_id in all_runs}
 
         for obj_id in config_objects: