deriva-dev 0.8.2__tar.gz → 0.8.3__tar.gz

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: deriva-dev
-Version: 0.8.2
+Version: 0.8.3
 Summary: Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks.
 Author: S. T. Grond
 Author-email: S. T. Grond <steven@deriva.dev>
 License-Expression: AGPL-3.0-or-later
 License-File: LICENSE
-Requires-Dist: deriva-studio==0.8.2
+Requires-Dist: deriva-studio==0.8.3
 Requires-Python: >=3.14
 Project-URL: Homepage, https://deriva.dev
 Project-URL: Documentation, https://docs.deriva.dev

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "deriva-dev"
-version = "0.8.2"
+version = "0.8.3"
 description = "Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks."
 readme = "README.md"
 requires-python = ">=3.14"
 license = "AGPL-3.0-or-later"
 license-files = ["LICENSE"]
-dependencies = ["deriva-studio==0.8.2"]
+dependencies = ["deriva-studio==0.8.3"]
 
 [[project.authors]]
 name = "S. T. Grond"

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "deriva-dev"
-version = "0.8.2"
+version = "0.8.3"
 description = "Deriva with benchmarking: the studio, the CLI and multi-model consistency benchmarks."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -8,7 +8,7 @@ license = "AGPL-3.0-or-later"
 # A copy of the repository LICENSE (a package cannot point outside its folder); tests/test_packaging.py keeps it identical
 license-files = ["LICENSE"]
 authors = [{ name = "S. T. Grond", email = "steven@deriva.dev" }]
-dependencies = ["deriva-studio==0.8.2"]
+dependencies = ["deriva-studio==0.8.3"]
 
 [project.urls]
 Homepage = "https://deriva.dev"

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/src/deriva/bench/benchmarking.py

@@ -297,13 +297,14 @@ class OCELStepContext:
         self._created_relationships.append(relationship_id)
 
     def _elapsed_seconds(self) -> float:
-        return round(time.perf_counter() - self._start_time, 2)
+        """The step's duration to the microsecond: the case log derives the start from it, so steps must not overlap."""
+        return round(time.perf_counter() - self._start_time, 6)
 
     def complete(self, message: str = "") -> None:
         """Mark step as completed and log to OCEL."""
         self._completed = True
         if self.logger.verbose:
-            print(f"    {self.step}: {self._elapsed_seconds()}s, {self.items_created} created")
+            print(f"    {self.step}: {self._elapsed_seconds():.2f}s, {self.items_created} created")
         config_type = "extraction" if self.logger._current_phase == "extraction" else "derivation"
         self.logger.log_config_result(
             config_type=config_type,
@@ -319,7 +320,7 @@ class OCELStepContext:
         """Mark step as errored."""
         self._completed = True
         if self.logger.verbose:
-            print(f"    {self.step}: {self._elapsed_seconds()}s, ERROR {error[:120]}")
+            print(f"    {self.step}: {self._elapsed_seconds():.2f}s, ERROR {error[:120]}")
         config_type = "extraction" if self.logger._current_phase == "extraction" else "derivation"
         self.logger.log_config_result(
             config_type=config_type,
@@ -506,6 +507,8 @@ class BenchmarkOrchestrator:
         self._call_logs: dict[str, LlmCallLog] = {}
         # LLM answers per decision per step, recorded in the session metadata
         self._llm_samples: dict[str, int] = {}
+        # The code commit and dirty flag when the session started, recorded in the session metadata
+        self._code: dict[str, Any] = {"commit": None, "dirty": None}
         self._current_model: str | None = None
         self._current_repo: str | None = None
 
@@ -1457,6 +1460,7 @@ class BenchmarkOrchestrator:
                 model_name=model_display_name,
                 properties=leveled.properties,
                 views=leveled.views,
+                relationship_properties=leveled.relationship_properties,
             )
 
             # JSON snapshot for consistency analysis: the XML carries no properties,
@@ -1478,11 +1482,18 @@ class BenchmarkOrchestrator:
                         "name": e.name,
                         "source": (e.properties or {}).get("source"),
                         "level": "architect",
-                        # The name came from an LLM choice among several candidates (origin split)
-                        "name_choice": bool((e.properties or {}).get("name_choice")) or len((e.properties or {}).get("name_candidates") or []) > 1,
+                        # The name came from an LLM choice among several candidates (origin split): as the naming
+                        # step recorded it, else (steps that record none) several candidates mean a choice
+                        "name_choice": bool((e.properties or {})["name_choice"])
+                        if "name_choice" in (e.properties or {})
+                        else len((e.properties or {}).get("name_candidates") or []) > 1,
                         # The structural name (first candidate) next to the chosen one: what LLM naming adds
                         # (no naming call: the name is the structural one)
                         "structure_name": (e.properties or {}).get("structure_name") or next(iter((e.properties or {}).get("name_candidates") or []), None) or e.name,
+                        # A closed-set name's candidates in order (None for a name that is no choice)
+                        "name_candidates": list((e.properties or {}).get("name_candidates") or []) or None,
+                        # The abstraction levels the abstraction_levels step wrote (None: not tagged, enterprise)
+                        "levels": list((e.properties or {})["levels"]) if isinstance((e.properties or {}).get("levels"), list) else None,
                     }
                     for e in elements
                 ],
@@ -1493,6 +1504,9 @@ class BenchmarkOrchestrator:
                         "type": r.relationship_type,
                         "derived_from": (r.properties or {}).get("derived_from"),
                         "confidence": (r.properties or {}).get("confidence"),
+                        # Its levels (those both ends share); a cross-level link (no shared level) counts in the union only
+                        "levels": list((r.properties or {})["levels"]) if isinstance((r.properties or {}).get("levels"), list) else None,
+                        "cross_level": bool((r.properties or {}).get("cross_level")),
                     }
                     for r in relationships
                 ],
@@ -1569,12 +1583,20 @@ class BenchmarkOrchestrator:
                 self.session_start.isoformat(),
             ],
         )
+        self._capture_code()
         try:
             self._write_session_inputs()
         except Exception as exc:
             # A record that cannot be written does not stop the session (as a failed model export does not fail its run)
             logger.warning("Could not record the inputs of session %s: %s", self.session_id, exc)
 
+    def _capture_code(self) -> None:
+        """The commit the session starts on, with a dirty flag (the code may change while the session runs)."""
+        try:
+            self._code = export_bundle.code_commit()
+        except Exception as exc:  # A commit that cannot be read does not stop the session
+            logger.warning("Could not read the code commit of session %s: %s", self.session_id, exc)
+
     def _write_session_inputs(self) -> None:
         """Everything that changes results, as it is when the session starts, into the session folder.
 
@@ -1703,6 +1725,8 @@ class BenchmarkOrchestrator:
         summary = {
             "session_id": self.session_id,
             "config": self.config.to_dict(),
+            # The code the session ran: its commit and whether the working tree had changes at the start
+            "code": self._code,
             "started_at": self.session_start.isoformat() if self.session_start else None,
             "completed_at": datetime.now().isoformat(),
             "total_events": len(self.ocel_log.events),
@@ -2176,13 +2200,25 @@ def model_quality_for_session(
                 row: dict[str, Any] = {"repository": repo, "model": model, "run": int(run), **compute_model_quality(elements, relationships).to_dict()}
                 snapshot_path = path.with_suffix(".json")
                 try:
-                    row["solve"] = json.loads(snapshot_path.read_text(encoding="utf-8")).get("solve") if snapshot_path.is_file() else None
+                    snapshot = json.loads(snapshot_path.read_text(encoding="utf-8")) if snapshot_path.is_file() else {}
                 except OSError, ValueError:
-                    row["solve"] = None
+                    snapshot = {}
+                row["solve"] = snapshot.get("solve")
                 row["reference"] = None
                 if reference:
-                    # Lineage-free: name + type, the user's manual matches and derivable labels next to the reference
-                    derived = [{"id": e.identifier, "name": e.name, "type": e.element_type} for e in elements]
+                    # Name + type, the user's accepted names and structural refs (sources and lineage from the run's
+                    # snapshot: the XML carries none) and derivable labels: the gate's matching (golden_mapping)
+                    structure = {str(e.get("identifier")): e for e in snapshot.get("elements", []) if isinstance(e, dict)}
+                    derived = [
+                        {
+                            "id": e.identifier,
+                            "name": e.name,
+                            "type": e.element_type,
+                            "source": structure.get(e.identifier, {}).get("source"),
+                            "lineage": structure.get(e.identifier, {}).get("lineage") or [],
+                        }
+                        for e in elements
+                    ]
                     aliases, derivable = session_metrics.manual_inputs(reference_models[repo])
                     row["reference"] = session_metrics.golden_comparison(derived, reference, aliases=aliases, derivable=derivable)
                 rows.append(row)

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/src/deriva/bench/cli.py

@@ -238,7 +238,7 @@ def benchmark_run(
                 typer.echo(f"  ... and {len(result.errors) - 5} more")
 
         typer.echo("\nTo analyze results:")
-        typer.echo(f"  deriva benchmark analyze {result.session_id}")
+        typer.echo(f"  deriva-cli benchmark analyze {result.session_id}")
 
         if not result.success:
             raise typer.Exit(1)
@@ -383,10 +383,10 @@ def benchmark_gate(
     reference: Annotated[str | None, typer.Option("--reference", help="Golden model folder (default: workspace/reference)")] = None,
     benchmarks: Annotated[str | None, typer.Option("--benchmarks", help="Benchmark folder (default: workspace/benchmarks)")] = None,
 ) -> None:
-    """Recall, precision and consistency by id per repository, and the verdict of a candidate against a baseline.
+    """Element and relationship recall and precision and consistency by id per repository, and the verdict of a candidate.
 
-    Accepted when recall or precision rises on at least two thirds of the measured repositories, none drops by more
-    than 2 points on either, and element and relationship consistency by id stay at 95% or more everywhere.
+    Accepted when one of the four measures rises by 1 point or more on at least two thirds of the measured repositories,
+    none of them drops by more than 2 points anywhere, and element and relationship consistency by id stay at 95% or more.
     """
     from deriva.bench import gate
 
@@ -399,17 +399,27 @@ def benchmark_gate(
             typer.echo(f"Error: no benchmark session {', '.join(missing)} in {root}", err=True)
             raise typer.Exit(1)
     try:
-        measures = {name: gate.measure([root / s for s in side], goldens_dir, reference_dir, model=model) for name, side in sides.items() if side}
+        measures = {
+            level: {name: gate.measure([root / s for s in side], goldens_dir, reference_dir, model=model, level=level) for name, side in sides.items() if side}
+            for level in gate.GATE_LEVELS
+        }
     except ValueError as exc:
         typer.echo(f"Error: {exc}", err=True)
         raise typer.Exit(1) from exc
     lines = [f"# Gate: {', '.join(sides['baseline'])}" + (f" vs {', '.join(sides['candidate'])}" if sides["candidate"] else ""), ""]
-    lines += gate.table_markdown(measures["baseline"], f"Baseline ({', '.join(sides['baseline'])})")
-    payload: dict = {"sessions": sides, "baseline": measures["baseline"]}
-    if "candidate" in measures:
-        lines += gate.table_markdown(measures["candidate"], f"Candidate ({', '.join(sides['candidate'])})")
-        payload |= {"candidate": measures["candidate"], "verdict": gate.verdict(measures["baseline"], measures["candidate"])}
-        lines += gate.verdict_markdown(payload["verdict"])
+    payload: dict = {"sessions": sides, "levels": {}}
+    for level, by_side in measures.items():
+        lines += [f"## {level.capitalize()} level", ""]
+        lines += gate.table_markdown(by_side["baseline"], f"Baseline ({', '.join(sides['baseline'])})")
+        payload["levels"][level] = {"baseline": by_side["baseline"]}
+        if "candidate" in by_side:
+            lines += gate.table_markdown(by_side["candidate"], f"Candidate ({', '.join(sides['candidate'])})")
+            payload["levels"][level] |= {"candidate": by_side["candidate"], "verdict": gate.verdict(by_side["baseline"], by_side["candidate"])}
+            lines += gate.verdict_markdown(payload["levels"][level]["verdict"])
+    if sides["candidate"]:
+        result, voting = gate.overall_verdict({level: entry["verdict"] for level, entry in payload["levels"].items()})
+        payload["verdict"] = {"verdict": result, "levels": voting}
+        lines += [f"**Overall: {result}** (levels that voted: {', '.join(voting) or 'none'})", ""]
     for line in lines:
         typer.echo(line)
     name = "_".join(sides["baseline"]) + (f"_vs_{'_'.join(sides['candidate'])}" if sides["candidate"] else "")

deriva_dev-0.8.3/src/deriva/bench/gate.py

@@ -0,0 +1,305 @@
+"""The gate for a change: recall, precision and consistency per repository, a candidate against a baseline.
+
+Per repository (read-only, from the sessions' run snapshots):
+
+- **A development reference** (``<goldens>/<name>.json``, the reference format of ``reference_eval``): element and
+  relationship recall and precision as ``reference_eval`` counts them, the mean over the runs.
+- **A golden model** (``<reference>/<repository>.archimate``): recall on the derivable subset the user labelled
+  (``<repository>.derivable.json``) with the user's accepted names (``<repository>.matches.json``), and precision.
+  A golden relationship is found when a derived relationship of its type and direction joins the derived elements
+  matched to its two ends; it is in the subset when both ends are. Without the user's files the ``.template.json``
+  versions are read and the result is provisional: the subset is then the elements of the derived types and the
+  relationships between two of them.
+- **Consistency by id**: the share of elements and of relationships in every run (``session_metrics``).
+
+A repository without either reference keeps its consistency and is left out of the count. The verdict: the change is
+accepted when one of the four measures (element or relationship recall or precision) rises by at least the given
+points on enough of the measured repositories, none of the four drops by more than the allowed points anywhere, and
+consistency stays at the minimum everywhere.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from deriva.bench import reference_eval, session_metrics
+from deriva.modules.analysis.semantic_matching import parse_archi_xml, parse_exchange_format_xml
+
+MEASURES = ("recall", "precision", "relationship_recall", "relationship_precision")
+GATE_LEVELS = ("enterprise", "solution", "union")  # Each is measured and judged; levels without a measure do not vote
+LABELS = {"recall": "recall", "precision": "precision", "relationship_recall": "relationship recall", "relationship_precision": "relationship precision"}
+
+
+@dataclass(frozen=True)
+class GateRules:
+    """The acceptance rule."""
+
+    better_share: float = 2 / 3  # Of the measured repositories, at least this share better on one of the measures (6 of 9)
+    min_rise: float = 1.0  # Points a measure must rise to count as better (a smaller move holds)
+    max_drop: float = 2.0  # Points any measure may drop on any repository
+    min_consistency: float = 0.95  # Element and relationship consistency by id on every repository
+
+
+def _user_file(folder: Path, repository: str, kind: str) -> tuple[dict[str, Any], bool]:
+    """The user's file (matches or derivable), else its template; True when the template was read."""
+    for name, template in ((f"{repository}.{kind}.json", False), (f"{repository}.{kind}.template.json", True)):
+        path = folder / name
+        if path.is_file():
+            try:
+                return json.loads(path.read_text(encoding="utf-8")), template
+            except ValueError:
+                return {}, True
+    return {}, True
+
+
+def _golden_model(path: Path) -> tuple[list[Any], list[Any]]:
+    elements, relationships = parse_archi_xml(path)
+    return (elements, relationships) if elements else parse_exchange_format_xml(path)
+
+
+def _mean(values: list[float | None]) -> float | None:
+    known = [v for v in values if v is not None]
+    return round(sum(known) / len(known), 3) if known else None
+
+
+def golden_relationships(
+    derived: list[dict[str, Any]],
+    relationships: list[dict[str, Any]],
+    reference: list[Any],
+    golden: list[Any],
+    aliases: dict[str, Any],
+    scope: set[str],
+) -> dict[str, Any]:
+    """Relationship recall over the golden relationships with both ends in ``scope``, and precision over all derived ones.
+
+    A derived element maps to golden elements as the element scores map it (``session_metrics.golden_mapping``: by
+    name, by the user's accepted names, then by structural refs); a derived relationship is right when a golden
+    relationship of its type joins the golden elements of its source and target, in that direction.
+    """
+    to_golden = session_metrics.golden_mapping(derived, reference, aliases)
+    wanted = {(g.source, g.relationship_type, g.target) for g in golden}
+    found: set[tuple[str, str, str]] = set()
+    correct = 0
+    for rel in relationships:
+        kind = str(rel["type"]).removeprefix("Model:")
+        pairs = {(a, kind, b) for a in to_golden.get(rel["source"], ()) for b in to_golden.get(rel["target"], ())} & wanted
+        found |= pairs
+        correct += bool(pairs)
+    required = [g for g in golden if g.source in scope and g.target in scope]
+    hits = sum(1 for g in required if (g.source, g.relationship_type, g.target) in found)
+    return {
+        "required": len(required),
+        "matched": hits,
+        "recall": round(hits / len(required), 3) if required else None,
+        "derived": len(relationships),
+        "correct": correct,
+        "precision": round(correct / len(relationships), 3) if relationships else None,
+    }
+
+
+def _golden_row(path: Path, reference_dir: Path, repo: str, snapshots: list[dict[str, Any]]) -> dict[str, Any]:
+    reference, golden_rels = _golden_model(path)
+    aliases, alias_template = _user_file(reference_dir, repo, "matches")
+    derivable, derivable_template = _user_file(reference_dir, repo, "derivable")
+    provisional = alias_template or derivable_template
+    if provisional:
+        scope = {r.identifier for r in reference if r.element_type in session_metrics.DERIVED_TYPES}
+    else:
+        scope = {r.identifier for r in reference if isinstance(derivable.get(r.identifier), dict) and derivable[r.identifier].get("derivable")}
+    elements, relationships = [], []
+    for snapshot in snapshots:
+        derived = [
+            {"id": e["identifier"], "name": e.get("name"), "type": str(e["type"]).removeprefix("Model:"), "source": e.get("source"), "lineage": e.get("lineage") or []}
+            for e in snapshot.get("elements", [])
+        ]
+        elements.append(session_metrics.golden_comparison(derived, reference, aliases=aliases, derivable=derivable))
+        relationships.append(golden_relationships(derived, snapshot.get("relationships", []), reference, golden_rels, aliases, scope))
+    recall_key = "derivable_types_recall" if provisional else "derivable_recall"
+    return {
+        "source": "golden",
+        "provisional": provisional,
+        "recall": _mean([r[recall_key] for r in elements]),
+        "precision": _mean([r["precision"] for r in elements]),
+        "relationship_recall": _mean([r["recall"] for r in relationships]),
+        "relationship_precision": _mean([r["precision"] for r in relationships]),
+    }
+
+
+def measure(sessions: list[Path], goldens_dir: Path, reference_dir: Path, model: str | None = None, *, level: str | None = None) -> dict[str, dict[str, Any]]:
+    """Per repository of the sessions: its model, runs, source of the measure, the four measures and consistency by id.
+
+    ``level`` (``enterprise``, ``solution`` or ``union``) measures one abstraction level; ``None`` measures everything,
+    as before levels existed. A reference with no element at the level has no measure there. A reference without level
+    tags, and a golden model (none has levels yet), is measured on the union only: untagged, every element would count
+    as enterprise and the solution elements a run derives would show as missing there (``note``: no level tags).
+    """
+    sessions = [Path(s) for s in sessions]
+    runs: dict[str, dict[str, list[dict[str, Any]]]] = {}
+    for session in sessions:
+        for repo, run_model, _, snapshot in session_metrics.run_snapshots(session):
+            runs.setdefault(repo, {}).setdefault(run_model, []).append(snapshot)
+    references = reference_eval.load_references(goldens_dir) if Path(goldens_dir).is_dir() else {}
+    consistency = {(r["repository"], r["model"]): r["identities"]["id"] for r in session_metrics.consistency_rows(sessions, level=level)}
+
+    out: dict[str, dict[str, Any]] = {}
+    for repo, by_model in sorted(runs.items()):
+        if model is not None:
+            snapshots = by_model.get(model, [])
+            if not snapshots:
+                continue
+            chosen = model
+        elif len(by_model) > 1:
+            raise ValueError(f"{repo} has runs of several models ({', '.join(sorted(by_model))}): name one with --model")
+        else:
+            ((chosen, snapshots),) = by_model.items()
+        row: dict[str, Any] = {"repository": repo, "model": chosen, "runs": len(snapshots), "source": None, "provisional": False, "note": None} | dict.fromkeys(MEASURES)
+        golden = Path(reference_dir) / f"{repo}.archimate"
+        single = level not in (None, "union")
+        has_level = not single or (repo in references and any(level in e.levels for e in references[repo].elements))
+        if single and ((repo in references and not references[repo].tagged) or (repo not in references and golden.is_file())):
+            row["note"] = "no level tags"  # measured on the union only
+        elif repo in references and not has_level:
+            pass  # no reference at this level
+        elif repo in references:
+            results = [reference_eval.evaluate(references[repo], s, level=level) for s in snapshots]
+            row |= {
+                "source": "reference",
+                "recall": _mean([r["element_recall"] for r in results]),
+                "precision": _mean([r["element_precision"] for r in results]),
+                "relationship_recall": _mean([r["relationship_recall"] for r in results]),
+                "relationship_precision": _mean([r["relationship_totals"]["precision"] for r in results]),
+            }
+        elif golden.is_file():
+            row |= _golden_row(golden, Path(reference_dir), repo, snapshots)
+        ids = consistency.get((repo, chosen))
+        row["consistency"] = {"elements": ids["elements"]["cons"], "relationships": ids["relationships"]["cons"]} if ids else None
+        out[repo] = row
+    return out
+
+
+def _points(before: float | None, after: float | None) -> float | None:
+    return None if before is None or after is None else round((after - before) * 100, 1)
+
+
+def _lowest(row: dict[str, Any]) -> float | None:
+    values = [v for v in (row.get("consistency") or {}).values() if v is not None]
+    return min(values) if values else None
+
+
+def verdict(baseline: dict[str, dict[str, Any]], candidate: dict[str, dict[str, Any]], rules: GateRules | None = None) -> dict[str, Any]:
+    """The candidate against the baseline, per repository and overall (PASS or FAIL, with the reasons)."""
+    rules = rules or GateRules()
+    rows, failures, unmeasured = [], [], []
+    for repo in sorted(set(baseline) | set(candidate)):
+        before, after = baseline.get(repo), candidate.get(repo)
+        measured = bool(before and after and before["recall"] is not None and after["recall"] is not None)
+        row: dict[str, Any] = {"repository": repo, "measured": measured, "baseline": before, "candidate": after, "better": False}
+        if measured and before and after:
+            row["points"] = {m: _points(before.get(m), after.get(m)) for m in MEASURES}
+            changes = {m: p for m, p in row["points"].items() if p is not None}
+            row["better"] = any(p >= rules.min_rise for p in changes.values())
+            row["drop"] = max([0.0] + [-p for p in changes.values()])
+            for m, p in changes.items():
+                if -p > rules.max_drop:
+                    failures.append(f"{repo} drops {-p:.1f} points on {LABELS[m]} (at most {rules.max_drop:.1f})")
+        else:
+            unmeasured.append(repo)
+        lowest = _lowest(after) if after else None
+        row["consistency"] = lowest
+        if lowest is not None and lowest < rules.min_consistency:
+            failures.append(f"{repo} consistency {lowest:.1%} (at least {rules.min_consistency:.0%})")
+        rows.append(row)
+    measured_rows = [r for r in rows if r["measured"]]
+    better = sum(1 for r in measured_rows if r["better"])
+    needed = math.ceil(rules.better_share * len(measured_rows) - 1e-9)
+    held: list[str] = []  # Too few better, nothing failing: the level holds (HELD), it does not fail
+    if not measured_rows:
+        failures.append("no repository has a measure in both the baseline and the candidate")
+    elif better < needed:
+        held.append(f"{better} of {len(measured_rows)} measured repositories better (at least {needed})")
+    consistencies = [r["consistency"] for r in rows if r["consistency"] is not None]
+    return {
+        "rows": rows,
+        "measured": len(measured_rows),
+        "better": better,
+        "needed": needed,
+        "max_drop": max((r["drop"] for r in measured_rows), default=0.0),
+        "min_consistency": min(consistencies) if consistencies else None,
+        "unmeasured": unmeasured,
+        "failures": failures,
+        "held": held,
+        "verdict": "FAIL" if failures else ("HELD" if held else "PASS"),
+        "rules": {"better_share": rules.better_share, "min_rise": rules.min_rise, "max_drop": rules.max_drop, "min_consistency": rules.min_consistency},
+    }
+
+
+def overall_verdict(by_level: dict[str, dict[str, Any]]) -> tuple[str, list[str]]:
+    """The levels with a measured repository vote: FAIL when one fails, PASS when none fails and at least one passes
+    (a level the change only holds does not block one it improves), else HELD. Returns the verdict and the voters."""
+    voting = [level for level, result in by_level.items() if result.get("measured")]
+    verdicts = {by_level[level]["verdict"] for level in voting}
+    if not voting or "FAIL" in verdicts:
+        return "FAIL", voting
+    return ("PASS" if "PASS" in verdicts else "HELD"), voting
+
+
+def _pct(value: float | None) -> str:
+    return "-" if value is None else f"{value:.1%}"
+
+
+def _source(row: dict[str, Any] | None) -> str:
+    if not row or row["source"] is None:
+        return (row or {}).get("note") or "no reference"
+    return f"{row['source']} (provisional)" if row["provisional"] else row["source"]
+
+
+def table_markdown(measures: dict[str, dict[str, Any]], title: str) -> list[str]:
+    """One row per repository: source, runs, the four measures, consistency by id."""
+    lines = [
+        f"## {title}",
+        "",
+        "| Repository | Measure | Runs | Recall | Precision | Rel. recall | Rel. precision | Elements by id | Relationships by id |",
+        "|---|---|---|---|---|---|---|---|---|",
+    ]
+    for repo, row in sorted(measures.items()):
+        cons = row.get("consistency") or {}
+        values = " | ".join(_pct(row.get(m)) for m in MEASURES)
+        lines.append(f"| {repo} | {_source(row)} | {row['runs']} | {values} | {_pct(cons.get('elements'))} | {_pct(cons.get('relationships'))} |")
+    return [*lines, ""]
+
+
+def verdict_markdown(result: dict[str, Any]) -> list[str]:
+    """The comparison table (before -> after per measure) and the verdict with its reasons."""
+    lines = [
+        "## Verdict",
+        "",
+        "| Repository | Measure | Recall | Precision | Rel. recall | Rel. precision | Consistency | Better |",
+        "|---|---|---|---|---|---|---|---|",
+    ]
+    for r in result["rows"]:
+        before, after = r["baseline"] or {}, r["candidate"] or {}
+        better = ("yes" if r["better"] else "held") if r["measured"] else "not measured"
+        cells = " | ".join(f"{_pct(before.get(m))} -> {_pct(after.get(m))}" for m in MEASURES)
+        lines.append(f"| {r['repository']} | {_source(r['candidate'] or r['baseline'])} | {cells} | {_pct(r['consistency'])} | {better} |")
+    rules = result["rules"]
+    lines += [
+        "",
+        f"**{result['verdict']}**: {result['better']}/{result['measured']} measured repositories better by {rules['min_rise']:.1f} points or more "
+        f"(at least {result['needed']}), largest drop {result['max_drop']:.1f} points, lowest consistency {_pct(result['min_consistency'])}.",
+    ]
+    if result["unmeasured"]:
+        lines.append(f"Counted out of the {result['measured']} repositories with a measure; not measured: {', '.join(result['unmeasured'])}.")
+    lines += [f"- {f}" for f in result["failures"] + result.get("held", [])]
+    return [*lines, ""]
+
+
+def write(out_dir: Path, payload: dict[str, Any], lines: list[str]) -> dict[str, str]:
+    """gate.json and gate.md in a folder."""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    (out_dir / "gate.json").write_text(json.dumps(payload, indent=1), encoding="utf-8")
+    (out_dir / "gate.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return {"json": str(out_dir / "gate.json"), "markdown": str(out_dir / "gate.md")}

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/src/deriva/bench/reference_eval.py

@@ -29,6 +29,7 @@ class RefElement:
     aliases: tuple[str, ...] = ()
     lineage: frozenset[str] = frozenset()
     optional: bool = False
+    levels: frozenset[str] = frozenset({"enterprise"})  # The abstraction levels it belongs to
 
 
 @dataclass(frozen=True)
@@ -38,6 +39,7 @@ class RefRelationship:
     type: str
     kind: str
     optional: bool = False
+    levels: frozenset[str] = frozenset({"enterprise"})
 
 
 @dataclass(frozen=True)
@@ -52,6 +54,7 @@ class Reference:
     elements: list[RefElement] = field(default_factory=list)
     relationships: list[RefRelationship] = field(default_factory=list)
     chains: list[Chain] = field(default_factory=list)
+    tagged: bool = False  # Whether any element or relationship names its levels; untagged, all count as enterprise
 
 
 def parse_reference(data: dict[str, Any]) -> Reference:
@@ -64,18 +67,66 @@ def parse_reference(data: dict[str, Any]) -> Reference:
             aliases=tuple(e.get("aliases", [])),
             lineage=frozenset(e.get("lineage", [])),
             optional=bool(e.get("optional", False)),
+            levels=_levels_of(e),
         )
         for e in data["elements"]
     ]
     keys = {e.key for e in elements}
     if len(keys) != len(elements):
         raise ValueError(f"{data['repository']}: element keys must be unique")
-    relationships = [RefRelationship(r["source"], r["target"], r["type"], r["kind"], bool(r.get("optional", False))) for r in data.get("relationships", [])]
+    relationships = [RefRelationship(r["source"], r["target"], r["type"], r["kind"], bool(r.get("optional", False)), _levels_of(r)) for r in data.get("relationships", [])]
     chains = [Chain(c["name"], tuple((s[0], s[1], s[2], len(s) > 3 and s[3] == "reverse") for s in c["steps"])) for c in data.get("chains", [])]
     ends = {k for r in relationships for k in (r.source, r.target)} | {k for c in chains for s in c.steps for k in (s[0], s[2])}
     if unknown := ends - keys:
         raise ValueError(f"{data['repository']}: unknown element keys {sorted(unknown)}")
-    return Reference(repository=data["repository"], elements=elements, relationships=relationships, chains=chains)
+    tagged = any("levels" in item or "cross_level" in item for item in [*data["elements"], *data.get("relationships", [])])
+    return Reference(repository=data["repository"], elements=elements, relationships=relationships, chains=chains, tagged=tagged)
+
+
+LEVELS = ("enterprise", "solution")
+
+
+def _levels_of(item: dict[str, Any]) -> frozenset[str]:
+    """The levels a reference or model item belongs to (``levels``); without them, the enterprise level.
+
+    A cross-level item (``cross_level``, or an explicit empty ``levels``: a roll-up between levels) belongs to no
+    single level and counts in the union only.
+    """
+    if item.get("cross_level"):
+        return frozenset()
+    levels = item.get("levels")
+    return frozenset(str(v) for v in levels) if isinstance(levels, list) else frozenset({"enterprise"})
+
+
+def at_level(reference: Reference, model: dict[str, Any], level: str | None) -> tuple[Reference, dict[str, Any]]:
+    """The reference and the model at one level (``enterprise`` or ``solution``); ``None`` or ``union``: all of both."""
+    if level is None or level == "union":
+        return reference, model
+    if level not in LEVELS:
+        raise ValueError(f"level is one of {', '.join(LEVELS)}, union or None, not {level!r}")
+    keys = {e.key for e in reference.elements if level in e.levels}
+    leveled = Reference(
+        repository=reference.repository,
+        elements=[e for e in reference.elements if e.key in keys],
+        relationships=[r for r in reference.relationships if level in r.levels and r.source in keys and r.target in keys],
+        chains=[c for c in reference.chains if all(a in keys and b in keys for a, _, b, _ in c.steps)],
+    )
+    return leveled, snapshot_at_level(model, level)
+
+
+def snapshot_at_level(model: dict[str, Any], level: str | None) -> dict[str, Any]:
+    """A run model at one level: its elements tagged with the level (no tag: enterprise) and the relationships on it
+    between them; ``None`` or ``union``: the whole model."""
+    if level is None or level == "union":
+        return model
+    if level not in LEVELS:
+        raise ValueError(f"level is one of {', '.join(LEVELS)}, union or None, not {level!r}")
+    ids = {e["identifier"] for e in model.get("elements", []) if level in _levels_of(e)}
+    return {
+        **model,
+        "elements": [e for e in model.get("elements", []) if e["identifier"] in ids],
+        "relationships": [r for r in model.get("relationships", []) if level in _levels_of(r) and r["source"] in ids and r["target"] in ids],
+    }
 
 
 def _type(value: str) -> str:
@@ -131,8 +182,13 @@ def naming(reference: Reference, model: dict[str, Any]) -> dict[str, dict[str, i
     return out
 
 
-def evaluate(reference: Reference, model: dict[str, Any]) -> dict[str, Any]:
-    """Recall and precision per element type and relationship kind, splits, chains, for one model."""
+def evaluate(reference: Reference, model: dict[str, Any], *, level: str | None = None) -> dict[str, Any]:
+    """Recall and precision per element type and relationship kind, splits, chains, for one model.
+
+    ``level`` (``enterprise``, ``solution`` or ``union``) scores one abstraction level of both; ``None`` keeps
+    every element and relationship, as before levels existed.
+    """
+    reference, model = at_level(reference, model, level)
     mapping = map_elements(reference, model)
     by_key = {r.key: r for r in reference.elements}
     covered: dict[str, list[str]] = defaultdict(list)  # reference key -> model identifiers

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/src/deriva/bench/router.py

@@ -59,8 +59,9 @@ def list_sessions(limit: int = Query(20, ge=1, le=500), session: Any = Depends(g
 
 @router.get("/models")
 def models(session: Any = Depends(get_benchmarks)) -> list[dict[str, Any]]:
-    """The configured benchmark models (name, provider, model; never keys)."""
-    return [{"name": name, "provider": cfg.provider, "model": cfg.model} for name, cfg in sorted(session.list_benchmark_models().items())]
+    """The configured benchmark models (name, provider, model; never keys), with the one Deriva uses marked ``default``."""
+    default = next((m.get("name") for m in session.list_model_configs() or [] if m.get("default")), None)
+    return [{"name": name, "provider": cfg.provider, "model": cfg.model, "default": name == default} for name, cfg in sorted(session.list_benchmark_models().items())]
 
 
 @router.post("", status_code=202)

{deriva_dev-0.8.2 → deriva_dev-0.8.3}/src/deriva/bench/session_metrics.py

@@ -141,12 +141,19 @@ def _identity_report(views: list[dict[str, Any]], identity: str) -> dict[str, An
     return report
 
 
-def consistency_rows(sessions: list[Path]) -> list[dict[str, Any]]:
-    """Per repository and model (and over all models when the sessions hold several), both identities."""
+def consistency_rows(sessions: list[Path], *, level: str | None = None) -> list[dict[str, Any]]:
+    """Per repository and model (and over all models when the sessions hold several), both identities.
+
+    ``level`` (``enterprise``, ``solution`` or ``union``) compares only that abstraction level of each run; ``None``
+    compares the whole model, as before levels existed.
+    """
+    from deriva.bench.reference_eval import snapshot_at_level
+
     grouped: dict[tuple[str, str], list[dict[str, Any]]] = {}
     for session in sessions:
         calls = _calls(session)
-        for repo, model, number, snapshot in run_snapshots(session):
+        for repo, model, number, whole in run_snapshots(session):
+            snapshot = snapshot_at_level(whole, level)
             classified = {str(c.get("step")) for c in calls.get((repo, model, number), []) if _decision_type(c).endswith(" classification")}
             grouped.setdefault((repo, model), []).append(_run_view(snapshot, classified))
     rows = []
@@ -298,6 +305,62 @@ NAME_MATCHES = frozenset({"exact", "fuzzy_name"})
 DERIVED_TYPES = frozenset(ELEMENT_TYPES) - {"Artifact"}  # Artifact belongs to the export's detail level
 
 
+def accepted(entry: Any) -> tuple[list[str], list[str]]:
+    """The names and structural refs the user accepted for a golden element: a list of names, or
+    ``{"names": [...], "structure_refs": [...]}``."""
+    if isinstance(entry, list):
+        return [str(n) for n in entry], []
+    if isinstance(entry, dict):
+        return [str(n) for n in entry.get("names") or []], [str(r) for r in entry.get("structure_refs") or []]
+    return [], []
+
+
+def _clean_path(path: str) -> str:
+    path = str(path).replace(chr(92), "/").strip()
+    while path.startswith("./"):
+        path = path[2:]
+    return path.strip("/")
+
+
+def _structural_match(element: dict[str, Any], refs: list[str]) -> bool:
+    """A structural ref names the element's source (or lineage) id, or (``path:``) the folder or file of that id itself."""
+    ids = [str(i) for i in [element.get("source"), *(element.get("lineage") or [])] if i]
+    paths = {_clean_path(i.split("::")[2]) for i in ids if len(i.split("::")) >= 3}
+    return any((_clean_path(r[5:]) in paths) if r.startswith("path:") else (r in ids) for r in refs)
+
+
+def golden_mapping(derived: list[dict[str, Any]], reference: list[Any], aliases: dict[str, Any] | None = None) -> dict[str, set[str]]:
+    """Derived element id -> the golden elements it is matched to.
+
+    First by name, as always: the element matcher (same type, the same or a similar name) and the names the user
+    accepted. Then a golden element still unmatched takes, through its structural refs, the first derived element of
+    its type (by id) that no golden element has yet, so one derived element matches one golden element there.
+    """
+    aliases = aliases or {}
+    mapping: dict[str, set[str]] = {}
+    for m in match_elements(derived, reference):
+        if m.match_type in NAME_MATCHES and m.derived_type == m.reference_type and m.reference_id:
+            mapping.setdefault(m.derived_id, set()).add(m.reference_id)
+    by_key: dict[tuple[str, str], list[str]] = {}
+    for d in derived:
+        by_key.setdefault((str(d.get("type")), name_key(str(d.get("name") or ""))), []).append(str(d["id"]))
+    for r in reference:
+        for alias in accepted(aliases.get(r.identifier))[0]:
+            for derived_id in by_key.get((r.element_type, name_key(alias)), []):
+                mapping.setdefault(derived_id, set()).add(r.identifier)
+    found = set().union(*mapping.values()) if mapping else set()
+    for r in sorted(reference, key=lambda r: r.identifier):
+        refs = accepted(aliases.get(r.identifier))[1]
+        if r.identifier in found or not refs:
+            continue
+        for d in sorted(derived, key=lambda d: str(d["id"])):
+            if str(d["id"]) not in mapping and str(d.get("type")) == r.element_type and _structural_match(d, refs):
+                mapping[str(d["id"])] = {r.identifier}
+                found.add(r.identifier)
+                break
+    return mapping
+
+
 def golden_comparison(
     derived: list[dict[str, Any]],
     reference: list[Any],
@@ -312,12 +375,16 @@ def golden_comparison(
     derived types. Precision counts each found reference element once, as the model-quality report does.
     """
     aliases, derivable = aliases or {}, derivable or {}
-    found = {m.reference_id for m in match_elements(derived, reference) if m.match_type in NAME_MATCHES and m.derived_type == m.reference_type}
+    by_name = {m.reference_id for m in match_elements(derived, reference) if m.match_type in NAME_MATCHES and m.derived_type == m.reference_type}
     by_type: dict[str, set[str]] = {}
     for d in derived:
         by_type.setdefault(str(d.get("type")), set()).add(name_key(str(d.get("name") or "")))
-    manual = {r.identifier for r in reference if r.identifier not in found and any(name_key(a) in by_type.get(r.element_type, set()) for a in aliases.get(r.identifier, []))}
-    found |= manual
+    manual = {
+        r.identifier for r in reference if r.identifier not in by_name and any(name_key(a) in by_type.get(r.element_type, set()) for a in accepted(aliases.get(r.identifier))[0])
+    }
+    mapped = golden_mapping(derived, reference, aliases)
+    found = set().union(*mapped.values()) if mapped else set()
+    structural = found - by_name - manual
 
     def share(part: int, whole: int) -> float | None:
         return round(part / whole, 3) if whole else None
@@ -340,6 +407,7 @@ def golden_comparison(
         "derivable_recall": share(sum(1 for r in can if r.identifier in found), len(can)),
         "derivable_labelled": len(labelled),
         "manual_matches": len(manual),
+        "structure_matches": len(structural),
     }
 
 

deriva_dev-0.8.2/src/deriva/bench/gate.py

@@ -1,216 +0,0 @@
-"""The gate for a change: recall, precision and consistency per repository, a candidate against a baseline.
-
-Per repository (read-only, from the sessions' run snapshots):
-
-- **A development reference** (``<goldens>/<name>.json``, the reference format of ``reference_eval``): element
-  recall and precision, the mean over the runs.
-- **A golden model** (``<reference>/<repository>.archimate``): recall on the derivable subset the user labelled
-  (``<repository>.derivable.json``) with the user's accepted names (``<repository>.matches.json``), and precision.
-  Without those files the ``.template.json`` versions are read and the result is provisional: recall then counts the
-  reference elements of the derived types.
-- **Consistency by id**: the share of elements and of relationships in every run (``session_metrics``).
-
-A repository without either reference keeps its consistency and is left out of the count. The verdict: the change is
-accepted when recall or precision rises on enough of the measured repositories, none drops by more than the allowed
-points on either, and consistency stays at the minimum everywhere.
-"""
-
-from __future__ import annotations
-
-import json
-import math
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-from deriva.bench import reference_eval, session_metrics
-from deriva.modules.analysis.semantic_matching import parse_archi_xml, parse_exchange_format_xml
-
-
-@dataclass(frozen=True)
-class GateRules:
-    """The acceptance rule."""
-
-    better_share: float = 2 / 3  # Of the measured repositories, at least this share better on recall or precision (6 of 9)
-    max_drop: float = 2.0  # Points either may drop on any repository
-    min_consistency: float = 0.95  # Element and relationship consistency by id on every repository
-
-
-def _user_file(folder: Path, repository: str, kind: str) -> tuple[dict[str, Any], bool]:
-    """The user's file (matches or derivable), else its template; True when the template was read."""
-    for name, template in ((f"{repository}.{kind}.json", False), (f"{repository}.{kind}.template.json", True)):
-        path = folder / name
-        if path.is_file():
-            try:
-                return json.loads(path.read_text(encoding="utf-8")), template
-            except ValueError:
-                return {}, True
-    return {}, True
-
-
-def _golden_elements(path: Path) -> list[Any]:
-    elements, _ = parse_archi_xml(path)
-    return elements or parse_exchange_format_xml(path)[0]
-
-
-def _mean(values: list[float | None]) -> float | None:
-    known = [v for v in values if v is not None]
-    return round(sum(known) / len(known), 3) if known else None
-
-
-def measure(sessions: list[Path], goldens_dir: Path, reference_dir: Path, model: str | None = None) -> dict[str, dict[str, Any]]:
-    """Per repository of the sessions: its model, runs, source of the measure, recall, precision and consistency by id."""
-    sessions = [Path(s) for s in sessions]
-    runs: dict[str, dict[str, list[dict[str, Any]]]] = {}
-    for session in sessions:
-        for repo, run_model, _, snapshot in session_metrics.run_snapshots(session):
-            runs.setdefault(repo, {}).setdefault(run_model, []).append(snapshot)
-    references = reference_eval.load_references(goldens_dir) if Path(goldens_dir).is_dir() else {}
-    consistency = {(r["repository"], r["model"]): r["identities"]["id"] for r in session_metrics.consistency_rows(sessions)}
-
-    out: dict[str, dict[str, Any]] = {}
-    for repo, by_model in sorted(runs.items()):
-        if model is not None:
-            snapshots = by_model.get(model, [])
-            if not snapshots:
-                continue
-            chosen = model
-        elif len(by_model) > 1:
-            raise ValueError(f"{repo} has runs of several models ({', '.join(sorted(by_model))}): name one with --model")
-        else:
-            ((chosen, snapshots),) = by_model.items()
-        row: dict[str, Any] = {"repository": repo, "model": chosen, "runs": len(snapshots), "source": None, "provisional": False, "recall": None, "precision": None}
-        golden = Path(reference_dir) / f"{repo}.archimate"
-        if repo in references:
-            results = [reference_eval.evaluate(references[repo], s) for s in snapshots]
-            row |= {"source": "reference", "recall": _mean([r["element_recall"] for r in results]), "precision": _mean([r["element_precision"] for r in results])}
-        elif golden.is_file():
-            reference = _golden_elements(golden)
-            aliases, alias_template = _user_file(Path(reference_dir), repo, "matches")
-            derivable, derivable_template = _user_file(Path(reference_dir), repo, "derivable")
-            provisional = alias_template or derivable_template
-            results = []
-            for snapshot in snapshots:
-                derived = [{"id": e["identifier"], "name": e.get("name"), "type": str(e["type"]).removeprefix("Model:")} for e in snapshot.get("elements", [])]
-                results.append(session_metrics.golden_comparison(derived, reference, aliases=aliases, derivable=derivable))
-            recall_key = "derivable_types_recall" if provisional else "derivable_recall"
-            row |= {
-                "source": "golden",
-                "provisional": provisional,
-                "recall": _mean([r[recall_key] for r in results]),
-                "precision": _mean([r["precision"] for r in results]),
-            }
-        ids = consistency.get((repo, chosen))
-        row["consistency"] = {"elements": ids["elements"]["cons"], "relationships": ids["relationships"]["cons"]} if ids else None
-        out[repo] = row
-    return out
-
-
-def _points(before: float | None, after: float | None) -> float | None:
-    return None if before is None or after is None else round((after - before) * 100, 1)
-
-
-def _lowest(row: dict[str, Any]) -> float | None:
-    values = [v for v in (row.get("consistency") or {}).values() if v is not None]
-    return min(values) if values else None
-
-
-def verdict(baseline: dict[str, dict[str, Any]], candidate: dict[str, dict[str, Any]], rules: GateRules | None = None) -> dict[str, Any]:
-    """The candidate against the baseline, per repository and overall (PASS or FAIL, with the reasons)."""
-    rules = rules or GateRules()
-    rows, failures, unmeasured = [], [], []
-    for repo in sorted(set(baseline) | set(candidate)):
-        before, after = baseline.get(repo), candidate.get(repo)
-        measured = bool(before and after and before["recall"] is not None and after["recall"] is not None)
-        row: dict[str, Any] = {"repository": repo, "measured": measured, "baseline": before, "candidate": after, "better": False}
-        if measured and before and after:
-            row["recall_points"] = _points(before["recall"], after["recall"])
-            row["precision_points"] = _points(before["precision"], after["precision"])
-            changes = [p for p in (row["recall_points"], row["precision_points"]) if p is not None]
-            row["better"] = any(p > 0 for p in changes)
-            row["drop"] = max([0.0] + [-p for p in changes])
-            if row["drop"] > rules.max_drop:
-                worst = "recall" if (row["recall_points"] or 0) <= (row["precision_points"] or 0) else "precision"
-                failures.append(f"{repo} drops {row['drop']:.1f} points on {worst} (at most {rules.max_drop:.1f})")
-        else:
-            unmeasured.append(repo)
-        lowest = _lowest(after) if after else None
-        row["consistency"] = lowest
-        if lowest is not None and lowest < rules.min_consistency:
-            failures.append(f"{repo} consistency {lowest:.1%} (at least {rules.min_consistency:.0%})")
-        rows.append(row)
-    measured_rows = [r for r in rows if r["measured"]]
-    better = sum(1 for r in measured_rows if r["better"])
-    needed = math.ceil(rules.better_share * len(measured_rows) - 1e-9)
-    if not measured_rows:
-        failures.append("no repository has a measure in both the baseline and the candidate")
-    elif better < needed:
-        failures.append(f"{better} of {len(measured_rows)} measured repositories better (at least {needed})")
-    consistencies = [r["consistency"] for r in rows if r["consistency"] is not None]
-    return {
-        "rows": rows,
-        "measured": len(measured_rows),
-        "better": better,
-        "needed": needed,
-        "max_drop": max((r["drop"] for r in measured_rows), default=0.0),
-        "min_consistency": min(consistencies) if consistencies else None,
-        "unmeasured": unmeasured,
-        "failures": failures,
-        "verdict": "FAIL" if failures else "PASS",
-        "rules": {"better_share": rules.better_share, "max_drop": rules.max_drop, "min_consistency": rules.min_consistency},
-    }
-
-
-def _pct(value: float | None) -> str:
-    return "-" if value is None else f"{value:.1%}"
-
-
-def _source(row: dict[str, Any] | None) -> str:
-    if not row or row["source"] is None:
-        return "no reference"
-    return f"{row['source']} (provisional)" if row["provisional"] else row["source"]
-
-
-def table_markdown(measures: dict[str, dict[str, Any]], title: str) -> list[str]:
-    """One row per repository: source, runs, recall, precision, consistency by id."""
-    lines = [f"## {title}", "", "| Repository | Measure | Runs | Recall | Precision | Elements by id | Relationships by id |", "|---|---|---|---|---|---|---|"]
-    for repo, row in sorted(measures.items()):
-        cons = row.get("consistency") or {}
-        lines.append(
-            f"| {repo} | {_source(row)} | {row['runs']} | {_pct(row['recall'])} | {_pct(row['precision'])} | {_pct(cons.get('elements'))} | {_pct(cons.get('relationships'))} |"
-        )
-    return [*lines, ""]
-
-
-def verdict_markdown(result: dict[str, Any]) -> list[str]:
-    """The comparison table and the verdict with its reasons."""
-    lines = [
-        "## Verdict",
-        "",
-        "| Repository | Measure | Recall before | Recall after | Precision before | Precision after | Consistency | Better |",
-        "|---|---|---|---|---|---|---|---|",
-    ]
-    for r in result["rows"]:
-        before, after = r["baseline"] or {}, r["candidate"] or {}
-        better = ("yes" if r["better"] else "no") if r["measured"] else "not measured"
-        lines.append(
-            f"| {r['repository']} | {_source(r['candidate'] or r['baseline'])} | {_pct(before.get('recall'))} | {_pct(after.get('recall'))} | "
-            f"{_pct(before.get('precision'))} | {_pct(after.get('precision'))} | {_pct(r['consistency'])} | {better} |"
-        )
-    lines += [
-        "",
-        f"**{result['verdict']}**: {result['better']}/{result['measured']} measured repositories better (at least {result['needed']}), "
-        f"largest drop {result['max_drop']:.1f} points, lowest consistency {_pct(result['min_consistency'])}.",
-    ]
-    if result["unmeasured"]:
-        lines.append(f"Counted out of the {result['measured']} repositories with a measure; not measured: {', '.join(result['unmeasured'])}.")
-    lines += [f"- {f}" for f in result["failures"]]
-    return [*lines, ""]
-
-
-def write(out_dir: Path, payload: dict[str, Any], lines: list[str]) -> dict[str, str]:
-    """gate.json and gate.md in a folder."""
-    out_dir.mkdir(parents=True, exist_ok=True)
-    (out_dir / "gate.json").write_text(json.dumps(payload, indent=1), encoding="utf-8")
-    (out_dir / "gate.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
-    return {"json": str(out_dir / "gate.json"), "markdown": str(out_dir / "gate.md")}