static-fire 0.0.0-stage → 0.54.1

package/src/core/coverage.ts

@@ -0,0 +1,84 @@
+// Which suites a view's index covers, and why the others are left out (DEC-019, DEC-020). The rule that picks the
+// default set is here, and so is the one line that `static-fire show` and the dashboard both print. No Node
+// imports: the dashboard bundles this file.
+
+/** A suite as the line names it. */
+export interface SuiteRef { id: string; title: string }
+
+/**
+ * Why a suite is not in view: the user did not pick it, no config has completed it, or too few configs have
+ * completed it together with the suites in view.
+ */
+export type LeftOutReason = "not-picked" | "none-completed" | "too-few";
+
+/** What a view's index covers. A view of tasks (no suites) has none. */
+export interface Coverage {
+  /** True when the suites were picked; false when the view chose them by the default rule. */
+  picked: boolean;
+  /** The suites in view, in catalog order: the Agent Performance Index is the mean of their Suite Indexes. */
+  covered: SuiteRef[];
+  /**
+   * Every other suite of the catalog, in catalog order, with the reason and how many configs in view have completed
+   * it. That count is null for a suite left out because it was not picked: the view does not read its trials.
+   */
+  leftOut: (SuiteRef & { reason: LeftOutReason; completed: number | null })[];
+  /** How many configs have completed every covered suite: the configs that have an index. */
+  complete: number;
+}
+
+/**
+ * The default set of suites (DEC-019). `suites` are the suite ids in catalog order; `completedBy` gives, for each,
+ * the configs that have a valid score on every task of it. Start with every suite and drop the suite the fewest
+ * configs have completed (on a tie, the one listed last), until at least two configs have completed all that
+ * remain. When no set has two, keep the largest set that one config has completed (on a tie, the set that comes
+ * first in catalog order). When no config has completed any suite, keep every suite. The result is in catalog order, and the same input always gives the same set.
+ */
+export function defaultSuites(suites: string[], completedBy: Map<string, Set<string>>): string[] {
+  const done = (id: string) => completedBy.get(id) ?? new Set<string>();
+  const completers = (ids: string[]) => {
+    if (!ids.length) return 0;
+    let all = new Set(done(ids[0]!));
+    for (const id of ids.slice(1)) all = new Set([...all].filter((c) => done(id).has(c)));
+    return all.size;
+  };
+  const left = [...suites];
+  while (left.length && completers(left) < 2) {
+    let drop = left[0]!;
+    for (const id of left) if (done(id).size <= done(drop).size) drop = id;   // <=: on a tie the later one
+    left.splice(left.indexOf(drop), 1);
+  }
+  if (left.length && completers(left) >= 2) return left;
+
+  // No set has two configs: the largest set one config has completed.
+  const configs = [...new Set([...completedBy.values()].flatMap((s) => [...s]))].sort();
+  const pos = (ids: string[]) => ids.map((id) => suites.indexOf(id));
+  const earlier = (a: string[], b: string[]) => { const pa = pos(a), pb = pos(b); for (let i = 0; i < pa.length; i++) if (pa[i] !== pb[i]) return pa[i]! < pb[i]!; return false; };
+  let best: string[] = [];
+  for (const c of configs) {
+    const set = suites.filter((id) => done(id).has(c));
+    if (set.length > best.length || (set.length === best.length && earlier(set, best))) best = set;
+  }
+  return best.length ? best : [...suites];
+}
+
+const list = (names: string[]) => names.length <= 1 ? names.join("") : `${names.slice(0, -1).join(", ")} and ${names.at(-1)}`;
+
+/**
+ * The one line a view gives about its index: which suites it covers and, grouped by reason, which it leaves out
+ * and why. `judge` is the label of the judge in view. Null for a view of tasks.
+ */
+export function coverageLine(coverage: Coverage | null, judge: string): string | null {
+  if (!coverage) return null;
+  const parts = [`Covers ${list(coverage.covered.map((s) => s.title)) || "no suite"}.`];
+  const why: Record<LeftOutReason, (many: boolean) => string> = {
+    "none-completed": () => `no config has a score on every task under judge ${judge}`,
+    "too-few": (many) => `fewer than two configs have completed ${many ? "them" : "it"} with the suites covered`,
+    "not-picked": () => "not picked",
+  };
+  const groups = (["none-completed", "too-few", "not-picked"] as const)
+    .map((r) => [r, coverage.leftOut.filter((s) => s.reason === r)] as const).filter(([, s]) => s.length)
+    .map(([r, s]) => `${list(s.map((x) => x.title))} (${why[r](s.length > 1)})`);
+  if (groups.length) parts.push(`Left out: ${groups.join("; ")}.`);
+  if (!coverage.complete && coverage.covered.length) parts.push(`No config has a score on every task of ${coverage.covered.length > 1 ? "these suites" : "it"} yet.`);
+  return parts.join(" ");
+}