pi-goal-list-loop-audit 0.25.1 → 0.25.2

package/extensions/goal-loop-core.ts

@@ -157,6 +157,10 @@ export interface Goal {
   };
   createdAt: string;
   updatedAt: string;
+  /** v0.25.2: per-goal telemetry for /glla stats premature-success
+   * detection. Bumped live: turns on agent_end, fileWrites/bashCalls on
+   * tool_result while the goal is active. */
+  telemetry?: { turns: number; fileWrites: number; bashCalls: number };
 }
 
 /**

package/extensions/goal-loop-stats.ts

@@ -0,0 +1,273 @@
+// pi-goal-list-loop-audit — v0.25.2
+// extensions/goal-loop-stats.ts
+//
+// /glla stats: per-project ledger rollups. Scans .pi-glla/active.jsonl
+// across every project on the rig and produces the cross-project table
+// the spec-driven verifier (v0.25 design) will be hardened against.
+// Pure helpers take strings/paths so tests drive them from tmpdirs —
+// no dependencies beyond node stdlib (contract boundary).
+
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+
+export interface LedgerEntry {
+  type: string;
+  at?: string;
+  value?: any;
+}
+
+export interface GoalTelemetry {
+  turns: number;
+  fileWrites: number;
+  bashCalls: number;
+}
+
+/** Minimal per-goal shape the rollup reasons about (the ledger's `state`
+ * snapshots carry the full goal object; archived goals keep their last). */
+export interface GoalRollupSource {
+  id: string;
+  status?: string;
+  createdAt?: string;
+  updatedAt?: string;
+  usage?: { tokensUsed?: number };
+  auditHistory?: Array<{ approved?: boolean; disapproved?: boolean; error?: string }>;
+  telemetry?: GoalTelemetry;
+}
+
+export interface ProjectRollup {
+  project: string;
+  goalsCreated: number;
+  auditsApproved: number;
+  auditsDisapproved: number;
+  auditsError: number;
+  avgTurns: number;
+  avgWrites: number;
+  prematureCount: number;
+  /** Total token usage across goals (cost in tokens — no price data on
+   * this rig; documented in INSTALL.md). */
+  totalCost: number;
+  lastActive: string;
+}
+
+/** Premature-success thresholds (spec-driven verifier design §3): an
+ * approved goal with almost no turns, no real editing, and no
+ * verification commands is a "claimed done in 12 turns with 0 file
+ * writes" pattern the auditor should have caught. */
+export const PREMATURE_THRESHOLDS = {
+  maxTurns: 50,
+  maxFileWrites: 5,
+  maxBashCalls: 8,
+} as const;
+
+export function parseLedgerEntries(jsonl: string): LedgerEntry[] {
+  const out: LedgerEntry[] = [];
+  for (const line of jsonl.split("\n")) {
+    const t = line.trim();
+    if (!t) continue;
+    try {
+      const e = JSON.parse(t);
+      if (e && typeof e === "object" && typeof e.type === "string") out.push(e as LedgerEntry);
+    } catch {
+      /* malformed line — skip */
+    }
+  }
+  return out;
+}
+
+/** Flag the "approved too easily" pattern. Goals without telemetry
+ * (archived before v0.25.2) are UNKNOWN, not premature — we do not
+ * back-convict historical goals on missing data. */
+export function detectPrematureSuccess(goal: GoalRollupSource): boolean {
+  const audits = goal.auditHistory ?? [];
+  const approved = audits.filter((a) => a.approved).length;
+  if (approved === 0) return false;
+  const t = goal.telemetry;
+  if (!t) return false;
+  return (
+    t.turns < PREMATURE_THRESHOLDS.maxTurns &&
+    t.fileWrites < PREMATURE_THRESHOLDS.maxFileWrites &&
+    t.bashCalls < PREMATURE_THRESHOLDS.maxBashCalls
+  );
+}
+
+/** Roll up one project's ledger. Pure over the parsed entries — the file
+ * read happens in rollupProject. */
+export function rollupEntries(project: string, entries: LedgerEntry[]): ProjectRollup {
+  let goalsCreated = 0;
+  let lastActive = "";
+  const finalGoal = new Map<string, GoalRollupSource>();
+  for (const e of entries) {
+    if (e.at && e.at > lastActive) lastActive = e.at;
+    if (e.type === "goal_created") goalsCreated++;
+    if (e.type === "state" && e.value?.goal?.id) {
+      finalGoal.set(String(e.value.goal.id), e.value.goal as GoalRollupSource);
+    }
+  }
+  let auditsApproved = 0;
+  let auditsDisapproved = 0;
+  let auditsError = 0;
+  let prematureCount = 0;
+  let totalCost = 0;
+  let turnsSum = 0;
+  let turnsN = 0;
+  let writesSum = 0;
+  let writesN = 0;
+  for (const goal of finalGoal.values()) {
+    for (const a of goal.auditHistory ?? []) {
+      if (a.approved) auditsApproved++;
+      else if (a.disapproved) auditsDisapproved++;
+      else if (a.error) auditsError++;
+    }
+    if (detectPrematureSuccess(goal)) prematureCount++;
+    totalCost += goal.usage?.tokensUsed ?? 0;
+    if (goal.telemetry) {
+      turnsSum += goal.telemetry.turns;
+      turnsN++;
+      writesSum += goal.telemetry.fileWrites;
+      writesN++;
+    }
+  }
+  return {
+    project,
+    goalsCreated,
+    auditsApproved,
+    auditsDisapproved,
+    auditsError,
+    avgTurns: turnsN > 0 ? Math.round((turnsSum / turnsN) * 10) / 10 : 0,
+    avgWrites: writesN > 0 ? Math.round((writesSum / writesN) * 10) / 10 : 0,
+    prematureCount,
+    totalCost,
+    lastActive,
+  };
+}
+
+export function rollupProject(projectPath: string): ProjectRollup | undefined {
+  const ledger = path.join(projectPath, ".pi-glla", "active.jsonl");
+  let raw: string;
+  try {
+    raw = fs.readFileSync(ledger, "utf-8");
+  } catch {
+    return undefined;
+  }
+  return rollupEntries(projectPath, parseLedgerEntries(raw));
+}
+
+/** Project discovery (contract item 6). Sources:
+ *  1. ~/.pi/agent/sessions/ — session dir names encode their cwd
+ *     (`--home-dracon-chat-` ≈ /home/dracon/chat); cheap, no file scans.
+ *  2. A bounded walk under ~ (maxdepth 6, pruning node_modules/.git and
+ *     hidden dirs) for .pi-glla/ directories — catches projects no
+ *     session has visited. (Deviation from the contract's one-level walk:
+ *     one level would miss the very projects cited — polis is depth 5.)
+ *  3. The current cwd.
+ * Only roots with a real .pi-glla/active.jsonl survive. Budget-guarded:
+ * the walk stops after `budgetMs`. */
+export function discoverGllaProjects(opts: { home?: string; cwd?: string; budgetMs?: number } = {}): string[] {
+  const home = opts.home ?? os.homedir();
+  const cwd = opts.cwd ?? process.cwd();
+  const budgetMs = opts.budgetMs ?? 2000;
+  const deadline = Date.now() + budgetMs;
+  const found = new Set<string>();
+
+  const hasLedger = (dir: string): boolean => {
+    try {
+      fs.accessSync(path.join(dir, ".pi-glla", "active.jsonl"));
+      return true;
+    } catch {
+      return false;
+    }
+  };
+
+  // Source 1: session dir names.
+  try {
+    const sessionsDir = path.join(home, ".pi", "agent", "sessions");
+    for (const name of fs.readdirSync(sessionsDir)) {
+      // --home-dracon-chat- → /home/dracon/chat (best-effort decode:
+      // strip leading/trailing dashes, then replace -- → /- and - → /).
+      const decoded = name.replace(/^-+|-+$/g, "").replace(/--/g, "\0").replace(/-/g, "/").replace(/\0/g, "-");
+      const candidate = "/" + decoded;
+      if (hasLedger(candidate)) found.add(candidate);
+    }
+  } catch {
+    /* no sessions dir */
+  }
+
+  // Source 2: bounded walk — targeted roots FIRST (~/Dev, ~/chat hold the
+  // rig's projects; polis sits at depth 5), the general home walk last
+  // with whatever budget remains. Deep/wide dirs that never hold projects
+  // are pruned.
+  const PRUNE = new Set(["node_modules", ".git", ".pi", ".cache", ".npm", ".local", ".config", "Downloads", "Pictures", "Videos", "Music", ".mozilla", ".vscode", "snap", ".steam", ".wine"]);
+  const walk = (dir: string, depth: number): void => {
+    if (depth > 6 || Date.now() > deadline) return;
+    if (hasLedger(dir)) {
+      found.add(dir);
+      return; // no nested projects below a project root
+    }
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      if (Date.now() > deadline) return;
+      if (!e.isDirectory()) continue;
+      if (PRUNE.has(e.name)) continue;
+      walk(path.join(dir, e.name), depth + 1);
+    }
+  };
+  for (const root of [path.join(home, "Dev"), path.join(home, "chat")]) {
+    if (Date.now() > deadline) break;
+    walk(root, 1);
+  }
+  walk(home, 0);
+
+  // Source 3: cwd.
+  if (hasLedger(cwd)) found.add(cwd);
+
+  return [...found].sort();
+}
+
+/** Contract item 4: premature filter — only projects with
+ * premature_count > 0, sorted by premature ratio descending. */
+export function filterPremature(rollups: ProjectRollup[]): ProjectRollup[] {
+  return rollups
+    .filter((r) => r.prematureCount > 0)
+    .sort((a, b) => b.prematureCount / Math.max(1, b.goalsCreated) - a.prematureCount / Math.max(1, a.goalsCreated));
+}
+
+function shortProject(p: string): string {
+  const home = os.homedir();
+  return p.startsWith(home) ? `~${p.slice(home.length)}` : p;
+}
+
+export function formatRollupTable(rollups: ProjectRollup[]): string {
+  const header = "| project | goals | approved | disapproved | errors | avg turns | avg writes | premature | tokens | last active |";
+  const sep = "|---|---|---|---|---|---|---|---|---|---|";
+  const rows = rollups.map(
+    (r) =>
+      `| ${shortProject(r.project)} | ${r.goalsCreated} | ${r.auditsApproved} | ${r.auditsDisapproved} | ${r.auditsError} | ${r.avgTurns} | ${r.avgWrites} | ${r.prematureCount} | ${r.totalCost.toLocaleString()} | ${r.lastActive ? r.lastActive.slice(0, 10) : "—"} |`,
+  );
+  return [header, sep, ...rows].join("\n");
+}
+
+/** JSON schema matches the table exactly (contract item 2). */
+export function formatRollupJson(rollups: ProjectRollup[]): string {
+  return JSON.stringify(
+    rollups.map((r) => ({
+      project: r.project,
+      goals_created: r.goalsCreated,
+      audits_approved: r.auditsApproved,
+      audits_disapproved: r.auditsDisapproved,
+      audits_error: r.auditsError,
+      avg_turns: r.avgTurns,
+      avg_writes: r.avgWrites,
+      premature_count: r.prematureCount,
+      total_cost: r.totalCost,
+      last_active: r.lastActive || null,
+    })),
+    null,
+    2,
+  );
+}

package/extensions/loops/goal.ts

@@ -90,6 +90,14 @@ import {
   settingsProvenance,
   type Settings,
 } from "../goal-settings.js";
+import {
+  discoverGllaProjects,
+  filterPremature,
+  formatRollupJson,
+  formatRollupTable,
+  rollupProject,
+  type ProjectRollup,
+} from "../goal-loop-stats.js";
 import { runGoalCompletionAuditor } from "../goal-loop-auditor.js";
 import {
   REPETITION,
@@ -2667,6 +2675,42 @@ async function openSettingsUI(ctx: ExtensionContext): Promise<void> {
   }
 }
 
+/**
+ * v0.25.2: /glla stats — one command, every project's rollup. Args:
+ *   (none)            markdown table, all discovered projects
+ *   json              machine-readable rollup (same schema as the table)
+ *   premature         only projects with premature_success > 0, ratio-sorted
+ *   project=<path>    limit the scan to one project
+ */
+function cmdStats(args: string, ctx: ExtensionContext): void {
+  const asJson = /\bjson\b/.test(args);
+  const prematureOnly = /\bpremature\b/.test(args);
+  const projectMatch = args.match(/project=(\S+)/);
+  let rollups: ProjectRollup[] = [];
+  if (projectMatch) {
+    const p = projectMatch[1]!.replace(/^~/, os.homedir());
+    const r = rollupProject(p);
+    if (!r) {
+      ctx.ui.notify(`/glla stats: no .pi-glla/active.jsonl under ${p}`, "warning");
+      return;
+    }
+    rollups = [r];
+  } else {
+    const projects = discoverGllaProjects({ cwd: ctx.cwd });
+    for (const p of projects) {
+      const r = rollupProject(p);
+      if (r) rollups.push(r);
+    }
+    if (rollups.length === 0) {
+      ctx.ui.notify("/glla stats: no projects with .pi-glla/active.jsonl found on this rig.", "info");
+      return;
+    }
+  }
+  if (prematureOnly) rollups = filterPremature(rollups);
+  const out = asJson ? formatRollupJson(rollups) : formatRollupTable(rollups);
+  ctx.ui.notify(`glla stats — ${rollups.length} project(s)${prematureOnly ? " (premature filter)" : ""}\n${out}`, "info");
+}
+
 async function cmdSettings(args: string, ctx: ExtensionContext): Promise<void> {
   // The plugin's ONE config surface — global by default, rarely opened.
   //   /glla                      show effective values + where each comes from
@@ -2678,7 +2722,13 @@ async function cmdSettings(args: string, ctx: ExtensionContext): Promise<void> {
   //   /glla auditfeedbackchars=800 cap executor-visible disapproval report (0=full, the default)
   //   /glla project model=...    write to PROJECT override (rare)
   //   /glla model=unset          remove key (from global; project model=unset for project)
+  //   /glla stats [json|premature|project=<path>]   per-project ledger rollups (v0.25.2)
   const trimmed = args.trim();
+  // v0.25.2: /glla stats sub-mode — cross-project telemetry rollups.
+  if (/^stats\b/.test(trimmed)) {
+    cmdStats(trimmed.slice("stats".length).trim(), ctx);
+    return;
+  }
   if (!trimmed) {
     if (ctx.hasUI) {
       await openSettingsUI(ctx);
@@ -2975,6 +3025,7 @@ export default function (pi: ExtensionAPI): void {
       ["aggressivemode=", "on: keep-going defaults — autoResume, cap 10, stuck 10, wedge off, quota auto-retry, cap→TODOs"],
       ["quotaretryminutes=", "N: minutes before auto-retrying a quota-exhausted auditor (default 60)"],
       ["stuckmax=", "N: consecutive stuck interventions before a loop stops (default 5)"],
+      ["stats", "per-project ledger rollups: /glla stats [json|premature|project=<path>]"],
       ["autoaccept=", "on: drafts activate without the Confirm dialog (unattended rigs)"],
       ["project", "write a project override: /glla project key=value"],
     ]),
@@ -3066,6 +3117,16 @@ export default function (pi: ExtensionAPI): void {
         loop.iterMetrics = metrics;
       }
     }
+    // v0.25.2: per-goal tool telemetry (/glla stats premature detection).
+    if (state.goal && state.goal.status === "active") {
+      const toolName = String(event?.toolName ?? "");
+      if (isLoopWriteTool(toolName) || toolName === "bash") {
+        const t = state.goal.telemetry ?? { turns: 0, fileWrites: 0, bashCalls: 0 };
+        if (isLoopWriteTool(toolName)) t.fileWrites++;
+        if (toolName === "bash") t.bashCalls++;
+        state.goal.telemetry = t;
+      }
+    }
     if (draftingTarget === null) return;
     if (askUserQuestionAnswered(String(event?.toolName ?? ""), event?.details)) {
       draftingUserReplies++;
@@ -3184,6 +3245,12 @@ export default function (pi: ExtensionAPI): void {
     // continuation loop.
     if (isForeignCtx(ctx)) return;
     noteActivity();
+    // v0.25.2: per-goal turn telemetry (/glla stats).
+    if (state.goal && state.goal.status === "active") {
+      const t = state.goal.telemetry ?? { turns: 0, fileWrites: 0, bashCalls: 0 };
+      t.turns++;
+      state.goal.telemetry = t;
+    }
     if (!registeredCtx) {
       registerAgentTools(pi, ctx);
       registeredCtx = ctx;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-goal-list-loop-audit",
-  "version": "0.25.1",
+  "version": "0.25.2",
   "description": "Goal. Loop. Audit. Done. — a pi-coding-agent extension that supervises long-running work, with isolated auditor on each completion. Beat bamboozling by design: the auditor runs in a fresh session with no extensions, no skills, no editor — only the read tools needed to verify your goal.",
   "license": "MIT",
   "author": "dracon",