tickmarkr 2.7.4 → 2.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -111,7 +111,7 @@ Before \`tickmarkr compile\` or \`tickmarkr run\`: run \`tickmarkr version\`, re
111
111
 
112
112
  ### Tip-verify-before-green
113
113
 
114
- A run is green only when ALL of these hold: the run-end event exists in the journal, tip verify is not "failed", the run-end summary's \`failed\`, \`human\`, \`blocked\` and \`pending\` buckets are every one of them empty, AND the owed checks in CURRENT \`tickmarkr status <runId>\` are outstanding empty AND known (\`outstanding 0\`; \`outstanding unknown\` is never empty). The shorter "run-end plus tip verify" form is NOT sufficient: a run ending \`done=[T1,T3,T4] human=[T2] tipVerify=passed\` is a closed PARTIAL run — three of four delivered, one parked — and calling it green is how a park becomes invisible. Empty execution buckets are not sufficient either (D-660): a run ending with every execution bucket empty and \`tipVerify=passed\` after an operator waived one review still owes that accepted-risk review check — status reads \`outstanding 1 (T7 review)\`, and \`tickmarkr run\` and \`resume\` exit 0 on execution alone — so it is execution complete, not green. Read the debt from CURRENT status, never from the run-end record: a \`tickmarkr verify --record <runId>\` discharge that lands after run-end moves current status to \`outstanding 0\` while the historical run-end row still carries 1, and only then is that run green; a legacy waiver without a recorded obligation reads \`outstanding unknown\` and is never green. Never report green to the operator, tab titles, or records until every clause holds; when one does not, name the bucket that is not empty, the outstanding checks, or the unknown debt.
114
+ A run is green only when ALL of these hold: the run-end event exists in the journal and is not fatal, tip verify is not "failed", the run-end summary's \`failed\`, \`human\`, \`blocked\` and \`pending\` buckets are every one of them empty, AND the owed checks in CURRENT \`tickmarkr status <runId>\` are outstanding empty AND known (\`outstanding 0\`; \`outstanding unknown\` is never empty). The shorter "run-end plus tip verify" form is NOT sufficient: a run ending \`done=[T1,T3,T4] human=[T2] tipVerify=passed\` is a closed PARTIAL run — three of four delivered, one parked — and calling it green is how a park becomes invisible. Empty execution buckets are not sufficient either (D-660): a run ending with every execution bucket empty and \`tipVerify=passed\` after an operator waived one review still owes that accepted-risk review check — status reads \`outstanding 1 (T7 review)\`, and \`tickmarkr run\` and \`resume\` exit 0 on execution alone — so it is execution complete, not green. Read the debt from CURRENT status, never from the run-end record: a \`tickmarkr verify --record <runId>\` discharge that lands after run-end moves current status to \`outstanding 0\` while the historical run-end row still carries 1, and only then is that run green; a legacy waiver without a recorded obligation reads \`outstanding unknown\` and is never green. A fatal run-end is never green either: the crash record (\`fatal: true\`) names its phase and error, every task it left running or gated sits in \`pending\` or \`blocked\`, and even with every bucket empty status and report lead with the crash and read \`outstanding unknown\` until the run is resumed; from then on the resumed engagement owns the debt. Never report green to the operator, tab titles, or records until every clause holds; when one does not, name the bucket that is not empty, the outstanding checks, or the unknown debt.
115
115
 
116
116
  ### Verified handoffs
117
117
 
@@ -11,7 +11,7 @@ import { compareRuns } from "../../report/compare.js";
11
11
  import { estimateCosts } from "../../report/cost.js";
12
12
  import { assignmentChannel, buildOperatorRecord, formatChannelMoney, formatChannelTokens, formatTokenUsage as fields, totalTokens as total, } from "../../report/operator-record.js";
13
13
  import { cellsOf, cellSummary } from "../../route/profile.js";
14
- import { effectiveEvents, foldOwedChecks, Journal, loadRoutingProfile, OWED_DISCHARGE_EVENT, } from "../../run/journal.js";
14
+ import { effectiveEvents, fatalRunEndCause, foldOwedChecks, Journal, loadRoutingProfile, OWED_DISCHARGE_EVENT, } from "../../run/journal.js";
15
15
  import { baselineProvenanceOf, fingerprintsOf, forgivenFingerprints, forgivenGateRow, formatBaselineProvenance, formatFingerprints, formatForgiven, formatTipProof, runEndTipProof, } from "../../run/daemon.js";
16
16
  import { formatSpan, WALL_PRIORITY, WALL_PRIORITY_TEXT, wallBudget, wallBudgetFacts } from "../../run/wall-budget.js";
17
17
  import { deriveRunCockpitData } from "../../tui/cockpit/derive.js";
@@ -241,8 +241,10 @@ const timeLead = (events) => {
241
241
  * comparability or supersession note); the rest is folded here from the journal and the current fold.
242
242
  */
243
243
  export function leadLines(events, owed, finished) {
244
+ // A fatal run-end leads with its crash: no tally after it may read as a finished run.
245
+ const crash = fatalRunEndCause(events);
244
246
  return [
245
- `finished ${[finished.tally, finished.tip, firstPassText(endToEndFirstPass(events)), ...(finished.notes ?? [])].join(" · ")}`,
247
+ `finished ${[...(crash ? [`run crashed — ${crash}`] : []), finished.tally, finished.tip, firstPassText(endToEndFirstPass(events)), ...(finished.notes ?? [])].join(" · ")}`,
246
248
  timeLead(events),
247
249
  needsYouText(events, owed),
248
250
  ];
@@ -13,7 +13,7 @@ import { projectOperatorSummary } from "../../run/operator-summary.js";
13
13
  import { trackJournalRows } from "../../run/protocol.js";
14
14
  import { newestPark, permittedDecisionVerbs } from "./approve.js";
15
15
  import { currentOwed, endToEndFirstPass, firstPassText, leadLines, leadRows, needsYouText, wallText } from "./report.js";
16
- import { Journal, bindingToken, effectiveEvents, formatJournalNarration, parseJournalText, physicalLine, engagementComparable, isQualityFailureParkKind, parseRunId, preservedRefsByTask, recordedTaskFailureKind, runHasEnded, upheldFeedbackByTask, } from "../../run/journal.js";
16
+ import { Journal, bindingToken, effectiveEvents, formatJournalNarration, parseJournalText, physicalLine, engagementComparable, isQualityFailureParkKind, parseRunId, preservedRefsByTask, fatalRunEndCause, recordedTaskFailureKind, runHasEnded, upheldFeedbackByTask, visibleControls, } from "../../run/journal.js";
17
17
  import { isPidLive, runLockRunId, runStatusLine } from "../../run/lock.js";
18
18
  import { normalizeGateOutcome } from "../../run/outcome.js";
19
19
  import { desiredPanes } from "../../run/reconcile.js";
@@ -649,10 +649,11 @@ const terminalFailureCause = (events) => {
649
649
  const e = events[i];
650
650
  if (e.event !== "run-end" || typeof e.data.error !== "string")
651
651
  continue;
652
- const phase = typeof e.data.phase === "string" && e.data.phase.trim()
653
- ? `${e.data.phase.trim()} failed`
652
+ // the recorded error is data: its terminal controls print escaped, never executed (the journal keeps them)
653
+ const phase = typeof e.data.phase === "string" && visibleControls(e.data.phase)
654
+ ? `${visibleControls(e.data.phase)} failed`
654
655
  : "run failed";
655
- return `${phase}: ${e.data.error.replace(/\s+/g, " ").trim()}`;
656
+ return `${phase}: ${visibleControls(e.data.error)}`;
656
657
  }
657
658
  return undefined;
658
659
  };
@@ -1429,8 +1430,10 @@ const oneLine = (cwd, namedRunId) => {
1429
1430
  // CG1: the lead's facts in one line — lineage first pass, the wall window, and what needs you from
1430
1431
  // the CURRENT fold, so a discharge after run-end moves this line too.
1431
1432
  const { events } = record;
1433
+ // A fatal run-end leads right after the run id: no completion fraction or verify state reads first.
1434
+ const crash = fatalRunEndCause(events);
1432
1435
  return sanitizeTaskText([
1433
- record.runId, ...claims, firstPassText(endToEndFirstPass(events)), wallText(events), needsYouText(events, currentOwed(events, cwd)),
1436
+ record.runId, ...(crash ? [`run crashed — ${crash}`] : []), ...claims, firstPassText(endToEndFirstPass(events)), wallText(events), needsYouText(events, currentOwed(events, cwd)),
1434
1437
  ].join(" · "));
1435
1438
  };
1436
1439
  export async function status(argv, cwd = process.cwd(), opts = {}) {
@@ -15,7 +15,7 @@ import { GATE_NAMES } from "../../graph/schema.js";
15
15
  import { executionSignal } from "../../run/execution-budget.js";
16
16
  import { addDetachedWorktree, linkNodeModules, removeWorktree, shGit, shGitOk } from "../../run/git.js";
17
17
  import { foldOwedChecks, integrationMapped, owedCriteria, Journal, OWED_DISCHARGE_EVENT } from "../../run/journal.js";
18
- import { withRepositoryLease } from "../../run/lease.js";
18
+ import { describeLeaseWait, withRepositoryLease } from "../../run/lease.js";
19
19
  import { isPidLive } from "../../run/lock.js";
20
20
  import { recordLaunchServices } from "../../run/launchservices-check.js";
21
21
  /**
@@ -529,9 +529,9 @@ export async function verify(argv, cwd = process.cwd(), options = {}) {
529
529
  return after ? rows.map((r) => ({ ...r, pass: false, details: `${refused(after)}\n${r.details}`, meta: { ...r.meta, subjectChanged: after } })) : rows;
530
530
  }, {
531
531
  signal: executionSignal(),
532
- onWait: (holder) => {
533
- console.error(`verify: waiting for the repository's runner lease held by pid ${holder.pid} in ${holder.cwd}`);
534
- recordJournal?.append("suite-wait", values.task ?? "VERIFY", { holderPid: holder.pid, holderCwd: holder.cwd });
532
+ onWait: (holder, blockers) => {
533
+ console.error(`verify: waiting for the repository's runner lease ${describeLeaseWait(holder, blockers)}`);
534
+ recordJournal?.append("suite-wait", values.task ?? "VERIFY", { holderPid: holder.pid, holderCwd: holder.cwd, ...(blockers ? { blockers } : {}) });
535
535
  },
536
536
  });
537
537
  results = [...results, ...suite].sort((a, b) => GATE_NAMES.indexOf(a.gate) - GATE_NAMES.indexOf(b.gate));
@@ -1858,14 +1858,18 @@ export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "s
1858
1858
  catch (readErr) {
1859
1859
  console.error(`tickmarkr ${runId}: journal read failed while recording the fatal run-end (${readErr instanceof Error ? readErr.message : String(readErr)}) — original error: ${original}`);
1860
1860
  }
1861
+ // A crash can land mid-dispatch. Resume re-pends every dispatched-but-unfinished task, so a running or
1862
+ // gated task is bucketed as the pending task resume will make it — pending, or blocked under a parked
1863
+ // dependency — and no task falls outside the five buckets every reader knows.
1864
+ const repended = graph && { ...graph, tasks: graph.tasks.map((t) => t.status === "running" || t.status === "gated" ? { ...t, status: "pending" } : t) };
1861
1865
  const record = {
1862
1866
  runId,
1863
1867
  branch,
1864
1868
  done: graph?.tasks.filter((t) => t.status === "done").map((t) => t.id) ?? [],
1865
1869
  failed: graph?.tasks.filter((t) => t.status === "failed").map((t) => t.id) ?? [],
1866
1870
  human: graph?.tasks.filter((t) => t.status === "human").map((t) => t.id) ?? [],
1867
- blocked: graph ? blockedTasks(graph).map((t) => t.id) : [],
1868
- pending: graph ? pendingTasks(graph).map((t) => t.id) : [],
1871
+ blocked: repended ? blockedTasks(repended).map((t) => t.id) : [],
1872
+ pending: repended ? pendingTasks(repended).map((t) => t.id) : [],
1869
1873
  phase,
1870
1874
  fatal: true,
1871
1875
  error: original,
@@ -191,6 +191,13 @@ export declare const GATE_FINGERPRINT_CAP = 2;
191
191
  export declare const receiptOrigin: (data: Record<string, unknown>) => string | undefined;
192
192
  export declare function identicalGateFailures(events: JournalEvent[], taskId: string, gate: string, normalized: string): number;
193
193
  export declare const readableExcerpt: (text: string) => string;
194
+ /**
195
+ * One-line display text for a recorded crash: every C0 control but tab and newline, DEL and every C1
196
+ * control prints as its visible escape (`\x1b`, `\u009b`), so no 7-bit or 8-bit OSC/CSI/ST sequence
197
+ * reaches a terminal and the reader still sees the text carried one; tabs and line breaks collapse to
198
+ * one space. The journal row keeps the raw bytes.
199
+ */
200
+ export declare const visibleControls: (text: string) => string;
194
201
  /** What each funded repair's next battery actually reached. A repair spends its budget only
195
202
  * when that battery reaches one of the gates it was funded to fix; an earlier red is evidence that
196
203
  * this repair never got its funded turn, not a charge against the repair ladder. */
@@ -379,6 +386,11 @@ export interface InterruptedAttempt {
379
386
  }
380
387
  export declare function resumeHarvestAuthor(events: JournalEvent[], taskId: string): Assignment | undefined;
381
388
  export declare function interruptedAttempt(events: JournalEvent[], taskId: string): InterruptedAttempt | undefined;
389
+ /**
390
+ * The crash a FATAL run-end records, when it closes the newest engagement: `<phase> failed: <error>`.
391
+ * A later run-start or run-resume supersedes it; a normal run-end is no crash.
392
+ */
393
+ export declare function fatalRunEndCause(events: readonly JournalEvent[]): string | undefined;
382
394
  export declare function runHasEnded(events: JournalEvent[]): boolean;
383
395
  /** OBS-53: classify worker-result failures so retries and routing see the true signal, not one lumped bucket. */
384
396
  export declare function classifyWorkerResultCause(opts: {
@@ -763,6 +763,16 @@ export function identicalGateFailures(events, taskId, gate, normalized) {
763
763
  // the failure fingerprint keep the raw text.
764
764
  const TERMINAL_CONTROL_RE = /\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?|\x1b\[[0-?]*[ -/]*[@-~]|\x1b[ -/]*[0-~]?|[\x00-\x08\x0b-\x1f\x7f]/g;
765
765
  export const readableExcerpt = (text) => text.replace(TERMINAL_CONTROL_RE, "");
766
+ /**
767
+ * One-line display text for a recorded crash: every C0 control but tab and newline, DEL and every C1
768
+ * control prints as its visible escape (`\x1b`, `\u009b`), so no 7-bit or 8-bit OSC/CSI/ST sequence
769
+ * reaches a terminal and the reader still sees the text carried one; tabs and line breaks collapse to
770
+ * one space. The journal row keeps the raw bytes.
771
+ */
772
+ export const visibleControls = (text) => text.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/gu, (c) => {
773
+ const code = c.charCodeAt(0).toString(16);
774
+ return c < "\x80" ? `\\x${code.padStart(2, "0")}` : `\\u${code.padStart(4, "0")}`;
775
+ }).replace(/\s+/gu, " ").trim();
766
776
  export function repairReachSinceApproval(events, taskId) {
767
777
  const repairs = [];
768
778
  for (const e of effectiveEvents(events)) { // OBS-1178: a refused or unsound approval resets no repair history
@@ -1367,6 +1377,28 @@ export function interruptedAttempt(events, taskId) {
1367
1377
  }
1368
1378
  // Runs can end and later resume in the same journal. The newest lifecycle marker decides whether
1369
1379
  // an unresolved task is still recoverable by this live daemon or belongs to an ended run.
1380
+ /**
1381
+ * The crash a FATAL run-end records, when it closes the newest engagement: `<phase> failed: <error>`.
1382
+ * A later run-start or run-resume supersedes it; a normal run-end is no crash.
1383
+ */
1384
+ export function fatalRunEndCause(events) {
1385
+ for (let i = events.length - 1; i >= 0; i--) {
1386
+ const e = events[i];
1387
+ if (e.event === "run-start" || e.event === "run-resume")
1388
+ return undefined;
1389
+ if (e.event !== "run-end")
1390
+ continue;
1391
+ if (e.data.fatal !== true)
1392
+ return undefined;
1393
+ // Display text only: terminal controls in a recorded error (OSC title, CSI clear) print escaped,
1394
+ // never executed, and the journal row keeps the raw bytes.
1395
+ const shown = (v) => typeof v === "string" ? visibleControls(v) : "";
1396
+ const phase = shown(e.data.phase) || "run";
1397
+ const error = shown(e.data.error);
1398
+ return `${phase} failed: ${error || "unrecorded error"}`;
1399
+ }
1400
+ return undefined;
1401
+ }
1370
1402
  export function runHasEnded(events) {
1371
1403
  for (let i = events.length - 1; i >= 0; i--) {
1372
1404
  const event = events[i].event;
@@ -1943,7 +1975,9 @@ export class Journal {
1943
1975
  // C1: the terminal record carries the owed-check fold as of this append, revalidated from
1944
1976
  // each discharge's artifact — a reopened journal re-reduces, it never copies a prior total.
1945
1977
  // Canonical (validated) rows are strict and keep their schema; the live producer is the tuple.
1946
- return decisionRow ? reduced : { ...reduced, owedChecks: foldOwedChecks(priorEvents, join(this.dir, "..", "..", "..")) };
1978
+ // A fatal record folds over itself, so its own owedChecks already read unknown with the crash.
1979
+ const folded = !decisionRow && data.fatal === true ? [...priorEvents, { ts: new Date().toISOString(), event, data }] : priorEvents;
1980
+ return decisionRow ? reduced : { ...reduced, owedChecks: foldOwedChecks(folded, join(this.dir, "..", "..", "..")) };
1947
1981
  })()
1948
1982
  : event === "resume-restore" && rowTaskId && upheldFeedbackByTask(effectiveEvents(priorEvents)).has(rowTaskId)
1949
1983
  ? {
@@ -2701,12 +2735,16 @@ function dischargeProblem(row, o, events, cwd, memo) {
2701
2735
  * across folds; every artifact, hash, criteria and reviewer condition is still re-read on every fold.
2702
2736
  */
2703
2737
  export function foldOwedChecks(events, cwd, memo) {
2738
+ let fold;
2704
2739
  try {
2705
- return foldOwedChecksOrThrow(events, cwd, memo);
2740
+ fold = foldOwedChecksOrThrow(events, cwd, memo);
2706
2741
  }
2707
2742
  catch (error) {
2708
- return { known: false, debt: "unknown", outstanding: [], acceptedRisk: [], discharged: [], unknown: [{ reason: `owed-check fold failed: ${String(error).split("\n")[0]}` }] };
2743
+ fold = { known: false, debt: "unknown", outstanding: [], acceptedRisk: [], discharged: [], unknown: [{ reason: `owed-check fold failed: ${String(error).split("\n")[0]}` }] };
2709
2744
  }
2745
+ // A fatal run-end never finished its checks: its debt is unknown and the crash leads the reasons, never 0.
2746
+ const fatal = fatalRunEndCause(events);
2747
+ return fatal ? { ...fold, known: false, debt: "unknown", unknown: [{ reason: `fatal run-end — ${fatal}` }, ...fold.unknown] } : fold;
2710
2748
  }
2711
2749
  function foldOwedChecksOrThrow(events, cwd, memo) {
2712
2750
  // v2.6.7 T4 (G2): checkout trust is re-derived at the fold boundary, before a memoised proof can answer —
@@ -35,11 +35,19 @@ export interface RepositoryLeaseHolder {
35
35
  roots?: number[];
36
36
  rootBirths?: Record<string, string>;
37
37
  }
38
+ /** Queue row 120: a live process keeping a dead or released holder's lease, as the wait names it: pid, executable and
39
+ * cwd. Its arguments and environment are never printed or read (`ps -o comm=`, the census's cwd-only probe). */
40
+ export interface LeaseBlocker {
41
+ pid: number;
42
+ executable: string;
43
+ cwd?: string;
44
+ }
38
45
  export interface RepositoryLeaseOptions {
39
46
  pollMs?: number;
40
47
  signal?: AbortSignal;
41
- /** Called once per distinct holder the waiter is queued behind. */
42
- onWait?: (holder: RepositoryLeaseHolder) => void;
48
+ /** Called once per distinct holder the waiter is queued behind, and again whenever the set of processes keeping a
49
+ * dead holder's lease changes; `blockers` names that set (queue row 120). */
50
+ onWait?: (holder: RepositoryLeaseHolder, blockers?: readonly LeaseBlocker[]) => void;
43
51
  /** The token a descendant inherited; defaults to REPOSITORY_LEASE_TOKEN_ENV. */
44
52
  inherited?: string;
45
53
  /** Manifest jobs bridge capabilities through explicit child environments, never process.env. */
@@ -82,6 +90,15 @@ export declare const readHolder: (path: string) => RepositoryLeaseHolder | undef
82
90
  export declare function tryReserve(path: string, mine: RepositoryLeaseHolder): boolean;
83
91
  /** Exposed for recovery regressions; the standalone verify uses the combined critical section. */
84
92
  export declare const reclaimDead: (path: string, identity: string, pollMs?: number) => Promise<boolean>;
93
+ /** Each blocker's pid, executable and working directory, and NO argument: `ps -o comm=` (a command line joins argv with
94
+ * spaces, so no redaction could tell where a secret value ends, D-1710). The cwd comes from the census's cwd-only probe
95
+ * (async, batched, never rejecting, no environment read; queue row 103: no synchronous lsof on a waiting loop), imported
96
+ * lazily because suite-census imports this module. Diagnostics only: an unreadable process reads "(executable
97
+ * unreadable)", and this never throws into the wait. */
98
+ export declare function describeLeaseBlockers(pids: readonly number[]): Promise<LeaseBlocker[]>;
99
+ /** The wait's own words: the holder alone while it lives; once it has exited or released the lease, every process still
100
+ * keeping it. */
101
+ export declare function describeLeaseWait(holder: RepositoryLeaseHolder, blockers?: readonly LeaseBlocker[]): string;
85
102
  export declare function protectedProcesses(holder: RepositoryLeaseHolder, tokenEnv?: string, ownedOnly?: boolean): Promise<number[]>;
86
103
  export declare const repositoryLeaseEnvironment: (env: NodeJS.ProcessEnv) => NodeJS.ProcessEnv;
87
104
  /** Called synchronously at spawn, before any user callback, and only on an owned generation. */
package/dist/run/lease.js CHANGED
@@ -305,14 +305,47 @@ const reclaimDeadLocked = (path, identity) => {
305
305
  export const reclaimDead = async (path, identity, pollMs = 10) => withMutationLock(path, pollMs, undefined, () => reclaimDeadLocked(path, identity));
306
306
  const inspectAndReserve = async (path, mine) => {
307
307
  let occupant = readOccupant(path);
308
- if (occupant && (!occupant.holder || (!holderAlive(occupant.holder) && !(occupant.holder.protectOrphans && (await protectedProcesses(occupant.holder)).length)))) {
308
+ const dead = occupant?.holder !== undefined && !holderAlive(occupant.holder);
309
+ // A dead holder's protected tree is what keeps the lease; the waiter is told which processes those are (queue row 120).
310
+ const blocking = dead && occupant.holder.protectOrphans ? await protectedProcesses(occupant.holder) : [];
311
+ if (occupant && (!occupant.holder || (dead && !blocking.length))) {
309
312
  reclaimDeadLocked(path, occupant.identity);
310
313
  occupant = readOccupant(path);
311
314
  }
312
315
  if (!occupant && tryReserve(path, mine))
313
316
  return { acquired: true };
314
- return { acquired: false, holder: occupant?.holder };
317
+ return { acquired: false, holder: occupant?.holder, ...(blocking.length ? { blocking } : {}) };
315
318
  };
319
+ /** Each blocker's pid, executable and working directory, and NO argument: `ps -o comm=` (a command line joins argv with
320
+ * spaces, so no redaction could tell where a secret value ends, D-1710). The cwd comes from the census's cwd-only probe
321
+ * (async, batched, never rejecting, no environment read; queue row 103: no synchronous lsof on a waiting loop), imported
322
+ * lazily because suite-census imports this module. Diagnostics only: an unreadable process reads "(executable
323
+ * unreadable)", and this never throws into the wait. */
324
+ export async function describeLeaseBlockers(pids) {
325
+ const rows = await new Promise((done) => {
326
+ try {
327
+ execFile("ps", ["-o", "pid=,comm=", "-p", pids.join(",")], { encoding: "utf8", timeout: 5_000 }, (_error, stdout) => done(stdout ?? ""));
328
+ }
329
+ catch {
330
+ done("");
331
+ }
332
+ });
333
+ const executables = new Map(rows.split("\n").flatMap((line) => { const row = /^\s*(\d+)\s+(.*)$/.exec(line); return row ? [[Number(row[1]), row[2].trim()]] : []; }));
334
+ const cwds = await import("./suite-census.js").then(({ batchedProcessCwds }) => batchedProcessCwds(pids)).catch(() => undefined);
335
+ return pids.map((pid) => {
336
+ const cwd = cwds?.get(pid);
337
+ return { pid, executable: executables.get(pid) || "(executable unreadable)", ...(cwd ? { cwd } : {}) };
338
+ });
339
+ }
340
+ /** The wait's own words: the holder alone while it lives; once it has exited or released the lease, every process still
341
+ * keeping it. */
342
+ export function describeLeaseWait(holder, blockers) {
343
+ const held = `held by pid ${holder.pid} in ${holder.cwd}`;
344
+ if (!blockers?.length)
345
+ return held;
346
+ const named = blockers.map((b) => `pid ${b.pid} ${b.executable}${b.cwd ? ` (cwd ${b.cwd})` : ""}`).join("; ");
347
+ return `${held}, which has exited or released it; ${blockers.length} process(es) it started still run and keep the lease until they exit: ${named}`;
348
+ }
316
349
  // A durable capability is also a crash owner: census checks the exact generation's inherited
317
350
  // environment and recorded process groups, including live orphans reparented after owner death.
318
351
  // An unreadable census refuses reclamation/release. No dead-pid shortcut admits over a live tree.
@@ -386,6 +419,8 @@ export function registerRepositoryChild(pid) {
386
419
  }
387
420
  export const withFreshCommandLease = (command, run) => ownership.run({ token: "", active: false }, () => withCommandLease(command, run));
388
421
  const repositoryQueues = new Map();
422
+ /** Queue row 120: the queue head's last view of what keeps a dead holder's lease, so in-process followers name it too. */
423
+ const repositoryBlocking = new Map();
389
424
  /** The standalone wrapper uses the same command-then-repository order when a scheduler exists.
390
425
  * Without one (CLI/global setup), this remains exactly the outer file reservation API. */
391
426
  export function withRepositoryLease(cwd, run, opts = {}) {
@@ -426,10 +461,11 @@ async function repositoryLease(cwd, run, opts) {
426
461
  const ticket = Symbol("repository job");
427
462
  queue.add(ticket);
428
463
  let waitingOn;
429
- const waitOn = (holder) => {
430
- if (holder && holder.token !== waitingOn) {
431
- waitingOn = holder.token;
432
- opts.onWait?.(holder);
464
+ const waitOn = async (holder, blocking = []) => {
465
+ const key = holder && `${holder.token}\0${[...blocking].sort((a, b) => a - b).join(",")}`;
466
+ if (holder && key !== waitingOn) {
467
+ waitingOn = key;
468
+ opts.onWait?.(holder, blocking.length ? await describeLeaseBlockers(blocking) : undefined);
433
469
  }
434
470
  };
435
471
  try {
@@ -437,12 +473,19 @@ async function repositoryLease(cwd, run, opts) {
437
473
  opts.signal?.throwIfAborted();
438
474
  if (queue.values().next().value === ticket) {
439
475
  const attempt = await withMutationLock(path, pollMs, opts.signal, () => inspectAndReserve(path, mine));
440
- if (attempt.acquired)
476
+ if (attempt.acquired) {
477
+ repositoryBlocking.delete(path);
441
478
  break;
442
- waitOn(attempt.holder);
479
+ }
480
+ if (attempt.holder)
481
+ repositoryBlocking.set(path, { token: attempt.holder.token, blocking: attempt.blocking ?? [] });
482
+ await waitOn(attempt.holder, attempt.blocking);
483
+ }
484
+ else {
485
+ const holder = readHolder(path);
486
+ const seen = repositoryBlocking.get(path);
487
+ await waitOn(holder, holder && seen?.token === holder.token ? seen.blocking : []);
443
488
  }
444
- else
445
- waitOn(readHolder(path));
446
489
  await new Promise(wake => setTimeout(wake, pollMs));
447
490
  }
448
491
  const exported = process.env[REPOSITORY_LEASE_TOKEN_ENV];
@@ -495,8 +538,10 @@ async function repositoryLease(cwd, run, opts) {
495
538
  }
496
539
  finally {
497
540
  queue.delete(ticket);
498
- if (!queue.size)
541
+ if (!queue.size) {
499
542
  repositoryQueues.delete(path);
543
+ repositoryBlocking.delete(path);
544
+ }
500
545
  // A waiter removes no repository generation, including cancellation before acquisition.
501
546
  }
502
547
  }
@@ -6,6 +6,9 @@
6
6
  * not already probed — the census asks for a parent's cwd too. The answers feed countLiveSuites' existing lookups.
7
7
  */
8
8
  export type ProbeExec = (file: string, args: readonly string[]) => Promise<string>;
9
+ /** Each pid's working directory: /proc first, then ONE batched lsof for the rest; never rejects, and reads no process
10
+ * environment (the runner-lease wait names blockers through it, queue row 120). Answers land in `cwds`. */
11
+ export declare function batchedProcessCwds(pids: readonly number[], exec?: ProbeExec, proc?: ProcReader, cwds?: Map<number, string>): Promise<Map<number, string>>;
9
12
  /** The pids of every live (non-zombie) row of a `ps -o pid=,ppid=,state=,command=` snapshot whose command is a runner. */
10
13
  export declare function runnerPids(snapshot: string): number[];
11
14
  /** The /proc reads, injectable so the Linux branch is testable anywhere; each throws where /proc has no entry. */
@@ -16,6 +16,42 @@ const probeExec = (file, args) => new Promise((resolve) => {
16
16
  resolve("");
17
17
  }
18
18
  });
19
+ // D-1567 batch poisoning: lsof and ps fail the WHOLE command for one -p member they cannot take (a marker of 22 digits
20
+ // prints as 1e+21, 310 nines as Infinity; ps refuses one above int32), erasing every other pid's answer. The base probed
21
+ // per pid, so such a pid lost only its own answer: it stays unanswered here and never joins a batch.
22
+ const probeable = (pid) => Number.isSafeInteger(pid) && pid > 0 && pid <= 2_147_483_647;
23
+ /** Each pid's working directory: /proc first, then ONE batched lsof for the rest; never rejects, and reads no process
24
+ * environment (the runner-lease wait names blockers through it, queue row 120). Answers land in `cwds`. */
25
+ export async function batchedProcessCwds(pids, exec = probeExec, proc = procReader, cwds = new Map()) {
26
+ const rest = [];
27
+ for (const pid of pids.filter(probeable)) {
28
+ try {
29
+ cwds.set(pid, proc.cwd(pid));
30
+ }
31
+ catch {
32
+ rest.push(pid);
33
+ } // Darwin has no /proc
34
+ }
35
+ if (rest.length === 0)
36
+ return cwds;
37
+ let listing = "";
38
+ try {
39
+ listing = await exec("lsof", ["-a", "-p", rest.join(","), "-d", "cwd", "-Fn"]);
40
+ }
41
+ catch { /* the census never rejects */ }
42
+ let pid;
43
+ for (const line of listing.split("\n")) {
44
+ if (line.startsWith("p"))
45
+ pid = Number(line.slice(1));
46
+ else if (line.startsWith("n") && pid !== undefined) {
47
+ try {
48
+ cwds.set(pid, realpathSync(line.slice(1)));
49
+ }
50
+ catch { /* the directory is gone */ }
51
+ }
52
+ }
53
+ return cwds;
54
+ }
19
55
  /** The pids of every live (non-zombie) row of a `ps -o pid=,ppid=,state=,command=` snapshot whose command is a runner. */
20
56
  export function runnerPids(snapshot) {
21
57
  const pids = [];
@@ -41,34 +77,7 @@ export async function batchedProcessProbes(pids, exec = probeExec, proc = procRe
41
77
  const cwds = new Map();
42
78
  const parents = new Map();
43
79
  const parentIn = (text) => new RegExp(`(?:^|\\s)${SUITE_PARENT_ENV}=(\\d+)(?:\\s|$)`).exec(text)?.[1];
44
- // D-1567 batch poisoning: lsof and ps fail the WHOLE command for one -p member they cannot take (a marker of 22 digits
45
- // prints as 1e+21, 310 nines as Infinity; ps refuses one above int32), erasing every other pid's answer. The base probed
46
- // per pid, so such a pid lost only its own answer: it stays unanswered here and never joins a batch.
47
- const probeable = (pid) => Number.isSafeInteger(pid) && pid > 0 && pid <= 2_147_483_647;
48
- const probeCwds = async (wanted) => {
49
- const rest = [];
50
- for (const pid of wanted.filter(probeable)) {
51
- try {
52
- cwds.set(pid, proc.cwd(pid));
53
- }
54
- catch {
55
- rest.push(pid);
56
- } // Darwin has no /proc
57
- }
58
- if (rest.length === 0)
59
- return;
60
- let pid;
61
- for (const line of (await ask("lsof", ["-a", "-p", rest.join(","), "-d", "cwd", "-Fn"])).split("\n")) {
62
- if (line.startsWith("p"))
63
- pid = Number(line.slice(1));
64
- else if (line.startsWith("n") && pid !== undefined) {
65
- try {
66
- cwds.set(pid, realpathSync(line.slice(1)));
67
- }
68
- catch { /* the directory is gone */ }
69
- }
70
- }
71
- };
80
+ const probeCwds = (wanted) => batchedProcessCwds(wanted, ask, proc, cwds);
72
81
  const probeParents = async (wanted) => {
73
82
  const rest = [];
74
83
  for (const pid of wanted.filter(probeable)) {
@@ -1,6 +1,6 @@
1
1
  import type { ShellReceipt } from "./protocol.js";
2
2
  import type { GateEvidenceOptions } from "../gates/baseline.js";
3
- import { type RepositoryLeaseHolder } from "./lease.js";
3
+ import { type LeaseBlocker, type RepositoryLeaseHolder } from "./lease.js";
4
4
  export interface VerificationPhase {
5
5
  phase: "discovery" | "first-pass" | "continuation-discovery" | "continuation";
6
6
  nonce: string;
@@ -33,7 +33,9 @@ export interface VerificationJobReport {
33
33
  releasedAt?: number;
34
34
  reservationReused?: boolean;
35
35
  reservation?: RepositoryLeaseHolder;
36
- waitingOn?: RepositoryLeaseHolder;
36
+ waitingOn?: RepositoryLeaseHolder & {
37
+ blockers?: readonly LeaseBlocker[];
38
+ };
37
39
  state: "queued" | "running" | "completed" | "failed" | "cancelled";
38
40
  phases: VerificationPhase[];
39
41
  }
@@ -132,7 +132,7 @@ export async function withVerificationJob(command, cwd, reportPath, evidence, ru
132
132
  }
133
133
  }));
134
134
  }, { isolated: true, protectOrphans: true, independent: !!jobs.getStore(), signal, pollMs: 20,
135
- onWait: holder => { report.waitingOn = { ...holder }; job.persist(); },
135
+ onWait: (holder, blockers) => { report.waitingOn = { ...holder, ...(blockers ? { blockers } : {}) }; job.persist(); },
136
136
  onAdmission: holder => { owner = holder; report.repositoryAdmittedAt = clock(); report.reservation = { ...holder }; job.persist(); },
137
137
  onRelease: () => { report.reapedAt ??= clock(); report.repositoryReleasedAt = clock(); job.persist(); } }));
138
138
  report.releasedAt = clock();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tickmarkr",
3
- "version": "2.7.4",
3
+ "version": "2.7.6",
4
4
  "description": "Spec in, verified work out.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -70,6 +70,18 @@ since_arm() { tail -n +$((base + 1)) "$1" 2>/dev/null; }
70
70
 
71
71
  field() { printf '%s' "$2" | sed -n "s/.*\"$1\":\"\([^\"]*\)\".*/\1/p" | head -1; }
72
72
  bucket() { printf '%s' "$2" | sed -n "s/.*\"$1\":\[\([^]]*\)\].*/\1/p" | head -1; }
73
+ # A row's data string value DECODED by JSON.parse (an escaped `\"` never cuts a quoted ref in a git error
74
+ # short), printed on one line as the product's own crash reason is (visibleControls): every C0 control
75
+ # but tab and newline, DEL and every C1 control prints as its visible escape (`\x1b`, `\u009b`), so a
76
+ # recorded 7-bit or 8-bit OSC/CSI/ST never reaches the terminal; tabs and line breaks collapse to a space.
77
+ jstr() {
78
+ printf '%s' "$2" | node -e '
79
+ let v; try { const row = JSON.parse(require("fs").readFileSync(0, "utf8")); v = (row.data ?? row)[process.argv[1]]; } catch {}
80
+ if (typeof v === "string") process.stdout.write(v.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/g, (c) => {
81
+ const h = c.charCodeAt(0).toString(16);
82
+ return c < "\x80" ? "\\x" + h.padStart(2, "0") : "\\u" + h.padStart(4, "0");
83
+ }).replace(/\s+/g, " ").trim());' "$1" 2>/dev/null
84
+ }
73
85
 
74
86
  report() {
75
87
  # $3 is the row's physical journal line: with its ts it is the token a decision binds to (OBS-1178)
@@ -77,8 +89,13 @@ report() {
77
89
  ev=$(field event "$line")
78
90
  case "$ev" in
79
91
  run-end)
80
- local tv done failed human blocked pending verdict
92
+ local tv done failed human blocked pending verdict fatal=""
81
93
  tv=$(field tipVerify "$line")
94
+ # A FATAL run-end is a crash, never green — even with every bucket empty (its tasks never finished).
95
+ case "$line" in *'"fatal":true'*)
96
+ local err; err=$(jstr error "$line")
97
+ fatal="$(jstr phase "$line") failed: ${err:0:160}" ;;
98
+ esac
82
99
  done=$(bucket done "$line"); failed=$(bucket failed "$line")
83
100
  human=$(bucket human "$line"); blocked=$(bucket blocked "$line")
84
101
  pending=$(bucket pending "$line")
@@ -87,16 +104,20 @@ report() {
87
104
  # clause open: a waived review is still owed (D-660), and a `verify --record` discharge that lands
88
105
  # AFTER run-end moves only CURRENT status. This row is history, so the best it can say is
89
106
  # EXECUTION COMPLETE; green is read from CURRENT `tickmarkr status <runId>` at `outstanding 0`.
90
- if [ "$tv" != "failed" ] && [ -z "$failed$human$blocked$pending" ]; then
107
+ if [ -z "$fatal" ] && [ "$tv" != "failed" ] && [ -z "$failed$human$blocked$pending" ]; then
91
108
  verdict="EXECUTION COMPLETE"
92
109
  else
93
110
  verdict="NOT GREEN"
94
111
  fi
95
- echo "RUN_END $run — $verdict (tipVerify=${tv:-unknown})"
112
+ echo "RUN_END $run — $verdict (${fatal:+fatal: $fatal; }tipVerify=${tv:-unknown})"
96
113
  echo " done=[${done}] failed=[${failed}] human=[${human}] blocked=[${blocked}] pending=[${pending}]"
97
- [ "$verdict" = "EXECUTION COMPLETE" ] \
98
- && echo " execution buckets empty and tip verify is not failed — not yet green: read CURRENT \`tickmarkr status $run\`; only \`outstanding 0\` is green, \`outstanding N (...)\` and \`outstanding unknown\` are not, and this run-end record's own count is historical" \
99
- || echo " a non-empty bucket or a failed tip above is the reason; name it, never report this run as green"
114
+ if [ -n "$fatal" ]; then
115
+ echo " the run CRASHED ($fatal) — a fatal run-end is never green; its debt reads unknown until the run is resumed, and from then on the resumed engagement owns it"
116
+ elif [ "$verdict" = "EXECUTION COMPLETE" ]; then
117
+ echo " execution buckets empty and tip verify is not failed — not yet green: read CURRENT \`tickmarkr status $run\`; only \`outstanding 0\` is green, \`outstanding N (...)\` and \`outstanding unknown\` are not, and this run-end record's own count is historical"
118
+ else
119
+ echo " a non-empty bucket or a failed tip above is the reason; name it, never report this run as green"
120
+ fi
100
121
  ;;
101
122
  task-human)
102
123
  echo "TASK_HUMAN $(field taskId "$line") — $run"
@@ -15,15 +15,23 @@
15
15
  # never hours; 900 is a generous default for a 7-task spec)
16
16
  # <overseer-pane> the pane that must hear about it (herdr pane id), e.g. wZ:p18S
17
17
  # [poll-s] poll interval, default 15
18
- # exit 0 LAUNCH_OK (lock seen; prints its contents) · exit 3 LAUNCH_OVERDUE (delivered) · exit 64 usage
18
+ # exit 0 LAUNCH_OK (a lock held by a LIVE pid; prints its contents) · exit 3 LAUNCH_OVERDUE (delivered) · exit 64 usage
19
+ #
20
+ # A lock is a launch only while its holder lives: a lock left by a crashed daemon (dead pid, or no pid
21
+ # at all) is stale, proves nothing about the new GO, and the watch keeps waiting for a live holder.
22
+ # `ps -p` answers for another user's process too, so EPERM never reads as death (lock.ts isPidLive).
19
23
  set -u
20
24
  LOCK="${1:-}"; DEADLINE="${2:-}"; PANE="${3:-}"; POLL="${4:-15}"
21
25
  [ -n "$LOCK" ] && [ -n "$DEADLINE" ] && [ -n "$PANE" ] || { echo "usage: watch-launch.sh <lock-path> <deadline-s> <overseer-pane> [poll-s]" >&2; exit 64; }
22
26
  start=$(date +%s)
23
27
  while :; do
24
28
  if [ -f "$LOCK" ]; then
25
- printf 'LAUNCH_OK %s %s\n' "$(date -u +%H:%M:%SZ)" "$(cat "$LOCK" 2>/dev/null | tr -d '\n')"
26
- exit 0
29
+ body=$(tr -d '\n' < "$LOCK" 2>/dev/null)
30
+ pid=$(printf '%s' "$body" | sed -n 's/.*"pid":\([0-9][0-9]*\).*/\1/p')
31
+ if [ -n "$pid" ] && ps -p "$pid" >/dev/null 2>&1; then
32
+ printf 'LAUNCH_OK %s %s\n' "$(date -u +%H:%M:%SZ)" "$body"
33
+ exit 0
34
+ fi
27
35
  fi
28
36
  now=$(date +%s)
29
37
  if [ $((now - start)) -ge "$DEADLINE" ]; then