tickmarkr 2.6.2 → 2.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/cli/commands/approve.d.ts +6 -3
- package/dist/cli/commands/approve.js +16 -4
- package/dist/cli/commands/doctor.d.ts +6 -2
- package/dist/cli/commands/fleet.js +46 -8
- package/dist/cli/commands/plan.js +13 -8
- package/dist/cli/commands/report.d.ts +2 -1
- package/dist/cli/commands/report.js +56 -6
- package/dist/cli/commands/status.js +19 -1
- package/dist/compile/native.js +7 -0
- package/dist/config/config.d.ts +17 -4
- package/dist/config/config.js +43 -6
- package/dist/config/fleet-overlay.d.ts +13 -3
- package/dist/config/fleet-overlay.js +12 -8
- package/dist/drivers/herdr.d.ts +12 -0
- package/dist/drivers/herdr.js +51 -0
- package/dist/drivers/orca.d.ts +9 -1
- package/dist/drivers/orca.js +29 -7
- package/dist/drivers/types.d.ts +2 -0
- package/dist/drivers/types.js +2 -2
- package/dist/gates/acceptance.d.ts +7 -0
- package/dist/gates/acceptance.js +27 -5
- package/dist/gates/baseline.d.ts +20 -1
- package/dist/gates/baseline.js +100 -20
- package/dist/gates/cache.d.ts +8 -0
- package/dist/gates/cache.js +12 -2
- package/dist/gates/llm.d.ts +6 -0
- package/dist/gates/llm.js +27 -8
- package/dist/gates/review.d.ts +6 -1
- package/dist/gates/review.js +122 -32
- package/dist/gates/run-gates.d.ts +54 -3
- package/dist/gates/run-gates.js +331 -45
- package/dist/gates/test-manifest.d.ts +42 -0
- package/dist/gates/test-manifest.js +69 -10
- package/dist/route/router.d.ts +12 -1
- package/dist/route/router.js +26 -9
- package/dist/run/consult.d.ts +3 -1
- package/dist/run/consult.js +4 -2
- package/dist/run/daemon.d.ts +2 -1
- package/dist/run/daemon.js +342 -77
- package/dist/run/interactive-seed.d.ts +4 -0
- package/dist/run/interactive-seed.js +35 -9
- package/dist/run/journal.d.ts +26 -0
- package/dist/run/journal.js +143 -15
- package/dist/run/lease.d.ts +13 -0
- package/dist/run/lease.js +45 -0
- package/dist/run/protocol.d.ts +15 -0
- package/dist/run/protocol.js +11 -1
- package/dist/run/receipt-resolver.d.ts +22 -0
- package/dist/run/receipt-resolver.js +40 -1
- package/dist/run/repair-selection.d.ts +11 -1
- package/dist/run/repair-selection.js +17 -9
- package/dist/run/wall-budget.d.ts +48 -0
- package/dist/run/wall-budget.js +280 -0
- package/dist/tui/cockpit/run-cockpit.js +2 -2
- package/dist/tui/cockpit/run-view.d.ts +2 -1
- package/dist/tui/cockpit/run-view.js +13 -9
- package/dist/tui/cockpit/setup-cockpit.d.ts +2 -0
- package/dist/tui/cockpit/setup-cockpit.js +4 -0
- package/package.json +2 -1
- package/schema/config.schema.json +8 -1
- package/skills/tickmarkr-loop/SKILL.md +7 -1
- package/skills/tickmarkr-overseer/scripts/classify-vitest-log.sh +4 -1
package/dist/run/daemon.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { HOST_PROBE_SAMPLE_MS, HOST_PROBE_SAMPLES, HostDegradedError, hostDegraded, observeHost } from "./host-health.js";
|
|
2
|
-
import { VITEST_CACHE_ENV, worktreeVitestCache } from "../gates/test-manifest.js";
|
|
2
|
+
import { runWithInterruptionSink, VITEST_CACHE_ENV, worktreeVitestCache } from "../gates/test-manifest.js";
|
|
3
3
|
import { COMMAND_LEASE_TOKEN_ENV, commandLeaseEnvironment, CommandLeases, currentCommandLeaseToken, isRunnerCommand, runWithCommandLease, withCommandLease } from "./lease.js";
|
|
4
4
|
import { execFileSync, spawn } from "node:child_process";
|
|
5
5
|
import { createHash, randomBytes } from "node:crypto";
|
|
@@ -21,7 +21,7 @@ import { driverEvidence } from "../drivers/index.js";
|
|
|
21
21
|
import { herdrSealShellPrefix, MAX_BUF, SubprocessDriver } from "../drivers/subprocess.js";
|
|
22
22
|
import { formatOwnedName } from "../drivers/types.js";
|
|
23
23
|
import { captureBaseline, detectGateCommands, detectVacuousOracles } from "../gates/baseline.js";
|
|
24
|
-
import { runGates } from "../gates/run-gates.js";
|
|
24
|
+
import { reobserveTestFiles, runGates } from "../gates/run-gates.js";
|
|
25
25
|
import { isInfraResult } from "../gates/cache.js";
|
|
26
26
|
import { filesGlob } from "../graph/files-glob.js";
|
|
27
27
|
import { addEvidence, attributeBlocked, batteryPriority, blockedTasks, getTask, graphDefinitionHash, loadGraph, pendingTasks, readyTasks, saveGraph, setStatus, taskContentDigest, tickmarkrDir } from "../graph/graph.js";
|
|
@@ -29,17 +29,17 @@ import { GATE_NAMES } from "../graph/schema.js";
|
|
|
29
29
|
import { distFingerprint } from "../cli/commands/version.js";
|
|
30
30
|
import { augmentRetryBrief, consult, renderRetryGuidance } from "./consult.js";
|
|
31
31
|
import { executionSignal, remainingExecutionMs, withExecutionBudget, withoutExecutionBudget, ExecutionBudgetExceeded } from "./execution-budget.js";
|
|
32
|
-
import { repairSelectionDecision } from "./repair-selection.js";
|
|
32
|
+
import { repairSelectionDecision, repairSelectionEnabled } from "./repair-selection.js";
|
|
33
33
|
import { failureDisposition, reserveInfrastructureRetry } from "./recovery.js";
|
|
34
34
|
import { runEnvironment } from "./environment.js";
|
|
35
35
|
import { changeRepresented, cleanupRunWorktrees, deriveForkCap, FORK_CAP_ENV, gitHead, linkNodeModules, npmDependencyInstallCommand, npmDependencyManifestChanged, PRESERVE_COMMIT_SUBJECT, PRESERVE_PRODUCER_TRAILER, preserveWorktree, producerFields, resolvedCapacity, runWithForkBudget, runWithVerificationBudget, sameCapacity, sameVerification, sh, shGit, SUITE_PARENT_ENV, verificationProtocol, WORKTREE_LAYOUT_CONTRACT, worktreePath } from "./git.js";
|
|
36
36
|
import { runInteractiveSeed } from "./interactive-seed.js";
|
|
37
37
|
import { classifyRepairDisposition, resolveScopeHints } from "./repair-disposition.js";
|
|
38
|
-
import { applyScopeAmendments, activeRetryBan, interruptedAttempt, RESUME_HARVEST_SOURCE, resumeHarvestAuthor, approvalAction, APPROVAL_REFUSED, bindingToken, effectiveEvents, foldDecisions, physicalLine, staleApprovals, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, outstandingReviewFindings, pendingApprovalActions, pendingRechecks, pendingRepairFindings, phaseForGate, readPriorRunEvidence, recordedTaskFailureKind, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, standingRulings, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
39
|
-
import { gateReviewerFloor, isDiffCapPark, pickReviewer } from "../gates/review.js";
|
|
38
|
+
import { applyScopeAmendments, activeRetryBan, interruptedAttempt, RESUME_HARVEST_SOURCE, resumeHarvestAuthor, approvalAction, APPROVAL_REFUSED, bindingToken, effectiveEvents, foldDecisions, physicalLine, staleApprovals, classifyTaskFailure, classifyWorkerResultCause, deferredReviewFindings, engagementComparable, formatPriorFindingEvidence, GATE_FINGERPRINT_CAP, GATE_SATISFIED_RELEASE, identicalGateFailures, isDeferredFinding, journaledFailureBrief, Journal, loadRoutingProfile, newRunId, normalizeGateFailure, outstandingConsultGuidance, readableExcerpt, receiptOrigin, outstandingReviewFindings, pendingApprovalActions, pendingRechecks, pendingRepairFindings, phaseForGate, priorJudgments, readPriorRunEvidence, recordedTaskFailureKind, reraisedReviewChains, RECHECK_RELEASE, renderStructuredReviewFinding, repairReachSinceApproval, repairsSinceApproval, reviewRoundsSinceApproval, runHasEnded, standingRulings, structuredFindings, upheldFeedbackByTask } from "./journal.js";
|
|
39
|
+
import { gateReviewerFloor, isDiffCapPark, isGarbageReview, pickReviewer } from "../gates/review.js";
|
|
40
40
|
import { acquireApprovalSerialization, acquireRunLock, isPidLive, releaseRunLock } from "./lock.js";
|
|
41
41
|
import { ensureIntegration, integrationBranch, integrationHead, mergeTask, reusedTipEvidence, verifyIntegrationTip } from "./merge.js";
|
|
42
|
-
import { climbChannel, marginalCostRank, nextChannel, route } from "../route/router.js";
|
|
42
|
+
import { climbChannel, inTaskPool, marginalCostRank, nextChannel, poolExhaustion, route, taskPool } from "../route/router.js";
|
|
43
43
|
import { desiredPanes } from "./reconcile.js";
|
|
44
44
|
import { readTierLiveness, readWatchBoard, supervisionPresencePath } from "./supervision.js";
|
|
45
45
|
import { harvestCpuFlatWindowMs, NUDGEABLE_ADAPTERS, PANE_READ_ROWS, StallProgressTracker, stallSnapshotBannerRows, WorkerTreeCpuAccountant, reapOwnedProcessGroup, readOwnedProcessGroup, } from "./stall.js";
|
|
@@ -370,6 +370,30 @@ function classifyInfraResult(g) {
|
|
|
370
370
|
return;
|
|
371
371
|
g.meta = { ...g.meta, classification: "infra", infra: true };
|
|
372
372
|
}
|
|
373
|
+
// OBS-1106 residual: a red whose assertion is a timeout or a wall-clock budget, and the runner-level
|
|
374
|
+
// diagnostics (never-started files, a worker RPC timeout) that make such a red infrastructure-SHAPED.
|
|
375
|
+
// Shape alone never parks anything: it only buys one isolated re-observation before a charge. A
|
|
376
|
+
// fresh-fingerprint section masks digits to `#`, so both spellings of a number are read.
|
|
377
|
+
const TIMEOUT_SHAPED_RE = /\b(?:Test|Hook) timed out in (?:\d+|#)\s*ms\b|\bexpected (?:\d+|#)(?:\.(?:\d+|#))? to be (?:less than|below)(?: or equal to)? (?:\d+|#)/i;
|
|
378
|
+
const RUNNER_INFRA_DIAGNOSTIC_RE = /\bnever-started (?:[1-9]\d*|#)(?![\d#])|\[vitest-worker\]: Timeout calling\b/;
|
|
379
|
+
const timeoutShaped = (details) => TIMEOUT_SHAPED_RE.test(readableExcerpt(details));
|
|
380
|
+
const ATTRIBUTED_FILE_RE = /^\s*(?:FAIL|×|✗)\s+(?:\|[^|\n]*\|\s+)?(\S+\.(?:test|spec)\.[cm]?[jt]sx?)\b/gm;
|
|
381
|
+
/** The failing files a red attributes to itself: the runner's own list, else its FAIL headlines. */
|
|
382
|
+
const attributedFailingFiles = (g) => [...new Set(Array.isArray(g.meta?.failingFiles) && g.meta.failingFiles.length
|
|
383
|
+
? g.meta.failingFiles.filter((f) => typeof f === "string")
|
|
384
|
+
: [...readableExcerpt(g.details).matchAll(ATTRIBUTED_FILE_RE)].map((m) => m[1].replace(/^\.\//, "")))].sort();
|
|
385
|
+
const metaFiles = (v) => Array.isArray(v) ? v.filter((f) => typeof f === "string") : undefined;
|
|
386
|
+
/** D-607: the files a rerun OBSERVABLY ran — the runner's manifest, the only complete selection
|
|
387
|
+
* evidence. Never the requested selection, and never the per-file lines a runner printed: a runner that
|
|
388
|
+
* ignores file arguments and reports only its failures looks isolated while it ran the whole suite.
|
|
389
|
+
* Undefined without a manifest, and the rerun is then ambiguous. */
|
|
390
|
+
const observedSelection = (g) => {
|
|
391
|
+
const ran = metaFiles(g.meta?.manifest);
|
|
392
|
+
return ran?.length ? [...new Set(ran)].sort() : undefined;
|
|
393
|
+
};
|
|
394
|
+
/** D-607: on the manifest path only the reporter's own attribution counts — a `FAIL path` line inside
|
|
395
|
+
* captured stdout is text, not attribution. Headline lexing is for runners without a reporter. */
|
|
396
|
+
const rerunFailingFiles = (g) => metaFiles(g.meta?.manifest) ? [...new Set(metaFiles(g.meta?.failingFiles) ?? [])].sort() : attributedFailingFiles(g);
|
|
373
397
|
// v1.85 T3: the gates whose failure IS a deterministic measurement — a machine re-ran a command over a
|
|
374
398
|
// tree and printed the same bytes. Those are the failures the fingerprint cap governs (the ruling names
|
|
375
399
|
// it a "deterministic-gate" cap): a third identical answer to a question already answered twice is the
|
|
@@ -460,18 +484,36 @@ function repairDiffFilesCut(diff, cap, names) {
|
|
|
460
484
|
function repairFindingsBrief(results) {
|
|
461
485
|
return results.filter(gateFailed).map((g) => {
|
|
462
486
|
const evidence = g.meta?.failureEvidence;
|
|
463
|
-
return `${g.gate}: ${g.details}` + (Array.isArray(evidence) && evidence.length
|
|
487
|
+
return `${g.gate}: ${readableExcerpt(g.details)}` + (Array.isArray(evidence) && evidence.length
|
|
464
488
|
? `\nAssertion evidence:\n${JSON.stringify(evidence, null, 2)}` : "");
|
|
465
489
|
}).join("\n\n");
|
|
466
490
|
}
|
|
467
|
-
|
|
468
|
-
|
|
491
|
+
// OBS-1019 add.2: the order a repeated review chain earns in place of the smallest-edit order.
|
|
492
|
+
const REPAIR_CLASS_SENTENCE = "Before editing a chain reraised more than once, enumerate the in-scope consumers,"
|
|
493
|
+
+ " bridges, operations and event sequences as a closed case table; repair every implicated member and verify"
|
|
494
|
+
+ " the accumulated diff against that table.";
|
|
495
|
+
/**
|
|
496
|
+
* The fix-only contract: the findings verbatim, then the diff content of the work already landed.
|
|
497
|
+
* OBS-1019 add.2: a first finding stays bounded to the smallest edit. A chain a review has re-raised
|
|
498
|
+
* more than once was chased edge by edge while every round re-judged the accumulated diff, so it owes
|
|
499
|
+
* the whole class instead: a closed case table inside the declared bounds, never beyond them.
|
|
500
|
+
*/
|
|
501
|
+
function repairBrief(findings, diff, baseRef, repeated = []) {
|
|
469
502
|
const repair = { findings, diff };
|
|
470
503
|
return [
|
|
471
|
-
"## Repair attempt — fix ONLY what these findings name",
|
|
504
|
+
repeated.length ? "## Repair attempt — repair the whole class these findings name" : "## Repair attempt — fix ONLY what these findings name",
|
|
472
505
|
"The commits from your prior attempt are already in this worktree and their diff is reproduced"
|
|
473
|
-
+ " below. Do NOT re-implement that work, do not start over, and do not revert it:
|
|
474
|
-
+ " smallest change that resolves every finding, then commit.",
|
|
506
|
+
+ " below. Do NOT re-implement that work, do not start over, and do not revert it: "
|
|
507
|
+
+ (repeated.length ? "resolve every member of the case table below, then commit." : "make the smallest change that resolves every finding, then commit."),
|
|
508
|
+
...(repeated.length ? [
|
|
509
|
+
"",
|
|
510
|
+
"### Repeated review chains — repair the class, not the next edge",
|
|
511
|
+
// Fingerprint and count only: the finding's prose is already quoted once, below or under its heading.
|
|
512
|
+
...repeated.map(({ finding, reraises }) => `- \`${finding.fingerprint}\` re-raised ${reraises} times`),
|
|
513
|
+
REPAIR_CLASS_SENTENCE,
|
|
514
|
+
"The table stays inside the task's declared write scope (files[]) and never narrows its goal or an"
|
|
515
|
+
+ " acceptance criterion; the reviewer judges the accumulated diff, not the table.",
|
|
516
|
+
] : []),
|
|
475
517
|
"",
|
|
476
518
|
"### Failing gate findings (verbatim)",
|
|
477
519
|
repair.findings,
|
|
@@ -881,7 +923,7 @@ export function formatTipProof(p) {
|
|
|
881
923
|
}
|
|
882
924
|
// Isolate the battery so cancellation can stop its control flow as well as its shell children.
|
|
883
925
|
// The child uses the same verifier and capacity; only the daemon writes lifecycle verdict rows.
|
|
884
|
-
async function cancellableTipBattery(intWt, commands, runDir, baseline, signal) {
|
|
926
|
+
async function cancellableTipBattery(intWt, commands, runDir, baseline, signal, evidence = {}) {
|
|
885
927
|
signal.throwIfAborted();
|
|
886
928
|
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
|
|
887
929
|
const script = `
|
|
@@ -930,9 +972,9 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
930
972
|
finally { process.send({ type: 'lease-release', id }); }
|
|
931
973
|
};
|
|
932
974
|
let input = ''; for await (const chunk of process.stdin) input += chunk;
|
|
933
|
-
const { intWt, commands, runDir, baseline, capacity } = JSON.parse(input);
|
|
975
|
+
const { intWt, commands, runDir, baseline, capacity, evidence } = JSON.parse(input);
|
|
934
976
|
try {
|
|
935
|
-
const result = await runWithVerificationBudget(capacity, () => runWithCommandLease(lease, () => verifyIntegrationTip(intWt, commands, runDir, baseline)));
|
|
977
|
+
const result = await runWithVerificationBudget(capacity, () => runWithCommandLease(lease, () => verifyIntegrationTip(intWt, commands, runDir, baseline, evidence)));
|
|
936
978
|
if (!cancelled) process.stdout.write(JSON.stringify(result.map(row => ({ ...row, capacity: [...commandCapacities].reverse().find(([command]) => command === row.cmd || command.startsWith(row.cmd + " "))?.[1] ?? capacity }))));
|
|
937
979
|
} catch (error) { if (!cancelled) throw error; }
|
|
938
980
|
finally { process.disconnect(); }
|
|
@@ -983,7 +1025,7 @@ async function cancellableTipBattery(intWt, commands, runDir, baseline, signal)
|
|
|
983
1025
|
});
|
|
984
1026
|
signal.addEventListener("abort", cancel, { once: true });
|
|
985
1027
|
child.stdin.on("error", () => { });
|
|
986
|
-
child.stdin.end(JSON.stringify({ intWt, commands, runDir, baseline, capacity: resolvedCapacity() }));
|
|
1028
|
+
child.stdin.end(JSON.stringify({ intWt, commands, runDir, baseline, capacity: resolvedCapacity(), evidence }));
|
|
987
1029
|
try {
|
|
988
1030
|
if (signal.aborted)
|
|
989
1031
|
cancel();
|
|
@@ -1044,9 +1086,9 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
1044
1086
|
return false;
|
|
1045
1087
|
}
|
|
1046
1088
|
let tipFailed = false;
|
|
1047
|
-
for (const r of await (opts.signal
|
|
1048
|
-
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal)
|
|
1049
|
-
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline))) {
|
|
1089
|
+
for (const r of await runWithInterruptionSink(journalInterruption(journal, undefined), () => opts.signal
|
|
1090
|
+
? cancellableTipBattery(intWt, commands, journal.dir, opts.baseline, opts.signal, opts.evidence)
|
|
1091
|
+
: verifyIntegrationTip(intWt, commands, journal.dir, opts.baseline, opts.evidence))) {
|
|
1050
1092
|
const measuredCapacity = r.capacity ?? capacity;
|
|
1051
1093
|
const evidence = r.reused ? reusedTipEvidence(r) : {
|
|
1052
1094
|
...(r.originRunRoot ? { originRunRoot: r.originRunRoot } : {}),
|
|
@@ -1080,6 +1122,12 @@ export async function verifyIntegrationTipCached(intWt, commands, journal, opts
|
|
|
1080
1122
|
}
|
|
1081
1123
|
return tipFailed;
|
|
1082
1124
|
}
|
|
1125
|
+
/** OBS-953: a detected wall-over-monotonic discontinuity journals `host-suspend` (its offset was
|
|
1126
|
+
* subtracted from active test time); an overdue poll whose clocks advanced together journals
|
|
1127
|
+
* `host-poll-overdue` (recorded, never subtracted). */
|
|
1128
|
+
const journalInterruption = (journal, taskId) => (interruption) => {
|
|
1129
|
+
journal.append(interruption.kind === "host-suspend" ? "host-suspend" : "host-poll-overdue", taskId, { ...interruption });
|
|
1130
|
+
};
|
|
1083
1131
|
async function commitsAheadOfRef(base, head, cwd) {
|
|
1084
1132
|
if (head === base)
|
|
1085
1133
|
return [];
|
|
@@ -1904,6 +1952,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1904
1952
|
// > repo > global > default. The resolved cfg carries mode-compiled floors; route() never sees the mode.
|
|
1905
1953
|
const rm = resolveRunMode(repoRoot, { flag: opts.mode, spec: graph.mode, globalDir: opts.globalDir });
|
|
1906
1954
|
const cfg = rm.cfg;
|
|
1955
|
+
// OBS-1199: the effective repair-selection default is resolved apart from the optional execution
|
|
1956
|
+
// budget, so an absent policy selects on every driver; a resume keeps what the run recorded.
|
|
1957
|
+
let repairSelection = repairSelectionEnabled(cfg);
|
|
1907
1958
|
if (opts.resume) {
|
|
1908
1959
|
const start = Journal.open(repoRoot, runId).read().find((e) => e.event === "run-start");
|
|
1909
1960
|
const snapshot = start?.data.effectivePolicy;
|
|
@@ -1915,6 +1966,11 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
1915
1966
|
// Historical runs did not opt in; current settings cannot grant them a new allowance.
|
|
1916
1967
|
cfg.executionPolicy = recorded?.config?.executionPolicy === undefined
|
|
1917
1968
|
? undefined : ExecutionPolicySchema.parse(recorded.config.executionPolicy);
|
|
1969
|
+
const recordedGates = recorded?.config?.gates;
|
|
1970
|
+
if (recordedGates?.repairSelection !== undefined && typeof recordedGates.repairSelection !== "boolean") {
|
|
1971
|
+
throw new Error("cannot restore malformed recorded repair selection");
|
|
1972
|
+
}
|
|
1973
|
+
repairSelection = repairSelectionEnabled({ executionPolicy: cfg.executionPolicy, gates: recordedGates });
|
|
1918
1974
|
}
|
|
1919
1975
|
if (cfg.executionPolicy && driver.id !== "subprocess") {
|
|
1920
1976
|
throw new Error("experimental executionPolicy currently requires the subprocess driver");
|
|
@@ -2366,6 +2422,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2366
2422
|
void promise.finally(() => hostChecks.delete(promise)).catch(() => { });
|
|
2367
2423
|
return promise;
|
|
2368
2424
|
};
|
|
2425
|
+
// OBS-1190: every admission poll still probes (the probe gates admission), but the journal keeps only
|
|
2426
|
+
// the first observation and each change of state or reference — healthy, degraded, unreadable,
|
|
2427
|
+
// recovered — the way suite-wait already dedupes by count. A steady wait no longer floods the journal.
|
|
2428
|
+
let journaledHost;
|
|
2429
|
+
const journalObservation = (taskId, observation, resuming) => {
|
|
2430
|
+
const state = observation.medianMs === null ? "unreadable"
|
|
2431
|
+
: hostReferenceMs !== undefined && hostDegraded(observation, hostReferenceMs) ? "degraded" : "healthy";
|
|
2432
|
+
const key = `${state}:${hostReferenceMs ?? ""}`;
|
|
2433
|
+
if (key === journaledHost)
|
|
2434
|
+
return;
|
|
2435
|
+
journaledHost = key;
|
|
2436
|
+
journal.append("host-observation", taskId, { ...observation, referenceMs: hostReferenceMs ?? null, resuming, state });
|
|
2437
|
+
};
|
|
2369
2438
|
const recordReference = (observation) => {
|
|
2370
2439
|
if (observation.medianMs === null)
|
|
2371
2440
|
return;
|
|
@@ -2389,7 +2458,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2389
2458
|
const probeBudget = HOST_PROBE_SAMPLE_MS * HOST_PROBE_SAMPLES;
|
|
2390
2459
|
if (!observation || (!first && remaining >= probeBudget)) {
|
|
2391
2460
|
observation = await observeHost(signal, remaining > 0 ? Math.min(probeBudget, remaining) : undefined);
|
|
2392
|
-
|
|
2461
|
+
journalObservation(taskId, observation, resuming);
|
|
2393
2462
|
}
|
|
2394
2463
|
first = false;
|
|
2395
2464
|
if (hostReferenceMs === undefined && observation.medianMs !== null)
|
|
@@ -2433,7 +2502,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2433
2502
|
const initializeHost = () => trackHost((async () => {
|
|
2434
2503
|
const signal = hostSignal();
|
|
2435
2504
|
const observation = await observeHost(signal);
|
|
2436
|
-
|
|
2505
|
+
journalObservation(undefined, observation, !!opts.resume);
|
|
2437
2506
|
if (hostReferenceMs === undefined)
|
|
2438
2507
|
recordReference(observation);
|
|
2439
2508
|
if (opts.resume || observation.medianMs === null)
|
|
@@ -2479,7 +2548,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2479
2548
|
waited = true;
|
|
2480
2549
|
journal.append("suite-wait", taskId, { count, ...(gate ? { gate } : {}) });
|
|
2481
2550
|
}, SUITE_POLL_MS, signal);
|
|
2482
|
-
}, async () => {
|
|
2551
|
+
}, () => runWithInterruptionSink(journalInterruption(journal, taskId), async () => {
|
|
2483
2552
|
try {
|
|
2484
2553
|
const result = await run();
|
|
2485
2554
|
// Some command oracles turn launch errors into results. Admission failure still parks infra.
|
|
@@ -2491,7 +2560,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2491
2560
|
if (taskId)
|
|
2492
2561
|
activeGatePhases.delete(taskId);
|
|
2493
2562
|
}
|
|
2494
|
-
});
|
|
2563
|
+
}));
|
|
2495
2564
|
};
|
|
2496
2565
|
let baseRef;
|
|
2497
2566
|
let baseline;
|
|
@@ -2710,24 +2779,39 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2710
2779
|
// owns listing/parsing/closing. Cosmetic by contract: failures are swallowed and subprocess has no
|
|
2711
2780
|
// reconcile (optional chain → no-op), so gates and the oracle suite never feel this. keepPanes
|
|
2712
2781
|
// "forever" is the keep-everything debug override — it disables the sweep entirely.
|
|
2713
|
-
|
|
2782
|
+
// OBS-1109: an interrupted attempt's pane is evidence — its nonce-bound trailer decides whether
|
|
2783
|
+
// the attempt is harvested, declined as foreign, or recovered — so the fold's run-resume clear
|
|
2784
|
+
// must not reach it before harvestInterruptedAttempt has read it and reaped its processes. Only a
|
|
2785
|
+
// resumed daemon can hold such an attempt: a fresh run's sweep is the baseline sweep.
|
|
2786
|
+
// OBS-1204: the set is FROZEN here, once, from the rows up to this daemon's own run-resume row: an
|
|
2787
|
+
// attempt launched after it is this daemon's own and reconciles exactly as in a fresh run. Each
|
|
2788
|
+
// frozen spare lapses for good once its harvest, decline or a superseding dispatch is journaled.
|
|
2789
|
+
const resumeInterrupted = new Map();
|
|
2790
|
+
if (opts.resume) {
|
|
2791
|
+
const rows = journal.read();
|
|
2792
|
+
let resumedAt = rows.length - 1;
|
|
2793
|
+
while (resumedAt >= 0 && rows[resumedAt].event !== "run-resume")
|
|
2794
|
+
resumedAt--;
|
|
2795
|
+
const prior = rows.slice(0, resumedAt + 1);
|
|
2796
|
+
for (const t of graph.tasks) {
|
|
2797
|
+
const owned = interruptedAttempt(prior, t.id);
|
|
2798
|
+
if (owned?.launch)
|
|
2799
|
+
resumeInterrupted.set(t.id, { attempt: owned.attempt, nonce: owned.launch.nonce });
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2714
2802
|
const reconcile = async (sweep) => {
|
|
2715
2803
|
if (keepForever)
|
|
2716
2804
|
return;
|
|
2717
2805
|
try {
|
|
2718
2806
|
const rows = journal.read();
|
|
2719
2807
|
const desired = desiredPanes(rows, runId);
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
const owned = interruptedAttempt(rows, t.id);
|
|
2728
|
-
if (owned?.launch)
|
|
2729
|
-
desired.add(formatOwnedName({ role: "worker", taskId: t.id, attempt: owned.attempt, runId }));
|
|
2730
|
-
}
|
|
2808
|
+
for (const [taskId, frozen] of resumeInterrupted) {
|
|
2809
|
+
const owned = interruptedAttempt(rows, taskId);
|
|
2810
|
+
if (owned?.launch?.nonce !== frozen.nonce)
|
|
2811
|
+
resumeInterrupted.delete(taskId);
|
|
2812
|
+
else
|
|
2813
|
+
desired.add(formatOwnedName({ role: "worker", taskId, attempt: frozen.attempt, runId }));
|
|
2814
|
+
}
|
|
2731
2815
|
// The watch pane is never the DRIVER sweep's candidate (panesToClose spares role "watch":
|
|
2732
2816
|
// herdr's watches bookkeeping lives in close(), and a raw pane-close in the sweep would
|
|
2733
2817
|
// leave narrator() a stale cache) — the driver always sees it as desired; its lifecycle is
|
|
@@ -2835,6 +2919,80 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2835
2919
|
await reconcile({ spareLiveLlm: true }); // task-human is a terminal event — sweep, sparing sibling tasks' live LLM panes
|
|
2836
2920
|
await driver.notify(`tickmarkr ${runId}: ${t.id} needs a human — ${reason}${token ? `\npark ${token} — bind the decision with \`--park ${token}\`` : ""}`, { tier: "attention" });
|
|
2837
2921
|
};
|
|
2922
|
+
// OBS-1106 residual: a retry that landed nothing on a timeout-class red re-observes it ONCE fresh:
|
|
2923
|
+
// neither a journal replay nor the verdict cache may re-state that red. Owed when the prior round's
|
|
2924
|
+
// test red sits on this very subject, is timeout-shaped, and has not yet been observed twice
|
|
2925
|
+
// independently — once it has, the fingerprint cap owns the next move. Read from the journal alone,
|
|
2926
|
+
// so the live gate loop and a resume battery decide alike.
|
|
2927
|
+
const owesTimeoutRefresh = (t, priorRows, commit) => {
|
|
2928
|
+
const red = priorRows.find((e) => e.event === "gate-result" && e.data.gate === "test" && e.data.pass === false
|
|
2929
|
+
&& e.data.skipped !== true && e.data.commit === commit && typeof e.data.details === "string" && timeoutShaped(e.data.details));
|
|
2930
|
+
return red !== undefined
|
|
2931
|
+
&& identicalGateFailures(journal.read(), t.id, "test", normalizeGateFailure(String(red.data.details))) < GATE_FINGERPRINT_CAP;
|
|
2932
|
+
};
|
|
2933
|
+
// OBS-1106 residual: before a timeout or wall-budget red accompanied by runner infra diagnostics is
|
|
2934
|
+
// charged, its attributed failing files are re-run ONCE, on the same checkout, under the suite
|
|
2935
|
+
// lease. Only a reproduced assertion is charged. A pass, an infra rerun, a rerun naming none of the
|
|
2936
|
+
// files, or an unattributable red parks with the ORIGINAL red journaled and unresolved: the rerun is
|
|
2937
|
+
// diagnostic, never a verdict, so it can neither green the gate nor permit a merge. Returns true
|
|
2938
|
+
// when it parked.
|
|
2939
|
+
const adjudicateInfraShapedRed = async (t, results, wt, subject, attempt, replayed, parkInfra) => {
|
|
2940
|
+
const red = results.find((g) => g.gate === "test" && gateFailed(g) && !isInfraResult(g)
|
|
2941
|
+
&& timeoutShaped(g.details) && RUNNER_INFRA_DIAGNOSTIC_RE.test(g.details));
|
|
2942
|
+
if (!red || commands.test === undefined)
|
|
2943
|
+
return false;
|
|
2944
|
+
// The observation this red states: its subject, its fingerprint and, when it carries one, its
|
|
2945
|
+
// receipt's invocation. A copy (journal replay, cache hit) is never re-run when THAT observation's
|
|
2946
|
+
// adjudication is on the ledger — its recorded outcome decides, so a crash after the adjudication
|
|
2947
|
+
// but before its park still parks. A crash before the adjudication leaves none, so the copy is
|
|
2948
|
+
// adjudicated now. A fresh red always is.
|
|
2949
|
+
const fingerprint = normalizeGateFailure(red.details);
|
|
2950
|
+
const observation = receiptOrigin({ evidenceReceipt: red.evidenceReceipt ?? red.meta?.evidenceReceipt,
|
|
2951
|
+
evidenceReceipts: red.evidenceReceipts ?? red.meta?.evidenceReceipts });
|
|
2952
|
+
const recorded = replayed || red.meta?.reused === true
|
|
2953
|
+
? journal.read().reverse().find((e) => e.taskId === t.id && e.event === "gate-reobserved"
|
|
2954
|
+
&& e.data.commit === subject && e.data.fingerprint === fingerprint
|
|
2955
|
+
&& (observation === undefined || e.data.observation === undefined || e.data.observation === observation))
|
|
2956
|
+
: undefined;
|
|
2957
|
+
let files = [];
|
|
2958
|
+
let outcome = "unattributed";
|
|
2959
|
+
let selection;
|
|
2960
|
+
if (recorded) {
|
|
2961
|
+
files = Array.isArray(recorded.data.files) ? recorded.data.files.filter((f) => typeof f === "string") : [];
|
|
2962
|
+
outcome = ["reproduced", "passed", "unattributed"].includes(String(recorded.data.outcome)) ? recorded.data.outcome : "ambiguous";
|
|
2963
|
+
}
|
|
2964
|
+
else {
|
|
2965
|
+
for (const file of attributedFailingFiles(red)) {
|
|
2966
|
+
if (posix.normalize(file) === file && !file.startsWith("../") && !file.startsWith("/") && existsSync(join(wt, file)))
|
|
2967
|
+
files.push(file);
|
|
2968
|
+
}
|
|
2969
|
+
let rerun;
|
|
2970
|
+
if (files.length) {
|
|
2971
|
+
rerun = await withCommandContext(t.id, () => reobserveTestFiles(wt, commands.test, baseline, files, journal.dir))
|
|
2972
|
+
.catch((error) => ({ gate: "test", pass: false, details: `infra: rerun failed: ${error instanceof Error ? error.message : String(error)}`, meta: { infra: true } }));
|
|
2973
|
+
classifyInfraResult(rerun);
|
|
2974
|
+
// Reproduced only by failure evidence the rerun ATTRIBUTES to one of exactly these files, from an
|
|
2975
|
+
// OBSERVED selection that was exactly these files (D-607): a path merely printed in output, a
|
|
2976
|
+
// positional filter that also collected another file, or a runner that ignored the file
|
|
2977
|
+
// arguments and reported no manifest of its selection, says nothing about the original red.
|
|
2978
|
+
const failing = rerunFailingFiles(rerun);
|
|
2979
|
+
selection = observedSelection(rerun);
|
|
2980
|
+
const isolated = selection !== undefined && selection.length === files.length
|
|
2981
|
+
&& files.every((file) => selection.includes(file)) && failing.every((file) => files.includes(file));
|
|
2982
|
+
outcome = rerun.pass ? "passed" : isolated && !isInfraResult(rerun) && failing.length ? "reproduced" : "ambiguous";
|
|
2983
|
+
}
|
|
2984
|
+
journal.append("gate-reobserved", t.id, {
|
|
2985
|
+
gate: "test", attempt, commit: subject, fingerprint, ...(observation ? { observation } : {}), files, outcome,
|
|
2986
|
+
...(selection ? { selection } : {}),
|
|
2987
|
+
...(rerun ? { pass: rerun.pass, details: rerun.details } : {}),
|
|
2988
|
+
...(rerun?.evidenceReceipt ? { evidenceReceipt: rerun.evidenceReceipt } : {}),
|
|
2989
|
+
});
|
|
2990
|
+
}
|
|
2991
|
+
if (outcome === "reproduced")
|
|
2992
|
+
return false;
|
|
2993
|
+
await parkInfra(`test: timeout-shaped red with runner infra diagnostics was not reproduced by one isolated rerun of ${files.length ? files.join(", ") : "its attributed files (none attributable)"} (${outcome}); the original red stands unresolved — recheck to re-measure:\n${readableExcerpt(red.details)}`);
|
|
2994
|
+
return true;
|
|
2995
|
+
};
|
|
2838
2996
|
// OBS-547: cross-reference a scope red against the prediction this run already computed. Every hard
|
|
2839
2997
|
// offender predicted ⇒ an AUTHORING defect whose repair is pre-written: journal the classification
|
|
2840
2998
|
// with the verbatim files[] lines and park unchargeable. It owes nothing to the quality machinery —
|
|
@@ -2853,7 +3011,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2853
3011
|
const verdict = scopeRed?.meta?.collateral;
|
|
2854
3012
|
const worker = journal.read().reverse().find((e) => e.taskId === t.id && e.event === "worker-result");
|
|
2855
3013
|
const disposition = classifyRepairDisposition({
|
|
2856
|
-
|
|
3014
|
+
// OBS-1072 add.2: hints are lexed from the readable excerpt, never from terminal control bytes.
|
|
3015
|
+
results: results.map((g) => ({ ...g, details: readableExcerpt(g.details) })), files: t.files, inventory: changed,
|
|
2857
3016
|
refusalSummary: worker?.data.ok === false && typeof worker.data.summary === "string" ? worker.data.summary : undefined,
|
|
2858
3017
|
});
|
|
2859
3018
|
for (const diagnostic of disposition.diagnostics) {
|
|
@@ -2965,7 +3124,13 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
2965
3124
|
if (changed)
|
|
2966
3125
|
saveGraph(repoRoot, graph);
|
|
2967
3126
|
};
|
|
3127
|
+
// OBS-1168(c): every gate callback is bound to the engagement that asked for it. Once the scheduler
|
|
3128
|
+
// settles that engagement (done, parked or failed), a late sibling's event is dropped, never journaled.
|
|
3129
|
+
const engagements = new Map();
|
|
2968
3130
|
const execTask = async (t) => {
|
|
3131
|
+
const engagement = {};
|
|
3132
|
+
engagements.set(t.id, engagement);
|
|
3133
|
+
const liveEngagement = () => engagements.get(t.id) === engagement;
|
|
2969
3134
|
const approvalAction = pendingDaemonApprovalActions(journal.read()).get(t.id);
|
|
2970
3135
|
const fundedRerun = approvalAction?.authority === "worker";
|
|
2971
3136
|
const startMs = Date.now();
|
|
@@ -3049,8 +3214,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3049
3214
|
// OBS-1161: no `attempts > 0` guard — every release already clears lastAssignment in the replay,
|
|
3050
3215
|
// and a lastAssignment at zero attempts is a first dispatch whose capacity requeue was taken back:
|
|
3051
3216
|
// the seat is still in force, so restore it instead of failing over its own tried[] entry early.
|
|
3217
|
+
// OBS-1187: a restored seat must still be a member of the task's pool (the operator may have closed it).
|
|
3052
3218
|
if (!hintsChanged && rs?.lastAssignment
|
|
3053
3219
|
&& channels.some((c) => channelKey(c) === channelKey(rs.lastAssignment))
|
|
3220
|
+
&& inTaskPool(t, cfg, channelKey(rs.lastAssignment))
|
|
3054
3221
|
&& !demotedChannels.has(channelKey(rs.lastAssignment))) {
|
|
3055
3222
|
assignment = rs.lastAssignment; // restore the consult-chosen assignment (bypasses route()'s static re-pick)
|
|
3056
3223
|
}
|
|
@@ -3079,7 +3246,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3079
3246
|
const lastApproval = taskHistory.map((e) => e.event).lastIndexOf("task-approved");
|
|
3080
3247
|
const priorClimb = taskHistory.slice(lastApproval + 1).reverse().find((e) => e.event === "tier-escalated");
|
|
3081
3248
|
if (!hintsChanged && priorClimb && taskHistory.indexOf(priorClimb) > taskHistory.map((e) => e.event).lastIndexOf("task-dispatch")) {
|
|
3082
|
-
const target = channels.find((c) => channelKey(c) === priorClimb.data.to && !demotedChannels.has(channelKey(c))
|
|
3249
|
+
const target = channels.find((c) => channelKey(c) === priorClimb.data.to && !demotedChannels.has(channelKey(c))
|
|
3250
|
+
&& inTaskPool(t, cfg, channelKey(c)));
|
|
3083
3251
|
if (target)
|
|
3084
3252
|
assignment = seatAssignment(target);
|
|
3085
3253
|
}
|
|
@@ -3122,7 +3290,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3122
3290
|
workerDispatchOrdinal: previousDispatch?.data.workerDispatchOrdinal ?? null,
|
|
3123
3291
|
});
|
|
3124
3292
|
if (bootstrapExhausted === channelKey(assignment)) { // a later tier-climb restore may have moved it off
|
|
3125
|
-
await park(t, `${bootstrapExhausted} failed at bootstrap and no eligible channel remains after resume`, "infra", assignment, rs?.attempts ?? 0, startMs, 0, 0, undefined, 0, "fresh", { cause: "bootstrap", channel: bootstrapExhausted });
|
|
3293
|
+
await park(t, `${bootstrapExhausted} failed at bootstrap and no eligible channel remains after resume${poolExhaustion(t, cfg)}`, "infra", assignment, rs?.attempts ?? 0, startMs, 0, 0, undefined, 0, "fresh", { cause: "bootstrap", channel: bootstrapExhausted });
|
|
3126
3294
|
return;
|
|
3127
3295
|
}
|
|
3128
3296
|
// Keep one live list: recovery retries must see exclusions added by onGate during the round.
|
|
@@ -3133,18 +3301,33 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3133
3301
|
if (!badReviewers.includes(e.payload.reviewer))
|
|
3134
3302
|
badReviewers.push(e.payload.reviewer);
|
|
3135
3303
|
}
|
|
3304
|
+
// OBS-1196: a seat whose same-subject re-emission delivered a valid verdict was a delivery flake,
|
|
3305
|
+
// not a bad reviewer — it stays seatable for this task's later rounds.
|
|
3306
|
+
if (e.name === "review-reemission" && e.payload.delivered === true && typeof e.payload.reviewer === "string") {
|
|
3307
|
+
const at = badReviewers.indexOf(e.payload.reviewer);
|
|
3308
|
+
if (at >= 0)
|
|
3309
|
+
badReviewers.splice(at, 1);
|
|
3310
|
+
}
|
|
3136
3311
|
};
|
|
3137
3312
|
// Keep the measured siblings, including a red judge, while replacing only a missing review.
|
|
3138
3313
|
// Every dispatched seat is excluded for this task, so the loop is bounded by the eligible pool.
|
|
3139
3314
|
const runReviewRecovery = async (task, ctx, allowSelection = true) => {
|
|
3140
3315
|
ctx.authorizeInfraRetry = cfg.executionPolicy?.boundedInfrastructure
|
|
3141
3316
|
? (subject, cause) => reserveInfrastructureRetry(journal.read(), task.id, subject, journal.append.bind(journal), cause) : undefined;
|
|
3142
|
-
if (
|
|
3317
|
+
if (repairSelection && allowSelection) {
|
|
3143
3318
|
const decision = repairSelectionDecision(journal.read(), task.id, true);
|
|
3144
3319
|
ctx.selectTests = decision.selectTests;
|
|
3145
3320
|
ctx.requiredRepairTests = decision.requiredFiles;
|
|
3146
3321
|
ctx.selectionReason = decision.reason;
|
|
3147
3322
|
}
|
|
3323
|
+
else if (repairSelection) {
|
|
3324
|
+
// A recheck runs the full suite; its recorded scope keeps a red there attributable.
|
|
3325
|
+
ctx.selectTests = false;
|
|
3326
|
+
ctx.selectionReason = "recheck-full-suite";
|
|
3327
|
+
}
|
|
3328
|
+
// OBS-1151: every round judges fresh; the journal's earlier judgments (this run and before a
|
|
3329
|
+
// resume) are only the subjects a reversed ruling is compared against.
|
|
3330
|
+
ctx.priorJudgments = priorJudgments(journal.read(), task.id);
|
|
3148
3331
|
// Resume and ordinary verification share this boundary. Count persisted rounds so a
|
|
3149
3332
|
// resumed daemon cannot reuse a previous round's identity within the same attempt.
|
|
3150
3333
|
ctx.buildReceiptIdentity = {
|
|
@@ -3247,9 +3430,9 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3247
3430
|
...(typeof g.meta.producer === "string" ? { producer: g.meta.producer } : {}),
|
|
3248
3431
|
...(typeof g.meta.producerAttempt === "number" ? { producerAttempt: g.meta.producerAttempt } : {}),
|
|
3249
3432
|
} : {}),
|
|
3250
|
-
...(cfg.executionPolicy && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
3433
|
+
...((cfg.executionPolicy || repairSelection) && !g.pass ? { disposition: failureDisposition(g) } : {}),
|
|
3251
3434
|
...Object.fromEntries(["runnerInfraRerun", "hostStarvedRerun", "recoveryBlocked", "failingFiles", "selectionDecision", "failureEvidence",
|
|
3252
|
-
"forgivenFingerprints", "freshFingerprints", "baselineProvenance"]
|
|
3435
|
+
"forgivenFingerprints", "freshFingerprints", "baselineProvenance", "judgment", "adjudication", "judgeDisagreement", "interruptions"]
|
|
3253
3436
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
3254
3437
|
// OBS-540: preserve terminal-vs-retryable infra exactly. normalizeGateOutcome deliberately
|
|
3255
3438
|
// defaults a legacy infra row to retryable, so dropping an explicit false here reverses the
|
|
@@ -3275,7 +3458,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3275
3458
|
} : {}),
|
|
3276
3459
|
...(g.gate === "review" && typeof g.meta?.reviewer === "string" ? {
|
|
3277
3460
|
reviewer: g.meta.reviewer,
|
|
3278
|
-
...Object.fromEntries(["cause", "seatAuthoredBytes", "bytes", "rawPath", "briefPath", "timeoutMs", "unparseable", "noVerdict", "resolved", "reraised", "reviewerFloor", "reviewerFloorCause", "reviewerTier"]
|
|
3461
|
+
...Object.fromEntries(["cause", "seatAuthoredBytes", "bytes", "rawPath", "briefPath", "timeoutMs", "unparseable", "noVerdict", "resolved", "reraised", "settledAnchors", "reviewerFloor", "reviewerFloorCause", "reviewerTier"]
|
|
3279
3462
|
.filter((key) => g.meta?.[key] !== undefined).map((key) => [key, g.meta[key]])),
|
|
3280
3463
|
...(typeof g.meta.vendor === "string" ? { vendor: g.meta.vendor } : {}),
|
|
3281
3464
|
...(typeof g.meta.provider === "string" ? { provider: g.meta.provider } : {}),
|
|
@@ -3409,7 +3592,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3409
3592
|
return;
|
|
3410
3593
|
const from = channelKey(assignment);
|
|
3411
3594
|
const to = channelKey(next);
|
|
3412
|
-
const poolBefore = channels.filter((c) => !tried.includes(channelKey(c)) && !demotedChannels.has(channelKey(c))
|
|
3595
|
+
const poolBefore = channels.filter((c) => !tried.includes(channelKey(c)) && !demotedChannels.has(channelKey(c))
|
|
3596
|
+
&& inTaskPool(t, cfg, channelKey(c))).map(channelKey);
|
|
3413
3597
|
journal.append("tier-escalated", t.id, {
|
|
3414
3598
|
attempt, cause, gate: gate.gate, fingerprint: normalizeGateFailure(gate.details).slice(0, 500),
|
|
3415
3599
|
from, to, fromTier: assignment.tier, toTier: next.tier, poolBefore,
|
|
@@ -3467,14 +3651,15 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3467
3651
|
// on artificial scarcity; MAX_ATTEMPTS and the review round cap are the real bounds. Only a
|
|
3468
3652
|
// fleet whose every channel is DEMOTED (verified dead: auth/setup/provider outage) has nothing
|
|
3469
3653
|
// left to spawn — that case alone still parks.
|
|
3654
|
+
// OBS-1187: shared with the demotion seam's pooled recycle so neither relaunches an escalated adapter.
|
|
3655
|
+
const recycleExcluded = () => [channelKey(assignment), ...channels.filter((c) => escalatedAdapters.has(c.adapter)).map(channelKey)];
|
|
3470
3656
|
const failoverOrRecycle = (site) => {
|
|
3471
3657
|
const next = failover(site);
|
|
3472
3658
|
if (next)
|
|
3473
3659
|
return next;
|
|
3474
3660
|
// OBS-1169: a vendor escalated away from stays out of the recycle pool until its release —
|
|
3475
3661
|
// recycling any of its channels, launched or not, would launch the broken bootstrap again.
|
|
3476
|
-
const
|
|
3477
|
-
const recycled = nextChannel(assignment, t, cfg, channels, recycleExcluded, profile, demotedChannels)
|
|
3662
|
+
const recycled = nextChannel(assignment, t, cfg, channels, recycleExcluded(), profile, demotedChannels)
|
|
3478
3663
|
?? (demotedChannels.has(channelKey(assignment)) ? null : assignment);
|
|
3479
3664
|
if (recycled)
|
|
3480
3665
|
journal.append("channel-recycle", t.id, { site, channel: channelKey(recycled) });
|
|
@@ -3493,7 +3678,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3493
3678
|
// D-07: consult panes self-clean when the verdict is read (keepLlm) — only "forever" keeps them.
|
|
3494
3679
|
// v1.54 T1: channels = this run's doctor-filtered live list — consult.prefer seat liveness
|
|
3495
3680
|
// is judged against it, never rebuilt from config (installed-but-unauthed seats would stall).
|
|
3496
|
-
{ keep: keepLlm, onSlot: keepLlm ? (s) => keptSlots.push(s) : undefined, onInvocation: (inv) => invocations.push(inv), runId, channels: pools.consult });
|
|
3681
|
+
{ keep: keepLlm, onSlot: keepLlm ? (s) => keptSlots.push(s) : undefined, onInvocation: (inv) => invocations.push(inv), runId, channels: pools.consult, health });
|
|
3497
3682
|
// A lone answering seat is already the row's adapter/model/vendor/effort; the list is journaled
|
|
3498
3683
|
// only when a seat failed, so the row never repeats itself and a failed seat never vanishes.
|
|
3499
3684
|
return invocations.some((inv) => inv.outcome === "failed") ? { ...v, invocations } : v;
|
|
@@ -3539,7 +3724,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3539
3724
|
tried.push(k);
|
|
3540
3725
|
return true;
|
|
3541
3726
|
}
|
|
3542
|
-
await park(t,
|
|
3727
|
+
await park(t, `consult said reroute but every channel is demoted (verified dead) — nothing left to spawn${poolExhaustion(t, cfg)}`, "reroute-exhausted", assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
3543
3728
|
return false;
|
|
3544
3729
|
}
|
|
3545
3730
|
await park(t, `consult verdict: ${v.action} — ${v.notes}`, trigger, assignment, attempts, startMs, gateFails, consults, tokens, metered, retryMode); // decompose|human
|
|
@@ -3620,20 +3805,19 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3620
3805
|
// died. A pane the driver cannot read is DECLINED, never classified: an unreadable pane proves
|
|
3621
3806
|
// neither a matching trailer nor the absence of a foreign one, so it is neither harvested nor
|
|
3622
3807
|
// gated as trailerless evidence. A driver whose terminals outlive it (Orca) adopts the owned
|
|
3623
|
-
// terminal: bound only on ownership evidence — the journaled owned
|
|
3624
|
-
// never a
|
|
3625
|
-
// driver numbers its slots from one again, so an interrupted task's old
|
|
3626
|
-
// terminal THIS instance bound under that id (another task's adoption or allocation).
|
|
3808
|
+
// terminal (Orca) or pane (Herdr, OBS-1203): bound only on ownership evidence — the journaled owned
|
|
3809
|
+
// name AND the task checkout — never a name alone. The journaled slot.id is NEVER trusted across
|
|
3810
|
+
// daemon instances: a fresh driver numbers its slots from one again, so an interrupted task's old
|
|
3811
|
+
// id can name whatever terminal THIS instance bound under that id (another task's adoption or allocation).
|
|
3627
3812
|
// A driver with no read-only adoption is declined BEFORE anything is bound: slot() allocates —
|
|
3628
3813
|
// herdr's reclaims the same-named pane (closing the evidence unread) and holds a dispatch lease
|
|
3629
3814
|
// only run() releases — so it is never an adoption fallback; that driver keeps ordinary recovery.
|
|
3630
|
-
|
|
3631
|
-
if (!adopt)
|
|
3815
|
+
if (!driver.adopt)
|
|
3632
3816
|
return decline("no read-only adoption on this driver");
|
|
3633
3817
|
let ownedSlot;
|
|
3634
3818
|
let pane;
|
|
3635
3819
|
try {
|
|
3636
|
-
ownedSlot = await adopt
|
|
3820
|
+
ownedSlot = await driver.adopt(slot);
|
|
3637
3821
|
pane = await driver.read(ownedSlot, PANE_READ_ROWS);
|
|
3638
3822
|
}
|
|
3639
3823
|
catch (error) {
|
|
@@ -3661,6 +3845,47 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3661
3845
|
return found.assignment;
|
|
3662
3846
|
};
|
|
3663
3847
|
const harvestAuthor = rs && !satisfiedGate && !replayedGates && !recheck ? await harvestInterruptedAttempt() : undefined;
|
|
3848
|
+
// OBS-1202: a recheck bound to a stall park that recorded a reapFailure re-verifies the parked
|
|
3849
|
+
// attempt's owned census through the same reapWorker a live harvest uses, BEFORE any gate reads
|
|
3850
|
+
// its tree. Only a complete, empty census releases the harvested commits to the battery; an
|
|
3851
|
+
// unreadable (null) or surviving census keeps the task parked on the same kind — a stall carrying
|
|
3852
|
+
// the fresh reapFailure, bound by its own new token — and no worker is funded either way. Plain
|
|
3853
|
+
// approve on that park keeps its dispatch semantics.
|
|
3854
|
+
if (recheck) {
|
|
3855
|
+
const approvalRow = effectiveEvents(journal.read()).reverse()
|
|
3856
|
+
.find((e) => e.taskId === t.id && e.event === "task-approved" && e.data.release === RECHECK_RELEASE);
|
|
3857
|
+
if (typeof approvalRow?.data.reapFailure === "string") {
|
|
3858
|
+
const launch = [...journal.read()].reverse().find((e) => e.taskId === t.id && e.event === "worker-launch");
|
|
3859
|
+
const owned = launch?.data.slot;
|
|
3860
|
+
let uncertain;
|
|
3861
|
+
if (typeof launch?.data.nonce !== "string" || typeof launch.data.dispatchScript !== "string"
|
|
3862
|
+
|| typeof owned?.name !== "string" || typeof owned.cwd !== "string") {
|
|
3863
|
+
uncertain = "the parked attempt recorded no launch ownership evidence";
|
|
3864
|
+
}
|
|
3865
|
+
else {
|
|
3866
|
+
const key = { ...owned };
|
|
3867
|
+
workerOwners.set(key, { taskId: t.id, attempt: Number(launch.data.attempt ?? 0),
|
|
3868
|
+
groupFile: `${launch.data.dispatchScript}.${launch.data.nonce}.pgid`, marker: launch.data.dispatchScript,
|
|
3869
|
+
identities: new Map(), descendants: new Map() });
|
|
3870
|
+
try {
|
|
3871
|
+
await reapWorker(key);
|
|
3872
|
+
// reapWorker returns normally for a null census when no ownership file was readable (a
|
|
3873
|
+
// missing .pgid/.session with no descendants); recovery needs a RECORDED empty census.
|
|
3874
|
+
const census = reapReports.get(key)?.survivors;
|
|
3875
|
+
if (!Array.isArray(census) || census.length > 0) {
|
|
3876
|
+
uncertain = `worker group ${reapReports.get(key)?.processGroup ?? "unknown"} census unrecorded`;
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
catch (error) {
|
|
3880
|
+
uncertain = error instanceof Error ? error.message : String(error);
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3883
|
+
if (uncertain !== undefined) {
|
|
3884
|
+
await park(t, `worker could not be reaped before recheck: ${uncertain}`, "stall", null, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode, { reapFailure: uncertain });
|
|
3885
|
+
return;
|
|
3886
|
+
}
|
|
3887
|
+
}
|
|
3888
|
+
}
|
|
3664
3889
|
resumeGateReplay: if (satisfiedGate || replayedGates || recheck || harvestAuthor) {
|
|
3665
3890
|
const taskBase = await integrationHead(intWt);
|
|
3666
3891
|
taskBases.set(t.id, taskBase);
|
|
@@ -3887,14 +4112,27 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3887
4112
|
}
|
|
3888
4113
|
await trackedDriver.project?.(t.id, "in-review");
|
|
3889
4114
|
await waitForBaseline(t.id);
|
|
4115
|
+
// OBS-1106 residual: the live loop's timeout refresh, decided again from the journal, so a crash
|
|
4116
|
+
// cannot launder the copy it forbids. A harvested retry that landed nothing owes it exactly as
|
|
4117
|
+
// the live loop would have; a crash inside a forced battery before its test row still owes it.
|
|
4118
|
+
const lastRound = (() => {
|
|
4119
|
+
const own = journal.read().filter((e) => e.taskId === t.id);
|
|
4120
|
+
return own.slice(own.map((e) => e.event === "phase-start" && e.data.phase === "gates").lastIndexOf(true) + 1);
|
|
4121
|
+
})();
|
|
4122
|
+
const freshTimeout = !recheck && !satisfiedGate && ((harvestAuthor !== undefined && owesTimeoutRefresh(t, lastRound, gateSubject.commit))
|
|
4123
|
+
|| (lastRound.some((e) => e.event === "gate-fresh-forced" && e.data.commit === gateSubject.commit)
|
|
4124
|
+
&& !lastRound.some((e) => e.event === "gate-result" && e.data.gate === "test")));
|
|
3890
4125
|
journal.phaseStart(t.id, "gates");
|
|
4126
|
+
if (freshTimeout) {
|
|
4127
|
+
journal.append("gate-fresh-forced", t.id, { gate: "test", attempt: gateSubject.attempt, commit: gateSubject.commit, reason: "no-commit-timeout-red", resumed: true });
|
|
4128
|
+
}
|
|
3891
4129
|
const { results } = await withCommandContext(t.id, async () => runReviewRecovery(resumedTask, {
|
|
3892
4130
|
carriedAuthors: await subjectAuthors(journal.read(), t.id, wt, taskBase),
|
|
3893
4131
|
producer: producerNow(),
|
|
3894
4132
|
carriedFindings: outstandingReviewFindings(journal.read(), t.id),
|
|
3895
4133
|
operatorContext,
|
|
3896
4134
|
worktree: wt, baseRef: taskBase, result: priorResult, author: gateAuthor,
|
|
3897
|
-
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
4135
|
+
commands, baseline, channels: pools.review, judgeChannels: pools.judge, health, adapters, cfg, artifactDir: journal.dir,
|
|
3898
4136
|
collateral: collateral.get(t.id) ?? [],
|
|
3899
4137
|
// a recheck re-verifies a human's release: it never selects tests down, it runs the suite.
|
|
3900
4138
|
via: cfg.visibility.llm === "pane"
|
|
@@ -3912,7 +4150,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3912
4150
|
// flaking seat is re-asked on every task.
|
|
3913
4151
|
reviewNoVerdicts,
|
|
3914
4152
|
recheck, // OBS-1055: a recheck discards cached reds — the battery re-measures what the operator questioned
|
|
4153
|
+
cachedRedBypass: freshTimeout ? "timeout-fresh" : undefined,
|
|
3915
4154
|
onGate: async (e) => {
|
|
4155
|
+
if (!liveEngagement())
|
|
4156
|
+
return;
|
|
3916
4157
|
if (e.phase === "start") {
|
|
3917
4158
|
notePhaseStart(e);
|
|
3918
4159
|
journal.phaseStart(t.id, phaseForGate(e.gate), { gate: e.gate, index: e.index, total: e.total, ...(e.parentAt === undefined ? {} : { parallel: true }) });
|
|
@@ -3928,10 +4169,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3928
4169
|
inParallelOrder(g.gate, () => {
|
|
3929
4170
|
journalGateResult(g);
|
|
3930
4171
|
noteReviewRetry(g);
|
|
3931
|
-
if (
|
|
3932
|
-
&& typeof g.meta?.reviewer === "string") {
|
|
4172
|
+
if (isGarbageReview(g))
|
|
3933
4173
|
badReviewers.push(g.meta.reviewer);
|
|
3934
|
-
}
|
|
3935
4174
|
});
|
|
3936
4175
|
},
|
|
3937
4176
|
}, false));
|
|
@@ -3974,6 +4213,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
3974
4213
|
// erase an EARLIER chargeable attempt and attribute the park to the assignment the rewind
|
|
3975
4214
|
// restored. Ask the journal which state this is, and take the classified dispatch's own
|
|
3976
4215
|
// assignment back with it.
|
|
4216
|
+
if (await adjudicateInfraShapedRed(t, results, wt, gateSubject.commit, rs?.attempts ?? 0, false, (reason) => park(t, reason, "infra", gateAuthor, rs?.attempts ?? 0, startMs, gateFails, consults, tokens, metered, retryMode)))
|
|
4217
|
+
return;
|
|
3977
4218
|
const classified = journal.classifiedDispatch(t.id);
|
|
3978
4219
|
if (await dispositionScopeRed(t, results, classified?.assignment ?? gateAuthor, classified ? (rs?.attempts ?? 0) : Math.max(0, (rs?.attempts ?? 0) - 1), startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
3979
4220
|
return;
|
|
@@ -4103,13 +4344,21 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4103
4344
|
}
|
|
4104
4345
|
// OBS-57: a demoted channel must not be re-dispatched on consult retry or provider requeue.
|
|
4105
4346
|
if (demotedChannels.has(channelKey(assignment))) {
|
|
4106
|
-
|
|
4347
|
+
// OBS-1187: a pooled task recycles a live pool member (OBS-202) before it may park naming the pool;
|
|
4348
|
+
// it never keeps a demoted seat and never leaves the pool.
|
|
4349
|
+
const pooled = taskPool(t, cfg) !== undefined;
|
|
4350
|
+
const next = nextChannel(assignment, t, cfg, channels, tried, profile, demotedChannels)
|
|
4351
|
+
?? (pooled ? nextChannel(assignment, t, cfg, channels, recycleExcluded(), profile, demotedChannels) : null);
|
|
4107
4352
|
if (next) {
|
|
4108
4353
|
assignment = next;
|
|
4109
4354
|
const k = channelKey(next);
|
|
4110
4355
|
if (!tried.includes(k))
|
|
4111
4356
|
tried.push(k);
|
|
4112
4357
|
}
|
|
4358
|
+
else if (pooled) {
|
|
4359
|
+
await park(t, `${channelKey(assignment)} is demoted (verified dead) and no live pool member remains${poolExhaustion(t, cfg)}`, "reroute-exhausted", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4360
|
+
return;
|
|
4361
|
+
}
|
|
4113
4362
|
}
|
|
4114
4363
|
// v1.29: consume the prior gate-failed session once. Same channel + known under-threshold context
|
|
4115
4364
|
// + adapter capability resumes; every other path is today's fresh dispatch.
|
|
@@ -4134,7 +4383,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4134
4383
|
gate: banned, from: channelKey(assignment), to: next ? channelKey(next) : null,
|
|
4135
4384
|
});
|
|
4136
4385
|
if (!next) {
|
|
4137
|
-
await park(t, gateFailApprovalReason(t.id, `identical ${banned} failure twice this engagement — an identical retry is banned and no untried channel is left`), "gate-fail", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4386
|
+
await park(t, gateFailApprovalReason(t.id, `identical ${banned} failure twice this engagement — an identical retry is banned and no untried channel is left${poolExhaustion(t, cfg)}`), "gate-fail", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
4138
4387
|
return;
|
|
4139
4388
|
}
|
|
4140
4389
|
assignment = next;
|
|
@@ -4348,10 +4597,18 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
4348
4597
|
? `${new TextDecoder().decode(rawBytes.subarray(0, cap))}\n… diff truncated at gates.diffCap (${cap} bytes)`
|
|
4349
4598
|
+ `\nPARTIAL DIFF: the diff above is incomplete; files cut by the cap: ${droppedFiles.join(", ") || "(unknown)"}`
|
|
4350
4599
|
: raw.stdout;
|
|
4351
|
-
const
|
|
4600
|
+
const repeated = reraisedReviewChains(journaledSoFar, t.id).filter((chain) => chain.reraises > 1);
|
|
4601
|
+
const brief = repairBrief(repairFindings, diff, taskBase, repeated);
|
|
4352
4602
|
// anything the live brief holds beyond the journaled findings (a consult's guidance) is kept:
|
|
4353
4603
|
// a repair adds the diff and the fix-only contract, it never subtracts what was already known.
|
|
4354
|
-
|
|
4604
|
+
// OBS-1195: a resumed upheld repair re-derives the same review bytes on both carries; when the
|
|
4605
|
+
// repair already quotes them verbatim they are said once, and the operator's uphold line stays.
|
|
4606
|
+
// Both carries had their deferrals cut above, so the quote is compared after the same cut.
|
|
4607
|
+
const upheldQuote = upheldFeedback ? withoutDeferrals(`review: ${upheldFeedback}`) : undefined;
|
|
4608
|
+
const rest = upheldQuote && brief.includes(upheldQuote)
|
|
4609
|
+
? feedback.replace(upheldQuote, "").replace(/\n{3,}/g, "\n\n").trim()
|
|
4610
|
+
: feedback;
|
|
4611
|
+
feedback = rest && !brief.includes(rest) ? `${brief}\n\n${rest}` : brief;
|
|
4355
4612
|
journal.append("repair-dispatch", t.id, { workerDispatchOrdinal, diffBytes: Buffer.byteLength(diff, "utf8"), capped, ...(capped ? { droppedFiles } : {}) });
|
|
4356
4613
|
}
|
|
4357
4614
|
if (feedback || priorNamed.length > 0) {
|
|
@@ -5675,7 +5932,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5675
5932
|
attempt--; // the dead bootstrap bought no worker turn
|
|
5676
5933
|
continue;
|
|
5677
5934
|
}
|
|
5678
|
-
await park(t, `${from} failed at bootstrap after ${retries} same-channel retry and no eligible channel remains`, "infra", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "bootstrap", channel: from, retries });
|
|
5935
|
+
await park(t, `${from} failed at bootstrap after ${retries} same-channel retry and no eligible channel remains${poolExhaustion(t, cfg)}`, "infra", assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "bootstrap", channel: from, retries });
|
|
5679
5936
|
return;
|
|
5680
5937
|
}
|
|
5681
5938
|
const preHarvestResult = result;
|
|
@@ -5787,7 +6044,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5787
6044
|
tried.push(channelKey(next));
|
|
5788
6045
|
continue;
|
|
5789
6046
|
}
|
|
5790
|
-
await park(t, `capacity exhausted on ${from} after ${requeues} requeues and no eligible channel at floor`, "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "capacity", channel: from, requeues });
|
|
6047
|
+
await park(t, `capacity exhausted on ${from} after ${requeues} requeues and no eligible channel at floor${poolExhaustion(t, cfg)}`, "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode, { cause: "capacity", channel: from, requeues });
|
|
5791
6048
|
return;
|
|
5792
6049
|
}
|
|
5793
6050
|
// quota exhaustion → failover within floor; does NOT consume the ladder (spec §4)
|
|
@@ -5850,7 +6107,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5850
6107
|
tried.push(channelKey(next));
|
|
5851
6108
|
continue;
|
|
5852
6109
|
}
|
|
5853
|
-
await park(t,
|
|
6110
|
+
await park(t, `quota exhausted on every eligible channel${poolExhaustion(t, cfg)}`, "quota", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5854
6111
|
return;
|
|
5855
6112
|
}
|
|
5856
6113
|
// v1.65 T1: typed dead-channel failure — the parse boundary classified this no-trailer result
|
|
@@ -5898,7 +6155,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5898
6155
|
continue;
|
|
5899
6156
|
}
|
|
5900
6157
|
const chargedAttempts = adapterStartupFailure || startupFailure ? attempt : attempt + 1;
|
|
5901
|
-
await park(t, `dead channel (${dead}) and no eligible channel remains`, "reroute-exhausted", assignment, chargedAttempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
6158
|
+
await park(t, `dead channel (${dead}) and no eligible channel remains${poolExhaustion(t, cfg)}`, "reroute-exhausted", assignment, chargedAttempts, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
5902
6159
|
return;
|
|
5903
6160
|
}
|
|
5904
6161
|
if (!finished) {
|
|
@@ -5943,6 +6200,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5943
6200
|
}
|
|
5944
6201
|
}
|
|
5945
6202
|
const onGate = async (e) => {
|
|
6203
|
+
if (!liveEngagement())
|
|
6204
|
+
return;
|
|
5946
6205
|
if (e.phase === "start") {
|
|
5947
6206
|
notePhaseStart(e);
|
|
5948
6207
|
journal.phaseStart(t.id, phaseForGate(e.gate), { gate: e.gate, index: e.index, total: e.total, ...(e.parentAt === undefined ? {} : { parallel: true }) });
|
|
@@ -5976,9 +6235,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5976
6235
|
journalGateResult(g);
|
|
5977
6236
|
noteReviewRetry(g);
|
|
5978
6237
|
// v1.1 failover: never re-ask a reviewer channel that produced garbage for this task
|
|
5979
|
-
if (
|
|
6238
|
+
if (isGarbageReview(g))
|
|
5980
6239
|
badReviewers.push(g.meta.reviewer);
|
|
5981
|
-
}
|
|
5982
6240
|
});
|
|
5983
6241
|
};
|
|
5984
6242
|
let results = [];
|
|
@@ -5994,12 +6252,16 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
5994
6252
|
// sees the replay, without buying another command or reviewer invocation.
|
|
5995
6253
|
const taskEvents = journal.read().filter((e) => e.taskId === t.id);
|
|
5996
6254
|
const previousRound = taskEvents.map((e) => e.event === "phase-start" && e.data.phase === "gates").lastIndexOf(true);
|
|
5997
|
-
const
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6255
|
+
const lastRoundRows = taskEvents.slice(previousRound + 1).filter((e) => e.event === "gate-result"
|
|
6256
|
+
&& e.data.attempt === attempt - 1);
|
|
6257
|
+
const previousRows = retryMode === "repair" ? lastRoundRows : [];
|
|
6258
|
+
// OBS-1106 residual: see owesTimeoutRefresh — the replay below and the verdict cache are both refused.
|
|
6259
|
+
const freshTimeout = !explicitlyFundedAttempt && owesTimeoutRefresh(t, lastRoundRows, gateSubject.commit);
|
|
6001
6260
|
journal.phaseStart(t.id, "gates");
|
|
6002
|
-
|
|
6261
|
+
if (freshTimeout) {
|
|
6262
|
+
journal.append("gate-fresh-forced", t.id, { gate: "test", attempt, priorAttempt: attempt - 1, commit: gateSubject.commit, reason: "no-commit-timeout-red" });
|
|
6263
|
+
}
|
|
6264
|
+
const replay = !explicitlyFundedAttempt && !freshTimeout && previousRows.length > 0
|
|
6003
6265
|
&& previousRows.every((e) => e.data.commit === gateSubject.commit)
|
|
6004
6266
|
&& previousRows.some((e) => e.data.pass === false && e.data.skipped !== true);
|
|
6005
6267
|
if (replay) {
|
|
@@ -6029,10 +6291,10 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6029
6291
|
carriedFindings: outstandingFindings,
|
|
6030
6292
|
operatorContext,
|
|
6031
6293
|
worktree: wt, baseRef: taskBase, result, author: assignment,
|
|
6032
|
-
commands, baseline, channels: pools.review, judgeChannels: pools.judge, adapters, cfg, artifactDir: journal.dir,
|
|
6294
|
+
commands, baseline, channels: pools.review, judgeChannels: pools.judge, health, adapters, cfg, artifactDir: journal.dir,
|
|
6033
6295
|
collateral: collateral.get(t.id) ?? [],
|
|
6034
6296
|
selectTests: !testGateFailed,
|
|
6035
|
-
cachedRedBypass: explicitlyFundedAttempt ? "operator-rerun" : undefined,
|
|
6297
|
+
cachedRedBypass: explicitlyFundedAttempt ? "operator-rerun" : freshTimeout ? "timeout-fresh" : undefined,
|
|
6036
6298
|
via: cfg.visibility.llm === "pane"
|
|
6037
6299
|
? {
|
|
6038
6300
|
driver: trackedDriver,
|
|
@@ -6139,6 +6401,8 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6139
6401
|
await park(t, `${capTrip.gate}: ${capTrip.details}`, "diff-cap", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode);
|
|
6140
6402
|
return;
|
|
6141
6403
|
}
|
|
6404
|
+
if (await adjudicateInfraShapedRed(t, results, wt, gateSubject.commit, attempt, gateSubject.replayedFromAttempt !== undefined, (reason) => park(t, reason, "infra", assignment, attempt + 1, startMs, gateFails, consults, tokens, metered, retryMode)))
|
|
6405
|
+
return;
|
|
6142
6406
|
// OBS-547: who pays for this red is decided by the run's collateral prediction (see
|
|
6143
6407
|
// dispositionScopeRed) — an authoring defect parks unchargeable before any accounting below.
|
|
6144
6408
|
if (await dispositionScopeRed(t, results, assignment, attempt, startMs, gateFails, consults, tokens, metered, retryMode, await treeOrDiffPaths(taskBase, wt)))
|
|
@@ -6371,6 +6635,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6371
6635
|
for (const t of ready) {
|
|
6372
6636
|
const p = executeWithPolicy(t)
|
|
6373
6637
|
.catch(async (err) => {
|
|
6638
|
+
engagements.delete(t.id);
|
|
6374
6639
|
const cleanupErrors = [];
|
|
6375
6640
|
for (const [slot, owner] of workerOwners) {
|
|
6376
6641
|
if (owner.taskId !== t.id || !liveSlots.has(slot))
|
|
@@ -6429,7 +6694,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6429
6694
|
journal.telemetry({ taskId: t.id, shape: t.shape, adapter: "-", model: "-", channel: "-", attempts: 0, outcome: "failed", durationMs: 0 });
|
|
6430
6695
|
await reconcile({ spareLiveLlm: true }); // task-failed is a terminal event
|
|
6431
6696
|
})
|
|
6432
|
-
.finally(() => inflight.delete(t.id));
|
|
6697
|
+
.finally(() => { inflight.delete(t.id); engagements.delete(t.id); });
|
|
6433
6698
|
inflight.set(t.id, p);
|
|
6434
6699
|
}
|
|
6435
6700
|
if (inflight.size === 0) {
|
|
@@ -6529,7 +6794,7 @@ export async function runDaemon(repoRoot, opts = {}) {
|
|
|
6529
6794
|
poll = setTimeout(pollApprovals, APPROVAL_POLL_MS);
|
|
6530
6795
|
let tipFailed;
|
|
6531
6796
|
try {
|
|
6532
|
-
const verification = withCommandContext(undefined, () => verifyIntegrationTipCached(intWt, commands, journal, { lastMergedTask, baseline, signal: controller.signal }), controller.signal);
|
|
6797
|
+
const verification = withCommandContext(undefined, () => verifyIntegrationTipCached(intWt, commands, journal, { lastMergedTask, baseline, signal: controller.signal, evidence: { quotaBytes: cfg.gates?.evidenceQuotaBytes } }), controller.signal);
|
|
6533
6798
|
activeTipVerify = { controller, settled: verification.then(() => { }, () => { }) };
|
|
6534
6799
|
tipFailed = await verification;
|
|
6535
6800
|
}
|