pi-goal-list-loop-audit 0.35.4 → 0.35.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7666 -0
- package/INSTALL.md +377 -0
- package/README.md +26 -13
- package/docs/DESIGN.md +31 -4
- package/docs/INDEX.md +32 -7
- package/examples/example-objective.md +119 -0
- package/extensions/faulty-objective-recovery.ts +13 -2
- package/extensions/goal-commands.ts +50 -6
- package/extensions/goal-heartbeat.ts +25 -21
- package/extensions/goal-loop-auditor-process.ts +184 -3
- package/extensions/goal-loop-core.ts +108 -15
- package/extensions/goal-loop-display.ts +82 -0
- package/extensions/goal-loop-shield.ts +55 -0
- package/extensions/goal-recovery.ts +317 -7
- package/extensions/goal-settings.ts +31 -1
- package/extensions/loops/goal-activation.ts +74 -7
- package/extensions/loops/goal-auditor-hooks.ts +30 -55
- package/extensions/loops/goal-list-queue.ts +18 -5
- package/extensions/loops/goal-orchestrator.ts +5 -0
- package/extensions/loops/goal-runtime-globals.ts +2 -0
- package/extensions/loops/goal-session.ts +101 -21
- package/extensions/loops/goal-settings-ui.ts +122 -63
- package/extensions/loops/goal-tools.ts +110 -36
- package/extensions/loops/goal-ui.ts +59 -2
- package/extensions/main-model-recovery.ts +24 -1
- package/extensions/model-picker.ts +3 -1
- package/extensions/model-selector.ts +2 -1
- package/extensions/multi-model-picker.ts +64 -7
- package/extensions/settings-menu.ts +16 -0
- package/package.json +4 -1
- package/prompts/goal-loop-continuation.md +11 -8
- package/prompts/goal-loop-draft.md +5 -5
- package/schemas/goal.schema.json +17 -0
|
@@ -22,7 +22,22 @@ import { state } from "./goal-state.js";
|
|
|
22
22
|
import { appendLedger, claimRecoveryNotice, nowIso, piGlaDir, isForbiddenModel, isStaleApiError, nextHourlyProbeMs, providerErrorFingerprint, providerErrorPresentation, sanitizeProviderDisplayText, writeGoalMd, type Goal, type MainModelRecovery, type PendingCompletion } from "./goal-loop-core.js";
|
|
23
23
|
import { persistStateLine } from "./goal-state.js";
|
|
24
24
|
import { cancelDetachedGoalCompletionAuditor } from "./goal-loop-auditor-process.js";
|
|
25
|
-
import {
|
|
25
|
+
import {
|
|
26
|
+
classifyMainModelFailure,
|
|
27
|
+
isContextOverflowError,
|
|
28
|
+
isMainModelFailbackAuto,
|
|
29
|
+
isMainModelFallbackFailure,
|
|
30
|
+
mainModelAutoRetryUntil,
|
|
31
|
+
mainModelFailureDelayMs,
|
|
32
|
+
mainModelPrimaryProbeDelayMs,
|
|
33
|
+
mainModelRetryDelayMs,
|
|
34
|
+
MAIN_MODEL_AUTO_RETRY_HORIZON_MS,
|
|
35
|
+
modelRef,
|
|
36
|
+
normalizeMainModelFallbackRefs,
|
|
37
|
+
requiresMainModelRecovery,
|
|
38
|
+
splitModelRef,
|
|
39
|
+
type MainModelFailure,
|
|
40
|
+
} from "./main-model-recovery.js";
|
|
26
41
|
import { ModelSelector, type ModelScope } from "./model-selector.js";
|
|
27
42
|
import { loadGlobalSettings, loadSettings } from "./goal-settings.js";
|
|
28
43
|
import { clearLoopTimer, scheduleLoopTick } from "./goal-loop.js";
|
|
@@ -361,6 +376,27 @@ export function mainModelFallbackRefs(ctx: ExtensionContext): string[] {
|
|
|
361
376
|
try { return normalizeMainModelFallbackRefs(loadGlobalSettings().mainModelFallbacks); } catch { return []; }
|
|
362
377
|
}
|
|
363
378
|
|
|
379
|
+
function mainModelFailbackEnabled(): boolean {
|
|
380
|
+
try { return isMainModelFailbackAuto(loadGlobalSettings().mainModelFailback); } catch { return true; }
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function mainModelPrimaryProbeDelay(): number {
|
|
384
|
+
try { return mainModelPrimaryProbeDelayMs(loadGlobalSettings().mainModelPrimaryProbeMinutes); } catch { return mainModelPrimaryProbeDelayMs(); }
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function sameModelRef(left: string | undefined, right: string | undefined): boolean {
|
|
388
|
+
return !!left && !!right && left.toLowerCase() === right.toLowerCase();
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function scheduleSupervisedPrimaryProbe(ctx: ExtensionContext, recovery: MainModelRecovery): void {
|
|
392
|
+
flags.continuationDispatchStoodDown = false;
|
|
393
|
+
if (recovery.kind === "goal" && state.goal?.status === "active") {
|
|
394
|
+
scheduleContinuation(ctx, true, 1_000);
|
|
395
|
+
} else if (recovery.kind === "loop" && state.loop?.active) {
|
|
396
|
+
scheduleLoopTick(ctx);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
364
400
|
export function holdMainModelRecovery(ctx: ExtensionContext, recovery: MainModelRecovery, why: string): void {
|
|
365
401
|
const normalizedBase = withMainModelRecoveryWindow(recovery);
|
|
366
402
|
const diagnostic = normalizedBase.providerErrorDiagnostic ?? normalizedBase.reason;
|
|
@@ -376,7 +412,13 @@ export function holdMainModelRecovery(ctx: ExtensionContext, recovery: MainModel
|
|
|
376
412
|
clearContinuationTimer();
|
|
377
413
|
clearLoopTimer();
|
|
378
414
|
flags.continuationDispatchStoodDown = true;
|
|
379
|
-
state.mainModelRecovery = {
|
|
415
|
+
state.mainModelRecovery = {
|
|
416
|
+
...normalized,
|
|
417
|
+
retryAt: undefined,
|
|
418
|
+
primaryProbeAt: undefined,
|
|
419
|
+
primaryProbeInFlight: undefined,
|
|
420
|
+
manualResumeRequired: true,
|
|
421
|
+
};
|
|
380
422
|
const resumeCmd = recoverySurfaceCommand(normalized.kind, "resume");
|
|
381
423
|
const pauseReason = `main model recovery — automatic probes stopped (${sanitizeProviderDisplayText(why)})`;
|
|
382
424
|
const action = `No automatic provider probes remain. Switch /model if desired, then ${resumeCmd} to start a fresh recovery window; ${activeGoalSurfaceCommand("cancel")} stops it.`;
|
|
@@ -503,6 +545,11 @@ export async function tryMainModelFallback(ctx: ExtensionContext, failure: MainM
|
|
|
503
545
|
const failureCopy = providerErrorPresentation(failure.raw, "main");
|
|
504
546
|
const recovery: MainModelRecovery = {
|
|
505
547
|
...baseRecovery,
|
|
548
|
+
// A provider failure while serving a fallback returns to the normal
|
|
549
|
+
// ordered recovery walk; the next successful fallback turn will arm a
|
|
550
|
+
// fresh preferred-primary probe.
|
|
551
|
+
primaryProbeAt: undefined,
|
|
552
|
+
primaryProbeInFlight: undefined,
|
|
506
553
|
reason: mainModelRecoveryReason(failure),
|
|
507
554
|
providerErrorDiagnostic: failureCopy.diagnostic,
|
|
508
555
|
recoveryEpisodeKey: baseRecovery.recoveryEpisodeKey ?? `${baseRecovery.firstFailureAt ?? nowIso()}:${failureCopy.fingerprint}`,
|
|
@@ -543,6 +590,8 @@ export async function tryMainModelFallback(ctx: ExtensionContext, failure: MainM
|
|
|
543
590
|
active: current,
|
|
544
591
|
reason: mainModelRecoveryReason(failure),
|
|
545
592
|
providerErrorDiagnostic: failureCopy.diagnostic,
|
|
593
|
+
primaryProbeAt: undefined,
|
|
594
|
+
primaryProbeInFlight: undefined,
|
|
546
595
|
skipped: recovery.skipped,
|
|
547
596
|
};
|
|
548
597
|
persistState(ctx);
|
|
@@ -593,7 +642,7 @@ export async function tryMainModelFallback(ctx: ExtensionContext, failure: MainM
|
|
|
593
642
|
state.mainModelRecovery = nextRecovery;
|
|
594
643
|
persistState(ctx);
|
|
595
644
|
appendLedger(ctx.cwd, "main_model_failover", { from: current, to: candidateRef, backupIndex, backupCount: refs.length, reason: failure.kind });
|
|
596
|
-
ctx.ui.notify(`Main session model failover: ${current} → backup ${backupIndex}/${refs.length} ${candidateRef}. The next supervised turn tests it; a
|
|
645
|
+
ctx.ui.notify(`Main session model failover: ${current} → backup ${backupIndex}/${refs.length} ${candidateRef}. The next supervised turn tests it; a healthy fallback schedules a preferred-primary failback probe.`, "warning");
|
|
597
646
|
return true;
|
|
598
647
|
} catch (err) {
|
|
599
648
|
// A user cancellation or host replacement may have cleared/replaced the
|
|
@@ -640,6 +689,8 @@ export function setMainModelRecoveryPause(ctx: ExtensionContext, recovery: MainM
|
|
|
640
689
|
state.mainModelRecovery = {
|
|
641
690
|
...normalized,
|
|
642
691
|
retryAt,
|
|
692
|
+
primaryProbeAt: undefined,
|
|
693
|
+
primaryProbeInFlight: undefined,
|
|
643
694
|
manualResumeRequired: undefined,
|
|
644
695
|
reason: sanitizeProviderDisplayText(normalized.reason),
|
|
645
696
|
};
|
|
@@ -703,6 +754,43 @@ export function scheduleMainModelRecoveryTimer(ctx: ExtensionContext, delayMs: n
|
|
|
703
754
|
scheduleHourlyProbe(ctx);
|
|
704
755
|
}
|
|
705
756
|
|
|
757
|
+
/** Keep the preferred-primary probe separate from the parked-recovery wait.
|
|
758
|
+
* `retryAt` means the goal/loop is paused; `primaryProbeAt` means the
|
|
759
|
+
* fallback is serving successfully and work may continue normally. */
|
|
760
|
+
export function scheduleMainModelPrimaryProbe(ctx: ExtensionContext, delayMs?: number): void {
|
|
761
|
+
const recovery = state.mainModelRecovery;
|
|
762
|
+
if (!recovery || recovery.manualResumeRequired === true || !mainModelFailbackEnabled()) return;
|
|
763
|
+
if (recovery.primaryProbeInFlight) return;
|
|
764
|
+
const configuredAt = recovery.primaryProbeAt ? Date.parse(recovery.primaryProbeAt) : Number.NaN;
|
|
765
|
+
const requestedDelay = delayMs === undefined
|
|
766
|
+
? Number.isFinite(configuredAt) ? Math.max(0, configuredAt - Date.now()) : mainModelPrimaryProbeDelay()
|
|
767
|
+
: Math.max(0, delayMs);
|
|
768
|
+
const probeAt = Number.isFinite(configuredAt) && delayMs === undefined
|
|
769
|
+
? recovery.primaryProbeAt
|
|
770
|
+
: new Date(Date.now() + requestedDelay).toISOString();
|
|
771
|
+
const changed = recovery.primaryProbeAt !== probeAt
|
|
772
|
+
|| recovery.retryAt !== undefined
|
|
773
|
+
|| recovery.pendingModelSwitch !== undefined;
|
|
774
|
+
if (changed) {
|
|
775
|
+
state.mainModelRecovery = {
|
|
776
|
+
...recovery,
|
|
777
|
+
primaryProbeAt: probeAt,
|
|
778
|
+
primaryProbeInFlight: undefined,
|
|
779
|
+
retryAt: undefined,
|
|
780
|
+
pendingModelSwitch: undefined,
|
|
781
|
+
resumeCurrent: undefined,
|
|
782
|
+
};
|
|
783
|
+
persistState(ctx);
|
|
784
|
+
appendLedger(ctx.cwd, "main_model_failback_scheduled", {
|
|
785
|
+
primary: recovery.primary,
|
|
786
|
+
active: recovery.active,
|
|
787
|
+
probeAt,
|
|
788
|
+
delayMs: requestedDelay,
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
scheduleMainModelRecoveryTimer(ctx, requestedDelay);
|
|
792
|
+
}
|
|
793
|
+
|
|
706
794
|
// =================================================================
|
|
707
795
|
// v0.34.142: hourly retry ticker — optional (default ON) extra retry at :00:30
|
|
708
796
|
// every hour while main-model recovery is parked. This is a blind retry slot:
|
|
@@ -815,6 +903,8 @@ export function manuallyResumeMainModelRecovery(ctx: ExtensionContext): boolean
|
|
|
815
903
|
firstFailureAt: new Date(now).toISOString(),
|
|
816
904
|
autoRetryUntil: mainModelAutoRetryUntil(now, MAIN_MODEL_AUTO_RETRY_HORIZON_MS),
|
|
817
905
|
retryAt: undefined,
|
|
906
|
+
primaryProbeAt: undefined,
|
|
907
|
+
primaryProbeInFlight: undefined,
|
|
818
908
|
manualResumeRequired: undefined,
|
|
819
909
|
resumeCurrent: undefined,
|
|
820
910
|
providerErrorDiagnostic: undefined,
|
|
@@ -863,6 +953,153 @@ export async function probeMainModelRecovery(ctx: ExtensionContext): Promise<voi
|
|
|
863
953
|
}
|
|
864
954
|
}
|
|
865
955
|
|
|
956
|
+
async function probePreferredPrimary(ctx: ExtensionContext, recovery: MainModelRecovery): Promise<void> {
|
|
957
|
+
if (!mainModelFailbackEnabled()) {
|
|
958
|
+
clearMainModelRecoveryTimer();
|
|
959
|
+
state.mainModelRecovery = undefined;
|
|
960
|
+
persistState(ctx);
|
|
961
|
+
appendLedger(ctx.cwd, "main_model_failback_cancelled", { reason: "sticky policy" });
|
|
962
|
+
return;
|
|
963
|
+
}
|
|
964
|
+
const generation = flags.sessionGeneration;
|
|
965
|
+
const primary = recovery.primary;
|
|
966
|
+
const current = modelRef(ctx.model);
|
|
967
|
+
|
|
968
|
+
// A host/session replacement may have committed the switch before the
|
|
969
|
+
// replacement process rehydrated its durable marker. Treat that as the
|
|
970
|
+
// same in-flight probe and wait for one real supervised turn before
|
|
971
|
+
// declaring the primary healthy.
|
|
972
|
+
if (recovery.primaryProbeInFlight && sameModelRef(current, primary)) {
|
|
973
|
+
const reconciled = {
|
|
974
|
+
...recovery,
|
|
975
|
+
active: current,
|
|
976
|
+
attempted: [primary],
|
|
977
|
+
primaryProbeAt: undefined,
|
|
978
|
+
primaryProbeInFlight: true,
|
|
979
|
+
pendingModelSwitch: undefined,
|
|
980
|
+
retryAt: undefined,
|
|
981
|
+
};
|
|
982
|
+
state.mainModelRecovery = reconciled;
|
|
983
|
+
persistState(ctx);
|
|
984
|
+
appendLedger(ctx.cwd, "main_model_failback_reconciled", { primary, generation });
|
|
985
|
+
scheduleSupervisedPrimaryProbe(ctx, reconciled);
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
// The timer can fire after the host has already selected the primary (for
|
|
990
|
+
// example, a restore event races the timer). Still require a supervised
|
|
991
|
+
// turn; setModel alone is not a provider health check.
|
|
992
|
+
if (!recovery.primaryProbeInFlight && sameModelRef(current, primary)) {
|
|
993
|
+
const inFlight = {
|
|
994
|
+
...recovery,
|
|
995
|
+
active: current,
|
|
996
|
+
attempted: [primary],
|
|
997
|
+
primaryProbeAt: undefined,
|
|
998
|
+
primaryProbeInFlight: true,
|
|
999
|
+
pendingModelSwitch: undefined,
|
|
1000
|
+
retryAt: undefined,
|
|
1001
|
+
};
|
|
1002
|
+
state.mainModelRecovery = inFlight;
|
|
1003
|
+
persistState(ctx);
|
|
1004
|
+
appendLedger(ctx.cwd, "main_model_failback_probe", { from: current, to: primary, mode: "already-selected" });
|
|
1005
|
+
scheduleSupervisedPrimaryProbe(ctx, inFlight);
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
if (isForbiddenModel(primary, loadGlobalSettings().forbiddenModels)) {
|
|
1010
|
+
clearMainModelRecoveryTimer();
|
|
1011
|
+
state.mainModelRecovery = undefined;
|
|
1012
|
+
persistState(ctx);
|
|
1013
|
+
appendLedger(ctx.cwd, "main_model_failback_cancelled", { primary, reason: "forbidden" });
|
|
1014
|
+
ctx.ui.notify(`Main-model failback skipped ${primary} because forbiddenModels excludes it; the current fallback remains selected.`, "warning");
|
|
1015
|
+
return;
|
|
1016
|
+
}
|
|
1017
|
+
const candidate = resolveMainModel(ctx, primary);
|
|
1018
|
+
if (!candidate) {
|
|
1019
|
+
const delay = mainModelPrimaryProbeDelay();
|
|
1020
|
+
state.mainModelRecovery = {
|
|
1021
|
+
...recovery,
|
|
1022
|
+
active: current ?? recovery.active,
|
|
1023
|
+
primaryProbeAt: new Date(Date.now() + delay).toISOString(),
|
|
1024
|
+
primaryProbeInFlight: undefined,
|
|
1025
|
+
pendingModelSwitch: undefined,
|
|
1026
|
+
retryAt: undefined,
|
|
1027
|
+
};
|
|
1028
|
+
persistState(ctx);
|
|
1029
|
+
appendLedger(ctx.cwd, "main_model_failback_unavailable", { primary, reason: "primary not in registry" });
|
|
1030
|
+
scheduleMainModelPrimaryProbe(ctx, delay);
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
if (flags.mainModelSwitchInFlight && modelSwitchOperationGeneration !== generation) {
|
|
1035
|
+
flags.mainModelSwitchInFlight = false;
|
|
1036
|
+
activeModelSwitchToken = 0;
|
|
1037
|
+
modelSwitchOperationGeneration = null;
|
|
1038
|
+
}
|
|
1039
|
+
if (flags.mainModelSwitchInFlight) return;
|
|
1040
|
+
const operationToken = ++modelSwitchOperationToken;
|
|
1041
|
+
activeModelSwitchToken = operationToken;
|
|
1042
|
+
modelSwitchOperationGeneration = generation;
|
|
1043
|
+
const pending = {
|
|
1044
|
+
...recovery,
|
|
1045
|
+
active: current ?? recovery.active,
|
|
1046
|
+
attempted: current ? [current] : recovery.attempted,
|
|
1047
|
+
primaryProbeAt: undefined,
|
|
1048
|
+
primaryProbeInFlight: true,
|
|
1049
|
+
pendingModelSwitch: primary,
|
|
1050
|
+
retryAt: new Date(Date.now() + 1_000).toISOString(),
|
|
1051
|
+
};
|
|
1052
|
+
state.mainModelRecovery = pending;
|
|
1053
|
+
persistState(ctx);
|
|
1054
|
+
flags.mainModelSwitchInFlight = true;
|
|
1055
|
+
try {
|
|
1056
|
+
if (generation !== flags.sessionGeneration || !freshCtxForGeneration(generation)) return;
|
|
1057
|
+
const accepted = await flags.extensionApi?.setModel(candidate);
|
|
1058
|
+
if (generation !== flags.sessionGeneration || !freshCtxForGeneration(generation)) return;
|
|
1059
|
+
if (state.mainModelRecovery?.pendingModelSwitch?.toLowerCase() !== primary.toLowerCase()) return;
|
|
1060
|
+
if (!accepted) throw new Error("no configured auth for preferred primary");
|
|
1061
|
+
const switched = {
|
|
1062
|
+
...state.mainModelRecovery!,
|
|
1063
|
+
active: primary,
|
|
1064
|
+
attempted: [primary],
|
|
1065
|
+
primaryProbeAt: undefined,
|
|
1066
|
+
primaryProbeInFlight: true,
|
|
1067
|
+
pendingModelSwitch: undefined,
|
|
1068
|
+
retryAt: undefined,
|
|
1069
|
+
};
|
|
1070
|
+
state.mainModelRecovery = switched;
|
|
1071
|
+
persistState(ctx);
|
|
1072
|
+
appendLedger(ctx.cwd, "main_model_failback", { from: current, to: primary });
|
|
1073
|
+
ctx.ui.notify(`Main session model failed back to ${primary} from ${current ?? "the fallback"}; the next supervised turn tests the primary.`, "info");
|
|
1074
|
+
scheduleSupervisedPrimaryProbe(ctx, switched);
|
|
1075
|
+
} catch (err) {
|
|
1076
|
+
if (generation !== flags.sessionGeneration || state.mainModelRecovery?.pendingModelSwitch?.toLowerCase() !== primary.toLowerCase()) return;
|
|
1077
|
+
const delay = mainModelPrimaryProbeDelay();
|
|
1078
|
+
const next = {
|
|
1079
|
+
...state.mainModelRecovery!,
|
|
1080
|
+
active: current ?? recovery.active,
|
|
1081
|
+
primaryProbeAt: new Date(Date.now() + delay).toISOString(),
|
|
1082
|
+
primaryProbeInFlight: undefined,
|
|
1083
|
+
pendingModelSwitch: undefined,
|
|
1084
|
+
retryAt: undefined,
|
|
1085
|
+
};
|
|
1086
|
+
state.mainModelRecovery = next;
|
|
1087
|
+
persistState(ctx);
|
|
1088
|
+
appendLedger(ctx.cwd, "main_model_failback_failed", {
|
|
1089
|
+
primary,
|
|
1090
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1091
|
+
nextProbeAt: next.primaryProbeAt,
|
|
1092
|
+
});
|
|
1093
|
+
scheduleMainModelPrimaryProbe(ctx, delay);
|
|
1094
|
+
} finally {
|
|
1095
|
+
if (activeModelSwitchToken === operationToken) {
|
|
1096
|
+
activeModelSwitchToken = 0;
|
|
1097
|
+
modelSwitchOperationGeneration = null;
|
|
1098
|
+
flags.mainModelSwitchInFlight = false;
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
|
|
866
1103
|
async function probeMainModelRecoveryImpl(ctx: ExtensionContext): Promise<void> {
|
|
867
1104
|
const generation = flags.sessionGeneration;
|
|
868
1105
|
const recovery = state.mainModelRecovery;
|
|
@@ -877,8 +1114,22 @@ async function probeMainModelRecoveryImpl(ctx: ExtensionContext): Promise<void>
|
|
|
877
1114
|
state.mainModelRecovery = { ...recovery, active: current, pendingModelSwitch: undefined, retryAt: undefined };
|
|
878
1115
|
persistState(ctx);
|
|
879
1116
|
}
|
|
1117
|
+
const preferredPrimaryRecovery = state.mainModelRecovery;
|
|
1118
|
+
if (preferredPrimaryRecovery?.primaryProbeAt || preferredPrimaryRecovery?.primaryProbeInFlight) {
|
|
1119
|
+
await probePreferredPrimary(ctx, preferredPrimaryRecovery);
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
880
1122
|
if (recovery.resumeCurrent && current) {
|
|
881
|
-
state.mainModelRecovery = {
|
|
1123
|
+
state.mainModelRecovery = {
|
|
1124
|
+
...recovery,
|
|
1125
|
+
active: current,
|
|
1126
|
+
attempted: [current],
|
|
1127
|
+
primaryProbeAt: undefined,
|
|
1128
|
+
primaryProbeInFlight: undefined,
|
|
1129
|
+
retryAt: undefined,
|
|
1130
|
+
resumeCurrent: undefined,
|
|
1131
|
+
pendingModelSwitch: undefined,
|
|
1132
|
+
};
|
|
882
1133
|
flags.continuationDispatchStoodDown = false;
|
|
883
1134
|
if (recovery.kind === "goal" && state.goal?.status === "paused" && (state.goal.pauseReason ?? "").startsWith("main model recovery")) {
|
|
884
1135
|
updateGoal({ status: "active", pauseKind: undefined, pauseResumeAt: undefined, pauseReason: undefined, pauseSuggestedAction: undefined, providerErrorDiagnostic: undefined, recoveryEpisodeKey: undefined, recoveryNoticeKeys: undefined }, ctx);
|
|
@@ -1038,7 +1289,15 @@ async function probeMainModelRecoveryImpl(ctx: ExtensionContext): Promise<void>
|
|
|
1038
1289
|
if (generation !== flags.sessionGeneration || !freshCtxForGeneration(generation)) return;
|
|
1039
1290
|
if (state.mainModelRecovery?.pendingModelSwitch?.toLowerCase() !== target.toLowerCase()) return;
|
|
1040
1291
|
if (!accepted) throw new Error(`no configured auth for ${target}`);
|
|
1041
|
-
state.mainModelRecovery = {
|
|
1292
|
+
state.mainModelRecovery = {
|
|
1293
|
+
...state.mainModelRecovery!,
|
|
1294
|
+
active: target,
|
|
1295
|
+
attempted,
|
|
1296
|
+
primaryProbeAt: undefined,
|
|
1297
|
+
primaryProbeInFlight: undefined,
|
|
1298
|
+
pendingModelSwitch: undefined,
|
|
1299
|
+
retryAt: undefined,
|
|
1300
|
+
};
|
|
1042
1301
|
persistState(ctx);
|
|
1043
1302
|
appendLedger(ctx.cwd, "main_model_probe", { from: current, to: target, tryLabel: targetTryLabel, attempts: recovery.attempts });
|
|
1044
1303
|
flags.continuationDispatchStoodDown = false;
|
|
@@ -1147,6 +1406,57 @@ export async function recoverMainModelFromSendStorm(ctx: ExtensionContext, kind:
|
|
|
1147
1406
|
export function mainModelRecoverySucceeded(ctx: ExtensionContext): void {
|
|
1148
1407
|
const recovery = state.mainModelRecovery;
|
|
1149
1408
|
if (!recovery) return;
|
|
1409
|
+
const current = modelRef(ctx.model);
|
|
1410
|
+
const preferredPrimary = recovery.primary;
|
|
1411
|
+
const servingFallback = !!current && !sameModelRef(current, preferredPrimary);
|
|
1412
|
+
|
|
1413
|
+
// A successful fallback turn proves only that the backup works. Keep the
|
|
1414
|
+
// original primary durable and schedule a reverse probe instead of settling
|
|
1415
|
+
// the episode permanently on the fallback.
|
|
1416
|
+
if (servingFallback && mainModelFailbackEnabled()) {
|
|
1417
|
+
const existingProbeAt = recovery.primaryProbeAt;
|
|
1418
|
+
const existingProbeMs = existingProbeAt ? Date.parse(existingProbeAt) : Number.NaN;
|
|
1419
|
+
const alreadyScheduled = Number.isFinite(existingProbeMs) && !recovery.primaryProbeInFlight;
|
|
1420
|
+
clearMainModelRecoveryTimer();
|
|
1421
|
+
cancelHourlyProbe();
|
|
1422
|
+
flags.lastMainModelFailure = null;
|
|
1423
|
+
flags.mainModelAbortForRecovery = false;
|
|
1424
|
+
flags.continuationDispatchStoodDown = false;
|
|
1425
|
+
flags.lastMainModelRecoveryResumeAt = Date.now();
|
|
1426
|
+
|
|
1427
|
+
const probeAt = alreadyScheduled
|
|
1428
|
+
? existingProbeAt!
|
|
1429
|
+
: new Date(Date.now() + mainModelPrimaryProbeDelay()).toISOString();
|
|
1430
|
+
const nextRecovery: MainModelRecovery = {
|
|
1431
|
+
...recovery,
|
|
1432
|
+
active: current,
|
|
1433
|
+
primaryProbeAt: probeAt,
|
|
1434
|
+
primaryProbeInFlight: undefined,
|
|
1435
|
+
retryAt: undefined,
|
|
1436
|
+
pendingModelSwitch: undefined,
|
|
1437
|
+
resumeCurrent: undefined,
|
|
1438
|
+
};
|
|
1439
|
+
state.mainModelRecovery = nextRecovery;
|
|
1440
|
+
const auditRetryStarted = recovery.kind === "goal"
|
|
1441
|
+
&& state.goal?.status === "paused"
|
|
1442
|
+
&& !!state.goal.pendingCompletion
|
|
1443
|
+
&& typeof maybeAutoRetryParkedCompletionAudit === "function"
|
|
1444
|
+
&& maybeAutoRetryParkedCompletionAudit("main-model-recovery");
|
|
1445
|
+
persistState(ctx);
|
|
1446
|
+
if (!alreadyScheduled) {
|
|
1447
|
+
appendLedger(ctx.cwd, "main_model_fallback_healthy", {
|
|
1448
|
+
model: current,
|
|
1449
|
+
primary: preferredPrimary,
|
|
1450
|
+
primaryProbeAt: probeAt,
|
|
1451
|
+
attempts: recovery.attempts,
|
|
1452
|
+
});
|
|
1453
|
+
ctx.ui.notify(`Main session fallback ${current} is healthy; the preferred primary ${preferredPrimary} will be probed automatically.`, "info");
|
|
1454
|
+
}
|
|
1455
|
+
if (auditRetryStarted) ctx.ui.notify("The stored completion audit is retrying after the healthy main-model fallback.", "info");
|
|
1456
|
+
scheduleMainModelRecoveryTimer(ctx, Math.max(0, Date.parse(probeAt) - Date.now()));
|
|
1457
|
+
return;
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1150
1460
|
clearMainModelRecoveryTimer();
|
|
1151
1461
|
cancelHourlyProbe(); // recovery settled — ticker stops
|
|
1152
1462
|
state.mainModelRecovery = undefined;
|
|
@@ -1169,7 +1479,7 @@ export function mainModelRecoverySucceeded(ctx: ExtensionContext): void {
|
|
|
1169
1479
|
: recovery.kind === "loop" && recoveryLoop
|
|
1170
1480
|
? "loop"
|
|
1171
1481
|
: undefined;
|
|
1172
|
-
appendLedger(ctx.cwd, "main_model_recovered", { model:
|
|
1482
|
+
appendLedger(ctx.cwd, "main_model_recovered", { model: current, attempts: recovery.attempts, resumed });
|
|
1173
1483
|
// A successful bounded provider recovery is also a healthy recovery event
|
|
1174
1484
|
// for a parked detached-auditor claim. The auditor hook owns the durable
|
|
1175
1485
|
// one-shot marker and generation/context fence; this call never bypasses
|
|
@@ -1195,7 +1505,7 @@ export function mainModelRecoverySucceeded(ctx: ExtensionContext): void {
|
|
|
1195
1505
|
} else {
|
|
1196
1506
|
persistState(ctx);
|
|
1197
1507
|
}
|
|
1198
|
-
ctx.ui.notify(`Main session model recovered on ${
|
|
1508
|
+
ctx.ui.notify(`Main session model recovered on ${current ?? "the active model"}; automatic recovery is cleared${resumed ? ` and the ${resumed} is resuming` : auditRetryStarted ? " and the stored completion audit is retrying" : ""}.`, "info");
|
|
1199
1509
|
}
|
|
1200
1510
|
|
|
1201
1511
|
/** Handle a provider error before loop/goal bookkeeping can mistake it for
|
|
@@ -5,6 +5,13 @@
|
|
|
5
5
|
// without importing the whole extension. Two-tier config (v0.7.0): GLOBAL
|
|
6
6
|
// is the normal home, PROJECT the rare local override. Resolution:
|
|
7
7
|
// project > global > defaults (per key).
|
|
8
|
+
//
|
|
9
|
+
// v0.35.6: this module owns the typed boundary for the long-term
|
|
10
|
+
// preferences policy (audit/LONG-TERM-PREFERENCES-POLICY-2026-08-19.md).
|
|
11
|
+
// The Settings interface is the ONLY schema accepted by saveSettings;
|
|
12
|
+
// natural-language conversation / completion / auditor / Explore
|
|
13
|
+
// transcripts are NEVER written here. A regression in
|
|
14
|
+
// tests/long-term-preferences-boundary.test.ts pins this boundary.
|
|
8
15
|
|
|
9
16
|
import * as fs from "node:fs";
|
|
10
17
|
import * as os from "node:os";
|
|
@@ -17,7 +24,10 @@ import {
|
|
|
17
24
|
piGlaDir,
|
|
18
25
|
} from "./goal-loop-core.ts";
|
|
19
26
|
import type { SubagentModelStrategy } from "./goal-loop-subagents.js";
|
|
20
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
DEFAULT_MAIN_MODEL_PRIMARY_PROBE_MINUTES,
|
|
29
|
+
normalizeMainModelFallbackRefs,
|
|
30
|
+
} from "./main-model-recovery.js";
|
|
21
31
|
|
|
22
32
|
export interface Settings {
|
|
23
33
|
/** v0.34.57: model refs/ids that must never be selected — the policy
|
|
@@ -56,6 +66,12 @@ export interface Settings {
|
|
|
56
66
|
subagentFallbacks?: Record<string, string[]>;
|
|
57
67
|
/** Global-only base minutes before main-session recovery; doubles per attempt, caps at 5h, and the automatic window ends at 24h. */
|
|
58
68
|
mainModelRetryMinutes?: number;
|
|
69
|
+
/** Global-only: after a fallback succeeds, automatically test the preferred
|
|
70
|
+
* primary again, or keep the fallback for the rest of the session. */
|
|
71
|
+
mainModelFailback?: "auto" | "sticky";
|
|
72
|
+
/** Global-only minutes between preferred-primary health probes while a
|
|
73
|
+
* fallback is serving successfully. */
|
|
74
|
+
mainModelPrimaryProbeMinutes?: number;
|
|
59
75
|
/** "provider/model-id" or bare "model-id". Unset → session model. */
|
|
60
76
|
auditorModel?: string;
|
|
61
77
|
/** v0.31.3/v0.34.25: next detached auditor candidate when the primary
|
|
@@ -181,6 +197,8 @@ export interface Settings {
|
|
|
181
197
|
const GLOBAL_MAIN_RECOVERY_KEYS: ReadonlySet<keyof Settings> = new Set([
|
|
182
198
|
"mainModelFallbacks",
|
|
183
199
|
"mainModelRetryMinutes",
|
|
200
|
+
"mainModelFailback",
|
|
201
|
+
"mainModelPrimaryProbeMinutes",
|
|
184
202
|
"hourlyRetryProbe",
|
|
185
203
|
"drafterModel",
|
|
186
204
|
"drafterThinkingLevel",
|
|
@@ -201,6 +219,8 @@ export const DEFAULT_SETTINGS: Settings = {
|
|
|
201
219
|
// vision CLI job, never a reason to switch models (note.md 2026-08-07).
|
|
202
220
|
visionAssist: true,
|
|
203
221
|
mainModelRetryMinutes: 15,
|
|
222
|
+
mainModelFailback: "auto",
|
|
223
|
+
mainModelPrimaryProbeMinutes: DEFAULT_MAIN_MODEL_PRIMARY_PROBE_MINUTES,
|
|
204
224
|
// Unset = inherit the session thinking level while the temporary drafter
|
|
205
225
|
// agent is active; the original session level is restored afterward.
|
|
206
226
|
drafterThinkingLevel: undefined,
|
|
@@ -260,6 +280,14 @@ function normalizeLoadedSettings(settings: Settings): Settings {
|
|
|
260
280
|
// all see the same bounded value.
|
|
261
281
|
settings.mainModelFallbacks = normalizeMainModelFallbackRefs(settings.mainModelFallbacks);
|
|
262
282
|
settings.drafterModelFallbacks = normalizeMainModelFallbackRefs(settings.drafterModelFallbacks);
|
|
283
|
+
if (settings.mainModelFailback !== "auto" && settings.mainModelFailback !== "sticky") {
|
|
284
|
+
settings.mainModelFailback = "auto";
|
|
285
|
+
}
|
|
286
|
+
if (typeof settings.mainModelPrimaryProbeMinutes !== "number"
|
|
287
|
+
|| !Number.isFinite(settings.mainModelPrimaryProbeMinutes)
|
|
288
|
+
|| settings.mainModelPrimaryProbeMinutes <= 0) {
|
|
289
|
+
settings.mainModelPrimaryProbeMinutes = DEFAULT_MAIN_MODEL_PRIMARY_PROBE_MINUTES;
|
|
290
|
+
}
|
|
263
291
|
// v0.34.142: these old policy knobs no longer control recovery. Drop
|
|
264
292
|
// them from the effective object so stale files cannot resurrect the old
|
|
265
293
|
// behavior or make the settings UI imply that quota inspection exists.
|
|
@@ -317,6 +345,8 @@ export const SETTINGS_KEYS: Array<keyof Settings> = [
|
|
|
317
345
|
"drafterThinkingLevel",
|
|
318
346
|
"drafterModelFallbacks",
|
|
319
347
|
"mainModelRetryMinutes",
|
|
348
|
+
"mainModelFailback",
|
|
349
|
+
"mainModelPrimaryProbeMinutes",
|
|
320
350
|
"forbiddenModels",
|
|
321
351
|
"blockForbiddenModelSwitches",
|
|
322
352
|
"visionAssist",
|