pi-taskflow 0.0.24 → 0.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -0
- package/extensions/cache.ts +6 -1
- package/extensions/flowir/hash.ts +97 -0
- package/extensions/flowir/index.ts +73 -0
- package/extensions/flowir/meta.ts +126 -0
- package/extensions/flowir/translate.ts +163 -0
- package/extensions/index.ts +292 -5
- package/extensions/interpolate.ts +17 -0
- package/extensions/runtime.ts +417 -49
- package/extensions/schema.ts +3 -1
- package/extensions/stale.ts +193 -0
- package/extensions/store.ts +25 -0
- package/package.json +1 -1
package/extensions/runtime.ts
CHANGED
|
@@ -20,6 +20,8 @@ import { type Budget, type CacheScope, dependenciesOf, finalPhase, LOOP_DEFAULT_
|
|
|
20
20
|
import { verifyTaskflow } from "./verify.ts";
|
|
21
21
|
import { hashInput, newRunId, type PhaseState, type RunState, runsDir } from "./store.ts";
|
|
22
22
|
import { CacheStore, resolveFingerprint } from "./cache.ts";
|
|
23
|
+
import { compileTaskflowToIR } from "./flowir/index.ts";
|
|
24
|
+
import { computeStaleFrontier, declaredReadMapOfDef, readMapOf } from "./stale.ts";
|
|
23
25
|
import { ctxDirFor, drainPendingSpawns, initCtxDir, registerNode, setNodeStatus, type SpawnAssignment } from "./context-store.ts";
|
|
24
26
|
import { allocateWorkspace, isWorkspaceKeyword, type Workspace } from "./workspace.ts";
|
|
25
27
|
|
|
@@ -55,6 +57,8 @@ export interface RuntimeDeps {
|
|
|
55
57
|
loadFlow?: (name: string) => Taskflow | undefined;
|
|
56
58
|
/** Cross-run memoization store. Omit to construct a default one for `deps.cwd`. */
|
|
57
59
|
cacheStore?: CacheStore;
|
|
60
|
+
/** Default cache scope for phases that don't specify one. */
|
|
61
|
+
cacheScopeDefault?: CacheScope;
|
|
58
62
|
/** Internal: sub-flow call stack, for recursion detection. */
|
|
59
63
|
_stack?: string[];
|
|
60
64
|
/** Internal: pre-resolved Shared Context Tree dir for this run (sub-flows inherit the parent's). */
|
|
@@ -74,6 +78,7 @@ function buildInterpolationContext(
|
|
|
74
78
|
state: RunState,
|
|
75
79
|
previousOutput: string | undefined,
|
|
76
80
|
locals?: Record<string, unknown>,
|
|
81
|
+
onRead?: (ref: string) => void,
|
|
77
82
|
): InterpolationContext {
|
|
78
83
|
const steps: Record<string, { output: string; json?: unknown }> = {};
|
|
79
84
|
for (const [id, ps] of Object.entries(state.phases)) {
|
|
@@ -90,7 +95,7 @@ function buildInterpolationContext(
|
|
|
90
95
|
}
|
|
91
96
|
}
|
|
92
97
|
}
|
|
93
|
-
return { args: state.args, steps, previousOutput, locals };
|
|
98
|
+
return { args: state.args, steps, previousOutput, locals, onRead };
|
|
94
99
|
}
|
|
95
100
|
|
|
96
101
|
function resultToPhaseState(id: string, r: RunResult, inputHash: string, parseJson: boolean): PhaseState { const failed = isFailed(r);
|
|
@@ -115,6 +120,27 @@ function resultToPhaseState(id: string, r: RunResult, inputHash: string, parseJs
|
|
|
115
120
|
};
|
|
116
121
|
}
|
|
117
122
|
|
|
123
|
+
/** Convert observed read refs (e.g. "steps.scout.output") into a structured
|
|
124
|
+
* readSet keyed by upstream phase id, tagging each with the version
|
|
125
|
+
* (= inputHash) that was current when read. Only `steps.*` refs are upstream
|
|
126
|
+
* phase dependencies; args/item/previous are invocation/loop values. */
|
|
127
|
+
function readRefsToReads(
|
|
128
|
+
refs: string[],
|
|
129
|
+
state: RunState,
|
|
130
|
+
): Array<{ stepId: string; version?: string }> {
|
|
131
|
+
const out: Array<{ stepId: string; version?: string }> = [];
|
|
132
|
+
const seen = new Set<string>();
|
|
133
|
+
for (const ref of refs) {
|
|
134
|
+
const m = /^steps\.([A-Za-z0-9_-]+)\b/.exec(ref);
|
|
135
|
+
if (!m) continue;
|
|
136
|
+
const stepId = m[1] as string;
|
|
137
|
+
if (seen.has(stepId)) continue;
|
|
138
|
+
seen.add(stepId);
|
|
139
|
+
out.push({ stepId, version: state.phases[stepId]?.inputHash });
|
|
140
|
+
}
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
|
|
118
144
|
/**
|
|
119
145
|
* Surface unresolved interpolation placeholders (the `missing[]` from
|
|
120
146
|
* `interpolate()`). Without this they are silently left intact in the task —
|
|
@@ -551,6 +577,15 @@ async function runSpawnedChildren(
|
|
|
551
577
|
* and tears it down afterwards. All allocation is fail-open: a failed allocation
|
|
552
578
|
* degrades to the base cwd so a phase never fails to run because of isolation.
|
|
553
579
|
*/
|
|
580
|
+
/** Optional per-invocation execution flags (e.g. M5 recompute forces a
|
|
581
|
+
* phase to re-run, bypassing the cross-run cache so the result refreshes). */
|
|
582
|
+
interface PhaseExecOpts {
|
|
583
|
+
/** Bypass the cache entirely (within-run prior AND cross-run store) and
|
|
584
|
+
* re-execute. Used by `/tf recompute` on the seeded phase so its new
|
|
585
|
+
* output — and only the downstream whose inputHash actually moves — refreshes. */
|
|
586
|
+
forceRerun?: boolean;
|
|
587
|
+
}
|
|
588
|
+
|
|
554
589
|
async function executePhase(
|
|
555
590
|
phase: Phase,
|
|
556
591
|
state: RunState,
|
|
@@ -558,10 +593,11 @@ async function executePhase(
|
|
|
558
593
|
prior: PhaseState | undefined,
|
|
559
594
|
emitProgress: () => void,
|
|
560
595
|
_retryDepth = 0,
|
|
596
|
+
opts?: PhaseExecOpts,
|
|
561
597
|
): Promise<PhaseState> {
|
|
562
598
|
// Non-keyword cwd (or none): no workspace lifecycle — run directly.
|
|
563
599
|
if (!isWorkspaceKeyword(phase.cwd)) {
|
|
564
|
-
return executePhaseInner(phase, state, deps, prior, emitProgress, _retryDepth);
|
|
600
|
+
return executePhaseInner(phase, state, deps, prior, emitProgress, _retryDepth, opts);
|
|
565
601
|
}
|
|
566
602
|
let ws: Workspace | undefined;
|
|
567
603
|
try {
|
|
@@ -576,7 +612,7 @@ async function executePhase(
|
|
|
576
612
|
}
|
|
577
613
|
const innerDeps: RuntimeDeps = ws ? { ...deps, _cwdOverride: ws.dir } : deps;
|
|
578
614
|
try {
|
|
579
|
-
const ps = await executePhaseInner(phase, state, innerDeps, prior, emitProgress, _retryDepth);
|
|
615
|
+
const ps = await executePhaseInner(phase, state, innerDeps, prior, emitProgress, _retryDepth, opts);
|
|
580
616
|
if (ws && (ws.kind !== "inherited" || ws.note)) {
|
|
581
617
|
const tag = ws.kind === "inherited" ? "workspace" : `workspace:${ws.kind}`;
|
|
582
618
|
const msg = ws.note ? `${tag} — ${ws.note}` : `${tag} at ${ws.dir}`;
|
|
@@ -599,6 +635,7 @@ async function executePhaseInner(
|
|
|
599
635
|
prior: PhaseState | undefined,
|
|
600
636
|
emitProgress: () => void,
|
|
601
637
|
_retryDepth = 0,
|
|
638
|
+
opts?: PhaseExecOpts,
|
|
602
639
|
): Promise<PhaseState> {
|
|
603
640
|
const type = phase.type ?? "agent";
|
|
604
641
|
const concurrency = phase.concurrency ?? state.def.concurrency ?? 8;
|
|
@@ -631,13 +668,49 @@ async function executePhaseInner(
|
|
|
631
668
|
// Resolve context pre-read files once, before any type branching.
|
|
632
669
|
// The content is prepended to every task so the subagent never spends
|
|
633
670
|
// turns on file exploration for files the flow author already knows.
|
|
634
|
-
|
|
671
|
+
// M3 observed-readSet: collect every upstream ref this phase resolves, so we
|
|
672
|
+
// can record what its result ACTUALLY depended on (not just its declared
|
|
673
|
+
// dependsOn). Shared by every interpolation in this phase (task / when / …).
|
|
674
|
+
const readRefs: string[] = [];
|
|
675
|
+
const onRead = (ref: string): void => {
|
|
676
|
+
readRefs.push(ref);
|
|
677
|
+
};
|
|
678
|
+
const ctx = buildInterpolationContext(state, previousOutput, undefined, onRead);
|
|
679
|
+
|
|
680
|
+
// M3 observed-readSet: when conditions are part of the phase's real
|
|
681
|
+
// dependencies. Evaluate them inside executePhaseInner so every upstream
|
|
682
|
+
// interpolation is captured by the shared onRead hook, not silently dropped
|
|
683
|
+
// by a separate out-of-band context.
|
|
684
|
+
if (phase.when !== undefined) {
|
|
685
|
+
if (!evaluateCondition(phase.when, ctx)) {
|
|
686
|
+
return {
|
|
687
|
+
id: phase.id,
|
|
688
|
+
status: "skipped",
|
|
689
|
+
error: `Condition not met: ${phase.when}`,
|
|
690
|
+
endedAt: Date.now(),
|
|
691
|
+
usage: emptyUsage(),
|
|
692
|
+
reads: readRefsToReads(readRefs, state),
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
|
|
635
697
|
const preRead = await resolvePhaseContext(phase, ctx);
|
|
636
698
|
|
|
637
699
|
// Resolve this phase's cache policy once. Default scope is "run-only" (the
|
|
638
700
|
// historical within-run resume behavior). Only "cross-run" phases resolve a
|
|
639
701
|
// fingerprint and consult the persistent store.
|
|
640
|
-
|
|
702
|
+
let cacheScope: CacheScope = (phase.cache?.scope ?? deps.cacheScopeDefault ?? "run-only") as CacheScope;
|
|
703
|
+
// Defense in depth: gate/approval/loop/tournament must produce a fresh result
|
|
704
|
+
// each run (schema already rejects explicit cross-run, but the default-scope
|
|
705
|
+
// path must also be blocked). If flowDefHash failed, cross-run is unsafe
|
|
706
|
+
// because the key degrades to flowName-only and reopens cross-flow collisions.
|
|
707
|
+
const CROSS_RUN_BLOCKED_TYPES = new Set(["gate", "approval", "loop", "tournament"]);
|
|
708
|
+
if (cacheScope === "cross-run" && CROSS_RUN_BLOCKED_TYPES.has(type)) {
|
|
709
|
+
cacheScope = "run-only";
|
|
710
|
+
}
|
|
711
|
+
if (state.flowDefHash === "failed" && cacheScope === "cross-run") {
|
|
712
|
+
cacheScope = "run-only";
|
|
713
|
+
}
|
|
641
714
|
const cc: PhaseCacheCtx = {
|
|
642
715
|
scope: cacheScope,
|
|
643
716
|
ttlMs: phase.cache?.ttl ? (parseTtlMs(phase.cache.ttl) ?? undefined) : undefined,
|
|
@@ -647,6 +720,8 @@ async function executePhaseInner(
|
|
|
647
720
|
phaseId: phase.id,
|
|
648
721
|
flowName: state.flowName,
|
|
649
722
|
runId: state.runId,
|
|
723
|
+
flowDefHash: state.flowDefHash === "failed" ? undefined : state.flowDefHash,
|
|
724
|
+
forceRerun: opts?.forceRerun,
|
|
650
725
|
thinking: phase.thinking,
|
|
651
726
|
tools: phase.tools,
|
|
652
727
|
preRead,
|
|
@@ -823,7 +898,7 @@ async function executePhaseInner(
|
|
|
823
898
|
if (type === "agent" || type === "gate" || type === "reduce") {
|
|
824
899
|
// Eval gate: zero-token machine checks before the LLM gate.
|
|
825
900
|
if (type === "gate" && Array.isArray(phase.eval) && phase.eval.length > 0) {
|
|
826
|
-
const evalCtx = buildInterpolationContext(state, previousOutput);
|
|
901
|
+
const evalCtx = buildInterpolationContext(state, previousOutput, undefined, onRead);
|
|
827
902
|
let allPassed = true;
|
|
828
903
|
for (const check of phase.eval) {
|
|
829
904
|
let expr = check;
|
|
@@ -848,7 +923,7 @@ async function executePhaseInner(
|
|
|
848
923
|
}
|
|
849
924
|
if (allPassed) {
|
|
850
925
|
// All evals passed — skip the LLM gate, return an auto-pass.
|
|
851
|
-
const inputHash =
|
|
926
|
+
const inputHash = cacheKeys(cc, [phase.id, "eval-skip"]).key;
|
|
852
927
|
const ps: PhaseState = {
|
|
853
928
|
id: phase.id,
|
|
854
929
|
status: "done",
|
|
@@ -858,6 +933,7 @@ async function executePhaseInner(
|
|
|
858
933
|
inputHash,
|
|
859
934
|
endedAt: Date.now(),
|
|
860
935
|
};
|
|
936
|
+
if (readRefs.length) ps.reads = readRefsToReads(readRefs, state);
|
|
861
937
|
recordCache(cc, ps);
|
|
862
938
|
return ps;
|
|
863
939
|
}
|
|
@@ -867,12 +943,14 @@ async function executePhaseInner(
|
|
|
867
943
|
const refWarning = warnUnresolvedRefs(phase.id, interp.missing);
|
|
868
944
|
const fullTask = preRead + text;
|
|
869
945
|
const agentName = resolveAgent(phase.agent, deps, state);
|
|
870
|
-
const
|
|
871
|
-
const
|
|
946
|
+
const ck = cacheKeys(cc, [phase.id, agentName, phase.model ?? "", fullTask]);
|
|
947
|
+
const inputHash = ck.key;
|
|
948
|
+
const cached = cachedPhase(cc, ck);
|
|
872
949
|
if (cached) return cached;
|
|
873
950
|
|
|
874
951
|
const r = await runOne(agentName, fullTask, liveSink(state, phase.id, emitProgress), nodeIdFor());
|
|
875
952
|
const ps = resultToPhaseState(phase.id, r, inputHash, parseJson);
|
|
953
|
+
if (readRefs.length) ps.reads = readRefsToReads(readRefs, state);
|
|
876
954
|
if (refWarning) ps.warnings = [...(ps.warnings ?? []), refWarning];
|
|
877
955
|
if (type === "gate" && ps.status === "done") ps.gate = parseGateVerdict(r.output);
|
|
878
956
|
|
|
@@ -919,14 +997,14 @@ async function executePhaseInner(
|
|
|
919
997
|
for (const depId of phase.dependsOn ?? []) {
|
|
920
998
|
const d = state.def.phases.find((p) => p.id === depId);
|
|
921
999
|
if (!d) continue;
|
|
922
|
-
const dPs = await executePhase(d, state, depsForUpstream, prior, emitProgress, _retryDepth + 1);
|
|
1000
|
+
const dPs = await executePhase(d, state, depsForUpstream, prior, emitProgress, _retryDepth + 1, undefined);
|
|
923
1001
|
state.phases[depId] = dPs;
|
|
924
1002
|
}
|
|
925
1003
|
}
|
|
926
1004
|
const retryCtx = buildInterpolationContext(state, lastCompletedOutput(state, phase));
|
|
927
1005
|
const retryText = interpolate(phase.task ?? "", retryCtx).text;
|
|
928
1006
|
const retryTask = preRead + retryText;
|
|
929
|
-
const retryIH =
|
|
1007
|
+
const retryIH = cacheKeys(cc, [phase.id, agentName, phase.model ?? "", retryTask]).key;
|
|
930
1008
|
const retryR = await runOne(agentName, retryTask, liveSink(state, phase.id, emitProgress));
|
|
931
1009
|
gatePs = resultToPhaseState(phase.id, retryR, retryIH, parseJson);
|
|
932
1010
|
if (gatePs.status === "done") gatePs.gate = parseGateVerdict(retryR.output);
|
|
@@ -948,12 +1026,14 @@ async function executePhaseInner(
|
|
|
948
1026
|
task: preRead + r.text,
|
|
949
1027
|
};
|
|
950
1028
|
});
|
|
951
|
-
const
|
|
952
|
-
const
|
|
1029
|
+
const ck = cacheKeys(cc, [phase.id, phase.model ?? "", JSON.stringify(branches)]);
|
|
1030
|
+
const inputHash = ck.key;
|
|
1031
|
+
const cached = cachedPhase(cc, ck);
|
|
953
1032
|
if (cached) return cached;
|
|
954
1033
|
|
|
955
1034
|
const results = await runFanout(branches);
|
|
956
1035
|
const ps = mergePhaseState(phase.id, results, inputHash, parseJson);
|
|
1036
|
+
if (readRefs.length) ps.reads = readRefsToReads(readRefs, state);
|
|
957
1037
|
recordCache(cc, ps);
|
|
958
1038
|
return ps;
|
|
959
1039
|
}
|
|
@@ -982,18 +1062,20 @@ async function executePhaseInner(
|
|
|
982
1062
|
}
|
|
983
1063
|
const loopVar = phase.as ?? "item";
|
|
984
1064
|
const tasks = arr.map((item) => {
|
|
985
|
-
const localCtx = buildInterpolationContext(state, previousOutput, { [loopVar]: item });
|
|
1065
|
+
const localCtx = buildInterpolationContext(state, previousOutput, { [loopVar]: item }, onRead);
|
|
986
1066
|
return {
|
|
987
1067
|
agent: resolveAgent(phase.agent, deps, state),
|
|
988
1068
|
task: preRead + interpolate(phase.task ?? "", localCtx).text,
|
|
989
1069
|
};
|
|
990
1070
|
});
|
|
991
|
-
const
|
|
992
|
-
const
|
|
1071
|
+
const ck = cacheKeys(cc, [phase.id, phase.model ?? "", JSON.stringify(tasks)]);
|
|
1072
|
+
const inputHash = ck.key;
|
|
1073
|
+
const cached = cachedPhase(cc, ck);
|
|
993
1074
|
if (cached) return cached;
|
|
994
1075
|
|
|
995
1076
|
const results = await runFanout(tasks);
|
|
996
1077
|
const ps = mergePhaseState(phase.id, results, inputHash, parseJson);
|
|
1078
|
+
if (readRefs.length) ps.reads = readRefsToReads(readRefs, state);
|
|
997
1079
|
if (mapTruncated) {
|
|
998
1080
|
ps.warnings = [...(ps.warnings ?? []), `map fan-out truncated to MAX_DYNAMIC_MAP_ITEMS (${MAX_DYNAMIC_MAP_ITEMS}) inside a dynamic sub-flow`];
|
|
999
1081
|
// NB: do NOT set ps.budgetTruncated — that field drives the run-level
|
|
@@ -1005,10 +1087,12 @@ async function executePhaseInner(
|
|
|
1005
1087
|
}
|
|
1006
1088
|
|
|
1007
1089
|
if (type === "approval") {
|
|
1008
|
-
const
|
|
1090
|
+
const readRefs: string[] = [];
|
|
1091
|
+
const ctx = buildInterpolationContext(state, previousOutput, undefined, (ref) => readRefs.push(ref));
|
|
1009
1092
|
const message = interpolate(phase.task ?? "Approve to continue?", ctx).text;
|
|
1010
|
-
const
|
|
1011
|
-
const
|
|
1093
|
+
const ck = cacheKeys(cc, [phase.id, phase.model ?? "", "approval", message]);
|
|
1094
|
+
const inputHash = ck.key;
|
|
1095
|
+
const cached = cachedPhase(cc, ck);
|
|
1012
1096
|
if (cached) return cached;
|
|
1013
1097
|
|
|
1014
1098
|
// Non-interactive (headless/CI/detached): auto-REJECT, fail-open, but record it.
|
|
@@ -1023,6 +1107,7 @@ async function executePhaseInner(
|
|
|
1023
1107
|
gate: { verdict: "block", reason: "(auto-rejected: no interactive approver available)" },
|
|
1024
1108
|
usage: emptyUsage(),
|
|
1025
1109
|
inputHash,
|
|
1110
|
+
reads: readRefsToReads(readRefs, state),
|
|
1026
1111
|
endedAt: Date.now(),
|
|
1027
1112
|
};
|
|
1028
1113
|
}
|
|
@@ -1035,6 +1120,7 @@ async function executePhaseInner(
|
|
|
1035
1120
|
approval: { decision: decision.decision, note },
|
|
1036
1121
|
usage: emptyUsage(),
|
|
1037
1122
|
inputHash,
|
|
1123
|
+
reads: readRefsToReads(readRefs, state),
|
|
1038
1124
|
endedAt: Date.now(),
|
|
1039
1125
|
};
|
|
1040
1126
|
// A rejection halts the flow via the same mechanism as a blocking gate.
|
|
@@ -1045,7 +1131,8 @@ async function executePhaseInner(
|
|
|
1045
1131
|
}
|
|
1046
1132
|
|
|
1047
1133
|
if (type === "flow") {
|
|
1048
|
-
const
|
|
1134
|
+
const readRefs: string[] = [];
|
|
1135
|
+
const ctx = buildInterpolationContext(state, previousOutput, undefined, (ref) => readRefs.push(ref));
|
|
1049
1136
|
const hasDef = (phase as { def?: unknown }).def !== undefined;
|
|
1050
1137
|
const stack = deps._stack ?? [];
|
|
1051
1138
|
|
|
@@ -1066,6 +1153,7 @@ async function executePhaseInner(
|
|
|
1066
1153
|
json: parseJson ? safeParse("") : undefined,
|
|
1067
1154
|
usage: emptyUsage(),
|
|
1068
1155
|
inputHash: hashInput(phase.id, `flow-def-error:${diag}`),
|
|
1156
|
+
reads: readRefsToReads(readRefs, state),
|
|
1069
1157
|
endedAt: Date.now(),
|
|
1070
1158
|
defError: diag,
|
|
1071
1159
|
});
|
|
@@ -1101,6 +1189,7 @@ async function executePhaseInner(
|
|
|
1101
1189
|
json: parseJson ? safeParse("") : undefined,
|
|
1102
1190
|
usage: emptyUsage(),
|
|
1103
1191
|
inputHash: hashInput(phase.id, "flow-def-empty"),
|
|
1192
|
+
reads: readRefsToReads(readRefs, state),
|
|
1104
1193
|
endedAt: Date.now(),
|
|
1105
1194
|
};
|
|
1106
1195
|
}
|
|
@@ -1147,8 +1236,9 @@ async function executePhaseInner(
|
|
|
1147
1236
|
// that a different generated plan yields a different key (and an identical plan
|
|
1148
1237
|
// hits cache). For saved flows the name is the identity (historical behavior).
|
|
1149
1238
|
const flowIdentity = hasDef ? `def:${JSON.stringify(subDef)}` : `flow:${name}`;
|
|
1150
|
-
const
|
|
1151
|
-
const
|
|
1239
|
+
const ck = cacheKeys(cc, [phase.id, flowIdentity, preRead, JSON.stringify(subArgs)]);
|
|
1240
|
+
const inputHash = ck.key;
|
|
1241
|
+
const cached = cachedPhase(cc, ck);
|
|
1152
1242
|
if (cached) return cached;
|
|
1153
1243
|
|
|
1154
1244
|
const live = state.phases[phase.id];
|
|
@@ -1222,6 +1312,7 @@ async function executePhaseInner(
|
|
|
1222
1312
|
},
|
|
1223
1313
|
error: subResult.ok ? undefined : `sub-flow '${name}' ${subResult.state.status}`,
|
|
1224
1314
|
inputHash,
|
|
1315
|
+
reads: readRefsToReads(readRefs, state),
|
|
1225
1316
|
endedAt: Date.now(),
|
|
1226
1317
|
};
|
|
1227
1318
|
recordCache(cc, flowPs);
|
|
@@ -1231,11 +1322,21 @@ async function executePhaseInner(
|
|
|
1231
1322
|
// loop-until-done: run the body repeatedly until `until` is truthy, the output
|
|
1232
1323
|
// converges to a fixed point, or maxIterations is hit (always terminates).
|
|
1233
1324
|
if (type === "loop") {
|
|
1325
|
+
const readRefs: string[] = [];
|
|
1234
1326
|
const agentName = resolveAgent(phase.agent, deps, state);
|
|
1235
1327
|
const rawMax = phase.maxIterations ?? LOOP_DEFAULT_MAX_ITERATIONS;
|
|
1236
1328
|
const maxIters = Math.max(1, Math.min(LOOP_HARD_MAX_ITERATIONS, Math.floor(rawMax)));
|
|
1237
1329
|
const convergence = phase.convergence ?? true;
|
|
1238
1330
|
|
|
1331
|
+
// Canonical first-iteration body for the cache key. It must fold in the
|
|
1332
|
+
// interpolated task/upstream refs so that a changed upstream changes the
|
|
1333
|
+
// key and recompute no longer silently reuses a stale loop (critic finding).
|
|
1334
|
+
const firstBodyCtx = buildInterpolationContext(state, previousOutput, {
|
|
1335
|
+
loop: { iteration: 1, lastOutput: "", maxIterations: maxIters },
|
|
1336
|
+
}, (ref) => readRefs.push(ref));
|
|
1337
|
+
const firstBody = preRead + interpolate(phase.task ?? "", firstBodyCtx).text;
|
|
1338
|
+
const inputHash = hashInput(phase.id, "loop", phase.until ?? "", firstBody, String(maxIters));
|
|
1339
|
+
|
|
1239
1340
|
const usages: UsageStats[] = [];
|
|
1240
1341
|
const loopWarnings: string[] = [];
|
|
1241
1342
|
let lastOutput = "";
|
|
@@ -1253,7 +1354,7 @@ async function executePhaseInner(
|
|
|
1253
1354
|
// The body sees its iteration number and the prior iteration's output.
|
|
1254
1355
|
const bodyCtx = buildInterpolationContext(state, previousOutput, {
|
|
1255
1356
|
loop: { iteration: i, lastOutput, maxIterations: maxIters },
|
|
1256
|
-
});
|
|
1357
|
+
}, (ref) => readRefs.push(ref));
|
|
1257
1358
|
const body = preRead + interpolate(phase.task ?? "", bodyCtx).text;
|
|
1258
1359
|
const r = await runOne(agentName, body, liveSink(state, phase.id, emitProgress));
|
|
1259
1360
|
usages.push(r.usage);
|
|
@@ -1270,7 +1371,7 @@ async function executePhaseInner(
|
|
|
1270
1371
|
// Loop locals ({loop.iteration} etc.) are available to the condition too.
|
|
1271
1372
|
const untilCtx = buildInterpolationContext(state, previousOutput, {
|
|
1272
1373
|
loop: { iteration: i, lastOutput, maxIterations: maxIters },
|
|
1273
|
-
});
|
|
1374
|
+
}, (ref) => readRefs.push(ref));
|
|
1274
1375
|
untilCtx.steps[phase.id] = { output: lastOutput, json: safeParse(lastOutput) };
|
|
1275
1376
|
const { value: done, error: condErr } = tryEvaluateCondition(phase.until ?? "", untilCtx);
|
|
1276
1377
|
// A malformed condition must not spin forever: stop and surface a warning
|
|
@@ -1301,7 +1402,8 @@ async function executePhaseInner(
|
|
|
1301
1402
|
error: failedResult?.errorMessage || failedResult?.stderr || (stop === "aborted" ? "Aborted" : `loop '${phase.id}' iteration ${iterations} failed`),
|
|
1302
1403
|
loop: { iterations, stop },
|
|
1303
1404
|
warnings: loopWarnings.length ? loopWarnings : undefined,
|
|
1304
|
-
inputHash
|
|
1405
|
+
inputHash,
|
|
1406
|
+
reads: readRefsToReads(readRefs, state),
|
|
1305
1407
|
endedAt: Date.now(),
|
|
1306
1408
|
};
|
|
1307
1409
|
}
|
|
@@ -1313,7 +1415,8 @@ async function executePhaseInner(
|
|
|
1313
1415
|
usage: aggUsage,
|
|
1314
1416
|
loop: { iterations, stop },
|
|
1315
1417
|
warnings: loopWarnings.length ? loopWarnings : undefined,
|
|
1316
|
-
inputHash
|
|
1418
|
+
inputHash,
|
|
1419
|
+
reads: readRefsToReads(readRefs, state),
|
|
1317
1420
|
endedAt: Date.now(),
|
|
1318
1421
|
};
|
|
1319
1422
|
}
|
|
@@ -1336,6 +1439,20 @@ async function executePhaseInner(
|
|
|
1336
1439
|
competitors = Array.from({ length: n }, () => ({ agent: resolveAgent(phase.agent, deps, state), task: body }));
|
|
1337
1440
|
}
|
|
1338
1441
|
|
|
1442
|
+
// The inputHash must fold in the resolved competitors (which embed the
|
|
1443
|
+
// interpolated task/upstream refs) and the judge rubric, otherwise a changed
|
|
1444
|
+
// upstream produces the same key and recompute silently reuses a stale
|
|
1445
|
+
// tournament (critic finding: unsound for cross-run/recompute).
|
|
1446
|
+
const rubric = interpolate(phase.judge ?? "", ctx).text.trim();
|
|
1447
|
+
const inputHash = hashInput(
|
|
1448
|
+
phase.id,
|
|
1449
|
+
"tournament",
|
|
1450
|
+
mode,
|
|
1451
|
+
String(competitors.length),
|
|
1452
|
+
JSON.stringify(competitors.map((c) => ({ agent: c.agent, task: c.task }))),
|
|
1453
|
+
rubric,
|
|
1454
|
+
);
|
|
1455
|
+
|
|
1339
1456
|
const results = await runFanout(competitors);
|
|
1340
1457
|
const ran = results.filter((r) => r.stopReason !== "budget-skipped");
|
|
1341
1458
|
const ok = ran.filter((r) => !isFailed(r));
|
|
@@ -1355,7 +1472,8 @@ async function executePhaseInner(
|
|
|
1355
1472
|
error: `tournament '${phase.id}': all ${competitors.length} variants failed`,
|
|
1356
1473
|
budgetTruncated: budgetSkipCount > 0 || undefined,
|
|
1357
1474
|
tournament: { variants: competitors.length, winner: 0, mode },
|
|
1358
|
-
inputHash
|
|
1475
|
+
inputHash,
|
|
1476
|
+
reads: readRefsToReads(readRefs, state),
|
|
1359
1477
|
endedAt: Date.now(),
|
|
1360
1478
|
};
|
|
1361
1479
|
}
|
|
@@ -1370,7 +1488,8 @@ async function executePhaseInner(
|
|
|
1370
1488
|
model: ok[0].model,
|
|
1371
1489
|
budgetTruncated: budgetSkipCount > 0 || undefined,
|
|
1372
1490
|
tournament: { variants: competitors.length, winner: ranIdx(ok[0]), mode, reason: "only surviving variant" },
|
|
1373
|
-
inputHash
|
|
1491
|
+
inputHash,
|
|
1492
|
+
reads: readRefsToReads(readRefs, state),
|
|
1374
1493
|
endedAt: Date.now(),
|
|
1375
1494
|
};
|
|
1376
1495
|
}
|
|
@@ -1387,7 +1506,8 @@ async function executePhaseInner(
|
|
|
1387
1506
|
budgetTruncated: budgetSkipCount > 0 || undefined,
|
|
1388
1507
|
warnings: ["judge skipped: run aborted or budget exceeded"],
|
|
1389
1508
|
tournament: { variants: competitors.length, winner: ranIdx(ok[0]), mode, reason: "judge skipped" },
|
|
1390
|
-
inputHash
|
|
1509
|
+
inputHash,
|
|
1510
|
+
reads: readRefsToReads(readRefs, state),
|
|
1391
1511
|
endedAt: Date.now(),
|
|
1392
1512
|
};
|
|
1393
1513
|
}
|
|
@@ -1396,14 +1516,14 @@ async function executePhaseInner(
|
|
|
1396
1516
|
const labelled = ran
|
|
1397
1517
|
.map((r, i) => `### Variant ${i + 1}${isFailed(r) ? " (failed — ineligible)" : ""}\n\n${r.output}`)
|
|
1398
1518
|
.join("\n\n---\n\n");
|
|
1399
|
-
const
|
|
1400
|
-
|
|
1519
|
+
const finalRubric =
|
|
1520
|
+
rubric ||
|
|
1401
1521
|
"You are judging competing answers to the same task. Pick the single best variant on correctness, completeness, and clarity.";
|
|
1402
1522
|
const directive =
|
|
1403
1523
|
mode === "best"
|
|
1404
1524
|
? `End your reply with a line exactly: WINNER: <number> (1–${ran.length}), choosing the strongest eligible variant.`
|
|
1405
1525
|
: `Synthesize the strongest possible answer by combining the best parts of the eligible variants. Then end with a line: WINNER: <number> indicating which variant contributed most.`;
|
|
1406
|
-
const judgeTask = `${
|
|
1526
|
+
const judgeTask = `${finalRubric}\n\nThe candidate variants:\n\n${labelled}\n\n${directive}`;
|
|
1407
1527
|
const judgeAgent = resolveAgent(phase.judgeAgent ?? phase.agent, deps, state);
|
|
1408
1528
|
const judgeRes = await runOne(judgeAgent, judgeTask, liveSink(state, phase.id, emitProgress));
|
|
1409
1529
|
const judgeUsage = aggregateUsage([variantUsage, judgeRes.usage]);
|
|
@@ -1421,7 +1541,8 @@ async function executePhaseInner(
|
|
|
1421
1541
|
budgetTruncated: budgetSkipCount > 0 || undefined,
|
|
1422
1542
|
warnings: [`judge failed (${judgeRes.errorMessage ?? "error"}); used variant ${ranIdx(ok[0])}`],
|
|
1423
1543
|
tournament: { variants: competitors.length, winner: ranIdx(ok[0]), mode, reason: "judge failed" },
|
|
1424
|
-
inputHash
|
|
1544
|
+
inputHash,
|
|
1545
|
+
reads: readRefsToReads(readRefs, state),
|
|
1425
1546
|
endedAt: Date.now(),
|
|
1426
1547
|
};
|
|
1427
1548
|
}
|
|
@@ -1444,7 +1565,8 @@ async function executePhaseInner(
|
|
|
1444
1565
|
budgetTruncated: budgetSkipCount > 0 || undefined,
|
|
1445
1566
|
warnings: winnerIneligible ? [`judge picked an ineligible variant; used variant ${winnerIdx}`] : undefined,
|
|
1446
1567
|
tournament: { variants: competitors.length, winner: winnerIdx, mode, reason },
|
|
1447
|
-
inputHash
|
|
1568
|
+
inputHash,
|
|
1569
|
+
reads: readRefsToReads(readRefs, state),
|
|
1448
1570
|
endedAt: Date.now(),
|
|
1449
1571
|
};
|
|
1450
1572
|
}
|
|
@@ -1490,7 +1612,7 @@ function lastCompletedOutput(state: RunState, phase: Phase): string | undefined
|
|
|
1490
1612
|
* scope, optional TTL, and a pre-resolved fingerprint string so each phase-type
|
|
1491
1613
|
* branch can fold it into its inputHash and consult the cross-run store uniformly.
|
|
1492
1614
|
*/
|
|
1493
|
-
interface PhaseCacheCtx {
|
|
1615
|
+
export interface PhaseCacheCtx {
|
|
1494
1616
|
scope: CacheScope;
|
|
1495
1617
|
ttlMs?: number;
|
|
1496
1618
|
fingerprint: string;
|
|
@@ -1509,22 +1631,62 @@ interface PhaseCacheCtx {
|
|
|
1509
1631
|
* whether a given branch happens to fold preRead into its task string
|
|
1510
1632
|
* (previously this was only incidentally true via `fullTask`). */
|
|
1511
1633
|
preRead?: string;
|
|
1634
|
+
/** Content fingerprint of the desugared flow definition — folded into the
|
|
1635
|
+
* key so two structurally-different flows that share a name can never
|
|
1636
|
+
* collide, and a changed flow never serves a stale cross-run hit. */
|
|
1637
|
+
flowDefHash?: string | "failed";
|
|
1638
|
+
/** Force this phase to re-execute, ignoring the within-run prior AND the
|
|
1639
|
+
* cross-run store (M5 recompute seed). Downstream phases are NOT forced —
|
|
1640
|
+
* they re-evaluate naturally: if the seed's new output changed their
|
|
1641
|
+
* inputHash they miss and re-run, otherwise they hit (early cutoff). */
|
|
1642
|
+
forceRerun?: boolean;
|
|
1512
1643
|
}
|
|
1513
1644
|
|
|
1514
1645
|
/** Fold the phase fingerprint into the base hash parts to form the final cache key. */
|
|
1515
|
-
|
|
1646
|
+
/** A computed cache identity: the new (versioned) key plus the read-only
|
|
1647
|
+
* fallback keys used to honor entries written by older releases. The `key`
|
|
1648
|
+
* is what we WRITE under and what `PhaseState.inputHash` carries; the
|
|
1649
|
+
* `legacyKey`/`bareKey` are consulted READ-ONLY on a miss so an upgrade
|
|
1650
|
+
* never produces a miss-storm. See docs/internal/cache-migration.md. */
|
|
1651
|
+
export interface CacheKeys {
|
|
1652
|
+
/** Current key: folds `v2:flowdef:<hash>` (the overstory content fingerprint). */
|
|
1653
|
+
key: string;
|
|
1654
|
+
/** Pre-flowDefHash-era key: the flowdef line OMITTED entirely. Read-only. */
|
|
1655
|
+
legacyKey: string;
|
|
1656
|
+
/** Bare (unversioned) `flowdef:` key — written by pre-H1 code that folded
|
|
1657
|
+
* the hash without a `v2:` prefix. Read-only. Removed in v0.1.0. */
|
|
1658
|
+
bareKey: string;
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
/** Fold the phase fingerprint into the base hash parts to form the cache keys.
|
|
1662
|
+
*
|
|
1663
|
+
* Three keys are produced for backward compatibility (see
|
|
1664
|
+
* docs/internal/cache-migration.md):
|
|
1665
|
+
* - `key` : `v2:flowdef:<hash>` — the current write key.
|
|
1666
|
+
* - `legacyKey`: the flowdef line omitted — pre-flowDefHash entries.
|
|
1667
|
+
* - `bareKey` : bare `flowdef:<hash>` (unversioned) — pre-H1 entries that
|
|
1668
|
+
* folded the hash without the `v2:` prefix.
|
|
1669
|
+
* `cachedPhase` consults all three READ-ONLY on a miss; `recordCache` writes
|
|
1670
|
+
* only `key`. This means an upgrade never produces a miss-storm: existing
|
|
1671
|
+
* entries (whichever shape) still hit, and new writes converge on `key`. */
|
|
1672
|
+
export function cacheKeys(cc: PhaseCacheCtx, baseParts: string[]): CacheKeys {
|
|
1516
1673
|
// Fold the full cache identity into the hash: flow name (prevents collisions
|
|
1517
1674
|
// across different flows that share a phase.id + task + model), the per-phase
|
|
1518
1675
|
// thinking/tools config (changing either changes the subagent's output), the
|
|
1519
1676
|
// resolved context pre-read content, and the world-state fingerprint.
|
|
1520
|
-
const
|
|
1521
|
-
`flow:${cc.flowName}`,
|
|
1677
|
+
const tail = [
|
|
1522
1678
|
...baseParts,
|
|
1523
1679
|
`think:${cc.thinking ?? ""}`,
|
|
1524
1680
|
`tools:${JSON.stringify(cc.tools ?? [])}`,
|
|
1525
1681
|
`ctx:${cc.preRead ?? ""}`,
|
|
1526
1682
|
];
|
|
1527
|
-
|
|
1683
|
+
const fold = (parts: string[]): string =>
|
|
1684
|
+
cc.fingerprint ? hashInput(...parts, cc.fingerprint) : hashInput(...parts);
|
|
1685
|
+
return {
|
|
1686
|
+
key: fold([`flow:${cc.flowName}`, `v2:flowdef:${cc.flowDefHash ?? ""}`, ...tail]),
|
|
1687
|
+
legacyKey: fold([`flow:${cc.flowName}`, ...tail]),
|
|
1688
|
+
bareKey: fold([`flow:${cc.flowName}`, `flowdef:${cc.flowDefHash ?? ""}`, ...tail]),
|
|
1689
|
+
};
|
|
1528
1690
|
}
|
|
1529
1691
|
|
|
1530
1692
|
/**
|
|
@@ -1533,23 +1695,39 @@ function cacheKey(cc: PhaseCacheCtx, baseParts: string[]): string {
|
|
|
1533
1695
|
* - "run-only": within-run resume only (historical behavior).
|
|
1534
1696
|
* - "cross-run": within-run first, then the persistent cross-run store.
|
|
1535
1697
|
* On a cross-run hit, usage is zeroed and `cacheHit` records the source.
|
|
1698
|
+
*
|
|
1699
|
+
* The cross-run read is THREE-TIER and READ-ONLY for fallback keys: it tries
|
|
1700
|
+
* `keys.key` (current `v2:flowdef:` shape) first, then `keys.bareKey` (pre-H1
|
|
1701
|
+
* bare `flowdef:`), then `keys.legacyKey` (pre-flowDefHash, no flowdef line).
|
|
1702
|
+
* A hit on ANY tier is restored as a cache hit; we do NOT write-through (no
|
|
1703
|
+
* re-store under the new key) so the cache size stays stable and the legacy
|
|
1704
|
+
* entry ages out naturally. See docs/internal/cache-migration.md.
|
|
1536
1705
|
*/
|
|
1537
|
-
function cachedPhase(cc: PhaseCacheCtx,
|
|
1706
|
+
function cachedPhase(cc: PhaseCacheCtx, keys: CacheKeys): PhaseState | null {
|
|
1538
1707
|
if (cc.scope === "off") return null;
|
|
1708
|
+
if (cc.forceRerun) return null;
|
|
1539
1709
|
|
|
1540
1710
|
// 1. within-run resume (fastest; always allowed unless scope is off)
|
|
1541
|
-
if (cc.prior && cc.prior.status === "done" && cc.prior.inputHash ===
|
|
1711
|
+
if (cc.prior && cc.prior.status === "done" && cc.prior.inputHash === keys.key) {
|
|
1542
1712
|
return { ...cc.prior, status: "done" };
|
|
1543
1713
|
}
|
|
1544
1714
|
|
|
1545
|
-
// 2. cross-run memoization (opt-in)
|
|
1715
|
+
// 2. cross-run memoization (opt-in) — three-tier read-only fallback.
|
|
1546
1716
|
if (cc.scope === "cross-run") {
|
|
1547
|
-
const
|
|
1548
|
-
|
|
1717
|
+
for (const k of [keys.key, keys.bareKey, keys.legacyKey]) {
|
|
1718
|
+
const e = cc.store.get(k, cc.ttlMs);
|
|
1719
|
+
if (!e) continue;
|
|
1720
|
+
// If we stored the full PhaseState, restore it (preserving gate,
|
|
1721
|
+
// approval, reads, loop/tournament metadata, warnings) and just mark
|
|
1722
|
+
// the cache hit + zero usage. Fallback to the legacy trimmed surface
|
|
1723
|
+
// for entries written before this change.
|
|
1724
|
+
if (e.state) {
|
|
1725
|
+
return { ...e.state, inputHash: keys.key, usage: emptyUsage(), cacheHit: "cross-run", endedAt: Date.now() };
|
|
1726
|
+
}
|
|
1549
1727
|
return {
|
|
1550
1728
|
id: cc.phaseId,
|
|
1551
1729
|
status: "done",
|
|
1552
|
-
inputHash,
|
|
1730
|
+
inputHash: keys.key,
|
|
1553
1731
|
output: e.output,
|
|
1554
1732
|
json: e.json,
|
|
1555
1733
|
model: e.model,
|
|
@@ -1573,6 +1751,7 @@ function recordCache(cc: PhaseCacheCtx, ps: PhaseState): void {
|
|
|
1573
1751
|
output: ps.output,
|
|
1574
1752
|
json: ps.json,
|
|
1575
1753
|
model: ps.model,
|
|
1754
|
+
state: ps,
|
|
1576
1755
|
flowName: cc.flowName,
|
|
1577
1756
|
phaseId: cc.phaseId,
|
|
1578
1757
|
runId: cc.runId,
|
|
@@ -1701,6 +1880,167 @@ function safeProgress(deps: RuntimeDeps, state: RunState): void {
|
|
|
1701
1880
|
/**
|
|
1702
1881
|
* Execute a full taskflow. Mutates and persists `state` as it progresses.
|
|
1703
1882
|
*/
|
|
1883
|
+
/** Result of a recompute: what was (or would be) re-executed vs reused.
|
|
1884
|
+
* `cutoff` is the prize — phases in the stale frontier whose inputHash did
|
|
1885
|
+
* NOT move, so they hit their cached result instead of re-running (early
|
|
1886
|
+
* cutoff). That is what makes recompute cheaper than a full re-run. */
|
|
1887
|
+
export interface RecomputeReport {
|
|
1888
|
+
readonly dryRun: boolean;
|
|
1889
|
+
readonly aborted: boolean;
|
|
1890
|
+
readonly seeds: readonly string[];
|
|
1891
|
+
/** Phases that were (dry-run: would be) re-executed, or whose result moved. */
|
|
1892
|
+
readonly rerun: readonly string[];
|
|
1893
|
+
/** Phases outside the frontier — untouched, reused verbatim. */
|
|
1894
|
+
readonly reused: readonly string[];
|
|
1895
|
+
/** Phases in the frontier whose inputHash did NOT move → cached result
|
|
1896
|
+
* reused, no re-execution (early cutoff). Empty in dry-run (unknowable). */
|
|
1897
|
+
readonly cutoff: readonly string[];
|
|
1898
|
+
}
|
|
1899
|
+
|
|
1900
|
+
/** Scan a flow for dependencies that cannot be observed through the readSet.
|
|
1901
|
+
* These include Shared Context Tree, sub-flows, context: file pre-reads, and
|
|
1902
|
+
* interpolation placeholders that do not resolve through `steps.*` (previous,
|
|
1903
|
+
* args, item). Recomputing flows with such deps with dryRun:false risks
|
|
1904
|
+
* silently reusing stale upstream state. */
|
|
1905
|
+
function hasUnobservedDependencies(state: RunState): boolean {
|
|
1906
|
+
const scan = (text: string): boolean => /\{(previous\.output|args\.|item\b|item\.)/.test(text);
|
|
1907
|
+
for (const p of state.def.phases) {
|
|
1908
|
+
if (p.shareContext === true) return true;
|
|
1909
|
+
if (state.def.contextSharing === true) return true;
|
|
1910
|
+
if (p.type === "flow") return true;
|
|
1911
|
+
if (p.context && p.context.length > 0) return true;
|
|
1912
|
+
if (scan(p.task ?? "")) return true;
|
|
1913
|
+
if (p.when && scan(p.when)) return true;
|
|
1914
|
+
if (p.until && scan(p.until)) return true;
|
|
1915
|
+
if (Array.isArray(p.eval) && p.eval.some(scan)) return true;
|
|
1916
|
+
}
|
|
1917
|
+
return false;
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
/** Recompute a completed run minimally: force-rerun the `seeds`, then walk
|
|
1921
|
+
* their stale frontier in topological order. The cache provides early cutoff
|
|
1922
|
+
* for free — a downstream whose inputHash didn't move (because the seed's new
|
|
1923
|
+
* output happened to equal the old) hits its prior and is reused rather than
|
|
1924
|
+
* re-executed. `dryRun` computes the worst-case frontier without spending a
|
|
1925
|
+
* token. Returns a fresh state + a report. Throws only when dryRun:false is
|
|
1926
|
+
* requested for a flow with unobserved dependencies; callers should surface
|
|
1927
|
+
* that as a user-facing error. */
|
|
1928
|
+
export async function recomputeTaskflow(
|
|
1929
|
+
state: RunState,
|
|
1930
|
+
deps: RuntimeDeps,
|
|
1931
|
+
seeds: readonly string[],
|
|
1932
|
+
// Fail-safe default: a real recompute overwrites the run and spends tokens.
|
|
1933
|
+
// The tool/command wrappers can explicitly opt into dryRun:false.
|
|
1934
|
+
opts: { dryRun?: boolean } = { dryRun: true },
|
|
1935
|
+
): Promise<{ report: RecomputeReport; state: RunState }> {
|
|
1936
|
+
// Never mutate the caller's RunState in-place. Recompute is a speculative
|
|
1937
|
+
// replay; only the caller decides whether to persist the new state.
|
|
1938
|
+
const newState = structuredClone(state) as RunState;
|
|
1939
|
+
const reads = readMapOf(newState.phases);
|
|
1940
|
+
// M2: derive the declared read-map fresh from the def so the frontier uses
|
|
1941
|
+
// the UNION (observed ∪ declared). Derived here (not read from the persisted
|
|
1942
|
+
// `RunState.declaredDeps`) so old runs — pre-H1, no persisted declaredDeps —
|
|
1943
|
+
// also get union semantics. The persisted field is audit/provenance only.
|
|
1944
|
+
const declared = declaredReadMapOfDef(newState.def);
|
|
1945
|
+
const frontier = computeStaleFrontier(reads, seeds, declared);
|
|
1946
|
+
const allIds = Object.keys(newState.phases);
|
|
1947
|
+
|
|
1948
|
+
if (opts.dryRun) {
|
|
1949
|
+
return {
|
|
1950
|
+
report: {
|
|
1951
|
+
dryRun: true,
|
|
1952
|
+
aborted: false,
|
|
1953
|
+
seeds,
|
|
1954
|
+
rerun: [...frontier],
|
|
1955
|
+
reused: allIds.filter((id) => !frontier.has(id)),
|
|
1956
|
+
cutoff: [],
|
|
1957
|
+
},
|
|
1958
|
+
state: newState,
|
|
1959
|
+
};
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
// Guard: observed readSet only tracks `{steps.X.*}` interpolation refs. It is
|
|
1963
|
+
// blind to Shared Context Tree (ctx_read/ctx_write), sub-flow internals,
|
|
1964
|
+
// context: file pre-reads, {previous.output}, and loop locals ({args.*},
|
|
1965
|
+
// {item.*}). Recomputing such a run with dryRun:false could silently skip
|
|
1966
|
+
// phases whose deps changed outside the observed frontier and then persist a
|
|
1967
|
+
// corrupted run over the original.
|
|
1968
|
+
if (hasUnobservedDependencies(newState)) {
|
|
1969
|
+
throw new Error(
|
|
1970
|
+
"recompute dryRun:false is unsafe for this run: it contains dependencies " +
|
|
1971
|
+
"(shareContext, flow/ctx_spawn, context: files, {previous.output}, {args.*}, or {item.*}) " +
|
|
1972
|
+
"that are not tracked by the observed readSet. Use dryRun:true to inspect " +
|
|
1973
|
+
"the frontier, or change the upstream phase and re-run the whole flow.",
|
|
1974
|
+
);
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
// Real recompute: topological order over the frontier so a downstream always
|
|
1978
|
+
// sees its (already-refreshed) upstreams when it re-evaluates its cache key.
|
|
1979
|
+
// The order must respect declared dependsOn, observed reads, AND declared
|
|
1980
|
+
// reads (M2 union): pi-taskflow allows interpolation refs without an
|
|
1981
|
+
// explicit dependsOn edge, and a declared-but-unobserved edge (e.g. a `when`
|
|
1982
|
+
// ref that never fired) must still order the reader after its upstream so
|
|
1983
|
+
// the reader evaluates its cache key against the refreshed upstream (no
|
|
1984
|
+
// false early-cutoff).
|
|
1985
|
+
const seedSet = new Set(seeds);
|
|
1986
|
+
function depsFor(phaseId: string): string[] {
|
|
1987
|
+
// A phase reading its own prior output (e.g. a loop `until` checking
|
|
1988
|
+
// `{steps.thisId.output}`) must not create a self-edge in the scheduling
|
|
1989
|
+
// graph — otherwise topoLayers would deadlock on the self-loop.
|
|
1990
|
+
const observed = (newState.phases[phaseId]?.reads ?? [])
|
|
1991
|
+
.map((r) => r.stepId)
|
|
1992
|
+
.filter((id) => id !== phaseId);
|
|
1993
|
+
const declared_ = (declared.get(phaseId) ?? []).filter((id) => id !== phaseId);
|
|
1994
|
+
return [...new Set([...observed, ...declared_])];
|
|
1995
|
+
}
|
|
1996
|
+
const augmentedPhases = newState.def.phases.map((p) => ({
|
|
1997
|
+
...p,
|
|
1998
|
+
dependsOn: [...new Set([...(p.dependsOn ?? []), ...depsFor(p.id)])],
|
|
1999
|
+
}));
|
|
2000
|
+
const order = topoLayers(augmentedPhases)
|
|
2001
|
+
.flat()
|
|
2002
|
+
.map((p) => p.id)
|
|
2003
|
+
.filter((id) => frontier.has(id));
|
|
2004
|
+
const rerun: string[] = [];
|
|
2005
|
+
const cutoff: string[] = [];
|
|
2006
|
+
const noop = () => {};
|
|
2007
|
+
let aborted = false;
|
|
2008
|
+
for (const id of order) {
|
|
2009
|
+
// A partial recompute must NOT be persisted over the original run — the
|
|
2010
|
+
// caller discards `state` when `aborted` is set.
|
|
2011
|
+
if (deps.signal?.aborted) {
|
|
2012
|
+
aborted = true;
|
|
2013
|
+
break;
|
|
2014
|
+
}
|
|
2015
|
+
const phase = newState.def.phases.find((p) => p.id === id);
|
|
2016
|
+
if (!phase) continue;
|
|
2017
|
+
const before = newState.phases[id]?.inputHash;
|
|
2018
|
+
const execOpts = seedSet.has(id) ? { forceRerun: true } : undefined;
|
|
2019
|
+
try {
|
|
2020
|
+
const ps = await executePhase(phase, newState, deps, newState.phases[id], noop, 0, execOpts);
|
|
2021
|
+
newState.phases[id] = ps;
|
|
2022
|
+
// A phase counts as "rerun" if it was a forced seed OR its result moved;
|
|
2023
|
+
// otherwise it hit its cache (inputHash unchanged) → early cutoff.
|
|
2024
|
+
if (seedSet.has(id) || ps.inputHash !== before) rerun.push(id);
|
|
2025
|
+
else cutoff.push(id);
|
|
2026
|
+
} catch {
|
|
2027
|
+
// A failing recompute phase is recorded as rerun (it was attempted).
|
|
2028
|
+
rerun.push(id);
|
|
2029
|
+
}
|
|
2030
|
+
}
|
|
2031
|
+
return {
|
|
2032
|
+
report: {
|
|
2033
|
+
dryRun: false,
|
|
2034
|
+
aborted,
|
|
2035
|
+
seeds,
|
|
2036
|
+
rerun,
|
|
2037
|
+
reused: allIds.filter((id) => !frontier.has(id)),
|
|
2038
|
+
cutoff,
|
|
2039
|
+
},
|
|
2040
|
+
state: newState,
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
|
|
1704
2044
|
export async function executeTaskflow(state: RunState, deps: RuntimeDeps): Promise<RuntimeResult> {
|
|
1705
2045
|
const def: Taskflow = state.def;
|
|
1706
2046
|
try {
|
|
@@ -1726,6 +2066,38 @@ export async function executeTaskflow(state: RunState, deps: RuntimeDeps): Promi
|
|
|
1726
2066
|
async function runTaskflowLayers(state: RunState, deps: RuntimeDeps): Promise<RuntimeResult> {
|
|
1727
2067
|
const def: Taskflow = state.def;
|
|
1728
2068
|
const layers = topoLayers(def.phases);
|
|
2069
|
+
// Content-fingerprint the desugared definition ONCE per run and fold it into
|
|
2070
|
+
// every phase's cache key (overstory hash algorithm; see ./flowir/hash.ts).
|
|
2071
|
+
// Reused by every phase, persisted on the RunState for audit/resume.
|
|
2072
|
+
// Never throws into the run — a hash failure leaves the field unset and the
|
|
2073
|
+
// cache key degrades to the legacy flowName-only shape.
|
|
2074
|
+
//
|
|
2075
|
+
// Routed through the FlowIR compile seam (M1): `compileTaskflowToIR`
|
|
2076
|
+
// produces the content-addressed IR whose `hash` (== flowDefHash in the
|
|
2077
|
+
// stub) folds into the cache key, and whose `meta.declaredDeps` (M2 declared
|
|
2078
|
+
// plane) is persisted for audit/provenance. The declared plane is also
|
|
2079
|
+
// derived fresh from `def` in recompute (so old runs get union semantics
|
|
2080
|
+
// too); the persisted copy is for display.
|
|
2081
|
+
if (state.flowDefHash === undefined) {
|
|
2082
|
+
try {
|
|
2083
|
+
const ir = await compileTaskflowToIR(def);
|
|
2084
|
+
state.flowDefHash = ir.hash ?? "failed";
|
|
2085
|
+
state.declaredDeps = ir.meta.declaredDeps;
|
|
2086
|
+
if (ir.errors.length) {
|
|
2087
|
+
console.warn(
|
|
2088
|
+
`[taskflow] IR compile errors for '${def.name}': ${ir.errors.map((e) => e.message).join("; ")}`,
|
|
2089
|
+
);
|
|
2090
|
+
}
|
|
2091
|
+
} catch (e) {
|
|
2092
|
+
// Fail-safe: warn loudly rather than silently degrading to the legacy
|
|
2093
|
+
// flowName-only key, which would reopen the cross-flow collision hole.
|
|
2094
|
+
console.warn(
|
|
2095
|
+
`[taskflow] flowDefHash failed for '${def.name}': ${e instanceof Error ? e.message : String(e)}. ` +
|
|
2096
|
+
"Cross-run cache is disabled for this run to prevent stale cross-flow hits.",
|
|
2097
|
+
);
|
|
2098
|
+
state.flowDefHash = "failed";
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
1729
2101
|
|
|
1730
2102
|
state.status = "running";
|
|
1731
2103
|
safeEmit(deps, state);
|
|
@@ -1770,10 +2142,6 @@ async function runTaskflowLayers(state: RunState, deps: RuntimeDeps): Promise<Ru
|
|
|
1770
2142
|
else if (budgetBlocked) skipReason = `Budget exceeded${budgetReason ? `: ${budgetReason}` : ""}`;
|
|
1771
2143
|
else if (!depsSatisfied)
|
|
1772
2144
|
skipReason = join === "any" ? "All dependencies failed or were skipped" : "Upstream dependency not satisfied";
|
|
1773
|
-
else if (phase.when !== undefined) {
|
|
1774
|
-
const condCtx = buildInterpolationContext(state, lastCompletedOutput(state, phase));
|
|
1775
|
-
if (!evaluateCondition(phase.when, condCtx)) skipReason = `Condition not met: ${phase.when}`;
|
|
1776
|
-
}
|
|
1777
2145
|
|
|
1778
2146
|
if (skipReason) {
|
|
1779
2147
|
if (skipReason.startsWith("Budget exceeded")) budgetBlocked = true;
|