smol-symphony 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +105 -38
- package/PRODUCT.md +2 -1
- package/README.md +195 -98
- package/SPEC.md +543 -1915
- package/WORKFLOW.md +654 -179
- package/WORKFLOW.template.md +761 -121
- package/dist/acp-bridge.js +324 -0
- package/dist/acp-bridge.js.map +1 -0
- package/dist/actions/cache.js +191 -0
- package/dist/actions/cache.js.map +1 -0
- package/dist/actions/effects.js +41 -0
- package/dist/actions/effects.js.map +1 -0
- package/dist/actions/executor.js +570 -0
- package/dist/actions/executor.js.map +1 -0
- package/dist/actions/index.js +13 -0
- package/dist/actions/index.js.map +1 -0
- package/dist/actions/parsing.js +273 -0
- package/dist/actions/parsing.js.map +1 -0
- package/dist/actions/predicate-env.js +27 -0
- package/dist/actions/predicate-env.js.map +1 -0
- package/dist/actions/predicates.js +49 -0
- package/dist/actions/predicates.js.map +1 -0
- package/dist/actions/templating.js +66 -0
- package/dist/actions/templating.js.map +1 -0
- package/dist/actions/types.js +15 -0
- package/dist/actions/types.js.map +1 -0
- package/dist/agent/acp.js +232 -63
- package/dist/agent/acp.js.map +1 -1
- package/dist/agent/adapter-names.js +159 -0
- package/dist/agent/adapter-names.js.map +1 -0
- package/dist/agent/adapters.js +338 -102
- package/dist/agent/adapters.js.map +1 -1
- package/dist/agent/credential-extractors.js +342 -0
- package/dist/agent/credential-extractors.js.map +1 -0
- package/dist/agent/credential-secrets.js +628 -0
- package/dist/agent/credential-secrets.js.map +1 -0
- package/dist/agent/credential-ticker.js +57 -0
- package/dist/agent/credential-ticker.js.map +1 -0
- package/dist/agent/gondolin-creds-staging.js +356 -0
- package/dist/agent/gondolin-creds-staging.js.map +1 -0
- package/dist/agent/gondolin-dispatch.js +375 -0
- package/dist/agent/gondolin-dispatch.js.map +1 -0
- package/dist/agent/gondolin.js +124 -0
- package/dist/agent/gondolin.js.map +1 -0
- package/dist/agent/runner-decisions.js +134 -0
- package/dist/agent/runner-decisions.js.map +1 -0
- package/dist/agent/runner.js +1352 -290
- package/dist/agent/runner.js.map +1 -1
- package/dist/agent/tool-call-summary.js +102 -0
- package/dist/agent/tool-call-summary.js.map +1 -0
- package/dist/agent/vm-acp-mapping.js +73 -0
- package/dist/agent/vm-acp-mapping.js.map +1 -0
- package/dist/agent/vm-guards.js +262 -0
- package/dist/agent/vm-guards.js.map +1 -0
- package/dist/agent/vm-port.js +22 -0
- package/dist/agent/vm-port.js.map +1 -0
- package/dist/agent/vm-process-registry.js +79 -0
- package/dist/agent/vm-process-registry.js.map +1 -0
- package/dist/bin/cli-args.js +105 -0
- package/dist/bin/cli-args.js.map +1 -0
- package/dist/bin/symphony.js +719 -130
- package/dist/bin/symphony.js.map +1 -1
- package/dist/errors.js +15 -0
- package/dist/errors.js.map +1 -0
- package/dist/http-disk.js +135 -0
- package/dist/http-disk.js.map +1 -0
- package/dist/http-handlers.js +180 -0
- package/dist/http-handlers.js.map +1 -0
- package/dist/http.js +1476 -764
- package/dist/http.js.map +1 -1
- package/dist/issues.js +178 -0
- package/dist/issues.js.map +1 -0
- package/dist/logging.js +163 -5
- package/dist/logging.js.map +1 -1
- package/dist/mcp.js +391 -163
- package/dist/mcp.js.map +1 -1
- package/dist/memory.js +85 -0
- package/dist/memory.js.map +1 -0
- package/dist/orchestrator-decisions.js +331 -0
- package/dist/orchestrator-decisions.js.map +1 -0
- package/dist/orchestrator.js +1189 -303
- package/dist/orchestrator.js.map +1 -1
- package/dist/prompt.js +5 -5
- package/dist/prompt.js.map +1 -1
- package/dist/reconciler/cache.js +65 -0
- package/dist/reconciler/cache.js.map +1 -0
- package/dist/reconciler/index.js +448 -0
- package/dist/reconciler/index.js.map +1 -0
- package/dist/reconciler/ledger.js +131 -0
- package/dist/reconciler/ledger.js.map +1 -0
- package/dist/reconciler/pr-adapters.js +174 -0
- package/dist/reconciler/pr-adapters.js.map +1 -0
- package/dist/reconciler/pr-decide.js +167 -0
- package/dist/reconciler/pr-decide.js.map +1 -0
- package/dist/reconciler/pr.js +422 -0
- package/dist/reconciler/pr.js.map +1 -0
- package/dist/reconciler/types.js +12 -0
- package/dist/reconciler/types.js.map +1 -0
- package/dist/reconciler/vm.js +243 -0
- package/dist/reconciler/vm.js.map +1 -0
- package/dist/reconciler/workspace-defaults.js +83 -0
- package/dist/reconciler/workspace-defaults.js.map +1 -0
- package/dist/reconciler/workspace.js +272 -0
- package/dist/reconciler/workspace.js.map +1 -0
- package/dist/runlog.js +403 -0
- package/dist/runlog.js.map +1 -0
- package/dist/scaffold.js +165 -0
- package/dist/scaffold.js.map +1 -0
- package/dist/trackers/local.js +234 -133
- package/dist/trackers/local.js.map +1 -1
- package/dist/trackers/types.js +1 -1
- package/dist/trackers/types.js.map +1 -1
- package/dist/types.js +1 -1
- package/dist/util/clock.js +12 -0
- package/dist/util/clock.js.map +1 -0
- package/dist/util/crypto.js +25 -0
- package/dist/util/crypto.js.map +1 -0
- package/dist/util/frontmatter.js +70 -0
- package/dist/util/frontmatter.js.map +1 -0
- package/dist/util/fs-issues.js +22 -0
- package/dist/util/fs-issues.js.map +1 -0
- package/dist/util/process.js +152 -0
- package/dist/util/process.js.map +1 -0
- package/dist/util/workspace-key.js +10 -0
- package/dist/util/workspace-key.js.map +1 -0
- package/dist/workflow-loader.js +147 -0
- package/dist/workflow-loader.js.map +1 -0
- package/dist/workflow.js +656 -219
- package/dist/workflow.js.map +1 -1
- package/dist/workspace-types.js +8 -0
- package/dist/workspace-types.js.map +1 -0
- package/dist/workspace.js +367 -120
- package/dist/workspace.js.map +1 -1
- package/package.json +14 -6
- package/scripts/vm-agent.mjs +211 -0
- package/dist/agent/codex.js +0 -439
- package/dist/agent/codex.js.map +0 -1
- package/dist/agent/smolvm.js +0 -174
- package/dist/agent/smolvm.js.map +0 -1
- package/scripts/build-vm.sh +0 -67
package/dist/orchestrator.js
CHANGED
|
@@ -1,11 +1,35 @@
|
|
|
1
|
-
// Orchestrator
|
|
2
|
-
//
|
|
3
|
-
import { validateDispatch, WorkflowError } from './workflow.js';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
1
|
+
// Orchestrator. Owns the single-authority runtime state and drives the
|
|
2
|
+
// poll-and-dispatch tick, retries, reconciliation, and worker exit handling.
|
|
3
|
+
import { deriveArmRouting, derivePrRouting, validateDispatch, WorkflowError, } from './workflow.js';
|
|
4
|
+
import { validateDispatchIo } from './workflow-loader.js';
|
|
5
|
+
import { writeIssueFile, pickHoldingState } from './issues.js';
|
|
6
|
+
import { resolveDispatchConfig } from './agent/runner.js';
|
|
7
|
+
import { codexCredentialAvailable, codexMissingCredentialMessage, hostClaudeCredentialPath, hostCodexCredentialPath, hostOpencodeCredentialPath, isKnownAdapter, opencodeCredentialAvailable, opencodeMissingCredentialMessage, } from './agent/adapter-names.js';
|
|
8
|
+
import { accessSync, constants as fsConstants, readFileSync } from 'node:fs';
|
|
9
|
+
import { activeStateNames, terminalStateNames } from './issues.js';
|
|
10
|
+
import { buildIssueDetailDto, classifyPrIntent, computeEligibilityReason, decideCircuitBreaker, decideExitRetry, decideReconcileForIssue, decideRetryAfterIneligible, decideSleepCycleArm, requiredAdapterIds, resolveActorString, sleepCycleArmNotes, } from './orchestrator-decisions.js';
|
|
11
|
+
import { resolveGithubRepo } from './workspace.js';
|
|
6
12
|
import { withIssue, log } from './logging.js';
|
|
13
|
+
import { openRunLog } from './runlog.js';
|
|
14
|
+
import { defaultMemProbe, computeMemoryAdmission } from './memory.js';
|
|
15
|
+
import { runProcess } from './util/process.js';
|
|
7
16
|
const CONTINUATION_DELAY_MS = 1_000;
|
|
8
17
|
const FAILURE_BASE_MS = 10_000;
|
|
18
|
+
// Actor stamped into the notes header when the orchestrator (not an agent)
|
|
19
|
+
// auto-arms the reflection issue, so the move is attributable on the dashboard
|
|
20
|
+
// and in the issue body.
|
|
21
|
+
const SLEEP_CYCLE_ACTOR = 'symphony/sleep-cycle';
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the base branch the autopilot should rebase against. Mirrors the
|
|
24
|
+
* canonical workspace-setup contract — the `SYMPHONY_BASE_BRANCH` env wins,
|
|
25
|
+
* else the parsed `workspace.base_branch` (which defaults to `main`).
|
|
26
|
+
*/
|
|
27
|
+
function baseBranchName(configBaseBranch) {
|
|
28
|
+
const env = process.env.SYMPHONY_BASE_BRANCH;
|
|
29
|
+
if (env && env.length > 0)
|
|
30
|
+
return env;
|
|
31
|
+
return configBaseBranch;
|
|
32
|
+
}
|
|
9
33
|
export class Orchestrator {
|
|
10
34
|
cfg;
|
|
11
35
|
workflowDef;
|
|
@@ -13,33 +37,81 @@ export class Orchestrator {
|
|
|
13
37
|
tracker;
|
|
14
38
|
workspaces;
|
|
15
39
|
runner;
|
|
40
|
+
memProbe;
|
|
41
|
+
reconciler;
|
|
16
42
|
running = new Map();
|
|
17
43
|
claimed = new Set();
|
|
18
44
|
retryAttempts = new Map();
|
|
45
|
+
// Per-issue circuit-breaker streak (issue 128): the last abnormal-exit reason
|
|
46
|
+
// (normalized) and how many consecutive attempts failed with it. Updated on
|
|
47
|
+
// every worker exit; cleared on a clean exit, on trip, and on claim release.
|
|
48
|
+
// In-memory only — a process restart resets the streak, but the *trip itself*
|
|
49
|
+
// is restart-safe because it physically moves the issue out of the active set.
|
|
50
|
+
circuitBreakers = new Map();
|
|
51
|
+
// Sleep-cycle auto-arm (issue 125). Count of terminal-state transitions
|
|
52
|
+
// observed since the reflection issue was last armed; the idle and
|
|
53
|
+
// done-threshold triggers both read it, and it resets to 0 on each arm. The
|
|
54
|
+
// in-flight guard stops two overlapping ticks from both firing the async
|
|
55
|
+
// Dormant → Reflect move. In-memory only — a process restart resets the
|
|
56
|
+
// streak (consistent with `circuitBreakers`).
|
|
57
|
+
doneSinceReflect = 0;
|
|
58
|
+
armingReflection = false;
|
|
19
59
|
completed = new Set();
|
|
20
|
-
|
|
60
|
+
// Per-state ledger of the most-recent action-list execution. Surfaced via
|
|
61
|
+
// `snapshot.reconciler.resources` so the dashboard can render "Done.actions:
|
|
62
|
+
// push_branch ok, create_pr_if_missing in_progress" without a separate
|
|
63
|
+
// first-class surface for action state (issue 36 AC5).
|
|
64
|
+
lastActionResults = new Map();
|
|
65
|
+
// Per-issue JSONL run log. Opened lazily on first dispatch for an issue, kept open across
|
|
66
|
+
// retries so the file is one chronological stream per issue, and closed only when the
|
|
67
|
+
// issue finally unwinds (terminal cleanup, claim release without redispatch, or stop()).
|
|
68
|
+
runLogs = new Map();
|
|
69
|
+
// Set of issue ids whose terminal cleanup (workspaces.remove) is still in
|
|
70
|
+
// flight. Used by closeRunLog to defer the close until the terminal-state
|
|
71
|
+
// actions capture has stopped writing; otherwise the retry-timer's "claim
|
|
72
|
+
// released" close fires ~1s after worker exit (before the actions finish)
|
|
73
|
+
// and we'd lose the action output lines in the JSONL log.
|
|
74
|
+
cleanupInFlight = new Set();
|
|
75
|
+
sessionTotals = {
|
|
21
76
|
input_tokens: 0,
|
|
22
77
|
output_tokens: 0,
|
|
23
78
|
total_tokens: 0,
|
|
24
79
|
seconds_running: 0,
|
|
25
80
|
};
|
|
26
|
-
|
|
81
|
+
rateLimits = null;
|
|
27
82
|
tickTimer = null;
|
|
28
83
|
stopped = false;
|
|
29
84
|
refreshRequested = false;
|
|
30
85
|
// Latest dispatch validation error, if any (operator-visible).
|
|
31
86
|
lastValidationError = null;
|
|
32
87
|
// Optional callback used to propagate reloaded config to components that hold their own
|
|
33
|
-
// tracker/runner/workspace state (so prompt body,
|
|
34
|
-
// on the next dispatch
|
|
88
|
+
// tracker/runner/workspace state (so prompt body, per-state actions, gondolin config, etc.,
|
|
89
|
+
// take effect on the next dispatch).
|
|
35
90
|
onConfigReloaded;
|
|
36
|
-
|
|
91
|
+
// Last clamp-active state observed by availableGlobalSlots. Used to log
|
|
92
|
+
// transitions (clamp_active true→false or false→true) at info level without
|
|
93
|
+
// spamming the log every tick while the cap stays clamped.
|
|
94
|
+
memoryClampActive = false;
|
|
95
|
+
constructor(cfg, workflowDef, workflowSrc, tracker, workspaces, runner,
|
|
96
|
+
// Memory probe used by the admission cap (issue 27). Defaults to reading
|
|
97
|
+
// /proc/meminfo synchronously; tests inject a stub that returns a controlled
|
|
98
|
+
// mem_available_mib so the clamp behavior is deterministic.
|
|
99
|
+
memProbe = defaultMemProbe,
|
|
100
|
+
// Reconciler (issue 32, 33) — owns managed external resources: the
|
|
101
|
+
// symphony-VM lifecycle reaper, the workspace janitor, and PR autopilot.
|
|
102
|
+
// Optional so tests that don't exercise reconciliation don't have to
|
|
103
|
+
// construct one; when absent, `Snapshot.reconciler` is null and stray VM
|
|
104
|
+
// reaping is skipped. Production wiring in bin/symphony.ts always passes
|
|
105
|
+
// one in.
|
|
106
|
+
reconciler = null) {
|
|
37
107
|
this.cfg = cfg;
|
|
38
108
|
this.workflowDef = workflowDef;
|
|
39
109
|
this.workflowSrc = workflowSrc;
|
|
40
110
|
this.tracker = tracker;
|
|
41
111
|
this.workspaces = workspaces;
|
|
42
112
|
this.runner = runner;
|
|
113
|
+
this.memProbe = memProbe;
|
|
114
|
+
this.reconciler = reconciler;
|
|
43
115
|
workflowSrc.onChange((next) => {
|
|
44
116
|
if ('error' in next) {
|
|
45
117
|
this.lastValidationError = next.error.message;
|
|
@@ -50,6 +122,10 @@ export class Orchestrator {
|
|
|
50
122
|
this.workflowDef = next.definition;
|
|
51
123
|
this.lastValidationError = null;
|
|
52
124
|
this.onConfigReloaded?.(next.config, next.definition);
|
|
125
|
+
// Issue 32: a config-watcher change is one of the reconciler's declared
|
|
126
|
+
// triggers. Re-binding the resource set picks up new managed-resource
|
|
127
|
+
// config (e.g. `gondolin.*` VM settings).
|
|
128
|
+
this.reconciler?.updateConfig(next.config);
|
|
53
129
|
log.info('runtime config reloaded', {
|
|
54
130
|
poll_interval_ms: next.config.polling.interval_ms,
|
|
55
131
|
max_concurrent_agents: next.config.agent.max_concurrent_agents,
|
|
@@ -61,33 +137,98 @@ export class Orchestrator {
|
|
|
61
137
|
this.onConfigReloaded = cb;
|
|
62
138
|
}
|
|
63
139
|
async start() {
|
|
64
|
-
const validation = validateDispatch(this.cfg);
|
|
140
|
+
const validation = validateDispatch(this.cfg) ?? validateDispatchIo(this.cfg);
|
|
65
141
|
if (validation) {
|
|
66
142
|
log.error('startup validation failed', { error: validation });
|
|
67
143
|
throw new WorkflowError('workflow_parse_error', validation);
|
|
68
144
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
// own credential plumbing, so skip the check in that branch.
|
|
72
|
-
if (this.cfg.acp.command === null && isKnownAdapter(this.cfg.acp.adapter)) {
|
|
73
|
-
const profile = ADAPTERS[this.cfg.acp.adapter];
|
|
74
|
-
try {
|
|
75
|
-
await assertHostCredentialReadable(profile);
|
|
76
|
-
}
|
|
77
|
-
catch (err) {
|
|
78
|
-
log.error('startup credential check failed', { error: err.message });
|
|
79
|
-
throw new WorkflowError('missing_host_credential', err.message);
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
await this.startupTerminalCleanup();
|
|
145
|
+
await this.assertAdapterCredentials();
|
|
146
|
+
await this.runStartupReconcile();
|
|
83
147
|
this.scheduleTick(0);
|
|
84
148
|
}
|
|
149
|
+
/**
|
|
150
|
+
* Fail fast when symphony will dispatch to an adapter whose host credential
|
|
151
|
+
* (substituted into the outbound request at Gondolin egress) is missing.
|
|
152
|
+
* Per-state overrides can change the adapter, so the set is the union of
|
|
153
|
+
* `cfg.acp.adapter` and every distinct `states.<name>.adapter`. claude needs
|
|
154
|
+
* `~/.claude/.credentials.json`; codex needs either a `~/.codex/auth.json`
|
|
155
|
+
* token or an `OPENAI_API_KEY` env var; opencode needs either a
|
|
156
|
+
* `github-copilot` token in `~/.local/share/opencode/auth.json` or a
|
|
157
|
+
* COPILOT_GITHUB_TOKEN/GH_TOKEN/GITHUB_TOKEN env var. A missing credential
|
|
158
|
+
* surfaces here as a clear startup error rather than an opaque per-request
|
|
159
|
+
* egress failure mid-dispatch.
|
|
160
|
+
*/
|
|
161
|
+
async assertAdapterCredentials() {
|
|
162
|
+
const ids = requiredAdapterIds(this.cfg, isKnownAdapter);
|
|
163
|
+
if (ids.has('claude'))
|
|
164
|
+
this.assertClaudeCredential();
|
|
165
|
+
if (ids.has('codex'))
|
|
166
|
+
this.assertCodexCredential();
|
|
167
|
+
if (ids.has('opencode'))
|
|
168
|
+
this.assertOpencodeCredential();
|
|
169
|
+
}
|
|
170
|
+
assertClaudeCredential() {
|
|
171
|
+
const credPath = hostClaudeCredentialPath();
|
|
172
|
+
try {
|
|
173
|
+
accessSync(credPath, fsConstants.R_OK);
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
const msg = `adapter "claude" requires a host credential at ${credPath}, but it is missing or unreadable: ${err.message}`;
|
|
177
|
+
log.error('startup credential check failed', { adapter: 'claude', error: msg });
|
|
178
|
+
throw new WorkflowError('missing_host_credential', msg);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
assertCodexCredential() {
|
|
182
|
+
let authText = null;
|
|
183
|
+
try {
|
|
184
|
+
authText = readFileSync(hostCodexCredentialPath(), 'utf8');
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
authText = null;
|
|
188
|
+
}
|
|
189
|
+
if (codexCredentialAvailable(authText, process.env))
|
|
190
|
+
return;
|
|
191
|
+
const msg = codexMissingCredentialMessage();
|
|
192
|
+
log.error('startup credential check failed', { adapter: 'codex', error: msg });
|
|
193
|
+
throw new WorkflowError('missing_host_credential', msg);
|
|
194
|
+
}
|
|
195
|
+
assertOpencodeCredential() {
|
|
196
|
+
let authText = null;
|
|
197
|
+
try {
|
|
198
|
+
authText = readFileSync(hostOpencodeCredentialPath(), 'utf8');
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
authText = null;
|
|
202
|
+
}
|
|
203
|
+
if (opencodeCredentialAvailable(authText, process.env))
|
|
204
|
+
return;
|
|
205
|
+
const msg = opencodeMissingCredentialMessage();
|
|
206
|
+
log.error('startup credential check failed', { adapter: 'opencode', error: msg });
|
|
207
|
+
throw new WorkflowError('missing_host_credential', msg);
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Initial workspace + VM reap and the first reconcile pass (issues 32-34).
|
|
211
|
+
* The `running` map is empty here, so the janitors converge to "remove
|
|
212
|
+
* anything orphaned by the previous process" and the bake (if any) starts
|
|
213
|
+
* before the first dispatch.
|
|
214
|
+
*/
|
|
215
|
+
async runStartupReconcile() {
|
|
216
|
+
if (!this.reconciler)
|
|
217
|
+
return;
|
|
218
|
+
await this.reconciler.reapWorkspaces();
|
|
219
|
+
await this.reconciler.reapVms();
|
|
220
|
+
this.reconciler.start();
|
|
221
|
+
void this.reconciler.reconcile().catch((err) => log.warn('initial reconcile pass failed', { error: err.message }));
|
|
222
|
+
}
|
|
85
223
|
async stop() {
|
|
86
224
|
this.stopped = true;
|
|
87
225
|
if (this.tickTimer) {
|
|
88
226
|
clearTimeout(this.tickTimer);
|
|
89
227
|
this.tickTimer = null;
|
|
90
228
|
}
|
|
229
|
+
if (this.reconciler) {
|
|
230
|
+
await this.reconciler.stop().catch(() => undefined);
|
|
231
|
+
}
|
|
91
232
|
for (const e of this.retryAttempts.values())
|
|
92
233
|
clearTimeout(e.timer_handle);
|
|
93
234
|
this.retryAttempts.clear();
|
|
@@ -96,8 +237,39 @@ export class Orchestrator {
|
|
|
96
237
|
e.cancel();
|
|
97
238
|
this.running.clear();
|
|
98
239
|
this.claimed.clear();
|
|
240
|
+
this.circuitBreakers.clear();
|
|
241
|
+
// Drain every open run log so the JSONL files are flushed before exit.
|
|
242
|
+
const closures = [];
|
|
243
|
+
for (const [issueId, rl] of this.runLogs) {
|
|
244
|
+
rl.system('runlog_closed', { reason: 'orchestrator_stopped' });
|
|
245
|
+
closures.push(rl.close());
|
|
246
|
+
this.runLogs.delete(issueId);
|
|
247
|
+
}
|
|
248
|
+
await Promise.all(closures);
|
|
249
|
+
// VM teardown lives in the reconciler `vm` resource (issue 52). stop() does NOT
|
|
250
|
+
// wait for in-flight workers to unwind before returning — the bin script then
|
|
251
|
+
// exits the process, which abruptly ends the per-VM Gondolin runners but can
|
|
252
|
+
// leave their session sockets behind. Without this backstop, every SIGTERM
|
|
253
|
+
// during an active run can leak one VM per running entry, and over enough
|
|
254
|
+
// operator restarts the host OOMs (issue 26). `running` is cleared above,
|
|
255
|
+
// so the reaper's intended set is ∅ and every `symphony-*` VM (live session +
|
|
256
|
+
// any orphaned socket) gets torn down.
|
|
257
|
+
if (this.reconciler) {
|
|
258
|
+
await this.reconciler.reapVms();
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Operator trigger for an immediate reconcile pass. Used by `symphony reconcile
|
|
263
|
+
* --force` (which invalidates the cache first via `force: true`) and by any
|
|
264
|
+
* future dashboard button that wants to re-evaluate the resource DAG without
|
|
265
|
+
* waiting for the backstop tick.
|
|
266
|
+
*/
|
|
267
|
+
async triggerReconcile(opts = {}) {
|
|
268
|
+
if (!this.reconciler)
|
|
269
|
+
return;
|
|
270
|
+
await this.reconciler.reconcile(opts);
|
|
99
271
|
}
|
|
100
|
-
/** Operator trigger for an immediate poll cycle (§
|
|
272
|
+
/** Operator trigger for an immediate poll cycle (§9.5 /refresh). */
|
|
101
273
|
triggerRefresh() {
|
|
102
274
|
if (this.refreshRequested)
|
|
103
275
|
return { queued: true, coalesced: true };
|
|
@@ -118,77 +290,113 @@ export class Orchestrator {
|
|
|
118
290
|
if (this.stopped)
|
|
119
291
|
return;
|
|
120
292
|
this.refreshRequested = false;
|
|
293
|
+
await this.reconcileSafely();
|
|
294
|
+
if (!this.applyDispatchValidation())
|
|
295
|
+
return;
|
|
296
|
+
const fetched = await this.fetchCandidatesForTick();
|
|
297
|
+
if (!fetched)
|
|
298
|
+
return;
|
|
299
|
+
if (this.gatedOnReconciler(fetched.issues.length))
|
|
300
|
+
return;
|
|
301
|
+
this.dispatchSorted(this.sortForDispatch(fetched.issues), fetched.root);
|
|
302
|
+
this.maybeArmSleepCycle(fetched.issues.length === 0);
|
|
303
|
+
this.scheduleTick(this.cfg.polling.interval_ms);
|
|
304
|
+
}
|
|
305
|
+
async reconcileSafely() {
|
|
121
306
|
try {
|
|
122
307
|
await this.reconcile();
|
|
123
308
|
}
|
|
124
309
|
catch (err) {
|
|
125
310
|
log.warn('reconcile error', { error: err.message });
|
|
126
311
|
}
|
|
127
|
-
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Run dispatch validation. Returns true to continue dispatch, false when the
|
|
315
|
+
* config is invalid (the tick is rescheduled and the caller must return).
|
|
316
|
+
*/
|
|
317
|
+
applyDispatchValidation() {
|
|
318
|
+
const validation = validateDispatch(this.cfg) ?? validateDispatchIo(this.cfg);
|
|
128
319
|
if (validation) {
|
|
129
320
|
this.lastValidationError = validation;
|
|
130
321
|
log.warn('dispatch validation failed; skipping dispatch', { error: validation });
|
|
131
322
|
this.scheduleTick(this.cfg.polling.interval_ms);
|
|
132
|
-
return;
|
|
323
|
+
return false;
|
|
133
324
|
}
|
|
134
325
|
this.lastValidationError = null;
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Atomic fetch: the tracker returns the issues AND the root it used during
|
|
330
|
+
* the scan. That's the snapshot we pin onto each RunningEntry, so a workflow
|
|
331
|
+
* reload that races the dispatch loop can't cause `transition` to operate
|
|
332
|
+
* against a different tracker root than where the issue lives. Returns
|
|
333
|
+
* null on tracker error (tick is rescheduled, caller must return).
|
|
334
|
+
*/
|
|
335
|
+
async fetchCandidatesForTick() {
|
|
138
336
|
try {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
// so a workflow reload that races the dispatch loop can't cause `mark_done`
|
|
142
|
-
// to operate against a different tracker config than where the issue lives.
|
|
143
|
-
const result = await this.tracker.fetchCandidateIssues();
|
|
144
|
-
candidates = result.issues;
|
|
145
|
-
snapshotTrackerRoot = result.root;
|
|
146
|
-
snapshotTerminalTarget = pickTerminalTarget(result.terminalStates);
|
|
337
|
+
const r = await this.tracker.fetchCandidateIssues();
|
|
338
|
+
return { issues: r.issues, root: r.root };
|
|
147
339
|
}
|
|
148
340
|
catch (err) {
|
|
149
341
|
log.warn('candidate fetch failed', { error: err.message });
|
|
150
342
|
this.scheduleTick(this.cfg.polling.interval_ms);
|
|
151
|
-
return;
|
|
343
|
+
return null;
|
|
152
344
|
}
|
|
153
|
-
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Reconciler gate (issue 32): refuse to dispatch any issue whose
|
|
348
|
+
* prerequisites haven't converged. When the gate is closed we kick a
|
|
349
|
+
* reconcile pass so the loop self-corrects on the next poll instead of
|
|
350
|
+
* waiting on the slower backstop tick. Returns true when dispatch must
|
|
351
|
+
* be skipped (caller must return).
|
|
352
|
+
*/
|
|
353
|
+
gatedOnReconciler(candidateCount) {
|
|
354
|
+
if (!this.reconciler || this.reconciler.dispatchReady())
|
|
355
|
+
return false;
|
|
356
|
+
log.debug('dispatch gated on reconciler', { candidate_count: candidateCount });
|
|
357
|
+
void this.reconciler.reconcile().catch((err) => log.debug('gated-reconcile failed', { error: err.message }));
|
|
358
|
+
this.scheduleTick(this.cfg.polling.interval_ms);
|
|
359
|
+
return true;
|
|
360
|
+
}
|
|
361
|
+
dispatchSorted(sorted, snapshotTrackerRoot) {
|
|
154
362
|
for (const issue of sorted) {
|
|
155
363
|
if (this.availableGlobalSlots() <= 0)
|
|
156
364
|
break;
|
|
157
365
|
if (!this.isEligible(issue))
|
|
158
366
|
continue;
|
|
159
|
-
void this.dispatchIssue(issue, null, {
|
|
160
|
-
trackerRoot: snapshotTrackerRoot,
|
|
161
|
-
terminalTarget: snapshotTerminalTarget,
|
|
162
|
-
});
|
|
367
|
+
void this.dispatchIssue(issue, null, { trackerRoot: snapshotTrackerRoot });
|
|
163
368
|
}
|
|
164
|
-
this.scheduleTick(this.cfg.polling.interval_ms);
|
|
165
369
|
}
|
|
166
|
-
/**
|
|
370
|
+
/** Stall detection + tracker state refresh for running issues. */
|
|
167
371
|
async reconcile() {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
372
|
+
this.detectStalls();
|
|
373
|
+
await this.refreshTrackerStates();
|
|
374
|
+
}
|
|
375
|
+
detectStalls() {
|
|
376
|
+
if (this.cfg.acp.stall_timeout_ms <= 0)
|
|
377
|
+
return;
|
|
378
|
+
const now = Date.now();
|
|
379
|
+
for (const [issueId, entry] of this.running) {
|
|
380
|
+
// Skip stall detection for issues awaiting human steering: the agent is
|
|
381
|
+
// intentionally paused while the human composes a reply, and the wait can
|
|
382
|
+
// legitimately exceed stall_timeout_ms. The cancel signal still applies
|
|
383
|
+
// (the runner's awaitSteeringReply respects it) for non-stall reasons like
|
|
384
|
+
// terminal-state transitions or operator-initiated cancels.
|
|
385
|
+
if (entry.steering_requested)
|
|
386
|
+
continue;
|
|
387
|
+
const ref = entry.last_event_at ?? entry.started_at;
|
|
388
|
+
const elapsed = now - Date.parse(ref);
|
|
389
|
+
if (Number.isFinite(elapsed) && elapsed > this.cfg.acp.stall_timeout_ms) {
|
|
390
|
+
log.warn('stall detected', {
|
|
391
|
+
issue_id: issueId,
|
|
392
|
+
issue_identifier: entry.identifier,
|
|
393
|
+
elapsed_ms: elapsed,
|
|
394
|
+
});
|
|
395
|
+
this.terminateRunning(issueId, false, `stalled after ${elapsed}ms`);
|
|
189
396
|
}
|
|
190
397
|
}
|
|
191
|
-
|
|
398
|
+
}
|
|
399
|
+
async refreshTrackerStates() {
|
|
192
400
|
const ids = [...this.running.keys()];
|
|
193
401
|
if (ids.length === 0)
|
|
194
402
|
return;
|
|
@@ -201,27 +409,18 @@ export class Orchestrator {
|
|
|
201
409
|
return;
|
|
202
410
|
}
|
|
203
411
|
const byId = new Map(refreshed.map((i) => [i.id, i]));
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const
|
|
214
|
-
if (
|
|
215
|
-
|
|
216
|
-
}
|
|
217
|
-
else if (active.has(s)) {
|
|
218
|
-
const entry = this.running.get(id);
|
|
219
|
-
if (entry)
|
|
220
|
-
entry.issue = fresh;
|
|
221
|
-
}
|
|
222
|
-
else {
|
|
223
|
-
this.terminateRunning(id, false, 'tracker_state_non_active');
|
|
224
|
-
}
|
|
412
|
+
for (const id of ids)
|
|
413
|
+
this.applyReconcileAction(id, byId.get(id));
|
|
414
|
+
}
|
|
415
|
+
applyReconcileAction(id, fresh) {
|
|
416
|
+
const decision = decideReconcileForIssue(fresh, this.cfg.states);
|
|
417
|
+
if (decision.kind === 'terminate') {
|
|
418
|
+
this.terminateRunning(id, decision.cleanup, decision.reason);
|
|
419
|
+
}
|
|
420
|
+
else if (decision.kind === 'refresh' && fresh) {
|
|
421
|
+
const entry = this.running.get(id);
|
|
422
|
+
if (entry)
|
|
423
|
+
entry.issue = fresh;
|
|
225
424
|
}
|
|
226
425
|
}
|
|
227
426
|
terminateRunning(issueId, cleanupWorkspace, reason) {
|
|
@@ -231,6 +430,10 @@ export class Orchestrator {
|
|
|
231
430
|
if (cleanupWorkspace)
|
|
232
431
|
entry.cleanup_workspace_on_exit = true;
|
|
233
432
|
entry.cancel();
|
|
433
|
+
this.runLogs.get(issueId)?.system('reconciliation_terminating', {
|
|
434
|
+
reason,
|
|
435
|
+
cleanup_workspace: cleanupWorkspace,
|
|
436
|
+
});
|
|
234
437
|
log.info('reconciliation terminating run', {
|
|
235
438
|
issue_id: issueId,
|
|
236
439
|
issue_identifier: entry.identifier,
|
|
@@ -238,7 +441,7 @@ export class Orchestrator {
|
|
|
238
441
|
cleanup_workspace: cleanupWorkspace,
|
|
239
442
|
});
|
|
240
443
|
}
|
|
241
|
-
/**
|
|
444
|
+
/** Candidate eligibility. */
|
|
242
445
|
isEligible(issue) {
|
|
243
446
|
return this.eligibilityReason(issue, /*ignoreOwnClaim*/ false) === null;
|
|
244
447
|
}
|
|
@@ -246,40 +449,110 @@ export class Orchestrator {
|
|
|
246
449
|
// form is used by the retry path so the issue's own claim/retry entry does not block
|
|
247
450
|
// its own redispatch.
|
|
248
451
|
eligibilityReason(issue, ignoreOwnClaim) {
|
|
249
|
-
|
|
250
|
-
return 'missing required issue fields';
|
|
251
|
-
}
|
|
252
|
-
const state = issue.state.toLowerCase();
|
|
253
|
-
const active = new Set(this.cfg.tracker.active_states.map((s) => s.toLowerCase()));
|
|
254
|
-
const terminal = new Set(this.cfg.tracker.terminal_states.map((s) => s.toLowerCase()));
|
|
255
|
-
if (!active.has(state) || terminal.has(state))
|
|
256
|
-
return 'state not active';
|
|
257
|
-
if (this.running.has(issue.id))
|
|
258
|
-
return 'already running';
|
|
259
|
-
if (!ignoreOwnClaim && this.claimed.has(issue.id))
|
|
260
|
-
return 'already claimed';
|
|
261
|
-
if (!this.hasPerStateSlot(issue.state))
|
|
262
|
-
return 'no per-state slot';
|
|
263
|
-
if (state === 'todo' && this.hasNonTerminalBlocker(issue))
|
|
264
|
-
return 'has non-terminal blocker';
|
|
265
|
-
return null;
|
|
452
|
+
return computeEligibilityReason(issue, ignoreOwnClaim, this.eligibilitySnapshot());
|
|
266
453
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
275
|
-
return false;
|
|
454
|
+
eligibilitySnapshot() {
|
|
455
|
+
return {
|
|
456
|
+
active: new Set(activeStateNames(this.cfg.states).map((s) => s.toLowerCase())),
|
|
457
|
+
terminal: new Set(terminalStateNames(this.cfg.states).map((s) => s.toLowerCase())),
|
|
458
|
+
running: new Set(this.running.keys()),
|
|
459
|
+
claimed: this.claimed,
|
|
460
|
+
perStateSlot: (state) => this.hasPerStateSlot(state),
|
|
461
|
+
};
|
|
276
462
|
}
|
|
277
463
|
availableGlobalSlots() {
|
|
278
|
-
|
|
464
|
+
// Pending continuations hold their slot. The continuation is the
|
|
465
|
+
// post-transition resume of an issue that just normal-exited (e.g.
|
|
466
|
+
// Todo→Review handoff); without this, a tick firing inside the 1s
|
|
467
|
+
// continuation window can dispatch a brand-new Todo and steal the slot
|
|
468
|
+
// the just-transitioned issue is about to reclaim, leaving it requeued
|
|
469
|
+
// with "no available orchestrator slots" until something else finishes.
|
|
470
|
+
// Failure-backoff retries do NOT hold slots: the orchestrator is free
|
|
471
|
+
// to run other work during the exponential-backoff window.
|
|
472
|
+
let pendingContinuations = 0;
|
|
473
|
+
for (const r of this.retryAttempts.values()) {
|
|
474
|
+
if (r.kind === 'continuation')
|
|
475
|
+
pendingContinuations++;
|
|
476
|
+
}
|
|
477
|
+
const admission = this.computeAdmission();
|
|
478
|
+
// Log a single line when the memory clamp transitions in or out of "active." This
|
|
479
|
+
// gives the operator the "why isn't this dispatching" signal in the log without
|
|
480
|
+
// spamming every tick while memory stays low.
|
|
481
|
+
if (admission.clamp_active !== this.memoryClampActive) {
|
|
482
|
+
this.memoryClampActive = admission.clamp_active;
|
|
483
|
+
if (admission.clamp_active) {
|
|
484
|
+
log.info('memory admission clamping concurrency', {
|
|
485
|
+
static_cap: admission.static_cap,
|
|
486
|
+
effective_cap: admission.effective_cap,
|
|
487
|
+
mem_available_mib: admission.mem_available_mib,
|
|
488
|
+
reserve_mib: admission.reserve_mib,
|
|
489
|
+
per_vm_mib: admission.per_vm_mib,
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
else {
|
|
493
|
+
log.info('memory admission cleared; full static cap available', {
|
|
494
|
+
static_cap: admission.static_cap,
|
|
495
|
+
mem_available_mib: admission.mem_available_mib,
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return Math.max(0, admission.effective_cap - this.running.size - pendingContinuations);
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Compute the current memory-admission snapshot. Reads `/proc/meminfo` via the injected
|
|
503
|
+
* probe (default reads the real file; tests inject a stub). Pure with respect to the
|
|
504
|
+
* orchestrator state — just folds running count + config + probe reading into the
|
|
505
|
+
* dynamic cap. Snapshot endpoint and slot accounting both call through here so they
|
|
506
|
+
* never desync.
|
|
507
|
+
*/
|
|
508
|
+
computeAdmission() {
|
|
509
|
+
const staticCap = this.cfg.agent.max_concurrent_agents;
|
|
510
|
+
const reserveMib = this.cfg.agent.host_memory_reserve_mib;
|
|
511
|
+
const perVmMib = this.cfg.gondolin.mem_mib;
|
|
512
|
+
const enabled = this.cfg.agent.memory_admission_enabled;
|
|
513
|
+
const probe = enabled
|
|
514
|
+
? this.memProbe()
|
|
515
|
+
: { mem_available_mib: null, supported: false };
|
|
516
|
+
const { effective_cap, admission_room, clamp_active } = computeMemoryAdmission({
|
|
517
|
+
enabled,
|
|
518
|
+
static_cap: staticCap,
|
|
519
|
+
running: this.running.size,
|
|
520
|
+
probe,
|
|
521
|
+
reserve_mib: reserveMib,
|
|
522
|
+
per_vm_mib: perVmMib,
|
|
523
|
+
});
|
|
524
|
+
return {
|
|
525
|
+
enabled,
|
|
526
|
+
probe_supported: probe.supported,
|
|
527
|
+
mem_available_mib: probe.mem_available_mib,
|
|
528
|
+
reserve_mib: reserveMib,
|
|
529
|
+
per_vm_mib: perVmMib,
|
|
530
|
+
static_cap: staticCap,
|
|
531
|
+
effective_cap,
|
|
532
|
+
admission_room,
|
|
533
|
+
clamp_active,
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Resolve a state's per-state concurrency cap (`states.<name>.max_concurrent`)
|
|
538
|
+
* with a case-insensitive name lookup, mirroring how issue states arrive from
|
|
539
|
+
* the tracker in arbitrary case. Returns undefined when the state is unknown
|
|
540
|
+
* or declares no cap.
|
|
541
|
+
*/
|
|
542
|
+
perStateConcurrencyCap(stateName) {
|
|
543
|
+
const lower = stateName.toLowerCase();
|
|
544
|
+
for (const [name, sc] of Object.entries(this.cfg.states)) {
|
|
545
|
+
if (name.toLowerCase() === lower)
|
|
546
|
+
return sc.max_concurrent;
|
|
547
|
+
}
|
|
548
|
+
return undefined;
|
|
279
549
|
}
|
|
280
|
-
/**
|
|
550
|
+
/** Per-state slot accounting using current running entries. */
|
|
281
551
|
hasPerStateSlot(stateName) {
|
|
282
|
-
|
|
552
|
+
// Per-state concurrency lives on the state (issue 137): read
|
|
553
|
+
// `states.<name>.max_concurrent` (case-insensitively). Undefined → no
|
|
554
|
+
// per-state cap, so only the global ceiling applies.
|
|
555
|
+
const cap = this.perStateConcurrencyCap(stateName);
|
|
283
556
|
if (!cap)
|
|
284
557
|
return this.availableGlobalSlots() > 0;
|
|
285
558
|
let inState = 0;
|
|
@@ -287,9 +560,17 @@ export class Orchestrator {
|
|
|
287
560
|
if (e.issue.state.toLowerCase() === stateName.toLowerCase())
|
|
288
561
|
inState++;
|
|
289
562
|
}
|
|
563
|
+
// Mirror the global rule for per-state caps: a pending continuation whose
|
|
564
|
+
// target state matches counts against the state's cap, so the resuming
|
|
565
|
+
// worker is guaranteed a slot when its timer fires.
|
|
566
|
+
for (const r of this.retryAttempts.values()) {
|
|
567
|
+
if (r.kind === 'continuation' && r.target_state.toLowerCase() === stateName.toLowerCase()) {
|
|
568
|
+
inState++;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
290
571
|
return inState < cap && this.availableGlobalSlots() > 0;
|
|
291
572
|
}
|
|
292
|
-
/**
|
|
573
|
+
/** Sort: priority ASC (null last), then created_at ASC, then identifier. */
|
|
293
574
|
sortForDispatch(issues) {
|
|
294
575
|
return [...issues].sort((a, b) => {
|
|
295
576
|
const pa = a.priority ?? Number.POSITIVE_INFINITY;
|
|
@@ -303,7 +584,7 @@ export class Orchestrator {
|
|
|
303
584
|
return a.identifier.localeCompare(b.identifier);
|
|
304
585
|
});
|
|
305
586
|
}
|
|
306
|
-
/**
|
|
587
|
+
/** Dispatch one issue. */
|
|
307
588
|
async dispatchIssue(issue, attempt, snapshot) {
|
|
308
589
|
if (this.running.has(issue.id))
|
|
309
590
|
return;
|
|
@@ -312,15 +593,26 @@ export class Orchestrator {
|
|
|
312
593
|
const cancel = { cancelled: false };
|
|
313
594
|
const startedAt = new Date().toISOString();
|
|
314
595
|
const workspacePath = this.workspaces.workspacePathFor(issue.identifier);
|
|
315
|
-
// Snapshot tracker.root
|
|
316
|
-
//
|
|
317
|
-
//
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
//
|
|
321
|
-
//
|
|
596
|
+
// Snapshot tracker.root BEFORE workspace setup or Gondolin VM
|
|
597
|
+
// bring-up. A WORKFLOW.md reload during that window (or even between
|
|
598
|
+
// fetchCandidateIssues returning and this iteration of the dispatch loop)
|
|
599
|
+
// can mutate the live tracker config; pinning here closes that window.
|
|
600
|
+
// When the caller supplies a snapshot (the tick/retry path does — it
|
|
601
|
+
// captured at the fetch atomically), prefer that value; the optional
|
|
602
|
+
// fallback reads the live config for completeness.
|
|
322
603
|
const trackerRootAtDispatch = snapshot?.trackerRoot ?? (this.tracker.currentRoot ? this.tracker.currentRoot() : null);
|
|
323
|
-
|
|
604
|
+
// Pin the effective repo + base branch at dispatch time (same window as
|
|
605
|
+
// tracker.root above) so a mid-run WORKFLOW.md reload can't drift the Done
|
|
606
|
+
// action context away from the values the workspace was set up with.
|
|
607
|
+
const githubRepoAtDispatch = resolveGithubRepo(this.cfg.workspace.github_repo);
|
|
608
|
+
const baseBranchAtDispatch = baseBranchName(this.cfg.workspace.base_branch);
|
|
609
|
+
// Resolve "<adapter>/<model or 'default'>" at dispatch time and pin it on
|
|
610
|
+
// the entry. The MCP transition tool stamps this into the notes-block
|
|
611
|
+
// header the next agent reads in `issue.description`. The helper folds any
|
|
612
|
+
// per-state override on top of the workflow defaults; an unknown state
|
|
613
|
+
// falls back to workflow defaults so an older test harness without a
|
|
614
|
+
// states map still produces a non-null actor string.
|
|
615
|
+
const resolvedActor = resolveActorString(this.cfg.states, this.cfg.acp.adapter, this.cfg.acp.model, issue.state);
|
|
324
616
|
const entry = {
|
|
325
617
|
issue_id: issue.id,
|
|
326
618
|
identifier: issue.identifier,
|
|
@@ -328,13 +620,13 @@ export class Orchestrator {
|
|
|
328
620
|
session_id: null,
|
|
329
621
|
thread_id: null,
|
|
330
622
|
turn_id: null,
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
623
|
+
adapter_pid: null,
|
|
624
|
+
last_event: null,
|
|
625
|
+
last_event_at: null,
|
|
626
|
+
last_message: null,
|
|
627
|
+
input_tokens: 0,
|
|
628
|
+
output_tokens: 0,
|
|
629
|
+
total_tokens: 0,
|
|
338
630
|
last_reported_input_tokens: 0,
|
|
339
631
|
last_reported_output_tokens: 0,
|
|
340
632
|
last_reported_total_tokens: 0,
|
|
@@ -350,25 +642,114 @@ export class Orchestrator {
|
|
|
350
642
|
cleanup_workspace_on_exit: false,
|
|
351
643
|
mcp_token: null,
|
|
352
644
|
tracker_root_at_dispatch: trackerRootAtDispatch,
|
|
353
|
-
|
|
354
|
-
|
|
645
|
+
github_repo_at_dispatch: githubRepoAtDispatch,
|
|
646
|
+
base_branch_at_dispatch: baseBranchAtDispatch,
|
|
647
|
+
resolved_actor: resolvedActor,
|
|
648
|
+
transitioned: false,
|
|
355
649
|
steering_requested: false,
|
|
356
650
|
steering_question: null,
|
|
357
651
|
steering_context: null,
|
|
652
|
+
last_transition: null,
|
|
358
653
|
};
|
|
359
654
|
this.running.set(issue.id, entry);
|
|
360
655
|
const logger = withIssue({ issue_id: issue.id, issue_identifier: issue.identifier });
|
|
656
|
+
const runLog = this.ensureRunLog(issue.id, issue.identifier);
|
|
657
|
+
if (runLog) {
|
|
658
|
+
runLog.setAttempt(attempt ?? 0);
|
|
659
|
+
runLog.system('attempt_started', {
|
|
660
|
+
attempt: attempt ?? 0,
|
|
661
|
+
issue_state: issue.state,
|
|
662
|
+
issue_title: issue.title,
|
|
663
|
+
workspace_path: workspacePath,
|
|
664
|
+
tracker_root: trackerRootAtDispatch,
|
|
665
|
+
// Pin the per-state turn budget so the run-summary reducer can report
|
|
666
|
+
// turns-used-vs-budget without re-resolving config (issue 123).
|
|
667
|
+
max_turns: this.resolveStateMaxTurns(issue.state),
|
|
668
|
+
});
|
|
669
|
+
}
|
|
361
670
|
logger.info('agent attempt started', { attempt });
|
|
362
|
-
void this.runWorker(issue, attempt, entry, cancel);
|
|
671
|
+
void this.runWorker(issue, attempt, entry, cancel, runLog);
|
|
363
672
|
}
|
|
364
|
-
|
|
673
|
+
/**
|
|
674
|
+
* Open (or return the existing) per-issue run log. Returns `undefined` only when log file
|
|
675
|
+
* opening throws — symphony should keep running even if logs can't be persisted, so the
|
|
676
|
+
* runner sees `undefined` and behaves exactly as before.
|
|
677
|
+
*
|
|
678
|
+
* `issueId` (tracker primary key) is stamped on every line and is the map key so the
|
|
679
|
+
* lifecycle survives identifier collisions or renames; `identifier` derives the filename.
|
|
680
|
+
*/
|
|
681
|
+
ensureRunLog(issueId, identifier) {
|
|
682
|
+
const existing = this.runLogs.get(issueId);
|
|
683
|
+
if (existing)
|
|
684
|
+
return existing;
|
|
685
|
+
try {
|
|
686
|
+
const rl = openRunLog(this.cfg.logs.root, issueId, identifier);
|
|
687
|
+
this.runLogs.set(issueId, rl);
|
|
688
|
+
return rl;
|
|
689
|
+
}
|
|
690
|
+
catch (err) {
|
|
691
|
+
log.warn('runlog open failed; continuing without run log', {
|
|
692
|
+
issue_id: issueId,
|
|
693
|
+
issue_identifier: identifier,
|
|
694
|
+
error: err.message,
|
|
695
|
+
});
|
|
696
|
+
return undefined;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Resolve the per-state turn budget for the run log's `attempt_started`
|
|
701
|
+
* event. Returns null on any resolution failure (unknown state) — the
|
|
702
|
+
* summary reducer treats a null budget as "unknown", never an error.
|
|
703
|
+
*/
|
|
704
|
+
resolveStateMaxTurns(state) {
|
|
705
|
+
try {
|
|
706
|
+
return resolveDispatchConfig(this.cfg, state).max_turns;
|
|
707
|
+
}
|
|
708
|
+
catch {
|
|
709
|
+
return null;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Record the end-of-attempt lifecycle events: the `transition` the agent (or
|
|
714
|
+
* an action reroute) performed during this attempt, if any, followed by
|
|
715
|
+
* `attempt_ended`. Both feed the run-summary reducer (issue 123); recording
|
|
716
|
+
* the transition here — once per attempt, off the hot path — is what makes
|
|
717
|
+
* the state path, rejection notes, and terminal outcome reconstructable.
|
|
718
|
+
*/
|
|
719
|
+
recordAttemptEnd(runLog, entry, ok, reason, turnsCompleted) {
|
|
720
|
+
if (entry.last_transition)
|
|
721
|
+
runLog?.system('transition', { ...entry.last_transition });
|
|
722
|
+
runLog?.system('attempt_ended', { ok, reason, turns_completed: turnsCompleted });
|
|
723
|
+
}
|
|
724
|
+
closeRunLog(issueId, fields, opts = {}) {
|
|
725
|
+
// If a terminal cleanup (`workspaces.remove`) is mid-flight for this issue, the
|
|
726
|
+
// terminal-state actions capture may still be writing to the run log. Closing the
|
|
727
|
+
// log here would truncate those lines on disk. Defer until the cleanup's .finally
|
|
728
|
+
// fires the close with `viaCleanup: true`.
|
|
729
|
+
if (!opts.viaCleanup && this.cleanupInFlight.has(issueId))
|
|
730
|
+
return;
|
|
731
|
+
const rl = this.runLogs.get(issueId);
|
|
732
|
+
if (!rl)
|
|
733
|
+
return;
|
|
734
|
+
if (fields)
|
|
735
|
+
rl.system('runlog_closed', fields);
|
|
736
|
+
// Emit the compact per-issue run summary (issue 123) at the terminal unwind,
|
|
737
|
+
// when the lifecycle accumulator holds the full trajectory. Pure over
|
|
738
|
+
// in-memory state, so it precedes (and does not depend on) the stream flush.
|
|
739
|
+
rl.writeSummary();
|
|
740
|
+
this.runLogs.delete(issueId);
|
|
741
|
+
void rl.close();
|
|
742
|
+
}
|
|
743
|
+
async runWorker(issue, attempt, entry, cancelSignal, runLog) {
|
|
365
744
|
const logger = withIssue({ issue_id: issue.id, issue_identifier: issue.identifier });
|
|
366
745
|
let ok = false;
|
|
367
746
|
let reason = 'unknown';
|
|
747
|
+
let turnsCompleted = 0;
|
|
368
748
|
try {
|
|
369
|
-
const result = await this.runner.runAttempt(issue, attempt, cancelSignal, entry);
|
|
749
|
+
const result = await this.runner.runAttempt(issue, attempt, cancelSignal, entry, runLog);
|
|
370
750
|
ok = result.ok;
|
|
371
751
|
reason = result.reason;
|
|
752
|
+
turnsCompleted = result.turnsCompleted;
|
|
372
753
|
if (result.threadId)
|
|
373
754
|
entry.thread_id = result.threadId;
|
|
374
755
|
entry.turn_count = result.turnsCompleted;
|
|
@@ -378,54 +759,272 @@ export class Orchestrator {
|
|
|
378
759
|
reason = err.message;
|
|
379
760
|
logger.error('worker threw', { error: reason });
|
|
380
761
|
}
|
|
762
|
+
this.recordAttemptEnd(runLog, entry, ok, reason, turnsCompleted);
|
|
381
763
|
this.onWorkerExit(issue.id, ok, reason, entry);
|
|
382
764
|
}
|
|
383
|
-
/**
|
|
765
|
+
/** on_worker_exit */
|
|
384
766
|
onWorkerExit(issueId, normal, reason, entry) {
|
|
385
767
|
this.running.delete(issueId);
|
|
768
|
+
// Issue 52: the reconciler `vm` resource is the sole owner of VM teardown.
|
|
769
|
+
// Every worker exit — clean or not — kicks the reaper so the intended set
|
|
770
|
+
// (now excluding this issue) converges in a single pass.
|
|
771
|
+
if (this.reconciler && !this.stopped) {
|
|
772
|
+
void this.reconciler.reapVms().catch((err) => log.debug('post-exit vm reap failed', { error: err.message }));
|
|
773
|
+
}
|
|
386
774
|
const elapsedMs = Date.now() - Date.parse(entry.started_at);
|
|
387
|
-
if (Number.isFinite(elapsedMs))
|
|
388
|
-
this.
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
// Workspace removal is deferred until the worker has fully unwound (including
|
|
394
|
-
// after_run hook execution) so we never delete the dir while the agent is still
|
|
395
|
-
// inside it.
|
|
396
|
-
this.workspaces
|
|
397
|
-
.remove(entry.identifier, this.cfg.hooks)
|
|
398
|
-
.catch((err) => logger.warn('workspace removal failed', { error: err.message }));
|
|
399
|
-
}
|
|
400
|
-
// §14.2: if the service was stopped while this worker was unwinding, do not schedule
|
|
775
|
+
if (Number.isFinite(elapsedMs))
|
|
776
|
+
this.sessionTotals.seconds_running += elapsedMs / 1000;
|
|
777
|
+
const logger = withIssue({ issue_id: issueId, issue_identifier: entry.identifier });
|
|
778
|
+
if (entry.cleanup_workspace_on_exit)
|
|
779
|
+
this.scheduleWorkspaceCleanup(issueId, entry, logger);
|
|
780
|
+
// If the service was stopped while this worker was unwinding, do not schedule
|
|
401
781
|
// a new retry — that would leave a live timer behind even though stop() was called.
|
|
402
782
|
if (this.stopped) {
|
|
403
783
|
this.claimed.delete(issueId);
|
|
784
|
+
this.circuitBreakers.delete(issueId);
|
|
404
785
|
return;
|
|
405
786
|
}
|
|
406
|
-
if (normal)
|
|
787
|
+
if (normal)
|
|
407
788
|
this.completed.add(issueId);
|
|
789
|
+
this.recordSleepCycleProgress(entry);
|
|
790
|
+
// Circuit breaker (issue 128): a deterministically-failing dispatch (same
|
|
791
|
+
// reason every attempt) would otherwise retry forever under backoff. Trip
|
|
792
|
+
// after the configured streak and route the issue to a holding state
|
|
793
|
+
// instead of scheduling another retry.
|
|
794
|
+
if (this.updateCircuitBreaker(issueId, normal, reason, entry))
|
|
795
|
+
return;
|
|
796
|
+
const plan = decideExitRetry({
|
|
797
|
+
normal,
|
|
798
|
+
reason,
|
|
799
|
+
priorAttempt: entry.retry_attempt,
|
|
800
|
+
targetState: entry.issue.state,
|
|
801
|
+
continuationDelayMs: CONTINUATION_DELAY_MS,
|
|
802
|
+
failureBaseMs: FAILURE_BASE_MS,
|
|
803
|
+
maxBackoffMs: this.cfg.agent.max_retry_backoff_ms,
|
|
804
|
+
});
|
|
805
|
+
if (normal) {
|
|
408
806
|
logger.info('worker exited (normal)', { reason });
|
|
409
|
-
this.scheduleRetry(issueId, {
|
|
410
|
-
identifier,
|
|
411
|
-
attempt: 1,
|
|
412
|
-
delayMs: CONTINUATION_DELAY_MS,
|
|
413
|
-
error: null,
|
|
414
|
-
});
|
|
415
807
|
}
|
|
416
808
|
else {
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
809
|
+
logger.warn('worker exited (abnormal)', {
|
|
810
|
+
reason,
|
|
811
|
+
next_attempt: plan.attempt,
|
|
812
|
+
delay_ms: plan.delayMs,
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
this.scheduleRetry(issueId, { identifier: entry.identifier, ...plan });
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Fold this exit into the per-issue circuit-breaker streak (issue 128) and
|
|
819
|
+
* return true when the breaker tripped — the caller must then NOT schedule a
|
|
820
|
+
* retry. A clean exit clears the streak; an abnormal exit either records the
|
|
821
|
+
* (normalized) failure or, on reaching `agent.circuit_breaker_threshold`
|
|
822
|
+
* consecutive identical failures, trips and fires the holding-state route.
|
|
823
|
+
* The pure `decideCircuitBreaker` owns the counting; this shell just persists
|
|
824
|
+
* the streak and dispatches the side effect.
|
|
825
|
+
*/
|
|
826
|
+
updateCircuitBreaker(issueId, normal, reason, entry) {
|
|
827
|
+
const decision = decideCircuitBreaker({
|
|
828
|
+
normal,
|
|
829
|
+
reason,
|
|
830
|
+
prior: this.circuitBreakers.get(issueId) ?? null,
|
|
831
|
+
threshold: this.cfg.agent.circuit_breaker_threshold,
|
|
832
|
+
});
|
|
833
|
+
if (decision.kind === 'continue') {
|
|
834
|
+
this.circuitBreakers.set(issueId, {
|
|
835
|
+
normalizedReason: decision.normalizedReason,
|
|
836
|
+
count: decision.count,
|
|
837
|
+
});
|
|
838
|
+
return false;
|
|
839
|
+
}
|
|
840
|
+
this.circuitBreakers.delete(issueId);
|
|
841
|
+
if (decision.kind === 'trip') {
|
|
842
|
+
void this.tripCircuitBreaker(issueId, entry, reason, decision.count);
|
|
843
|
+
return true;
|
|
844
|
+
}
|
|
845
|
+
return false;
|
|
846
|
+
}
|
|
847
|
+
/**
|
|
848
|
+
* Stop retrying a circuit-broken issue and move it into a holding state so a
|
|
849
|
+
* human sees "stuck on identical failure" on the dashboard rather than a
|
|
850
|
+
* silent multi-hour loop. The move is restart-safe (the file leaves the
|
|
851
|
+
* active set on disk, so the loop cannot resume on the next process start).
|
|
852
|
+
* If routing fails — no `holding` state declared, or the tracker can't write
|
|
853
|
+
* — we keep the issue's dispatch claim so the tick's `already claimed` gate
|
|
854
|
+
* still halts the loop for this session, and log loudly.
|
|
855
|
+
*/
|
|
856
|
+
async tripCircuitBreaker(issueId, entry, reason, count) {
|
|
857
|
+
const logger = withIssue({ issue_id: issueId, issue_identifier: entry.identifier });
|
|
858
|
+
this.runLogs.get(issueId)?.system('circuit_breaker_tripped', {
|
|
859
|
+
reason,
|
|
860
|
+
consecutive_failures: count,
|
|
861
|
+
});
|
|
862
|
+
logger.error('circuit breaker tripped; halting retries', { reason, consecutive_failures: count });
|
|
863
|
+
let holdingState;
|
|
864
|
+
try {
|
|
865
|
+
holdingState = pickHoldingState(this.cfg.states);
|
|
866
|
+
}
|
|
867
|
+
catch {
|
|
868
|
+
logger.error('no holding state declared; retaining claim to halt the retry loop', { reason });
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
871
|
+
const moved = await this.routeToHolding(issueId, entry, holdingState, reason, count);
|
|
872
|
+
if (moved) {
|
|
873
|
+
this.claimed.delete(issueId);
|
|
874
|
+
this.closeRunLog(issueId, { reason: 'circuit_breaker_tripped' });
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Move `entry`'s tracker file into `holdingState`, appending a diagnostic
|
|
879
|
+
* note (rendered into the issue body before the rename) so the operator sees
|
|
880
|
+
* why it stopped. Returns false when the tracker can't perform the move so
|
|
881
|
+
* the caller can fall back to retaining the claim. Modelled on the runner's
|
|
882
|
+
* action-reroute path; the orchestrator owns this move because it is
|
|
883
|
+
* state-machine behavior, not repo-local glue.
|
|
884
|
+
*/
|
|
885
|
+
async routeToHolding(issueId, entry, holdingState, reason, count) {
|
|
886
|
+
if (!this.tracker.moveIssueToState)
|
|
887
|
+
return false;
|
|
888
|
+
const notes = [
|
|
889
|
+
`**Circuit breaker tripped** — routed to \`${holdingState}\` for human inspection.`,
|
|
890
|
+
'',
|
|
891
|
+
`Symphony stopped retrying after **${count} consecutive attempts failed with the same error**, to avoid an unbounded dispatch loop (issue 128).`,
|
|
892
|
+
'',
|
|
893
|
+
`**Last failure reason:** ${reason}`,
|
|
894
|
+
'',
|
|
895
|
+
`Resolve the underlying cause, then move the issue back into an active state to resume dispatch.`,
|
|
896
|
+
].join('\n');
|
|
897
|
+
try {
|
|
898
|
+
await this.tracker.moveIssueToState(issueId, holdingState, {
|
|
899
|
+
fromRoot: entry.tracker_root_at_dispatch ?? undefined,
|
|
900
|
+
fromState: entry.issue.state,
|
|
901
|
+
notes,
|
|
902
|
+
actor: entry.resolved_actor,
|
|
425
903
|
});
|
|
904
|
+
return true;
|
|
905
|
+
}
|
|
906
|
+
catch (err) {
|
|
907
|
+
withIssue({ issue_id: issueId, issue_identifier: entry.identifier }).error('circuit breaker route to holding failed; retaining claim', { error: err.message });
|
|
908
|
+
return false;
|
|
426
909
|
}
|
|
427
910
|
}
|
|
428
|
-
/**
|
|
911
|
+
/**
|
|
912
|
+
* Sleep-cycle auto-arm (issue 125): fold a finished attempt into the
|
|
913
|
+
* terminal-transition counter the triggers read. Only the work the reflector
|
|
914
|
+
* mines counts — a transition into a `role: terminal` state (Done/Cancelled)
|
|
915
|
+
* — and the reflection issue's own moves (it can only go to its holding
|
|
916
|
+
* dormant state) are excluded so a reflection run never counts toward arming
|
|
917
|
+
* the next one. No-op when the block is disabled.
|
|
918
|
+
*/
|
|
919
|
+
recordSleepCycleProgress(entry) {
|
|
920
|
+
const arm = deriveArmRouting(this.cfg.states);
|
|
921
|
+
if (!arm.issue)
|
|
922
|
+
return;
|
|
923
|
+
if (entry.issue_id === arm.issue || entry.identifier === arm.issue)
|
|
924
|
+
return;
|
|
925
|
+
if (entry.last_transition?.terminal)
|
|
926
|
+
this.doneSinceReflect += 1;
|
|
927
|
+
}
|
|
928
|
+
/**
|
|
929
|
+
* Sleep-cycle auto-arm (issue 125): decide — purely — whether to arm the
|
|
930
|
+
* reflection cycle this tick, and fire the async Dormant → Reflect move when
|
|
931
|
+
* a trigger fires. `noActiveCandidates` is true when this poll surfaced no
|
|
932
|
+
* active-state issues; combined with empty running/claimed/retry sets that is
|
|
933
|
+
* the orchestrator-idle signal. The in-flight guard prevents two overlapping
|
|
934
|
+
* ticks from both launching the move.
|
|
935
|
+
*/
|
|
936
|
+
maybeArmSleepCycle(noActiveCandidates) {
|
|
937
|
+
const arm = deriveArmRouting(this.cfg.states);
|
|
938
|
+
if (!arm.armState || this.armingReflection)
|
|
939
|
+
return;
|
|
940
|
+
const idle = noActiveCandidates &&
|
|
941
|
+
this.running.size === 0 &&
|
|
942
|
+
this.claimed.size === 0 &&
|
|
943
|
+
this.retryAttempts.size === 0;
|
|
944
|
+
const trigger = decideSleepCycleArm({
|
|
945
|
+
enabled: true,
|
|
946
|
+
issueId: arm.issue,
|
|
947
|
+
armOnIdle: arm.onIdle,
|
|
948
|
+
armAfterDone: arm.afterTerminal,
|
|
949
|
+
doneSinceReflect: this.doneSinceReflect,
|
|
950
|
+
idle,
|
|
951
|
+
});
|
|
952
|
+
if (!trigger)
|
|
953
|
+
return;
|
|
954
|
+
this.armingReflection = true;
|
|
955
|
+
void this.armReflection(arm, trigger).finally(() => {
|
|
956
|
+
this.armingReflection = false;
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
/**
|
|
960
|
+
* Move the reflection issue from its dormant (holding) state into the active
|
|
961
|
+
* reflect state. Guarded so it is a no-op unless the issue currently lives in
|
|
962
|
+
* `dormant_state` (so we never yank it out of Reflect mid-run or relocate it
|
|
963
|
+
* from some other state), and the counter only resets on a successful move so
|
|
964
|
+
* a failed arm doesn't silently discard the accumulated terminal count.
|
|
965
|
+
*/
|
|
966
|
+
async armReflection(arm, trigger) {
|
|
967
|
+
if (!this.tracker.moveIssueToState || !arm.issue || !arm.from || !arm.armState)
|
|
968
|
+
return;
|
|
969
|
+
const current = await this.fetchReflectionIssue(arm.issue);
|
|
970
|
+
if (!current)
|
|
971
|
+
return;
|
|
972
|
+
if (current.state.toLowerCase() !== arm.from.toLowerCase())
|
|
973
|
+
return;
|
|
974
|
+
if (this.running.has(current.id) || this.claimed.has(current.id))
|
|
975
|
+
return;
|
|
976
|
+
const count = this.doneSinceReflect;
|
|
977
|
+
try {
|
|
978
|
+
await this.tracker.moveIssueToState(current.id, arm.armState, {
|
|
979
|
+
fromState: arm.from,
|
|
980
|
+
notes: sleepCycleArmNotes(trigger, count, arm.afterTerminal),
|
|
981
|
+
actor: SLEEP_CYCLE_ACTOR,
|
|
982
|
+
});
|
|
983
|
+
this.doneSinceReflect = 0;
|
|
984
|
+
log.info('sleep cycle armed reflection', {
|
|
985
|
+
issue_id: current.id,
|
|
986
|
+
trigger,
|
|
987
|
+
done_since_reflect: count,
|
|
988
|
+
reflect_state: arm.armState,
|
|
989
|
+
});
|
|
990
|
+
}
|
|
991
|
+
catch (err) {
|
|
992
|
+
log.warn('sleep cycle arm failed', {
|
|
993
|
+
issue_id: current.id,
|
|
994
|
+
error: err.message,
|
|
995
|
+
});
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
/** Look up the reflection issue's current tracker state; undefined on miss/error. */
|
|
999
|
+
async fetchReflectionIssue(issueId) {
|
|
1000
|
+
try {
|
|
1001
|
+
const found = await this.tracker.fetchIssueStatesByIds([issueId]);
|
|
1002
|
+
return found.find((i) => i.id === issueId || i.identifier === issueId);
|
|
1003
|
+
}
|
|
1004
|
+
catch (err) {
|
|
1005
|
+
log.debug('sleep cycle reflection-issue lookup failed', {
|
|
1006
|
+
issue_id: issueId,
|
|
1007
|
+
error: err.message,
|
|
1008
|
+
});
|
|
1009
|
+
return undefined;
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
/**
|
|
1013
|
+
* Workspace removal deferred until the worker has fully unwound (including the
|
|
1014
|
+
* terminal-state `actions:` block) so we never delete the dir while the agent
|
|
1015
|
+
* is still inside it.
|
|
1016
|
+
*/
|
|
1017
|
+
scheduleWorkspaceCleanup(issueId, entry, logger) {
|
|
1018
|
+
this.cleanupInFlight.add(issueId);
|
|
1019
|
+
this.workspaces
|
|
1020
|
+
.remove(entry.identifier)
|
|
1021
|
+
.catch((err) => logger.warn('workspace removal failed', { error: err.message }))
|
|
1022
|
+
.finally(() => {
|
|
1023
|
+
this.cleanupInFlight.delete(issueId);
|
|
1024
|
+
this.closeRunLog(issueId, { reason: 'cleanup_on_exit' }, { viaCleanup: true });
|
|
1025
|
+
});
|
|
1026
|
+
}
|
|
1027
|
+
/** Retry queue. */
|
|
429
1028
|
scheduleRetry(issueId, sched) {
|
|
430
1029
|
if (this.stopped)
|
|
431
1030
|
return;
|
|
@@ -441,10 +1040,12 @@ export class Orchestrator {
|
|
|
441
1040
|
due_at_ms: dueAt,
|
|
442
1041
|
timer_handle: handle,
|
|
443
1042
|
error: sched.error,
|
|
1043
|
+
kind: sched.kind,
|
|
1044
|
+
target_state: sched.target_state,
|
|
444
1045
|
});
|
|
445
1046
|
this.claimed.add(issueId);
|
|
446
1047
|
}
|
|
447
|
-
/**
|
|
1048
|
+
/** on_retry_timer */
|
|
448
1049
|
async onRetryTimer(issueId) {
|
|
449
1050
|
if (this.stopped)
|
|
450
1051
|
return;
|
|
@@ -452,14 +1053,30 @@ export class Orchestrator {
|
|
|
452
1053
|
if (!entry)
|
|
453
1054
|
return;
|
|
454
1055
|
this.retryAttempts.delete(issueId);
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
1056
|
+
const fetched = await this.fetchRetryCandidates(issueId, entry);
|
|
1057
|
+
if (!fetched)
|
|
1058
|
+
return;
|
|
1059
|
+
const issue = fetched.issues.find((i) => i.id === issueId);
|
|
1060
|
+
if (!issue) {
|
|
1061
|
+
this.releaseRetryClaim(issueId, entry.identifier, 'not_in_candidates');
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
1064
|
+
const reason = this.eligibilityReason(issue, true);
|
|
1065
|
+
if (reason !== null) {
|
|
1066
|
+
this.handleRetryIneligible(issue, entry, reason);
|
|
1067
|
+
return;
|
|
1068
|
+
}
|
|
1069
|
+
void this.dispatchIssue(issue, entry.attempt, { trackerRoot: fetched.root });
|
|
1070
|
+
}
|
|
1071
|
+
/**
|
|
1072
|
+
* Tracker poll for the retry timer. Returns the snapshot or null when the
|
|
1073
|
+
* fetch failed (a failure-shaped retry is rescheduled internally so the
|
|
1074
|
+
* caller can just bail).
|
|
1075
|
+
*/
|
|
1076
|
+
async fetchRetryCandidates(issueId, entry) {
|
|
458
1077
|
try {
|
|
459
|
-
const
|
|
460
|
-
|
|
461
|
-
snapshotTrackerRoot = result.root;
|
|
462
|
-
snapshotTerminalTarget = pickTerminalTarget(result.terminalStates);
|
|
1078
|
+
const r = await this.tracker.fetchCandidateIssues();
|
|
1079
|
+
return { issues: r.issues, root: r.root };
|
|
463
1080
|
}
|
|
464
1081
|
catch (err) {
|
|
465
1082
|
log.debug('retry poll failed', {
|
|
@@ -472,96 +1089,355 @@ export class Orchestrator {
|
|
|
472
1089
|
attempt: entry.attempt + 1,
|
|
473
1090
|
delayMs: Math.min(FAILURE_BASE_MS * Math.pow(2, entry.attempt), this.cfg.agent.max_retry_backoff_ms),
|
|
474
1091
|
error: 'retry poll failed',
|
|
1092
|
+
kind: 'failure',
|
|
1093
|
+
target_state: entry.target_state,
|
|
475
1094
|
});
|
|
476
|
-
return;
|
|
1095
|
+
return null;
|
|
477
1096
|
}
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Re-applied candidate eligibility came back non-null. The pure
|
|
1100
|
+
* `decideRetryAfterIneligible` picks between rescheduling (only `no
|
|
1101
|
+
* per-state slot`, which is genuine contention) and releasing the claim
|
|
1102
|
+
* (everything else: blocker, missing fields, non-active state).
|
|
1103
|
+
*/
|
|
1104
|
+
handleRetryIneligible(issue, entry, reason) {
|
|
1105
|
+
const action = decideRetryAfterIneligible({
|
|
1106
|
+
reason,
|
|
1107
|
+
priorAttempt: entry.attempt,
|
|
1108
|
+
targetState: issue.state,
|
|
1109
|
+
failureBaseMs: FAILURE_BASE_MS,
|
|
1110
|
+
maxBackoffMs: this.cfg.agent.max_retry_backoff_ms,
|
|
1111
|
+
});
|
|
1112
|
+
if (action.kind === 'release') {
|
|
1113
|
+
this.releaseRetryClaim(issue.id, entry.identifier, `ineligible:${reason}`, reason);
|
|
485
1114
|
return;
|
|
486
1115
|
}
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
1116
|
+
this.scheduleRetry(issue.id, { identifier: issue.identifier, ...action.plan });
|
|
1117
|
+
}
|
|
1118
|
+
releaseRetryClaim(issueId, identifier, closeTag, ineligibleReason) {
|
|
1119
|
+
this.claimed.delete(issueId);
|
|
1120
|
+
this.circuitBreakers.delete(issueId);
|
|
1121
|
+
log.info(ineligibleReason
|
|
1122
|
+
? 'retry releasing claim (ineligible)'
|
|
1123
|
+
: 'retry releasing claim (not in candidates)', { issue_id: issueId, issue_identifier: identifier, ...(ineligibleReason ? { reason: ineligibleReason } : {}) });
|
|
1124
|
+
this.closeRunLog(issueId, { reason: `claim_released_${closeTag}` });
|
|
1125
|
+
}
|
|
1126
|
+
/**
|
|
1127
|
+
* Implements {@link IntendedVmProvider}. Returns the set of `symphony-*` VM
|
|
1128
|
+
* names the orchestrator currently intends to keep alive — one per running
|
|
1129
|
+
* dispatch. Used by the reconciler's vm resource to compute the orphan set
|
|
1130
|
+
* to reap. `running.set` happens BEFORE the runner calls `createVm`, so a VM
|
|
1131
|
+
* that exists in Gondolin's session registry as part of an in-flight create
|
|
1132
|
+
* is always already represented
|
|
1133
|
+
* here. The reaper sees it as intended and leaves it alone, closing the
|
|
1134
|
+
* "creating-but-not-yet-active" race the issue body calls out.
|
|
1135
|
+
*/
|
|
1136
|
+
intendedVmNames() {
|
|
1137
|
+
const out = new Set();
|
|
1138
|
+
for (const entry of this.running.values()) {
|
|
1139
|
+
out.add(this.runner.vmNameFor(entry.issue));
|
|
1140
|
+
}
|
|
1141
|
+
return out;
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Implements {@link WorkspaceIntendedProvider}. Returns the map of
|
|
1145
|
+
* identifier → state the reconciler should preserve workspaces for. Two
|
|
1146
|
+
* sources are unioned:
|
|
1147
|
+
*
|
|
1148
|
+
* • Tracker view: every issue file in a non-terminal state. Anything
|
|
1149
|
+
* terminal (Done, Cancelled) is fair game for removal — this replaces
|
|
1150
|
+
* the old `startupTerminalCleanup` sweep with a continuous pass.
|
|
1151
|
+
* • In-flight allocations: running entries plus claimed/pending retries.
|
|
1152
|
+
* The window between dispatch claiming an issue and the tracker
|
|
1153
|
+
* reflecting it is brief but real; without this, a fresh dispatch's
|
|
1154
|
+
* workspace could be reaped seconds after creation.
|
|
1155
|
+
*
|
|
1156
|
+
* The state value is carried so the reconciler's `create` callback can apply
|
|
1157
|
+
* the merge-state guard (it skips eager recreation of a workspace whose issue
|
|
1158
|
+
* sits in the autopilot's merge state).
|
|
1159
|
+
*
|
|
1160
|
+
* Tracker errors propagate. Catching them here would cause an empty set
|
|
1161
|
+
* to be returned, which the reconciler would treat as authoritative and
|
|
1162
|
+
* reap every workspace — the regression this contract closes. The
|
|
1163
|
+
* resource's `reconcile()` catches the throw and leaves on-disk state
|
|
1164
|
+
* untouched until the next pass.
|
|
1165
|
+
*
|
|
1166
|
+
* Mirrors `intendedVmNames()` in shape so the reconciler's race-condition
|
|
1167
|
+
* reasoning is the same across both janitors.
|
|
1168
|
+
*/
|
|
1169
|
+
async activeIdentifiers() {
|
|
1170
|
+
const out = new Map();
|
|
1171
|
+
const nonTerminal = [];
|
|
1172
|
+
for (const [name, cfg] of Object.entries(this.cfg.states)) {
|
|
1173
|
+
if (cfg.role !== 'terminal')
|
|
1174
|
+
nonTerminal.push(name);
|
|
1175
|
+
}
|
|
1176
|
+
const issues = await this.tracker.fetchIssuesByStates(nonTerminal);
|
|
1177
|
+
for (const i of issues)
|
|
1178
|
+
out.set(i.identifier, i.state);
|
|
1179
|
+
// Issue 38/139: when the PR engine is enabled, the merge-state issues'
|
|
1180
|
+
// workspaces are owned by the pr resource (it rebases inside them and cleans
|
|
1181
|
+
// them up post-merge). Include those identifiers in the desired set so the
|
|
1182
|
+
// workspace janitor doesn't reap a workspace the pr resource is actively
|
|
1183
|
+
// driving. The merge state is derived by scanning states for `pr.auto_merge`
|
|
1184
|
+
// (no named-string sibling block). The orchestrator's `createWorkspace`
|
|
1185
|
+
// callback declines to eagerly recreate a missing merge-state workspace — so
|
|
1186
|
+
// adding it here is safe: the workspace either already exists from the
|
|
1187
|
+
// dispatch that ran the issue into the merge state, or it doesn't and the
|
|
1188
|
+
// autopilot just skips the rebase step for that PR.
|
|
1189
|
+
if (this.cfg.pr.enabled) {
|
|
1190
|
+
const mergeState = derivePrRouting(this.cfg.states).mergeState;
|
|
1191
|
+
if (mergeState) {
|
|
1192
|
+
const mergeIssues = await this.tracker.fetchIssuesByStates([mergeState]);
|
|
1193
|
+
for (const i of mergeIssues)
|
|
1194
|
+
out.set(i.identifier, i.state);
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
return out;
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Identifiers the orchestrator has claimed for dispatch but the tracker may
|
|
1201
|
+
* not yet reflect as active, with the state the eager workspace create should
|
|
1202
|
+
* see (used for the merge-state guard). Running entries carry the state the
|
|
1203
|
+
* dispatch was claimed from (`issue.state`); pending retries carry their
|
|
1204
|
+
* `target_state` (where the next attempt will run).
|
|
1205
|
+
*/
|
|
1206
|
+
inFlightIdentifiers() {
|
|
1207
|
+
const out = new Map();
|
|
1208
|
+
for (const e of this.running.values())
|
|
1209
|
+
out.set(e.identifier, e.issue.state);
|
|
1210
|
+
for (const r of this.retryAttempts.values())
|
|
1211
|
+
out.set(r.identifier, r.target_state);
|
|
1212
|
+
return out;
|
|
1213
|
+
}
|
|
1214
|
+
/**
|
|
1215
|
+
* Implements {@link ProposeFollowupSink} for the action executor's
|
|
1216
|
+
* `propose_followup` action (issue 36). Same tracker shape as the MCP
|
|
1217
|
+
* `propose_issue` tool — file lands in the first declared `holding` state,
|
|
1218
|
+
* with `proposed_by` set to the parent issue's identifier. Uses the live
|
|
1219
|
+
* tracker root (passed-in parent identifier is the canonical attribution).
|
|
1220
|
+
*/
|
|
1221
|
+
async proposeFollowup(input) {
|
|
1222
|
+
const root = this.tracker.currentRoot ? this.tracker.currentRoot() : this.cfg.tracker.root;
|
|
1223
|
+
if (!root) {
|
|
1224
|
+
throw new Error('tracker root not available; cannot file propose_followup');
|
|
1225
|
+
}
|
|
1226
|
+
const landingState = pickHoldingState(this.cfg.states);
|
|
1227
|
+
const result = await writeIssueFile({
|
|
1228
|
+
trackerRoot: root,
|
|
1229
|
+
state: landingState,
|
|
1230
|
+
title: input.title,
|
|
1231
|
+
description: input.description ?? '',
|
|
1232
|
+
priority: input.priority ?? null,
|
|
1233
|
+
labels: input.labels ?? [],
|
|
1234
|
+
now: () => Date.now(),
|
|
1235
|
+
extra_front_matter: {
|
|
1236
|
+
proposed_by: input.parent_identifier,
|
|
1237
|
+
proposed_at: new Date().toISOString(),
|
|
1238
|
+
},
|
|
1239
|
+
});
|
|
1240
|
+
log.info('action propose_followup', {
|
|
1241
|
+
proposed_by: input.parent_identifier,
|
|
1242
|
+
identifier: result.identifier,
|
|
1243
|
+
state: result.state,
|
|
1244
|
+
});
|
|
1245
|
+
return { identifier: result.identifier };
|
|
1246
|
+
}
|
|
1247
|
+
/**
|
|
1248
|
+
* Receive a per-attempt action ledger from the runner's cleanup pass. The
|
|
1249
|
+
* snapshot is keyed by state so the dashboard can render "Done.actions:
|
|
1250
|
+
* push_branch ok, create_pr_if_missing rate-limited, retrying" without the
|
|
1251
|
+
* orchestrator having to know about specific action kinds.
|
|
1252
|
+
*
|
|
1253
|
+
* `id` should follow the `actions:<StateName>` convention so it sorts
|
|
1254
|
+
* predictably next to reconciler-resource rows in
|
|
1255
|
+
* `snapshot.reconciler.resources`.
|
|
1256
|
+
*/
|
|
1257
|
+
recordActionResult(id, snapshot) {
|
|
1258
|
+
this.lastActionResults.set(id, snapshot);
|
|
1259
|
+
}
|
|
1260
|
+
/**
|
|
1261
|
+
* Workspace removal callback the reconciler invokes for stale dirs (issue
|
|
1262
|
+
* 34). Defers to `WorkspaceManager.remove` (a best-effort `rm -rf`).
|
|
1263
|
+
* Failures are logged at warn (the reconciler's action ledger also records
|
|
1264
|
+
* them).
|
|
1265
|
+
*/
|
|
1266
|
+
async removeWorkspace(identifier) {
|
|
1267
|
+
await this.workspaces.remove(identifier);
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* Workspace create callback the reconciler invokes for non-terminal issues
|
|
1271
|
+
* whose dirs are not yet on disk (issue 34). Delegates to
|
|
1272
|
+
* `WorkspaceManager.ensureFor` so the same canonical clone+branch+remote
|
|
1273
|
+
* setup the dispatch path runs also fires here.
|
|
1274
|
+
*
|
|
1275
|
+
* The intended-set provider supplies the state alongside the identifier so
|
|
1276
|
+
* the merge-state guard below can fire; the `null` fallback is defensive —
|
|
1277
|
+
* production callers always pass a state.
|
|
1278
|
+
*
|
|
1279
|
+
* Race with the runner's dispatch-time `ensureFor` is handled inside
|
|
1280
|
+
* `WorkspaceManager` via a per-identifier in-flight promise lock: both
|
|
1281
|
+
* callers coalesce into one setup pass, so the canonical setup runs exactly
|
|
1282
|
+
* once whether the reconciler or the runner wins the race.
|
|
1283
|
+
*/
|
|
1284
|
+
async createWorkspace(identifier, state) {
|
|
1285
|
+
// Issue 38: refuse to eagerly recreate a missing workspace for an issue
|
|
1286
|
+
// in the autopilot's merge state. Those workspaces only exist as
|
|
1287
|
+
// leftovers from a prior dispatch; recreating one from scratch would
|
|
1288
|
+
// miss the agent's local commits (the agent's branch is on the remote,
|
|
1289
|
+
// but a fresh clone would still need a separate fetch to pick it up).
|
|
1290
|
+
// Operators who genuinely want a recreated workspace can cancel the
|
|
1291
|
+
// issue (Cancelled triggers the close path + normal cleanup) and refile.
|
|
1292
|
+
if (this.cfg.pr.enabled && state !== null) {
|
|
1293
|
+
const mergeState = derivePrRouting(this.cfg.states).mergeState;
|
|
1294
|
+
if (mergeState && state.toLowerCase() === mergeState.toLowerCase()) {
|
|
499
1295
|
return;
|
|
500
1296
|
}
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
1297
|
+
}
|
|
1298
|
+
await this.workspaces.ensureFor(identifier);
|
|
1299
|
+
}
|
|
1300
|
+
/**
|
|
1301
|
+
* Implements {@link PrIntendedProvider} (issue 38). Returns the set of
|
|
1302
|
+
* terminal-state issues the PR autopilot should manage:
|
|
1303
|
+
*
|
|
1304
|
+
* • Issues in the configured `merge_state` (default `Done`) become
|
|
1305
|
+
* `kind: 'merge'` intents. The autopilot rebases them on
|
|
1306
|
+
* `origin/<base>` and arms GitHub auto-merge.
|
|
1307
|
+
* • Issues in the configured `close_state` (default `Cancelled`) become
|
|
1308
|
+
* `kind: 'close'` intents. The autopilot closes the PR without merge
|
|
1309
|
+
* and best-effort-deletes the remote branch. The workspace is NOT
|
|
1310
|
+
* supplied — Cancelled cleanup goes through the orchestrator's
|
|
1311
|
+
* standard terminal path.
|
|
1312
|
+
*
|
|
1313
|
+
* Both queries hit the tracker; failures bubble (the pr resource catches
|
|
1314
|
+
* and surfaces in last_error so a transient tracker hiccup doesn't blank
|
|
1315
|
+
* the autopilot's intended set).
|
|
1316
|
+
*
|
|
1317
|
+
* When `pr.enabled` is false this method is never invoked (the reconciler
|
|
1318
|
+
* skips its pr pass entirely), but the early return keeps the public surface
|
|
1319
|
+
* idempotent. The merge/close targets are derived by scanning states for the
|
|
1320
|
+
* per-state `pr:` field (issue 139), not read from named strings.
|
|
1321
|
+
*/
|
|
1322
|
+
async prIntended() {
|
|
1323
|
+
if (!this.cfg.pr.enabled)
|
|
1324
|
+
return [];
|
|
1325
|
+
const { mergeState, closeState } = derivePrRouting(this.cfg.states);
|
|
1326
|
+
if (!mergeState && !closeState)
|
|
1327
|
+
return [];
|
|
1328
|
+
const baseBranch = baseBranchName(this.cfg.workspace.base_branch);
|
|
1329
|
+
const fetchStates = [mergeState, closeState].filter((s) => s !== null);
|
|
1330
|
+
const issues = await this.tracker.fetchIssuesByStates(fetchStates);
|
|
1331
|
+
const out = [];
|
|
1332
|
+
for (const issue of issues) {
|
|
1333
|
+
const intent = classifyPrIntent({
|
|
1334
|
+
issue,
|
|
1335
|
+
mergeState: mergeState ?? '',
|
|
1336
|
+
closeState,
|
|
1337
|
+
baseBranch,
|
|
1338
|
+
mergeWorkspacePath: this.workspaces.workspacePathFor(issue.identifier),
|
|
508
1339
|
});
|
|
509
|
-
|
|
1340
|
+
if (intent)
|
|
1341
|
+
out.push(intent);
|
|
510
1342
|
}
|
|
511
|
-
|
|
512
|
-
trackerRoot: snapshotTrackerRoot,
|
|
513
|
-
terminalTarget: snapshotTerminalTarget,
|
|
514
|
-
});
|
|
1343
|
+
return out;
|
|
515
1344
|
}
|
|
516
|
-
/**
|
|
517
|
-
|
|
1345
|
+
/**
|
|
1346
|
+
* Tracker-side transition the PR autopilot uses to route a conflict-rebasing
|
|
1347
|
+
* issue back into the implementing state (or, after exceeding the attempt
|
|
1348
|
+
* limit, into the holding state). Same shape as the MCP transition tool —
|
|
1349
|
+
* the tracker handles atomic notes-append + cross-directory rename.
|
|
1350
|
+
*
|
|
1351
|
+
* No workspace flag is touched here: the target state's `role` decides
|
|
1352
|
+
* cleanup at the transition's own level (active/holding never trigger
|
|
1353
|
+
* cleanup), and the workspace was preserved across the move into the merge
|
|
1354
|
+
* state in the first place because the PR engine suppresses the terminal
|
|
1355
|
+
* cleanup for that target. See {@link McpRegistry.performTransition} for the
|
|
1356
|
+
* role-driven rule.
|
|
1357
|
+
*/
|
|
1358
|
+
async routeIssueForAutopilot(input) {
|
|
1359
|
+
if (!this.tracker.moveIssueToState) {
|
|
1360
|
+
throw new Error('tracker does not support state transitions');
|
|
1361
|
+
}
|
|
1362
|
+
// The tracker file's `id` may diverge from the identifier when the file
|
|
1363
|
+
// sets an explicit front-matter `id`. Resolve via a candidate scan so the
|
|
1364
|
+
// tracker can find the right file even with that aliasing.
|
|
1365
|
+
let issueId = null;
|
|
518
1366
|
try {
|
|
519
|
-
const
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
}
|
|
524
|
-
catch (err) {
|
|
525
|
-
log.warn('terminal cleanup failed for issue', {
|
|
526
|
-
issue_identifier: issue.identifier,
|
|
527
|
-
error: err.message,
|
|
528
|
-
});
|
|
529
|
-
}
|
|
530
|
-
}
|
|
1367
|
+
const candidates = await this.tracker.fetchIssuesByStates([input.fromState]);
|
|
1368
|
+
const match = candidates.find((c) => c.identifier === input.identifier);
|
|
1369
|
+
if (match)
|
|
1370
|
+
issueId = match.id;
|
|
531
1371
|
}
|
|
532
|
-
catch
|
|
533
|
-
|
|
1372
|
+
catch {
|
|
1373
|
+
// Fall through to identifier fallback below.
|
|
534
1374
|
}
|
|
1375
|
+
if (issueId === null)
|
|
1376
|
+
issueId = input.identifier;
|
|
1377
|
+
await this.tracker.moveIssueToState(issueId, input.toState, {
|
|
1378
|
+
fromState: input.fromState,
|
|
1379
|
+
notes: input.notes,
|
|
1380
|
+
actor: input.actor,
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Implements {@link BaseRefProvider}. Returns the configured base branch
|
|
1385
|
+
* name AND its current SHA in the source repo (workflow_dir by default).
|
|
1386
|
+
* Returns null when the SHA can't be resolved (no `.git`, base branch
|
|
1387
|
+
* missing, etc.) — drift detection skips the pass.
|
|
1388
|
+
*
|
|
1389
|
+
* Why both fields: the reconciler's drift check compares the workspace's
|
|
1390
|
+
* own copy of `<branch>` (frozen at clone time) against this SHA. Returning
|
|
1391
|
+
* the branch name keeps the source-of-truth in one place; the inspector
|
|
1392
|
+
* uses it to run `git rev-parse <branch>` inside the workspace.
|
|
1393
|
+
*
|
|
1394
|
+
* The base branch (`workspace.base_branch`, or the `SYMPHONY_BASE_BRANCH` env
|
|
1395
|
+
* override, default `main`) is resolved the same way the dispatch-time clone
|
|
1396
|
+
* does, so the drift check compares against the same ref the workspace was
|
|
1397
|
+
* originally cloned from.
|
|
1398
|
+
*/
|
|
1399
|
+
async currentBaseRef() {
|
|
1400
|
+
const branch = process.env.SYMPHONY_BASE_BRANCH && process.env.SYMPHONY_BASE_BRANCH.length > 0
|
|
1401
|
+
? process.env.SYMPHONY_BASE_BRANCH
|
|
1402
|
+
: this.cfg.workspace.base_branch;
|
|
1403
|
+
const sourceRepo = process.env.SYMPHONY_SOURCE_REPO && process.env.SYMPHONY_SOURCE_REPO.length > 0
|
|
1404
|
+
? process.env.SYMPHONY_SOURCE_REPO
|
|
1405
|
+
: this.cfg.workflow_dir;
|
|
1406
|
+
const r = await runProcess('git', ['rev-parse', branch], { cwd: sourceRepo });
|
|
1407
|
+
if (r.exit_code !== 0)
|
|
1408
|
+
return null;
|
|
1409
|
+
const sha = r.stdout.trim();
|
|
1410
|
+
return sha.length > 0 ? { branch, sha } : null;
|
|
535
1411
|
}
|
|
536
|
-
// Public
|
|
1412
|
+
// Public callbacks the runner uses to feed events back.
|
|
537
1413
|
reportTokenUsage(issueId, usage) {
|
|
538
1414
|
const e = this.running.get(issueId);
|
|
539
1415
|
if (!e)
|
|
540
1416
|
return;
|
|
541
|
-
// §
|
|
1417
|
+
// §9.4: prefer absolute totals; track deltas to avoid double-counting.
|
|
542
1418
|
const dIn = Math.max(0, usage.input_tokens - e.last_reported_input_tokens);
|
|
543
1419
|
const dOut = Math.max(0, usage.output_tokens - e.last_reported_output_tokens);
|
|
544
1420
|
const dTot = Math.max(0, usage.total_tokens - e.last_reported_total_tokens);
|
|
545
|
-
e.
|
|
546
|
-
e.
|
|
547
|
-
e.
|
|
1421
|
+
e.input_tokens = usage.input_tokens;
|
|
1422
|
+
e.output_tokens = usage.output_tokens;
|
|
1423
|
+
e.total_tokens = usage.total_tokens;
|
|
548
1424
|
e.last_reported_input_tokens = usage.input_tokens;
|
|
549
1425
|
e.last_reported_output_tokens = usage.output_tokens;
|
|
550
1426
|
e.last_reported_total_tokens = usage.total_tokens;
|
|
551
|
-
this.
|
|
552
|
-
this.
|
|
553
|
-
this.
|
|
1427
|
+
this.sessionTotals.input_tokens += dIn;
|
|
1428
|
+
this.sessionTotals.output_tokens += dOut;
|
|
1429
|
+
this.sessionTotals.total_tokens += dTot;
|
|
554
1430
|
}
|
|
555
1431
|
reportRateLimits(_issueId, snapshot) {
|
|
556
|
-
this.
|
|
1432
|
+
this.rateLimits = snapshot;
|
|
557
1433
|
}
|
|
558
1434
|
reportRuntimeEvent(issueId, ev) {
|
|
559
1435
|
const e = this.running.get(issueId);
|
|
560
1436
|
if (!e)
|
|
561
1437
|
return;
|
|
562
|
-
e.
|
|
563
|
-
e.
|
|
564
|
-
e.
|
|
1438
|
+
e.last_event = ev.event;
|
|
1439
|
+
e.last_event_at = ev.at;
|
|
1440
|
+
e.last_message = ev.message;
|
|
565
1441
|
e.recent_events.push(ev);
|
|
566
1442
|
if (e.recent_events.length > 50)
|
|
567
1443
|
e.recent_events.shift();
|
|
@@ -572,7 +1448,7 @@ export class Orchestrator {
|
|
|
572
1448
|
return;
|
|
573
1449
|
e.session_id = info.sessionId;
|
|
574
1450
|
e.thread_id = info.threadId;
|
|
575
|
-
e.
|
|
1451
|
+
e.adapter_pid = info.pid;
|
|
576
1452
|
}
|
|
577
1453
|
reportTurnStarted(issueId, turnNumber) {
|
|
578
1454
|
const e = this.running.get(issueId);
|
|
@@ -580,7 +1456,7 @@ export class Orchestrator {
|
|
|
580
1456
|
return;
|
|
581
1457
|
e.turn_count = turnNumber;
|
|
582
1458
|
}
|
|
583
|
-
/** §
|
|
1459
|
+
/** §9.3 snapshot. */
|
|
584
1460
|
snapshot() {
|
|
585
1461
|
const generatedAt = new Date().toISOString();
|
|
586
1462
|
const liveExtraSeconds = [...this.running.values()]
|
|
@@ -597,19 +1473,19 @@ export class Orchestrator {
|
|
|
597
1473
|
state: e.issue.state,
|
|
598
1474
|
session_id: e.session_id,
|
|
599
1475
|
turn_count: e.turn_count,
|
|
600
|
-
last_event: e.
|
|
601
|
-
last_message: e.
|
|
1476
|
+
last_event: e.last_event,
|
|
1477
|
+
last_message: e.last_message,
|
|
602
1478
|
started_at: e.started_at,
|
|
603
|
-
last_event_at: e.
|
|
1479
|
+
last_event_at: e.last_event_at,
|
|
604
1480
|
tokens: {
|
|
605
|
-
input_tokens: e.
|
|
606
|
-
output_tokens: e.
|
|
607
|
-
total_tokens: e.
|
|
1481
|
+
input_tokens: e.input_tokens,
|
|
1482
|
+
output_tokens: e.output_tokens,
|
|
1483
|
+
total_tokens: e.total_tokens,
|
|
608
1484
|
},
|
|
609
1485
|
steering_requested: e.steering_requested,
|
|
610
1486
|
steering_question: e.steering_question,
|
|
611
1487
|
steering_context: e.steering_context,
|
|
612
|
-
|
|
1488
|
+
transitioned: e.transitioned,
|
|
613
1489
|
})),
|
|
614
1490
|
retrying: [...this.retryAttempts.values()].map((r) => ({
|
|
615
1491
|
issue_id: r.issue_id,
|
|
@@ -617,67 +1493,77 @@ export class Orchestrator {
|
|
|
617
1493
|
attempt: r.attempt,
|
|
618
1494
|
due_at: new Date(r.due_at_ms).toISOString(),
|
|
619
1495
|
error: r.error,
|
|
1496
|
+
kind: r.kind,
|
|
620
1497
|
})),
|
|
621
|
-
|
|
622
|
-
...this.
|
|
623
|
-
seconds_running: this.
|
|
1498
|
+
session_totals: {
|
|
1499
|
+
...this.sessionTotals,
|
|
1500
|
+
seconds_running: this.sessionTotals.seconds_running + liveExtraSeconds,
|
|
624
1501
|
},
|
|
625
|
-
rate_limits: this.
|
|
1502
|
+
rate_limits: this.rateLimits,
|
|
1503
|
+
memory_admission: this.computeAdmission(),
|
|
1504
|
+
reconciler: this.buildReconcilerSnapshot(),
|
|
626
1505
|
};
|
|
627
1506
|
}
|
|
1507
|
+
/**
|
|
1508
|
+
* Combine the reconciler's resource snapshot with the most recent action
|
|
1509
|
+
* results so the dashboard sees both surfaces under one `reconciler.resources`
|
|
1510
|
+
* list. When neither side has anything to report (no reconciler wired AND no
|
|
1511
|
+
* actions ever ran), the field is null to preserve the existing test
|
|
1512
|
+
* harness's "no reconciler" shape.
|
|
1513
|
+
*/
|
|
1514
|
+
buildReconcilerSnapshot() {
|
|
1515
|
+
const base = this.reconciler ? this.reconciler.snapshot() : null;
|
|
1516
|
+
if (!base && this.lastActionResults.size === 0)
|
|
1517
|
+
return null;
|
|
1518
|
+
const resources = base ? [...base.resources] : [];
|
|
1519
|
+
for (const snap of this.lastActionResults.values())
|
|
1520
|
+
resources.push(snap);
|
|
1521
|
+
return { resources };
|
|
1522
|
+
}
|
|
628
1523
|
/** Issue-detail view used by the HTTP /api/v1/<identifier> endpoint. */
|
|
629
1524
|
detailByIdentifier(identifier) {
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
1525
|
+
const entry = this.findRunningByIdentifier(identifier);
|
|
1526
|
+
const retry = this.findRetryByIdentifier(identifier);
|
|
1527
|
+
return buildIssueDetailDto(identifier, entry
|
|
1528
|
+
? {
|
|
1529
|
+
issue_id: entry.issue_id,
|
|
1530
|
+
identifier: entry.identifier,
|
|
1531
|
+
workspace_path: entry.workspace_path,
|
|
1532
|
+
session_id: entry.session_id,
|
|
1533
|
+
turn_count: entry.turn_count,
|
|
1534
|
+
state: entry.issue.state,
|
|
1535
|
+
started_at: entry.started_at,
|
|
1536
|
+
last_event: entry.last_event,
|
|
1537
|
+
last_message: entry.last_message,
|
|
1538
|
+
last_event_at: entry.last_event_at,
|
|
1539
|
+
input_tokens: entry.input_tokens,
|
|
1540
|
+
output_tokens: entry.output_tokens,
|
|
1541
|
+
total_tokens: entry.total_tokens,
|
|
1542
|
+
recent_events: entry.recent_events,
|
|
1543
|
+
last_error: entry.last_error,
|
|
635
1544
|
}
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
retry
|
|
641
|
-
|
|
1545
|
+
: null, retry
|
|
1546
|
+
? {
|
|
1547
|
+
issue_id: retry.issue_id,
|
|
1548
|
+
identifier: retry.identifier,
|
|
1549
|
+
attempt: retry.attempt,
|
|
1550
|
+
due_at_ms: retry.due_at_ms,
|
|
1551
|
+
error: retry.error,
|
|
1552
|
+
kind: retry.kind,
|
|
642
1553
|
}
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
session_id: entry.session_id,
|
|
657
|
-
turn_count: entry.turn_count,
|
|
658
|
-
state: entry.issue.state,
|
|
659
|
-
started_at: entry.started_at,
|
|
660
|
-
last_event: entry.last_codex_event,
|
|
661
|
-
last_message: entry.last_codex_message,
|
|
662
|
-
last_event_at: entry.last_codex_timestamp,
|
|
663
|
-
tokens: {
|
|
664
|
-
input_tokens: entry.codex_input_tokens,
|
|
665
|
-
output_tokens: entry.codex_output_tokens,
|
|
666
|
-
total_tokens: entry.codex_total_tokens,
|
|
667
|
-
},
|
|
668
|
-
}
|
|
669
|
-
: null,
|
|
670
|
-
retry: retry
|
|
671
|
-
? {
|
|
672
|
-
attempt: retry.attempt,
|
|
673
|
-
due_at: new Date(retry.due_at_ms).toISOString(),
|
|
674
|
-
error: retry.error,
|
|
675
|
-
}
|
|
676
|
-
: null,
|
|
677
|
-
recent_events: entry?.recent_events ?? [],
|
|
678
|
-
last_error: entry?.last_error ?? retry?.error ?? null,
|
|
679
|
-
tracked: {},
|
|
680
|
-
};
|
|
1554
|
+
: null);
|
|
1555
|
+
}
|
|
1556
|
+
findRunningByIdentifier(identifier) {
|
|
1557
|
+
for (const e of this.running.values())
|
|
1558
|
+
if (e.identifier === identifier)
|
|
1559
|
+
return e;
|
|
1560
|
+
return null;
|
|
1561
|
+
}
|
|
1562
|
+
findRetryByIdentifier(identifier) {
|
|
1563
|
+
for (const r of this.retryAttempts.values())
|
|
1564
|
+
if (r.identifier === identifier)
|
|
1565
|
+
return r;
|
|
1566
|
+
return null;
|
|
681
1567
|
}
|
|
682
1568
|
}
|
|
683
1569
|
//# sourceMappingURL=orchestrator.js.map
|