smol-symphony 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/AGENTS.md +105 -38
  2. package/PRODUCT.md +2 -1
  3. package/README.md +195 -98
  4. package/SPEC.md +543 -1915
  5. package/WORKFLOW.md +654 -179
  6. package/WORKFLOW.template.md +761 -121
  7. package/dist/acp-bridge.js +324 -0
  8. package/dist/acp-bridge.js.map +1 -0
  9. package/dist/actions/cache.js +191 -0
  10. package/dist/actions/cache.js.map +1 -0
  11. package/dist/actions/effects.js +41 -0
  12. package/dist/actions/effects.js.map +1 -0
  13. package/dist/actions/executor.js +570 -0
  14. package/dist/actions/executor.js.map +1 -0
  15. package/dist/actions/index.js +13 -0
  16. package/dist/actions/index.js.map +1 -0
  17. package/dist/actions/parsing.js +273 -0
  18. package/dist/actions/parsing.js.map +1 -0
  19. package/dist/actions/predicate-env.js +27 -0
  20. package/dist/actions/predicate-env.js.map +1 -0
  21. package/dist/actions/predicates.js +49 -0
  22. package/dist/actions/predicates.js.map +1 -0
  23. package/dist/actions/templating.js +66 -0
  24. package/dist/actions/templating.js.map +1 -0
  25. package/dist/actions/types.js +15 -0
  26. package/dist/actions/types.js.map +1 -0
  27. package/dist/agent/acp.js +232 -63
  28. package/dist/agent/acp.js.map +1 -1
  29. package/dist/agent/adapter-names.js +159 -0
  30. package/dist/agent/adapter-names.js.map +1 -0
  31. package/dist/agent/adapters.js +338 -102
  32. package/dist/agent/adapters.js.map +1 -1
  33. package/dist/agent/credential-extractors.js +342 -0
  34. package/dist/agent/credential-extractors.js.map +1 -0
  35. package/dist/agent/credential-secrets.js +628 -0
  36. package/dist/agent/credential-secrets.js.map +1 -0
  37. package/dist/agent/credential-ticker.js +57 -0
  38. package/dist/agent/credential-ticker.js.map +1 -0
  39. package/dist/agent/gondolin-creds-staging.js +356 -0
  40. package/dist/agent/gondolin-creds-staging.js.map +1 -0
  41. package/dist/agent/gondolin-dispatch.js +375 -0
  42. package/dist/agent/gondolin-dispatch.js.map +1 -0
  43. package/dist/agent/gondolin.js +124 -0
  44. package/dist/agent/gondolin.js.map +1 -0
  45. package/dist/agent/runner-decisions.js +134 -0
  46. package/dist/agent/runner-decisions.js.map +1 -0
  47. package/dist/agent/runner.js +1352 -290
  48. package/dist/agent/runner.js.map +1 -1
  49. package/dist/agent/tool-call-summary.js +102 -0
  50. package/dist/agent/tool-call-summary.js.map +1 -0
  51. package/dist/agent/vm-acp-mapping.js +73 -0
  52. package/dist/agent/vm-acp-mapping.js.map +1 -0
  53. package/dist/agent/vm-guards.js +262 -0
  54. package/dist/agent/vm-guards.js.map +1 -0
  55. package/dist/agent/vm-port.js +22 -0
  56. package/dist/agent/vm-port.js.map +1 -0
  57. package/dist/agent/vm-process-registry.js +79 -0
  58. package/dist/agent/vm-process-registry.js.map +1 -0
  59. package/dist/bin/cli-args.js +105 -0
  60. package/dist/bin/cli-args.js.map +1 -0
  61. package/dist/bin/symphony.js +719 -130
  62. package/dist/bin/symphony.js.map +1 -1
  63. package/dist/errors.js +15 -0
  64. package/dist/errors.js.map +1 -0
  65. package/dist/http-disk.js +135 -0
  66. package/dist/http-disk.js.map +1 -0
  67. package/dist/http-handlers.js +180 -0
  68. package/dist/http-handlers.js.map +1 -0
  69. package/dist/http.js +1476 -764
  70. package/dist/http.js.map +1 -1
  71. package/dist/issues.js +178 -0
  72. package/dist/issues.js.map +1 -0
  73. package/dist/logging.js +163 -5
  74. package/dist/logging.js.map +1 -1
  75. package/dist/mcp.js +391 -163
  76. package/dist/mcp.js.map +1 -1
  77. package/dist/memory.js +85 -0
  78. package/dist/memory.js.map +1 -0
  79. package/dist/orchestrator-decisions.js +331 -0
  80. package/dist/orchestrator-decisions.js.map +1 -0
  81. package/dist/orchestrator.js +1189 -303
  82. package/dist/orchestrator.js.map +1 -1
  83. package/dist/prompt.js +5 -5
  84. package/dist/prompt.js.map +1 -1
  85. package/dist/reconciler/cache.js +65 -0
  86. package/dist/reconciler/cache.js.map +1 -0
  87. package/dist/reconciler/index.js +448 -0
  88. package/dist/reconciler/index.js.map +1 -0
  89. package/dist/reconciler/ledger.js +131 -0
  90. package/dist/reconciler/ledger.js.map +1 -0
  91. package/dist/reconciler/pr-adapters.js +174 -0
  92. package/dist/reconciler/pr-adapters.js.map +1 -0
  93. package/dist/reconciler/pr-decide.js +167 -0
  94. package/dist/reconciler/pr-decide.js.map +1 -0
  95. package/dist/reconciler/pr.js +422 -0
  96. package/dist/reconciler/pr.js.map +1 -0
  97. package/dist/reconciler/types.js +12 -0
  98. package/dist/reconciler/types.js.map +1 -0
  99. package/dist/reconciler/vm.js +243 -0
  100. package/dist/reconciler/vm.js.map +1 -0
  101. package/dist/reconciler/workspace-defaults.js +83 -0
  102. package/dist/reconciler/workspace-defaults.js.map +1 -0
  103. package/dist/reconciler/workspace.js +272 -0
  104. package/dist/reconciler/workspace.js.map +1 -0
  105. package/dist/runlog.js +403 -0
  106. package/dist/runlog.js.map +1 -0
  107. package/dist/scaffold.js +165 -0
  108. package/dist/scaffold.js.map +1 -0
  109. package/dist/trackers/local.js +234 -133
  110. package/dist/trackers/local.js.map +1 -1
  111. package/dist/trackers/types.js +1 -1
  112. package/dist/trackers/types.js.map +1 -1
  113. package/dist/types.js +1 -1
  114. package/dist/util/clock.js +12 -0
  115. package/dist/util/clock.js.map +1 -0
  116. package/dist/util/crypto.js +25 -0
  117. package/dist/util/crypto.js.map +1 -0
  118. package/dist/util/frontmatter.js +70 -0
  119. package/dist/util/frontmatter.js.map +1 -0
  120. package/dist/util/fs-issues.js +22 -0
  121. package/dist/util/fs-issues.js.map +1 -0
  122. package/dist/util/process.js +152 -0
  123. package/dist/util/process.js.map +1 -0
  124. package/dist/util/workspace-key.js +10 -0
  125. package/dist/util/workspace-key.js.map +1 -0
  126. package/dist/workflow-loader.js +147 -0
  127. package/dist/workflow-loader.js.map +1 -0
  128. package/dist/workflow.js +656 -219
  129. package/dist/workflow.js.map +1 -1
  130. package/dist/workspace-types.js +8 -0
  131. package/dist/workspace-types.js.map +1 -0
  132. package/dist/workspace.js +367 -120
  133. package/dist/workspace.js.map +1 -1
  134. package/package.json +14 -6
  135. package/scripts/vm-agent.mjs +211 -0
  136. package/dist/agent/codex.js +0 -439
  137. package/dist/agent/codex.js.map +0 -1
  138. package/dist/agent/smolvm.js +0 -174
  139. package/dist/agent/smolvm.js.map +0 -1
  140. package/scripts/build-vm.sh +0 -67
@@ -1,11 +1,35 @@
1
- // Orchestrator (SPEC §7, §8, §14, §16). Owns the single-authority runtime state and
2
- // drives the poll-and-dispatch tick, retries, reconciliation, and worker exit handling.
3
- import { validateDispatch, WorkflowError } from './workflow.js';
4
- import { ADAPTERS, assertHostCredentialReadable, isKnownAdapter } from './agent/adapters.js';
5
- import { pickTerminalTarget } from './mcp.js';
1
+ // Orchestrator. Owns the single-authority runtime state and drives the
2
+ // poll-and-dispatch tick, retries, reconciliation, and worker exit handling.
3
+ import { deriveArmRouting, derivePrRouting, validateDispatch, WorkflowError, } from './workflow.js';
4
+ import { validateDispatchIo } from './workflow-loader.js';
5
+ import { writeIssueFile, pickHoldingState } from './issues.js';
6
+ import { resolveDispatchConfig } from './agent/runner.js';
7
+ import { codexCredentialAvailable, codexMissingCredentialMessage, hostClaudeCredentialPath, hostCodexCredentialPath, hostOpencodeCredentialPath, isKnownAdapter, opencodeCredentialAvailable, opencodeMissingCredentialMessage, } from './agent/adapter-names.js';
8
+ import { accessSync, constants as fsConstants, readFileSync } from 'node:fs';
9
+ import { activeStateNames, terminalStateNames } from './issues.js';
10
+ import { buildIssueDetailDto, classifyPrIntent, computeEligibilityReason, decideCircuitBreaker, decideExitRetry, decideReconcileForIssue, decideRetryAfterIneligible, decideSleepCycleArm, requiredAdapterIds, resolveActorString, sleepCycleArmNotes, } from './orchestrator-decisions.js';
11
+ import { resolveGithubRepo } from './workspace.js';
6
12
  import { withIssue, log } from './logging.js';
13
+ import { openRunLog } from './runlog.js';
14
+ import { defaultMemProbe, computeMemoryAdmission } from './memory.js';
15
+ import { runProcess } from './util/process.js';
7
16
  const CONTINUATION_DELAY_MS = 1_000;
8
17
  const FAILURE_BASE_MS = 10_000;
18
+ // Actor stamped into the notes header when the orchestrator (not an agent)
19
+ // auto-arms the reflection issue, so the move is attributable on the dashboard
20
+ // and in the issue body.
21
+ const SLEEP_CYCLE_ACTOR = 'symphony/sleep-cycle';
22
+ /**
23
+ * Resolve the base branch the autopilot should rebase against. Mirrors the
24
+ * canonical workspace-setup contract — the `SYMPHONY_BASE_BRANCH` env wins,
25
+ * else the parsed `workspace.base_branch` (which defaults to `main`).
26
+ */
27
+ function baseBranchName(configBaseBranch) {
28
+ const env = process.env.SYMPHONY_BASE_BRANCH;
29
+ if (env && env.length > 0)
30
+ return env;
31
+ return configBaseBranch;
32
+ }
9
33
  export class Orchestrator {
10
34
  cfg;
11
35
  workflowDef;
@@ -13,33 +37,81 @@ export class Orchestrator {
13
37
  tracker;
14
38
  workspaces;
15
39
  runner;
40
+ memProbe;
41
+ reconciler;
16
42
  running = new Map();
17
43
  claimed = new Set();
18
44
  retryAttempts = new Map();
45
+ // Per-issue circuit-breaker streak (issue 128): the last abnormal-exit reason
46
+ // (normalized) and how many consecutive attempts failed with it. Updated on
47
+ // every worker exit; cleared on a clean exit, on trip, and on claim release.
48
+ // In-memory only — a process restart resets the streak, but the *trip itself*
49
+ // is restart-safe because it physically moves the issue out of the active set.
50
+ circuitBreakers = new Map();
51
+ // Sleep-cycle auto-arm (issue 125). Count of terminal-state transitions
52
+ // observed since the reflection issue was last armed; the idle and
53
+ // done-threshold triggers both read it, and it resets to 0 on each arm. The
54
+ // in-flight guard stops two overlapping ticks from both firing the async
55
+ // Dormant → Reflect move. In-memory only — a process restart resets the
56
+ // streak (consistent with `circuitBreakers`).
57
+ doneSinceReflect = 0;
58
+ armingReflection = false;
19
59
  completed = new Set();
20
- codexTotals = {
60
+ // Per-state ledger of the most-recent action-list execution. Surfaced via
61
+ // `snapshot.reconciler.resources` so the dashboard can render "Done.actions:
62
+ // push_branch ok, create_pr_if_missing in_progress" without a separate
63
+ // first-class surface for action state (issue 36 AC5).
64
+ lastActionResults = new Map();
65
+ // Per-issue JSONL run log. Opened lazily on first dispatch for an issue, kept open across
66
+ // retries so the file is one chronological stream per issue, and closed only when the
67
+ // issue finally unwinds (terminal cleanup, claim release without redispatch, or stop()).
68
+ runLogs = new Map();
69
+ // Set of issue ids whose terminal cleanup (workspaces.remove) is still in
70
+ // flight. Used by closeRunLog to defer the close until the terminal-state
71
+ // actions capture has stopped writing; otherwise the retry-timer's "claim
72
+ // released" close fires ~1s after worker exit (before the actions finish)
73
+ // and we'd lose the action output lines in the JSONL log.
74
+ cleanupInFlight = new Set();
75
+ sessionTotals = {
21
76
  input_tokens: 0,
22
77
  output_tokens: 0,
23
78
  total_tokens: 0,
24
79
  seconds_running: 0,
25
80
  };
26
- codexRateLimits = null;
81
+ rateLimits = null;
27
82
  tickTimer = null;
28
83
  stopped = false;
29
84
  refreshRequested = false;
30
85
  // Latest dispatch validation error, if any (operator-visible).
31
86
  lastValidationError = null;
32
87
  // Optional callback used to propagate reloaded config to components that hold their own
33
- // tracker/runner/workspace state (so prompt body, hooks, smolvm config, etc., take effect
34
- // on the next dispatch — see §6.2).
88
+ // tracker/runner/workspace state (so prompt body, per-state actions, gondolin config, etc.,
89
+ // take effect on the next dispatch).
35
90
  onConfigReloaded;
36
- constructor(cfg, workflowDef, workflowSrc, tracker, workspaces, runner) {
91
+ // Last clamp-active state observed by availableGlobalSlots. Used to log
92
+ // transitions (clamp_active true→false or false→true) at info level without
93
+ // spamming the log every tick while the cap stays clamped.
94
+ memoryClampActive = false;
95
+ constructor(cfg, workflowDef, workflowSrc, tracker, workspaces, runner,
96
+ // Memory probe used by the admission cap (issue 27). Defaults to reading
97
+ // /proc/meminfo synchronously; tests inject a stub that returns a controlled
98
+ // mem_available_mib so the clamp behavior is deterministic.
99
+ memProbe = defaultMemProbe,
100
+ // Reconciler (issue 32, 33) — owns managed external resources: the
101
+ // symphony-VM lifecycle reaper, the workspace janitor, and PR autopilot.
102
+ // Optional so tests that don't exercise reconciliation don't have to
103
+ // construct one; when absent, `Snapshot.reconciler` is null and stray VM
104
+ // reaping is skipped. Production wiring in bin/symphony.ts always passes
105
+ // one in.
106
+ reconciler = null) {
37
107
  this.cfg = cfg;
38
108
  this.workflowDef = workflowDef;
39
109
  this.workflowSrc = workflowSrc;
40
110
  this.tracker = tracker;
41
111
  this.workspaces = workspaces;
42
112
  this.runner = runner;
113
+ this.memProbe = memProbe;
114
+ this.reconciler = reconciler;
43
115
  workflowSrc.onChange((next) => {
44
116
  if ('error' in next) {
45
117
  this.lastValidationError = next.error.message;
@@ -50,6 +122,10 @@ export class Orchestrator {
50
122
  this.workflowDef = next.definition;
51
123
  this.lastValidationError = null;
52
124
  this.onConfigReloaded?.(next.config, next.definition);
125
+ // Issue 32: a config-watcher change is one of the reconciler's declared
126
+ // triggers. Re-binding the resource set picks up new managed-resource
127
+ // config (e.g. `gondolin.*` VM settings).
128
+ this.reconciler?.updateConfig(next.config);
53
129
  log.info('runtime config reloaded', {
54
130
  poll_interval_ms: next.config.polling.interval_ms,
55
131
  max_concurrent_agents: next.config.agent.max_concurrent_agents,
@@ -61,33 +137,98 @@ export class Orchestrator {
61
137
  this.onConfigReloaded = cb;
62
138
  }
63
139
  async start() {
64
- const validation = validateDispatch(this.cfg);
140
+ const validation = validateDispatch(this.cfg) ?? validateDispatchIo(this.cfg);
65
141
  if (validation) {
66
142
  log.error('startup validation failed', { error: validation });
67
143
  throw new WorkflowError('workflow_parse_error', validation);
68
144
  }
69
- // Fail fast when symphony will auto-stage credentials but the host file the
70
- // adapter needs is missing. Operators who set acp.command explicitly own their
71
- // own credential plumbing, so skip the check in that branch.
72
- if (this.cfg.acp.command === null && isKnownAdapter(this.cfg.acp.adapter)) {
73
- const profile = ADAPTERS[this.cfg.acp.adapter];
74
- try {
75
- await assertHostCredentialReadable(profile);
76
- }
77
- catch (err) {
78
- log.error('startup credential check failed', { error: err.message });
79
- throw new WorkflowError('missing_host_credential', err.message);
80
- }
81
- }
82
- await this.startupTerminalCleanup();
145
+ await this.assertAdapterCredentials();
146
+ await this.runStartupReconcile();
83
147
  this.scheduleTick(0);
84
148
  }
149
+ /**
150
+ * Fail fast when symphony will dispatch to an adapter whose host credential
151
+ * (substituted into the outbound request at Gondolin egress) is missing.
152
+ * Per-state overrides can change the adapter, so the set is the union of
153
+ * `cfg.acp.adapter` and every distinct `states.<name>.adapter`. claude needs
154
+ * `~/.claude/.credentials.json`; codex needs either a `~/.codex/auth.json`
155
+ * token or an `OPENAI_API_KEY` env var; opencode needs either a
156
+ * `github-copilot` token in `~/.local/share/opencode/auth.json` or a
157
+ * COPILOT_GITHUB_TOKEN/GH_TOKEN/GITHUB_TOKEN env var. A missing credential
158
+ * surfaces here as a clear startup error rather than an opaque per-request
159
+ * egress failure mid-dispatch.
160
+ */
161
+ async assertAdapterCredentials() {
162
+ const ids = requiredAdapterIds(this.cfg, isKnownAdapter);
163
+ if (ids.has('claude'))
164
+ this.assertClaudeCredential();
165
+ if (ids.has('codex'))
166
+ this.assertCodexCredential();
167
+ if (ids.has('opencode'))
168
+ this.assertOpencodeCredential();
169
+ }
170
+ assertClaudeCredential() {
171
+ const credPath = hostClaudeCredentialPath();
172
+ try {
173
+ accessSync(credPath, fsConstants.R_OK);
174
+ }
175
+ catch (err) {
176
+ const msg = `adapter "claude" requires a host credential at ${credPath}, but it is missing or unreadable: ${err.message}`;
177
+ log.error('startup credential check failed', { adapter: 'claude', error: msg });
178
+ throw new WorkflowError('missing_host_credential', msg);
179
+ }
180
+ }
181
+ assertCodexCredential() {
182
+ let authText = null;
183
+ try {
184
+ authText = readFileSync(hostCodexCredentialPath(), 'utf8');
185
+ }
186
+ catch {
187
+ authText = null;
188
+ }
189
+ if (codexCredentialAvailable(authText, process.env))
190
+ return;
191
+ const msg = codexMissingCredentialMessage();
192
+ log.error('startup credential check failed', { adapter: 'codex', error: msg });
193
+ throw new WorkflowError('missing_host_credential', msg);
194
+ }
195
+ assertOpencodeCredential() {
196
+ let authText = null;
197
+ try {
198
+ authText = readFileSync(hostOpencodeCredentialPath(), 'utf8');
199
+ }
200
+ catch {
201
+ authText = null;
202
+ }
203
+ if (opencodeCredentialAvailable(authText, process.env))
204
+ return;
205
+ const msg = opencodeMissingCredentialMessage();
206
+ log.error('startup credential check failed', { adapter: 'opencode', error: msg });
207
+ throw new WorkflowError('missing_host_credential', msg);
208
+ }
209
+ /**
210
+ * Initial workspace + VM reap and the first reconcile pass (issues 32-34).
211
+ * The `running` map is empty here, so the janitors converge to "remove
212
+ * anything orphaned by the previous process" and the bake (if any) starts
213
+ * before the first dispatch.
214
+ */
215
+ async runStartupReconcile() {
216
+ if (!this.reconciler)
217
+ return;
218
+ await this.reconciler.reapWorkspaces();
219
+ await this.reconciler.reapVms();
220
+ this.reconciler.start();
221
+ void this.reconciler.reconcile().catch((err) => log.warn('initial reconcile pass failed', { error: err.message }));
222
+ }
85
223
  async stop() {
86
224
  this.stopped = true;
87
225
  if (this.tickTimer) {
88
226
  clearTimeout(this.tickTimer);
89
227
  this.tickTimer = null;
90
228
  }
229
+ if (this.reconciler) {
230
+ await this.reconciler.stop().catch(() => undefined);
231
+ }
91
232
  for (const e of this.retryAttempts.values())
92
233
  clearTimeout(e.timer_handle);
93
234
  this.retryAttempts.clear();
@@ -96,8 +237,39 @@ export class Orchestrator {
96
237
  e.cancel();
97
238
  this.running.clear();
98
239
  this.claimed.clear();
240
+ this.circuitBreakers.clear();
241
+ // Drain every open run log so the JSONL files are flushed before exit.
242
+ const closures = [];
243
+ for (const [issueId, rl] of this.runLogs) {
244
+ rl.system('runlog_closed', { reason: 'orchestrator_stopped' });
245
+ closures.push(rl.close());
246
+ this.runLogs.delete(issueId);
247
+ }
248
+ await Promise.all(closures);
249
+ // VM teardown lives in the reconciler `vm` resource (issue 52). stop() does NOT
250
+ // wait for in-flight workers to unwind before returning — the bin script then
251
+ // exits the process, which abruptly ends the per-VM Gondolin runners but can
252
+ // leave their session sockets behind. Without this backstop, every SIGTERM
253
+ // during an active run can leak one VM per running entry, and over enough
254
+ // operator restarts the host OOMs (issue 26). `running` is cleared above,
255
+ // so the reaper's intended set is ∅ and every `symphony-*` VM (live session +
256
+ // any orphaned socket) gets torn down.
257
+ if (this.reconciler) {
258
+ await this.reconciler.reapVms();
259
+ }
260
+ }
261
+ /**
262
+ * Operator trigger for an immediate reconcile pass. Used by `symphony reconcile
263
+ * --force` (which invalidates the cache first via `force: true`) and by any
264
+ * future dashboard button that wants to re-evaluate the resource DAG without
265
+ * waiting for the backstop tick.
266
+ */
267
+ async triggerReconcile(opts = {}) {
268
+ if (!this.reconciler)
269
+ return;
270
+ await this.reconciler.reconcile(opts);
99
271
  }
100
- /** Operator trigger for an immediate poll cycle (§13.7 /refresh). */
272
+ /** Operator trigger for an immediate poll cycle (§9.5 /refresh). */
101
273
  triggerRefresh() {
102
274
  if (this.refreshRequested)
103
275
  return { queued: true, coalesced: true };
@@ -118,77 +290,113 @@ export class Orchestrator {
118
290
  if (this.stopped)
119
291
  return;
120
292
  this.refreshRequested = false;
293
+ await this.reconcileSafely();
294
+ if (!this.applyDispatchValidation())
295
+ return;
296
+ const fetched = await this.fetchCandidatesForTick();
297
+ if (!fetched)
298
+ return;
299
+ if (this.gatedOnReconciler(fetched.issues.length))
300
+ return;
301
+ this.dispatchSorted(this.sortForDispatch(fetched.issues), fetched.root);
302
+ this.maybeArmSleepCycle(fetched.issues.length === 0);
303
+ this.scheduleTick(this.cfg.polling.interval_ms);
304
+ }
305
+ async reconcileSafely() {
121
306
  try {
122
307
  await this.reconcile();
123
308
  }
124
309
  catch (err) {
125
310
  log.warn('reconcile error', { error: err.message });
126
311
  }
127
- const validation = validateDispatch(this.cfg);
312
+ }
313
+ /**
314
+ * Run dispatch validation. Returns true to continue dispatch, false when the
315
+ * config is invalid (the tick is rescheduled and the caller must return).
316
+ */
317
+ applyDispatchValidation() {
318
+ const validation = validateDispatch(this.cfg) ?? validateDispatchIo(this.cfg);
128
319
  if (validation) {
129
320
  this.lastValidationError = validation;
130
321
  log.warn('dispatch validation failed; skipping dispatch', { error: validation });
131
322
  this.scheduleTick(this.cfg.polling.interval_ms);
132
- return;
323
+ return false;
133
324
  }
134
325
  this.lastValidationError = null;
135
- let candidates;
136
- let snapshotTrackerRoot;
137
- let snapshotTerminalTarget;
326
+ return true;
327
+ }
328
+ /**
329
+ * Atomic fetch: the tracker returns the issues AND the root it used during
330
+ * the scan. That's the snapshot we pin onto each RunningEntry, so a workflow
331
+ * reload that races the dispatch loop can't cause `transition` to operate
332
+ * against a different tracker root than where the issue lives. Returns
333
+ * null on tracker error (tick is rescheduled, caller must return).
334
+ */
335
+ async fetchCandidatesForTick() {
138
336
  try {
139
- // Atomic fetch: the tracker returns the issues AND the root/terminal_states
140
- // it used during the scan. That's the snapshot we pin onto each RunningEntry,
141
- // so a workflow reload that races the dispatch loop can't cause `mark_done`
142
- // to operate against a different tracker config than where the issue lives.
143
- const result = await this.tracker.fetchCandidateIssues();
144
- candidates = result.issues;
145
- snapshotTrackerRoot = result.root;
146
- snapshotTerminalTarget = pickTerminalTarget(result.terminalStates);
337
+ const r = await this.tracker.fetchCandidateIssues();
338
+ return { issues: r.issues, root: r.root };
147
339
  }
148
340
  catch (err) {
149
341
  log.warn('candidate fetch failed', { error: err.message });
150
342
  this.scheduleTick(this.cfg.polling.interval_ms);
151
- return;
343
+ return null;
152
344
  }
153
- const sorted = this.sortForDispatch(candidates);
345
+ }
346
+ /**
347
+ * Reconciler gate (issue 32): refuse to dispatch any issue whose
348
+ * prerequisites haven't converged. When the gate is closed we kick a
349
+ * reconcile pass so the loop self-corrects on the next poll instead of
350
+ * waiting on the slower backstop tick. Returns true when dispatch must
351
+ * be skipped (caller must return).
352
+ */
353
+ gatedOnReconciler(candidateCount) {
354
+ if (!this.reconciler || this.reconciler.dispatchReady())
355
+ return false;
356
+ log.debug('dispatch gated on reconciler', { candidate_count: candidateCount });
357
+ void this.reconciler.reconcile().catch((err) => log.debug('gated-reconcile failed', { error: err.message }));
358
+ this.scheduleTick(this.cfg.polling.interval_ms);
359
+ return true;
360
+ }
361
+ dispatchSorted(sorted, snapshotTrackerRoot) {
154
362
  for (const issue of sorted) {
155
363
  if (this.availableGlobalSlots() <= 0)
156
364
  break;
157
365
  if (!this.isEligible(issue))
158
366
  continue;
159
- void this.dispatchIssue(issue, null, {
160
- trackerRoot: snapshotTrackerRoot,
161
- terminalTarget: snapshotTerminalTarget,
162
- });
367
+ void this.dispatchIssue(issue, null, { trackerRoot: snapshotTrackerRoot });
163
368
  }
164
- this.scheduleTick(this.cfg.polling.interval_ms);
165
369
  }
166
- /** §8.5: stall detection + tracker state refresh for running issues. */
370
+ /** Stall detection + tracker state refresh for running issues. */
167
371
  async reconcile() {
168
- // Part A: stall detection.
169
- if (this.cfg.acp.stall_timeout_ms > 0) {
170
- const now = Date.now();
171
- for (const [issueId, entry] of this.running) {
172
- // Skip stall detection for issues awaiting human steering: the agent is
173
- // intentionally paused while the human composes a reply, and the wait can
174
- // legitimately exceed stall_timeout_ms. The cancel signal still applies
175
- // (the runner's awaitSteeringReply respects it) for non-stall reasons like
176
- // terminal-state transitions or operator-initiated cancels.
177
- if (entry.steering_requested)
178
- continue;
179
- const ref = entry.last_codex_timestamp ?? entry.started_at;
180
- const elapsed = now - Date.parse(ref);
181
- if (Number.isFinite(elapsed) && elapsed > this.cfg.acp.stall_timeout_ms) {
182
- log.warn('stall detected', {
183
- issue_id: issueId,
184
- issue_identifier: entry.identifier,
185
- elapsed_ms: elapsed,
186
- });
187
- this.terminateRunning(issueId, false, `stalled after ${elapsed}ms`);
188
- }
372
+ this.detectStalls();
373
+ await this.refreshTrackerStates();
374
+ }
375
+ detectStalls() {
376
+ if (this.cfg.acp.stall_timeout_ms <= 0)
377
+ return;
378
+ const now = Date.now();
379
+ for (const [issueId, entry] of this.running) {
380
+ // Skip stall detection for issues awaiting human steering: the agent is
381
+ // intentionally paused while the human composes a reply, and the wait can
382
+ // legitimately exceed stall_timeout_ms. The cancel signal still applies
383
+ // (the runner's awaitSteeringReply respects it) for non-stall reasons like
384
+ // terminal-state transitions or operator-initiated cancels.
385
+ if (entry.steering_requested)
386
+ continue;
387
+ const ref = entry.last_event_at ?? entry.started_at;
388
+ const elapsed = now - Date.parse(ref);
389
+ if (Number.isFinite(elapsed) && elapsed > this.cfg.acp.stall_timeout_ms) {
390
+ log.warn('stall detected', {
391
+ issue_id: issueId,
392
+ issue_identifier: entry.identifier,
393
+ elapsed_ms: elapsed,
394
+ });
395
+ this.terminateRunning(issueId, false, `stalled after ${elapsed}ms`);
189
396
  }
190
397
  }
191
- // Part B: tracker state refresh.
398
+ }
399
+ async refreshTrackerStates() {
192
400
  const ids = [...this.running.keys()];
193
401
  if (ids.length === 0)
194
402
  return;
@@ -201,27 +409,18 @@ export class Orchestrator {
201
409
  return;
202
410
  }
203
411
  const byId = new Map(refreshed.map((i) => [i.id, i]));
204
- const terminal = new Set(this.cfg.tracker.terminal_states.map((s) => s.toLowerCase()));
205
- const active = new Set(this.cfg.tracker.active_states.map((s) => s.toLowerCase()));
206
- for (const id of ids) {
207
- const fresh = byId.get(id);
208
- if (!fresh) {
209
- // Missing from tracker — non-active, no cleanup (§8.5 part B "neither" branch).
210
- this.terminateRunning(id, false, 'tracker_state_missing');
211
- continue;
212
- }
213
- const s = fresh.state.toLowerCase();
214
- if (terminal.has(s)) {
215
- this.terminateRunning(id, true, 'tracker_state_terminal');
216
- }
217
- else if (active.has(s)) {
218
- const entry = this.running.get(id);
219
- if (entry)
220
- entry.issue = fresh;
221
- }
222
- else {
223
- this.terminateRunning(id, false, 'tracker_state_non_active');
224
- }
412
+ for (const id of ids)
413
+ this.applyReconcileAction(id, byId.get(id));
414
+ }
415
+ applyReconcileAction(id, fresh) {
416
+ const decision = decideReconcileForIssue(fresh, this.cfg.states);
417
+ if (decision.kind === 'terminate') {
418
+ this.terminateRunning(id, decision.cleanup, decision.reason);
419
+ }
420
+ else if (decision.kind === 'refresh' && fresh) {
421
+ const entry = this.running.get(id);
422
+ if (entry)
423
+ entry.issue = fresh;
225
424
  }
226
425
  }
227
426
  terminateRunning(issueId, cleanupWorkspace, reason) {
@@ -231,6 +430,10 @@ export class Orchestrator {
231
430
  if (cleanupWorkspace)
232
431
  entry.cleanup_workspace_on_exit = true;
233
432
  entry.cancel();
433
+ this.runLogs.get(issueId)?.system('reconciliation_terminating', {
434
+ reason,
435
+ cleanup_workspace: cleanupWorkspace,
436
+ });
234
437
  log.info('reconciliation terminating run', {
235
438
  issue_id: issueId,
236
439
  issue_identifier: entry.identifier,
@@ -238,7 +441,7 @@ export class Orchestrator {
238
441
  cleanup_workspace: cleanupWorkspace,
239
442
  });
240
443
  }
241
- /** §8.2 candidate eligibility. */
444
+ /** Candidate eligibility. */
242
445
  isEligible(issue) {
243
446
  return this.eligibilityReason(issue, /*ignoreOwnClaim*/ false) === null;
244
447
  }
@@ -246,40 +449,110 @@ export class Orchestrator {
246
449
  // form is used by the retry path so the issue's own claim/retry entry does not block
247
450
  // its own redispatch.
248
451
  eligibilityReason(issue, ignoreOwnClaim) {
249
- if (!issue.id || !issue.identifier || !issue.title || !issue.state) {
250
- return 'missing required issue fields';
251
- }
252
- const state = issue.state.toLowerCase();
253
- const active = new Set(this.cfg.tracker.active_states.map((s) => s.toLowerCase()));
254
- const terminal = new Set(this.cfg.tracker.terminal_states.map((s) => s.toLowerCase()));
255
- if (!active.has(state) || terminal.has(state))
256
- return 'state not active';
257
- if (this.running.has(issue.id))
258
- return 'already running';
259
- if (!ignoreOwnClaim && this.claimed.has(issue.id))
260
- return 'already claimed';
261
- if (!this.hasPerStateSlot(issue.state))
262
- return 'no per-state slot';
263
- if (state === 'todo' && this.hasNonTerminalBlocker(issue))
264
- return 'has non-terminal blocker';
265
- return null;
452
+ return computeEligibilityReason(issue, ignoreOwnClaim, this.eligibilitySnapshot());
266
453
  }
267
- hasNonTerminalBlocker(issue) {
268
- const terminal = new Set(this.cfg.tracker.terminal_states.map((s) => s.toLowerCase()));
269
- for (const b of issue.blocked_by) {
270
- if (!b.state)
271
- return true;
272
- if (!terminal.has(b.state.toLowerCase()))
273
- return true;
274
- }
275
- return false;
454
+ eligibilitySnapshot() {
455
+ return {
456
+ active: new Set(activeStateNames(this.cfg.states).map((s) => s.toLowerCase())),
457
+ terminal: new Set(terminalStateNames(this.cfg.states).map((s) => s.toLowerCase())),
458
+ running: new Set(this.running.keys()),
459
+ claimed: this.claimed,
460
+ perStateSlot: (state) => this.hasPerStateSlot(state),
461
+ };
276
462
  }
277
463
  availableGlobalSlots() {
278
- return Math.max(0, this.cfg.agent.max_concurrent_agents - this.running.size);
464
+ // Pending continuations hold their slot. The continuation is the
465
+ // post-transition resume of an issue that just normal-exited (e.g.
466
+ // Todo→Review handoff); without this, a tick firing inside the 1s
467
+ // continuation window can dispatch a brand-new Todo and steal the slot
468
+ // the just-transitioned issue is about to reclaim, leaving it requeued
469
+ // with "no available orchestrator slots" until something else finishes.
470
+ // Failure-backoff retries do NOT hold slots: the orchestrator is free
471
+ // to run other work during the exponential-backoff window.
472
+ let pendingContinuations = 0;
473
+ for (const r of this.retryAttempts.values()) {
474
+ if (r.kind === 'continuation')
475
+ pendingContinuations++;
476
+ }
477
+ const admission = this.computeAdmission();
478
+ // Log a single line when the memory clamp transitions in or out of "active." This
479
+ // gives the operator the "why isn't this dispatching" signal in the log without
480
+ // spamming every tick while memory stays low.
481
+ if (admission.clamp_active !== this.memoryClampActive) {
482
+ this.memoryClampActive = admission.clamp_active;
483
+ if (admission.clamp_active) {
484
+ log.info('memory admission clamping concurrency', {
485
+ static_cap: admission.static_cap,
486
+ effective_cap: admission.effective_cap,
487
+ mem_available_mib: admission.mem_available_mib,
488
+ reserve_mib: admission.reserve_mib,
489
+ per_vm_mib: admission.per_vm_mib,
490
+ });
491
+ }
492
+ else {
493
+ log.info('memory admission cleared; full static cap available', {
494
+ static_cap: admission.static_cap,
495
+ mem_available_mib: admission.mem_available_mib,
496
+ });
497
+ }
498
+ }
499
+ return Math.max(0, admission.effective_cap - this.running.size - pendingContinuations);
500
+ }
501
+ /**
502
+ * Compute the current memory-admission snapshot. Reads `/proc/meminfo` via the injected
503
+ * probe (default reads the real file; tests inject a stub). Pure with respect to the
504
+ * orchestrator state — just folds running count + config + probe reading into the
505
+ * dynamic cap. Snapshot endpoint and slot accounting both call through here so they
506
+ * never desync.
507
+ */
508
+ computeAdmission() {
509
+ const staticCap = this.cfg.agent.max_concurrent_agents;
510
+ const reserveMib = this.cfg.agent.host_memory_reserve_mib;
511
+ const perVmMib = this.cfg.gondolin.mem_mib;
512
+ const enabled = this.cfg.agent.memory_admission_enabled;
513
+ const probe = enabled
514
+ ? this.memProbe()
515
+ : { mem_available_mib: null, supported: false };
516
+ const { effective_cap, admission_room, clamp_active } = computeMemoryAdmission({
517
+ enabled,
518
+ static_cap: staticCap,
519
+ running: this.running.size,
520
+ probe,
521
+ reserve_mib: reserveMib,
522
+ per_vm_mib: perVmMib,
523
+ });
524
+ return {
525
+ enabled,
526
+ probe_supported: probe.supported,
527
+ mem_available_mib: probe.mem_available_mib,
528
+ reserve_mib: reserveMib,
529
+ per_vm_mib: perVmMib,
530
+ static_cap: staticCap,
531
+ effective_cap,
532
+ admission_room,
533
+ clamp_active,
534
+ };
535
+ }
536
+ /**
537
+ * Resolve a state's per-state concurrency cap (`states.<name>.max_concurrent`)
538
+ * with a case-insensitive name lookup, mirroring how issue states arrive from
539
+ * the tracker in arbitrary case. Returns undefined when the state is unknown
540
+ * or declares no cap.
541
+ */
542
+ perStateConcurrencyCap(stateName) {
543
+ const lower = stateName.toLowerCase();
544
+ for (const [name, sc] of Object.entries(this.cfg.states)) {
545
+ if (name.toLowerCase() === lower)
546
+ return sc.max_concurrent;
547
+ }
548
+ return undefined;
279
549
  }
280
- /** §8.3: per-state slot accounting using current running entries. */
550
+ /** Per-state slot accounting using current running entries. */
281
551
  hasPerStateSlot(stateName) {
282
- const cap = this.cfg.agent.max_concurrent_agents_by_state[stateName.toLowerCase()];
552
+ // Per-state concurrency lives on the state (issue 137): read
553
+ // `states.<name>.max_concurrent` (case-insensitively). Undefined → no
554
+ // per-state cap, so only the global ceiling applies.
555
+ const cap = this.perStateConcurrencyCap(stateName);
283
556
  if (!cap)
284
557
  return this.availableGlobalSlots() > 0;
285
558
  let inState = 0;
@@ -287,9 +560,17 @@ export class Orchestrator {
287
560
  if (e.issue.state.toLowerCase() === stateName.toLowerCase())
288
561
  inState++;
289
562
  }
563
+ // Mirror the global rule for per-state caps: a pending continuation whose
564
+ // target state matches counts against the state's cap, so the resuming
565
+ // worker is guaranteed a slot when its timer fires.
566
+ for (const r of this.retryAttempts.values()) {
567
+ if (r.kind === 'continuation' && r.target_state.toLowerCase() === stateName.toLowerCase()) {
568
+ inState++;
569
+ }
570
+ }
290
571
  return inState < cap && this.availableGlobalSlots() > 0;
291
572
  }
292
- /** §8.2 sort: priority ASC (null last), then created_at ASC, then identifier. */
573
+ /** Sort: priority ASC (null last), then created_at ASC, then identifier. */
293
574
  sortForDispatch(issues) {
294
575
  return [...issues].sort((a, b) => {
295
576
  const pa = a.priority ?? Number.POSITIVE_INFINITY;
@@ -303,7 +584,7 @@ export class Orchestrator {
303
584
  return a.identifier.localeCompare(b.identifier);
304
585
  });
305
586
  }
306
- /** §16.4 dispatch_issue */
587
+ /** Dispatch one issue. */
307
588
  async dispatchIssue(issue, attempt, snapshot) {
308
589
  if (this.running.has(issue.id))
309
590
  return;
@@ -312,15 +593,26 @@ export class Orchestrator {
312
593
  const cancel = { cancelled: false };
313
594
  const startedAt = new Date().toISOString();
314
595
  const workspacePath = this.workspaces.workspacePathFor(issue.identifier);
315
- // Snapshot tracker.root and the terminal target BEFORE workspace setup,
316
- // before_run, or smolvm bring-up. A WORKFLOW.md reload during that window
317
- // (or even between fetchCandidateIssues returning and this iteration of
318
- // the dispatch loop) can mutate the live tracker config; pinning here closes
319
- // that window. When the caller supplies a snapshot (the tick/retry path
320
- // does — it captured at the fetch atomically), prefer those values; the
321
- // optional fallback reads the live config for completeness.
596
+ // Snapshot tracker.root BEFORE workspace setup or Gondolin VM
597
+ // bring-up. A WORKFLOW.md reload during that window (or even between
598
+ // fetchCandidateIssues returning and this iteration of the dispatch loop)
599
+ // can mutate the live tracker config; pinning here closes that window.
600
+ // When the caller supplies a snapshot (the tick/retry path does — it
601
+ // captured at the fetch atomically), prefer that value; the optional
602
+ // fallback reads the live config for completeness.
322
603
  const trackerRootAtDispatch = snapshot?.trackerRoot ?? (this.tracker.currentRoot ? this.tracker.currentRoot() : null);
323
- const terminalTargetAtDispatch = snapshot?.terminalTarget ?? pickTerminalTarget(this.cfg.tracker.terminal_states);
604
+ // Pin the effective repo + base branch at dispatch time (same window as
605
+ // tracker.root above) so a mid-run WORKFLOW.md reload can't drift the Done
606
+ // action context away from the values the workspace was set up with.
607
+ const githubRepoAtDispatch = resolveGithubRepo(this.cfg.workspace.github_repo);
608
+ const baseBranchAtDispatch = baseBranchName(this.cfg.workspace.base_branch);
609
+ // Resolve "<adapter>/<model or 'default'>" at dispatch time and pin it on
610
+ // the entry. The MCP transition tool stamps this into the notes-block
611
+ // header the next agent reads in `issue.description`. The helper folds any
612
+ // per-state override on top of the workflow defaults; an unknown state
613
+ // falls back to workflow defaults so an older test harness without a
614
+ // states map still produces a non-null actor string.
615
+ const resolvedActor = resolveActorString(this.cfg.states, this.cfg.acp.adapter, this.cfg.acp.model, issue.state);
324
616
  const entry = {
325
617
  issue_id: issue.id,
326
618
  identifier: issue.identifier,
@@ -328,13 +620,13 @@ export class Orchestrator {
328
620
  session_id: null,
329
621
  thread_id: null,
330
622
  turn_id: null,
331
- codex_app_server_pid: null,
332
- last_codex_event: null,
333
- last_codex_timestamp: null,
334
- last_codex_message: null,
335
- codex_input_tokens: 0,
336
- codex_output_tokens: 0,
337
- codex_total_tokens: 0,
623
+ adapter_pid: null,
624
+ last_event: null,
625
+ last_event_at: null,
626
+ last_message: null,
627
+ input_tokens: 0,
628
+ output_tokens: 0,
629
+ total_tokens: 0,
338
630
  last_reported_input_tokens: 0,
339
631
  last_reported_output_tokens: 0,
340
632
  last_reported_total_tokens: 0,
@@ -350,25 +642,114 @@ export class Orchestrator {
350
642
  cleanup_workspace_on_exit: false,
351
643
  mcp_token: null,
352
644
  tracker_root_at_dispatch: trackerRootAtDispatch,
353
- terminal_target_at_dispatch: terminalTargetAtDispatch,
354
- marked_done: false,
645
+ github_repo_at_dispatch: githubRepoAtDispatch,
646
+ base_branch_at_dispatch: baseBranchAtDispatch,
647
+ resolved_actor: resolvedActor,
648
+ transitioned: false,
355
649
  steering_requested: false,
356
650
  steering_question: null,
357
651
  steering_context: null,
652
+ last_transition: null,
358
653
  };
359
654
  this.running.set(issue.id, entry);
360
655
  const logger = withIssue({ issue_id: issue.id, issue_identifier: issue.identifier });
656
+ const runLog = this.ensureRunLog(issue.id, issue.identifier);
657
+ if (runLog) {
658
+ runLog.setAttempt(attempt ?? 0);
659
+ runLog.system('attempt_started', {
660
+ attempt: attempt ?? 0,
661
+ issue_state: issue.state,
662
+ issue_title: issue.title,
663
+ workspace_path: workspacePath,
664
+ tracker_root: trackerRootAtDispatch,
665
+ // Pin the per-state turn budget so the run-summary reducer can report
666
+ // turns-used-vs-budget without re-resolving config (issue 123).
667
+ max_turns: this.resolveStateMaxTurns(issue.state),
668
+ });
669
+ }
361
670
  logger.info('agent attempt started', { attempt });
362
- void this.runWorker(issue, attempt, entry, cancel);
671
+ void this.runWorker(issue, attempt, entry, cancel, runLog);
363
672
  }
364
- async runWorker(issue, attempt, entry, cancelSignal) {
673
+ /**
674
+ * Open (or return the existing) per-issue run log. Returns `undefined` only when log file
675
+ * opening throws — symphony should keep running even if logs can't be persisted, so the
676
+ * runner sees `undefined` and behaves exactly as before.
677
+ *
678
+ * `issueId` (tracker primary key) is stamped on every line and is the map key so the
679
+ * lifecycle survives identifier collisions or renames; `identifier` derives the filename.
680
+ */
681
+ ensureRunLog(issueId, identifier) {
682
+ const existing = this.runLogs.get(issueId);
683
+ if (existing)
684
+ return existing;
685
+ try {
686
+ const rl = openRunLog(this.cfg.logs.root, issueId, identifier);
687
+ this.runLogs.set(issueId, rl);
688
+ return rl;
689
+ }
690
+ catch (err) {
691
+ log.warn('runlog open failed; continuing without run log', {
692
+ issue_id: issueId,
693
+ issue_identifier: identifier,
694
+ error: err.message,
695
+ });
696
+ return undefined;
697
+ }
698
+ }
699
+ /**
700
+ * Resolve the per-state turn budget for the run log's `attempt_started`
701
+ * event. Returns null on any resolution failure (unknown state) — the
702
+ * summary reducer treats a null budget as "unknown", never an error.
703
+ */
704
+ resolveStateMaxTurns(state) {
705
+ try {
706
+ return resolveDispatchConfig(this.cfg, state).max_turns;
707
+ }
708
+ catch {
709
+ return null;
710
+ }
711
+ }
712
+ /**
713
+ * Record the end-of-attempt lifecycle events: the `transition` the agent (or
714
+ * an action reroute) performed during this attempt, if any, followed by
715
+ * `attempt_ended`. Both feed the run-summary reducer (issue 123); recording
716
+ * the transition here — once per attempt, off the hot path — is what makes
717
+ * the state path, rejection notes, and terminal outcome reconstructable.
718
+ */
719
+ recordAttemptEnd(runLog, entry, ok, reason, turnsCompleted) {
720
+ if (entry.last_transition)
721
+ runLog?.system('transition', { ...entry.last_transition });
722
+ runLog?.system('attempt_ended', { ok, reason, turns_completed: turnsCompleted });
723
+ }
724
+ closeRunLog(issueId, fields, opts = {}) {
725
+ // If a terminal cleanup (`workspaces.remove`) is mid-flight for this issue, the
726
+ // terminal-state actions capture may still be writing to the run log. Closing the
727
+ // log here would truncate those lines on disk. Defer until the cleanup's .finally
728
+ // fires the close with `viaCleanup: true`.
729
+ if (!opts.viaCleanup && this.cleanupInFlight.has(issueId))
730
+ return;
731
+ const rl = this.runLogs.get(issueId);
732
+ if (!rl)
733
+ return;
734
+ if (fields)
735
+ rl.system('runlog_closed', fields);
736
+ // Emit the compact per-issue run summary (issue 123) at the terminal unwind,
737
+ // when the lifecycle accumulator holds the full trajectory. Pure over
738
+ // in-memory state, so it precedes (and does not depend on) the stream flush.
739
+ rl.writeSummary();
740
+ this.runLogs.delete(issueId);
741
+ void rl.close();
742
+ }
743
+ async runWorker(issue, attempt, entry, cancelSignal, runLog) {
365
744
  const logger = withIssue({ issue_id: issue.id, issue_identifier: issue.identifier });
366
745
  let ok = false;
367
746
  let reason = 'unknown';
747
+ let turnsCompleted = 0;
368
748
  try {
369
- const result = await this.runner.runAttempt(issue, attempt, cancelSignal, entry);
749
+ const result = await this.runner.runAttempt(issue, attempt, cancelSignal, entry, runLog);
370
750
  ok = result.ok;
371
751
  reason = result.reason;
752
+ turnsCompleted = result.turnsCompleted;
372
753
  if (result.threadId)
373
754
  entry.thread_id = result.threadId;
374
755
  entry.turn_count = result.turnsCompleted;
@@ -378,54 +759,272 @@ export class Orchestrator {
378
759
  reason = err.message;
379
760
  logger.error('worker threw', { error: reason });
380
761
  }
762
+ this.recordAttemptEnd(runLog, entry, ok, reason, turnsCompleted);
381
763
  this.onWorkerExit(issue.id, ok, reason, entry);
382
764
  }
383
- /** §16.6 on_worker_exit */
765
+ /** on_worker_exit */
384
766
  onWorkerExit(issueId, normal, reason, entry) {
385
767
  this.running.delete(issueId);
768
+ // Issue 52: the reconciler `vm` resource is the sole owner of VM teardown.
769
+ // Every worker exit — clean or not — kicks the reaper so the intended set
770
+ // (now excluding this issue) converges in a single pass.
771
+ if (this.reconciler && !this.stopped) {
772
+ void this.reconciler.reapVms().catch((err) => log.debug('post-exit vm reap failed', { error: err.message }));
773
+ }
386
774
  const elapsedMs = Date.now() - Date.parse(entry.started_at);
387
- if (Number.isFinite(elapsedMs)) {
388
- this.codexTotals.seconds_running += elapsedMs / 1000;
389
- }
390
- const identifier = entry.identifier;
391
- const logger = withIssue({ issue_id: issueId, issue_identifier: identifier });
392
- if (entry.cleanup_workspace_on_exit) {
393
- // Workspace removal is deferred until the worker has fully unwound (including
394
- // after_run hook execution) so we never delete the dir while the agent is still
395
- // inside it.
396
- this.workspaces
397
- .remove(entry.identifier, this.cfg.hooks)
398
- .catch((err) => logger.warn('workspace removal failed', { error: err.message }));
399
- }
400
- // §14.2: if the service was stopped while this worker was unwinding, do not schedule
775
+ if (Number.isFinite(elapsedMs))
776
+ this.sessionTotals.seconds_running += elapsedMs / 1000;
777
+ const logger = withIssue({ issue_id: issueId, issue_identifier: entry.identifier });
778
+ if (entry.cleanup_workspace_on_exit)
779
+ this.scheduleWorkspaceCleanup(issueId, entry, logger);
780
+ // If the service was stopped while this worker was unwinding, do not schedule
401
781
  // a new retry — that would leave a live timer behind even though stop() was called.
402
782
  if (this.stopped) {
403
783
  this.claimed.delete(issueId);
784
+ this.circuitBreakers.delete(issueId);
404
785
  return;
405
786
  }
406
- if (normal) {
787
+ if (normal)
407
788
  this.completed.add(issueId);
789
+ this.recordSleepCycleProgress(entry);
790
+ // Circuit breaker (issue 128): a deterministically-failing dispatch (same
791
+ // reason every attempt) would otherwise retry forever under backoff. Trip
792
+ // after the configured streak and route the issue to a holding state
793
+ // instead of scheduling another retry.
794
+ if (this.updateCircuitBreaker(issueId, normal, reason, entry))
795
+ return;
796
+ const plan = decideExitRetry({
797
+ normal,
798
+ reason,
799
+ priorAttempt: entry.retry_attempt,
800
+ targetState: entry.issue.state,
801
+ continuationDelayMs: CONTINUATION_DELAY_MS,
802
+ failureBaseMs: FAILURE_BASE_MS,
803
+ maxBackoffMs: this.cfg.agent.max_retry_backoff_ms,
804
+ });
805
+ if (normal) {
408
806
  logger.info('worker exited (normal)', { reason });
409
- this.scheduleRetry(issueId, {
410
- identifier,
411
- attempt: 1,
412
- delayMs: CONTINUATION_DELAY_MS,
413
- error: null,
414
- });
415
807
  }
416
808
  else {
417
- const nextAttempt = entry.retry_attempt !== null && entry.retry_attempt !== undefined ? entry.retry_attempt + 1 : 1;
418
- const delayMs = Math.min(FAILURE_BASE_MS * Math.pow(2, nextAttempt - 1), this.cfg.agent.max_retry_backoff_ms);
419
- logger.warn('worker exited (abnormal)', { reason, next_attempt: nextAttempt, delay_ms: delayMs });
420
- this.scheduleRetry(issueId, {
421
- identifier,
422
- attempt: nextAttempt,
423
- delayMs,
424
- error: reason,
809
+ logger.warn('worker exited (abnormal)', {
810
+ reason,
811
+ next_attempt: plan.attempt,
812
+ delay_ms: plan.delayMs,
813
+ });
814
+ }
815
+ this.scheduleRetry(issueId, { identifier: entry.identifier, ...plan });
816
+ }
817
+ /**
818
+ * Fold this exit into the per-issue circuit-breaker streak (issue 128) and
819
+ * return true when the breaker tripped — the caller must then NOT schedule a
820
+ * retry. A clean exit clears the streak; an abnormal exit either records the
821
+ * (normalized) failure or, on reaching `agent.circuit_breaker_threshold`
822
+ * consecutive identical failures, trips and fires the holding-state route.
823
+ * The pure `decideCircuitBreaker` owns the counting; this shell just persists
824
+ * the streak and dispatches the side effect.
825
+ */
826
+ updateCircuitBreaker(issueId, normal, reason, entry) {
827
+ const decision = decideCircuitBreaker({
828
+ normal,
829
+ reason,
830
+ prior: this.circuitBreakers.get(issueId) ?? null,
831
+ threshold: this.cfg.agent.circuit_breaker_threshold,
832
+ });
833
+ if (decision.kind === 'continue') {
834
+ this.circuitBreakers.set(issueId, {
835
+ normalizedReason: decision.normalizedReason,
836
+ count: decision.count,
837
+ });
838
+ return false;
839
+ }
840
+ this.circuitBreakers.delete(issueId);
841
+ if (decision.kind === 'trip') {
842
+ void this.tripCircuitBreaker(issueId, entry, reason, decision.count);
843
+ return true;
844
+ }
845
+ return false;
846
+ }
847
+ /**
848
+ * Stop retrying a circuit-broken issue and move it into a holding state so a
849
+ * human sees "stuck on identical failure" on the dashboard rather than a
850
+ * silent multi-hour loop. The move is restart-safe (the file leaves the
851
+ * active set on disk, so the loop cannot resume on the next process start).
852
+ * If routing fails — no `holding` state declared, or the tracker can't write
853
+ * — we keep the issue's dispatch claim so the tick's `already claimed` gate
854
+ * still halts the loop for this session, and log loudly.
855
+ */
856
+ async tripCircuitBreaker(issueId, entry, reason, count) {
857
+ const logger = withIssue({ issue_id: issueId, issue_identifier: entry.identifier });
858
+ this.runLogs.get(issueId)?.system('circuit_breaker_tripped', {
859
+ reason,
860
+ consecutive_failures: count,
861
+ });
862
+ logger.error('circuit breaker tripped; halting retries', { reason, consecutive_failures: count });
863
+ let holdingState;
864
+ try {
865
+ holdingState = pickHoldingState(this.cfg.states);
866
+ }
867
+ catch {
868
+ logger.error('no holding state declared; retaining claim to halt the retry loop', { reason });
869
+ return;
870
+ }
871
+ const moved = await this.routeToHolding(issueId, entry, holdingState, reason, count);
872
+ if (moved) {
873
+ this.claimed.delete(issueId);
874
+ this.closeRunLog(issueId, { reason: 'circuit_breaker_tripped' });
875
+ }
876
+ }
877
+ /**
878
+ * Move `entry`'s tracker file into `holdingState`, appending a diagnostic
879
+ * note (rendered into the issue body before the rename) so the operator sees
880
+ * why it stopped. Returns false when the tracker can't perform the move so
881
+ * the caller can fall back to retaining the claim. Modelled on the runner's
882
+ * action-reroute path; the orchestrator owns this move because it is
883
+ * state-machine behavior, not repo-local glue.
884
+ */
885
+ async routeToHolding(issueId, entry, holdingState, reason, count) {
886
+ if (!this.tracker.moveIssueToState)
887
+ return false;
888
+ const notes = [
889
+ `**Circuit breaker tripped** — routed to \`${holdingState}\` for human inspection.`,
890
+ '',
891
+ `Symphony stopped retrying after **${count} consecutive attempts failed with the same error**, to avoid an unbounded dispatch loop (issue 128).`,
892
+ '',
893
+ `**Last failure reason:** ${reason}`,
894
+ '',
895
+ `Resolve the underlying cause, then move the issue back into an active state to resume dispatch.`,
896
+ ].join('\n');
897
+ try {
898
+ await this.tracker.moveIssueToState(issueId, holdingState, {
899
+ fromRoot: entry.tracker_root_at_dispatch ?? undefined,
900
+ fromState: entry.issue.state,
901
+ notes,
902
+ actor: entry.resolved_actor,
425
903
  });
904
+ return true;
905
+ }
906
+ catch (err) {
907
+ withIssue({ issue_id: issueId, issue_identifier: entry.identifier }).error('circuit breaker route to holding failed; retaining claim', { error: err.message });
908
+ return false;
426
909
  }
427
910
  }
428
- /** §8.4 retry queue. */
911
+ /**
912
+ * Sleep-cycle auto-arm (issue 125): fold a finished attempt into the
913
+ * terminal-transition counter the triggers read. Only the work the reflector
914
+ * mines counts — a transition into a `role: terminal` state (Done/Cancelled)
915
+ * — and the reflection issue's own moves (it can only go to its holding
916
+ * dormant state) are excluded so a reflection run never counts toward arming
917
+ * the next one. No-op when the block is disabled.
918
+ */
919
+ recordSleepCycleProgress(entry) {
920
+ const arm = deriveArmRouting(this.cfg.states);
921
+ if (!arm.issue)
922
+ return;
923
+ if (entry.issue_id === arm.issue || entry.identifier === arm.issue)
924
+ return;
925
+ if (entry.last_transition?.terminal)
926
+ this.doneSinceReflect += 1;
927
+ }
928
+ /**
929
+ * Sleep-cycle auto-arm (issue 125): decide — purely — whether to arm the
930
+ * reflection cycle this tick, and fire the async Dormant → Reflect move when
931
+ * a trigger fires. `noActiveCandidates` is true when this poll surfaced no
932
+ * active-state issues; combined with empty running/claimed/retry sets that is
933
+ * the orchestrator-idle signal. The in-flight guard prevents two overlapping
934
+ * ticks from both launching the move.
935
+ */
936
+ maybeArmSleepCycle(noActiveCandidates) {
937
+ const arm = deriveArmRouting(this.cfg.states);
938
+ if (!arm.armState || this.armingReflection)
939
+ return;
940
+ const idle = noActiveCandidates &&
941
+ this.running.size === 0 &&
942
+ this.claimed.size === 0 &&
943
+ this.retryAttempts.size === 0;
944
+ const trigger = decideSleepCycleArm({
945
+ enabled: true,
946
+ issueId: arm.issue,
947
+ armOnIdle: arm.onIdle,
948
+ armAfterDone: arm.afterTerminal,
949
+ doneSinceReflect: this.doneSinceReflect,
950
+ idle,
951
+ });
952
+ if (!trigger)
953
+ return;
954
+ this.armingReflection = true;
955
+ void this.armReflection(arm, trigger).finally(() => {
956
+ this.armingReflection = false;
957
+ });
958
+ }
959
+ /**
960
+ * Move the reflection issue from its dormant (holding) state into the active
961
+ * reflect state. Guarded so it is a no-op unless the issue currently lives in
962
+ * `dormant_state` (so we never yank it out of Reflect mid-run or relocate it
963
+ * from some other state), and the counter only resets on a successful move so
964
+ * a failed arm doesn't silently discard the accumulated terminal count.
965
+ */
966
+ async armReflection(arm, trigger) {
967
+ if (!this.tracker.moveIssueToState || !arm.issue || !arm.from || !arm.armState)
968
+ return;
969
+ const current = await this.fetchReflectionIssue(arm.issue);
970
+ if (!current)
971
+ return;
972
+ if (current.state.toLowerCase() !== arm.from.toLowerCase())
973
+ return;
974
+ if (this.running.has(current.id) || this.claimed.has(current.id))
975
+ return;
976
+ const count = this.doneSinceReflect;
977
+ try {
978
+ await this.tracker.moveIssueToState(current.id, arm.armState, {
979
+ fromState: arm.from,
980
+ notes: sleepCycleArmNotes(trigger, count, arm.afterTerminal),
981
+ actor: SLEEP_CYCLE_ACTOR,
982
+ });
983
+ this.doneSinceReflect = 0;
984
+ log.info('sleep cycle armed reflection', {
985
+ issue_id: current.id,
986
+ trigger,
987
+ done_since_reflect: count,
988
+ reflect_state: arm.armState,
989
+ });
990
+ }
991
+ catch (err) {
992
+ log.warn('sleep cycle arm failed', {
993
+ issue_id: current.id,
994
+ error: err.message,
995
+ });
996
+ }
997
+ }
998
+ /** Look up the reflection issue's current tracker state; undefined on miss/error. */
999
+ async fetchReflectionIssue(issueId) {
1000
+ try {
1001
+ const found = await this.tracker.fetchIssueStatesByIds([issueId]);
1002
+ return found.find((i) => i.id === issueId || i.identifier === issueId);
1003
+ }
1004
+ catch (err) {
1005
+ log.debug('sleep cycle reflection-issue lookup failed', {
1006
+ issue_id: issueId,
1007
+ error: err.message,
1008
+ });
1009
+ return undefined;
1010
+ }
1011
+ }
1012
+ /**
1013
+ * Workspace removal deferred until the worker has fully unwound (including the
1014
+ * terminal-state `actions:` block) so we never delete the dir while the agent
1015
+ * is still inside it.
1016
+ */
1017
+ scheduleWorkspaceCleanup(issueId, entry, logger) {
1018
+ this.cleanupInFlight.add(issueId);
1019
+ this.workspaces
1020
+ .remove(entry.identifier)
1021
+ .catch((err) => logger.warn('workspace removal failed', { error: err.message }))
1022
+ .finally(() => {
1023
+ this.cleanupInFlight.delete(issueId);
1024
+ this.closeRunLog(issueId, { reason: 'cleanup_on_exit' }, { viaCleanup: true });
1025
+ });
1026
+ }
1027
+ /** Retry queue. */
429
1028
  scheduleRetry(issueId, sched) {
430
1029
  if (this.stopped)
431
1030
  return;
@@ -441,10 +1040,12 @@ export class Orchestrator {
441
1040
  due_at_ms: dueAt,
442
1041
  timer_handle: handle,
443
1042
  error: sched.error,
1043
+ kind: sched.kind,
1044
+ target_state: sched.target_state,
444
1045
  });
445
1046
  this.claimed.add(issueId);
446
1047
  }
447
- /** §16.6 on_retry_timer */
1048
+ /** on_retry_timer */
448
1049
  async onRetryTimer(issueId) {
449
1050
  if (this.stopped)
450
1051
  return;
@@ -452,14 +1053,30 @@ export class Orchestrator {
452
1053
  if (!entry)
453
1054
  return;
454
1055
  this.retryAttempts.delete(issueId);
455
- let candidates;
456
- let snapshotTrackerRoot;
457
- let snapshotTerminalTarget;
1056
+ const fetched = await this.fetchRetryCandidates(issueId, entry);
1057
+ if (!fetched)
1058
+ return;
1059
+ const issue = fetched.issues.find((i) => i.id === issueId);
1060
+ if (!issue) {
1061
+ this.releaseRetryClaim(issueId, entry.identifier, 'not_in_candidates');
1062
+ return;
1063
+ }
1064
+ const reason = this.eligibilityReason(issue, true);
1065
+ if (reason !== null) {
1066
+ this.handleRetryIneligible(issue, entry, reason);
1067
+ return;
1068
+ }
1069
+ void this.dispatchIssue(issue, entry.attempt, { trackerRoot: fetched.root });
1070
+ }
1071
+ /**
1072
+ * Tracker poll for the retry timer. Returns the snapshot or null when the
1073
+ * fetch failed (a failure-shaped retry is rescheduled internally so the
1074
+ * caller can just bail).
1075
+ */
1076
+ async fetchRetryCandidates(issueId, entry) {
458
1077
  try {
459
- const result = await this.tracker.fetchCandidateIssues();
460
- candidates = result.issues;
461
- snapshotTrackerRoot = result.root;
462
- snapshotTerminalTarget = pickTerminalTarget(result.terminalStates);
1078
+ const r = await this.tracker.fetchCandidateIssues();
1079
+ return { issues: r.issues, root: r.root };
463
1080
  }
464
1081
  catch (err) {
465
1082
  log.debug('retry poll failed', {
@@ -472,96 +1089,355 @@ export class Orchestrator {
472
1089
  attempt: entry.attempt + 1,
473
1090
  delayMs: Math.min(FAILURE_BASE_MS * Math.pow(2, entry.attempt), this.cfg.agent.max_retry_backoff_ms),
474
1091
  error: 'retry poll failed',
1092
+ kind: 'failure',
1093
+ target_state: entry.target_state,
475
1094
  });
476
- return;
1095
+ return null;
477
1096
  }
478
- const issue = candidates.find((i) => i.id === issueId);
479
- if (!issue) {
480
- this.claimed.delete(issueId);
481
- log.info('retry releasing claim (not in candidates)', {
482
- issue_id: issueId,
483
- issue_identifier: entry.identifier,
484
- });
1097
+ }
1098
+ /**
1099
+ * Re-applied candidate eligibility came back non-null. The pure
1100
+ * `decideRetryAfterIneligible` picks between rescheduling (only `no
1101
+ * per-state slot`, which is genuine contention) and releasing the claim
1102
+ * (everything else: blocker, missing fields, non-active state).
1103
+ */
1104
+ handleRetryIneligible(issue, entry, reason) {
1105
+ const action = decideRetryAfterIneligible({
1106
+ reason,
1107
+ priorAttempt: entry.attempt,
1108
+ targetState: issue.state,
1109
+ failureBaseMs: FAILURE_BASE_MS,
1110
+ maxBackoffMs: this.cfg.agent.max_retry_backoff_ms,
1111
+ });
1112
+ if (action.kind === 'release') {
1113
+ this.releaseRetryClaim(issue.id, entry.identifier, `ineligible:${reason}`, reason);
485
1114
  return;
486
1115
  }
487
- // Re-apply full candidate eligibility, ignoring this issue's own claim. This catches
488
- // late-breaking issues like a new non-terminal blocker on a Todo or a state change to
489
- // an inactive value that slipped past the candidate filter on edge tracker shapes.
490
- const reason = this.eligibilityReason(issue, true);
491
- if (reason !== null) {
492
- if (reason === 'no per-state slot') {
493
- this.scheduleRetry(issueId, {
494
- identifier: issue.identifier,
495
- attempt: entry.attempt + 1,
496
- delayMs: Math.min(FAILURE_BASE_MS * Math.pow(2, entry.attempt), this.cfg.agent.max_retry_backoff_ms),
497
- error: 'no available orchestrator slots',
498
- });
1116
+ this.scheduleRetry(issue.id, { identifier: issue.identifier, ...action.plan });
1117
+ }
1118
+ releaseRetryClaim(issueId, identifier, closeTag, ineligibleReason) {
1119
+ this.claimed.delete(issueId);
1120
+ this.circuitBreakers.delete(issueId);
1121
+ log.info(ineligibleReason
1122
+ ? 'retry releasing claim (ineligible)'
1123
+ : 'retry releasing claim (not in candidates)', { issue_id: issueId, issue_identifier: identifier, ...(ineligibleReason ? { reason: ineligibleReason } : {}) });
1124
+ this.closeRunLog(issueId, { reason: `claim_released_${closeTag}` });
1125
+ }
1126
+ /**
1127
+ * Implements {@link IntendedVmProvider}. Returns the set of `symphony-*` VM
1128
+ * names the orchestrator currently intends to keep alive — one per running
1129
+ * dispatch. Used by the reconciler's vm resource to compute the orphan set
1130
+ * to reap. `running.set` happens BEFORE the runner calls `createVm`, so a VM
1131
+ * that exists in Gondolin's session registry as part of an in-flight create
1132
+ * is always already represented
1133
+ * here. The reaper sees it as intended and leaves it alone, closing the
1134
+ * "creating-but-not-yet-active" race the issue body calls out.
1135
+ */
1136
+ intendedVmNames() {
1137
+ const out = new Set();
1138
+ for (const entry of this.running.values()) {
1139
+ out.add(this.runner.vmNameFor(entry.issue));
1140
+ }
1141
+ return out;
1142
+ }
1143
+ /**
1144
+ * Implements {@link WorkspaceIntendedProvider}. Returns the map of
1145
+ * identifier → state the reconciler should preserve workspaces for. Two
1146
+ * sources are unioned:
1147
+ *
1148
+ * • Tracker view: every issue file in a non-terminal state. Anything
1149
+ * terminal (Done, Cancelled) is fair game for removal — this replaces
1150
+ * the old `startupTerminalCleanup` sweep with a continuous pass.
1151
+ * • In-flight allocations: running entries plus claimed/pending retries.
1152
+ * The window between dispatch claiming an issue and the tracker
1153
+ * reflecting it is brief but real; without this, a fresh dispatch's
1154
+ * workspace could be reaped seconds after creation.
1155
+ *
1156
+ * The state value is carried so the reconciler's `create` callback can apply
1157
+ * the merge-state guard (it skips eager recreation of a workspace whose issue
1158
+ * sits in the autopilot's merge state).
1159
+ *
1160
+ * Tracker errors propagate. Catching them here would cause an empty set
1161
+ * to be returned, which the reconciler would treat as authoritative and
1162
+ * reap every workspace — the regression this contract closes. The
1163
+ * resource's `reconcile()` catches the throw and leaves on-disk state
1164
+ * untouched until the next pass.
1165
+ *
1166
+ * Mirrors `intendedVmNames()` in shape so the reconciler's race-condition
1167
+ * reasoning is the same across both janitors.
1168
+ */
1169
+ async activeIdentifiers() {
1170
+ const out = new Map();
1171
+ const nonTerminal = [];
1172
+ for (const [name, cfg] of Object.entries(this.cfg.states)) {
1173
+ if (cfg.role !== 'terminal')
1174
+ nonTerminal.push(name);
1175
+ }
1176
+ const issues = await this.tracker.fetchIssuesByStates(nonTerminal);
1177
+ for (const i of issues)
1178
+ out.set(i.identifier, i.state);
1179
+ // Issue 38/139: when the PR engine is enabled, the merge-state issues'
1180
+ // workspaces are owned by the pr resource (it rebases inside them and cleans
1181
+ // them up post-merge). Include those identifiers in the desired set so the
1182
+ // workspace janitor doesn't reap a workspace the pr resource is actively
1183
+ // driving. The merge state is derived by scanning states for `pr.auto_merge`
1184
+ // (no named-string sibling block). The orchestrator's `createWorkspace`
1185
+ // callback declines to eagerly recreate a missing merge-state workspace — so
1186
+ // adding it here is safe: the workspace either already exists from the
1187
+ // dispatch that ran the issue into the merge state, or it doesn't and the
1188
+ // autopilot just skips the rebase step for that PR.
1189
+ if (this.cfg.pr.enabled) {
1190
+ const mergeState = derivePrRouting(this.cfg.states).mergeState;
1191
+ if (mergeState) {
1192
+ const mergeIssues = await this.tracker.fetchIssuesByStates([mergeState]);
1193
+ for (const i of mergeIssues)
1194
+ out.set(i.identifier, i.state);
1195
+ }
1196
+ }
1197
+ return out;
1198
+ }
1199
+ /**
1200
+ * Identifiers the orchestrator has claimed for dispatch but the tracker may
1201
+ * not yet reflect as active, with the state the eager workspace create should
1202
+ * see (used for the merge-state guard). Running entries carry the state the
1203
+ * dispatch was claimed from (`issue.state`); pending retries carry their
1204
+ * `target_state` (where the next attempt will run).
1205
+ */
1206
+ inFlightIdentifiers() {
1207
+ const out = new Map();
1208
+ for (const e of this.running.values())
1209
+ out.set(e.identifier, e.issue.state);
1210
+ for (const r of this.retryAttempts.values())
1211
+ out.set(r.identifier, r.target_state);
1212
+ return out;
1213
+ }
1214
+ /**
1215
+ * Implements {@link ProposeFollowupSink} for the action executor's
1216
+ * `propose_followup` action (issue 36). Same tracker shape as the MCP
1217
+ * `propose_issue` tool — file lands in the first declared `holding` state,
1218
+ * with `proposed_by` set to the parent issue's identifier. Uses the live
1219
+ * tracker root (passed-in parent identifier is the canonical attribution).
1220
+ */
1221
+ async proposeFollowup(input) {
1222
+ const root = this.tracker.currentRoot ? this.tracker.currentRoot() : this.cfg.tracker.root;
1223
+ if (!root) {
1224
+ throw new Error('tracker root not available; cannot file propose_followup');
1225
+ }
1226
+ const landingState = pickHoldingState(this.cfg.states);
1227
+ const result = await writeIssueFile({
1228
+ trackerRoot: root,
1229
+ state: landingState,
1230
+ title: input.title,
1231
+ description: input.description ?? '',
1232
+ priority: input.priority ?? null,
1233
+ labels: input.labels ?? [],
1234
+ now: () => Date.now(),
1235
+ extra_front_matter: {
1236
+ proposed_by: input.parent_identifier,
1237
+ proposed_at: new Date().toISOString(),
1238
+ },
1239
+ });
1240
+ log.info('action propose_followup', {
1241
+ proposed_by: input.parent_identifier,
1242
+ identifier: result.identifier,
1243
+ state: result.state,
1244
+ });
1245
+ return { identifier: result.identifier };
1246
+ }
1247
+ /**
1248
+ * Receive a per-attempt action ledger from the runner's cleanup pass. The
1249
+ * snapshot is keyed by state so the dashboard can render "Done.actions:
1250
+ * push_branch ok, create_pr_if_missing rate-limited, retrying" without the
1251
+ * orchestrator having to know about specific action kinds.
1252
+ *
1253
+ * `id` should follow the `actions:<StateName>` convention so it sorts
1254
+ * predictably next to reconciler-resource rows in
1255
+ * `snapshot.reconciler.resources`.
1256
+ */
1257
+ recordActionResult(id, snapshot) {
1258
+ this.lastActionResults.set(id, snapshot);
1259
+ }
1260
+ /**
1261
+ * Workspace removal callback the reconciler invokes for stale dirs (issue
1262
+ * 34). Defers to `WorkspaceManager.remove` (a best-effort `rm -rf`).
1263
+ * Failures are logged at warn (the reconciler's action ledger also records
1264
+ * them).
1265
+ */
1266
+ async removeWorkspace(identifier) {
1267
+ await this.workspaces.remove(identifier);
1268
+ }
1269
+ /**
1270
+ * Workspace create callback the reconciler invokes for non-terminal issues
1271
+ * whose dirs are not yet on disk (issue 34). Delegates to
1272
+ * `WorkspaceManager.ensureFor` so the same canonical clone+branch+remote
1273
+ * setup the dispatch path runs also fires here.
1274
+ *
1275
+ * The intended-set provider supplies the state alongside the identifier so
1276
+ * the merge-state guard below can fire; the `null` fallback is defensive —
1277
+ * production callers always pass a state.
1278
+ *
1279
+ * Race with the runner's dispatch-time `ensureFor` is handled inside
1280
+ * `WorkspaceManager` via a per-identifier in-flight promise lock: both
1281
+ * callers coalesce into one setup pass, so the canonical setup runs exactly
1282
+ * once whether the reconciler or the runner wins the race.
1283
+ */
1284
+ async createWorkspace(identifier, state) {
1285
+ // Issue 38: refuse to eagerly recreate a missing workspace for an issue
1286
+ // in the autopilot's merge state. Those workspaces only exist as
1287
+ // leftovers from a prior dispatch; recreating one from scratch would
1288
+ // miss the agent's local commits (the agent's branch is on the remote,
1289
+ // but a fresh clone would still need a separate fetch to pick it up).
1290
+ // Operators who genuinely want a recreated workspace can cancel the
1291
+ // issue (Cancelled triggers the close path + normal cleanup) and refile.
1292
+ if (this.cfg.pr.enabled && state !== null) {
1293
+ const mergeState = derivePrRouting(this.cfg.states).mergeState;
1294
+ if (mergeState && state.toLowerCase() === mergeState.toLowerCase()) {
499
1295
  return;
500
1296
  }
501
- // For non-slot reasons (blocker, missing fields, non-active state), the right action
502
- // is to release the claim rather than spin on it.
503
- this.claimed.delete(issueId);
504
- log.info('retry releasing claim (ineligible)', {
505
- issue_id: issueId,
506
- issue_identifier: entry.identifier,
507
- reason,
1297
+ }
1298
+ await this.workspaces.ensureFor(identifier);
1299
+ }
1300
+ /**
1301
+ * Implements {@link PrIntendedProvider} (issue 38). Returns the set of
1302
+ * terminal-state issues the PR autopilot should manage:
1303
+ *
1304
+ * • Issues in the configured `merge_state` (default `Done`) become
1305
+ * `kind: 'merge'` intents. The autopilot rebases them on
1306
+ * `origin/<base>` and arms GitHub auto-merge.
1307
+ * • Issues in the configured `close_state` (default `Cancelled`) become
1308
+ * `kind: 'close'` intents. The autopilot closes the PR without merge
1309
+ * and best-effort-deletes the remote branch. The workspace is NOT
1310
+ * supplied — Cancelled cleanup goes through the orchestrator's
1311
+ * standard terminal path.
1312
+ *
1313
+ * Both queries hit the tracker; failures bubble (the pr resource catches
1314
+ * and surfaces in last_error so a transient tracker hiccup doesn't blank
1315
+ * the autopilot's intended set).
1316
+ *
1317
+ * When `pr.enabled` is false this method is never invoked (the reconciler
1318
+ * skips its pr pass entirely), but the early return keeps the public surface
1319
+ * idempotent. The merge/close targets are derived by scanning states for the
1320
+ * per-state `pr:` field (issue 139), not read from named strings.
1321
+ */
1322
+ async prIntended() {
1323
+ if (!this.cfg.pr.enabled)
1324
+ return [];
1325
+ const { mergeState, closeState } = derivePrRouting(this.cfg.states);
1326
+ if (!mergeState && !closeState)
1327
+ return [];
1328
+ const baseBranch = baseBranchName(this.cfg.workspace.base_branch);
1329
+ const fetchStates = [mergeState, closeState].filter((s) => s !== null);
1330
+ const issues = await this.tracker.fetchIssuesByStates(fetchStates);
1331
+ const out = [];
1332
+ for (const issue of issues) {
1333
+ const intent = classifyPrIntent({
1334
+ issue,
1335
+ mergeState: mergeState ?? '',
1336
+ closeState,
1337
+ baseBranch,
1338
+ mergeWorkspacePath: this.workspaces.workspacePathFor(issue.identifier),
508
1339
  });
509
- return;
1340
+ if (intent)
1341
+ out.push(intent);
510
1342
  }
511
- void this.dispatchIssue(issue, entry.attempt, {
512
- trackerRoot: snapshotTrackerRoot,
513
- terminalTarget: snapshotTerminalTarget,
514
- });
1343
+ return out;
515
1344
  }
516
- /** §8.6 startup terminal workspace cleanup. */
517
- async startupTerminalCleanup() {
1345
+ /**
1346
+ * Tracker-side transition the PR autopilot uses to route a conflict-rebasing
1347
+ * issue back into the implementing state (or, after exceeding the attempt
1348
+ * limit, into the holding state). Same shape as the MCP transition tool —
1349
+ * the tracker handles atomic notes-append + cross-directory rename.
1350
+ *
1351
+ * No workspace flag is touched here: the target state's `role` decides
1352
+ * cleanup at the transition's own level (active/holding never trigger
1353
+ * cleanup), and the workspace was preserved across the move into the merge
1354
+ * state in the first place because the PR engine suppresses the terminal
1355
+ * cleanup for that target. See {@link McpRegistry.performTransition} for the
1356
+ * role-driven rule.
1357
+ */
1358
+ async routeIssueForAutopilot(input) {
1359
+ if (!this.tracker.moveIssueToState) {
1360
+ throw new Error('tracker does not support state transitions');
1361
+ }
1362
+ // The tracker file's `id` may diverge from the identifier when the file
1363
+ // sets an explicit front-matter `id`. Resolve via a candidate scan so the
1364
+ // tracker can find the right file even with that aliasing.
1365
+ let issueId = null;
518
1366
  try {
519
- const terminals = await this.tracker.fetchIssuesByStates(this.cfg.tracker.terminal_states);
520
- for (const issue of terminals) {
521
- try {
522
- await this.workspaces.remove(issue.identifier, this.cfg.hooks);
523
- }
524
- catch (err) {
525
- log.warn('terminal cleanup failed for issue', {
526
- issue_identifier: issue.identifier,
527
- error: err.message,
528
- });
529
- }
530
- }
1367
+ const candidates = await this.tracker.fetchIssuesByStates([input.fromState]);
1368
+ const match = candidates.find((c) => c.identifier === input.identifier);
1369
+ if (match)
1370
+ issueId = match.id;
531
1371
  }
532
- catch (err) {
533
- log.warn('startup terminal cleanup fetch failed', { error: err.message });
1372
+ catch {
1373
+ // Fall through to identifier fallback below.
534
1374
  }
1375
+ if (issueId === null)
1376
+ issueId = input.identifier;
1377
+ await this.tracker.moveIssueToState(issueId, input.toState, {
1378
+ fromState: input.fromState,
1379
+ notes: input.notes,
1380
+ actor: input.actor,
1381
+ });
1382
+ }
1383
+ /**
1384
+ * Implements {@link BaseRefProvider}. Returns the configured base branch
1385
+ * name AND its current SHA in the source repo (workflow_dir by default).
1386
+ * Returns null when the SHA can't be resolved (no `.git`, base branch
1387
+ * missing, etc.) — drift detection skips the pass.
1388
+ *
1389
+ * Why both fields: the reconciler's drift check compares the workspace's
1390
+ * own copy of `<branch>` (frozen at clone time) against this SHA. Returning
1391
+ * the branch name keeps the source-of-truth in one place; the inspector
1392
+ * uses it to run `git rev-parse <branch>` inside the workspace.
1393
+ *
1394
+ * The base branch (`workspace.base_branch`, or the `SYMPHONY_BASE_BRANCH` env
1395
+ * override, default `main`) is resolved the same way the dispatch-time clone
1396
+ * does, so the drift check compares against the same ref the workspace was
1397
+ * originally cloned from.
1398
+ */
1399
+ async currentBaseRef() {
1400
+ const branch = process.env.SYMPHONY_BASE_BRANCH && process.env.SYMPHONY_BASE_BRANCH.length > 0
1401
+ ? process.env.SYMPHONY_BASE_BRANCH
1402
+ : this.cfg.workspace.base_branch;
1403
+ const sourceRepo = process.env.SYMPHONY_SOURCE_REPO && process.env.SYMPHONY_SOURCE_REPO.length > 0
1404
+ ? process.env.SYMPHONY_SOURCE_REPO
1405
+ : this.cfg.workflow_dir;
1406
+ const r = await runProcess('git', ['rev-parse', branch], { cwd: sourceRepo });
1407
+ if (r.exit_code !== 0)
1408
+ return null;
1409
+ const sha = r.stdout.trim();
1410
+ return sha.length > 0 ? { branch, sha } : null;
535
1411
  }
536
- // Public hooks the runner uses to feed events back.
1412
+ // Public callbacks the runner uses to feed events back.
537
1413
  reportTokenUsage(issueId, usage) {
538
1414
  const e = this.running.get(issueId);
539
1415
  if (!e)
540
1416
  return;
541
- // §13.5: prefer absolute totals; track deltas to avoid double-counting.
1417
+ // §9.4: prefer absolute totals; track deltas to avoid double-counting.
542
1418
  const dIn = Math.max(0, usage.input_tokens - e.last_reported_input_tokens);
543
1419
  const dOut = Math.max(0, usage.output_tokens - e.last_reported_output_tokens);
544
1420
  const dTot = Math.max(0, usage.total_tokens - e.last_reported_total_tokens);
545
- e.codex_input_tokens = usage.input_tokens;
546
- e.codex_output_tokens = usage.output_tokens;
547
- e.codex_total_tokens = usage.total_tokens;
1421
+ e.input_tokens = usage.input_tokens;
1422
+ e.output_tokens = usage.output_tokens;
1423
+ e.total_tokens = usage.total_tokens;
548
1424
  e.last_reported_input_tokens = usage.input_tokens;
549
1425
  e.last_reported_output_tokens = usage.output_tokens;
550
1426
  e.last_reported_total_tokens = usage.total_tokens;
551
- this.codexTotals.input_tokens += dIn;
552
- this.codexTotals.output_tokens += dOut;
553
- this.codexTotals.total_tokens += dTot;
1427
+ this.sessionTotals.input_tokens += dIn;
1428
+ this.sessionTotals.output_tokens += dOut;
1429
+ this.sessionTotals.total_tokens += dTot;
554
1430
  }
555
1431
  reportRateLimits(_issueId, snapshot) {
556
- this.codexRateLimits = snapshot;
1432
+ this.rateLimits = snapshot;
557
1433
  }
558
1434
  reportRuntimeEvent(issueId, ev) {
559
1435
  const e = this.running.get(issueId);
560
1436
  if (!e)
561
1437
  return;
562
- e.last_codex_event = ev.event;
563
- e.last_codex_timestamp = ev.at;
564
- e.last_codex_message = ev.message;
1438
+ e.last_event = ev.event;
1439
+ e.last_event_at = ev.at;
1440
+ e.last_message = ev.message;
565
1441
  e.recent_events.push(ev);
566
1442
  if (e.recent_events.length > 50)
567
1443
  e.recent_events.shift();
@@ -572,7 +1448,7 @@ export class Orchestrator {
572
1448
  return;
573
1449
  e.session_id = info.sessionId;
574
1450
  e.thread_id = info.threadId;
575
- e.codex_app_server_pid = info.pid;
1451
+ e.adapter_pid = info.pid;
576
1452
  }
577
1453
  reportTurnStarted(issueId, turnNumber) {
578
1454
  const e = this.running.get(issueId);
@@ -580,7 +1456,7 @@ export class Orchestrator {
580
1456
  return;
581
1457
  e.turn_count = turnNumber;
582
1458
  }
583
- /** §13.3 snapshot. */
1459
+ /** §9.3 snapshot. */
584
1460
  snapshot() {
585
1461
  const generatedAt = new Date().toISOString();
586
1462
  const liveExtraSeconds = [...this.running.values()]
@@ -597,19 +1473,19 @@ export class Orchestrator {
597
1473
  state: e.issue.state,
598
1474
  session_id: e.session_id,
599
1475
  turn_count: e.turn_count,
600
- last_event: e.last_codex_event,
601
- last_message: e.last_codex_message,
1476
+ last_event: e.last_event,
1477
+ last_message: e.last_message,
602
1478
  started_at: e.started_at,
603
- last_event_at: e.last_codex_timestamp,
1479
+ last_event_at: e.last_event_at,
604
1480
  tokens: {
605
- input_tokens: e.codex_input_tokens,
606
- output_tokens: e.codex_output_tokens,
607
- total_tokens: e.codex_total_tokens,
1481
+ input_tokens: e.input_tokens,
1482
+ output_tokens: e.output_tokens,
1483
+ total_tokens: e.total_tokens,
608
1484
  },
609
1485
  steering_requested: e.steering_requested,
610
1486
  steering_question: e.steering_question,
611
1487
  steering_context: e.steering_context,
612
- marked_done: e.marked_done,
1488
+ transitioned: e.transitioned,
613
1489
  })),
614
1490
  retrying: [...this.retryAttempts.values()].map((r) => ({
615
1491
  issue_id: r.issue_id,
@@ -617,67 +1493,77 @@ export class Orchestrator {
617
1493
  attempt: r.attempt,
618
1494
  due_at: new Date(r.due_at_ms).toISOString(),
619
1495
  error: r.error,
1496
+ kind: r.kind,
620
1497
  })),
621
- codex_totals: {
622
- ...this.codexTotals,
623
- seconds_running: this.codexTotals.seconds_running + liveExtraSeconds,
1498
+ session_totals: {
1499
+ ...this.sessionTotals,
1500
+ seconds_running: this.sessionTotals.seconds_running + liveExtraSeconds,
624
1501
  },
625
- rate_limits: this.codexRateLimits,
1502
+ rate_limits: this.rateLimits,
1503
+ memory_admission: this.computeAdmission(),
1504
+ reconciler: this.buildReconcilerSnapshot(),
626
1505
  };
627
1506
  }
1507
+ /**
1508
+ * Combine the reconciler's resource snapshot with the most recent action
1509
+ * results so the dashboard sees both surfaces under one `reconciler.resources`
1510
+ * list. When neither side has anything to report (no reconciler wired AND no
1511
+ * actions ever ran), the field is null to preserve the existing test
1512
+ * harness's "no reconciler" shape.
1513
+ */
1514
+ buildReconcilerSnapshot() {
1515
+ const base = this.reconciler ? this.reconciler.snapshot() : null;
1516
+ if (!base && this.lastActionResults.size === 0)
1517
+ return null;
1518
+ const resources = base ? [...base.resources] : [];
1519
+ for (const snap of this.lastActionResults.values())
1520
+ resources.push(snap);
1521
+ return { resources };
1522
+ }
628
1523
  /** Issue-detail view used by the HTTP /api/v1/<identifier> endpoint. */
629
1524
  detailByIdentifier(identifier) {
630
- let entry = null;
631
- for (const e of this.running.values()) {
632
- if (e.identifier === identifier) {
633
- entry = e;
634
- break;
1525
+ const entry = this.findRunningByIdentifier(identifier);
1526
+ const retry = this.findRetryByIdentifier(identifier);
1527
+ return buildIssueDetailDto(identifier, entry
1528
+ ? {
1529
+ issue_id: entry.issue_id,
1530
+ identifier: entry.identifier,
1531
+ workspace_path: entry.workspace_path,
1532
+ session_id: entry.session_id,
1533
+ turn_count: entry.turn_count,
1534
+ state: entry.issue.state,
1535
+ started_at: entry.started_at,
1536
+ last_event: entry.last_event,
1537
+ last_message: entry.last_message,
1538
+ last_event_at: entry.last_event_at,
1539
+ input_tokens: entry.input_tokens,
1540
+ output_tokens: entry.output_tokens,
1541
+ total_tokens: entry.total_tokens,
1542
+ recent_events: entry.recent_events,
1543
+ last_error: entry.last_error,
635
1544
  }
636
- }
637
- let retry = null;
638
- for (const r of this.retryAttempts.values()) {
639
- if (r.identifier === identifier) {
640
- retry = r;
641
- break;
1545
+ : null, retry
1546
+ ? {
1547
+ issue_id: retry.issue_id,
1548
+ identifier: retry.identifier,
1549
+ attempt: retry.attempt,
1550
+ due_at_ms: retry.due_at_ms,
1551
+ error: retry.error,
1552
+ kind: retry.kind,
642
1553
  }
643
- }
644
- if (!entry && !retry)
645
- return null;
646
- return {
647
- issue_identifier: identifier,
648
- issue_id: entry?.issue_id ?? retry?.issue_id ?? null,
649
- status: entry ? 'running' : 'retrying',
650
- workspace: entry ? { path: entry.workspace_path } : null,
651
- attempts: {
652
- current_retry_attempt: retry?.attempt ?? null,
653
- },
654
- running: entry
655
- ? {
656
- session_id: entry.session_id,
657
- turn_count: entry.turn_count,
658
- state: entry.issue.state,
659
- started_at: entry.started_at,
660
- last_event: entry.last_codex_event,
661
- last_message: entry.last_codex_message,
662
- last_event_at: entry.last_codex_timestamp,
663
- tokens: {
664
- input_tokens: entry.codex_input_tokens,
665
- output_tokens: entry.codex_output_tokens,
666
- total_tokens: entry.codex_total_tokens,
667
- },
668
- }
669
- : null,
670
- retry: retry
671
- ? {
672
- attempt: retry.attempt,
673
- due_at: new Date(retry.due_at_ms).toISOString(),
674
- error: retry.error,
675
- }
676
- : null,
677
- recent_events: entry?.recent_events ?? [],
678
- last_error: entry?.last_error ?? retry?.error ?? null,
679
- tracked: {},
680
- };
1554
+ : null);
1555
+ }
1556
+ findRunningByIdentifier(identifier) {
1557
+ for (const e of this.running.values())
1558
+ if (e.identifier === identifier)
1559
+ return e;
1560
+ return null;
1561
+ }
1562
+ findRetryByIdentifier(identifier) {
1563
+ for (const r of this.retryAttempts.values())
1564
+ if (r.identifier === identifier)
1565
+ return r;
1566
+ return null;
681
1567
  }
682
1568
  }
683
1569
  //# sourceMappingURL=orchestrator.js.map