steroids-cli 0.12.5 → 0.12.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +52 -0
  2. package/dist/commands/gc.d.ts.map +1 -1
  3. package/dist/commands/gc.js +10 -0
  4. package/dist/commands/gc.js.map +1 -1
  5. package/dist/commands/llm-content.d.ts +2 -0
  6. package/dist/commands/llm-content.d.ts.map +1 -0
  7. package/dist/commands/llm-content.js +319 -0
  8. package/dist/commands/llm-content.js.map +1 -0
  9. package/dist/commands/llm.d.ts.map +1 -1
  10. package/dist/commands/llm.js +22 -290
  11. package/dist/commands/llm.js.map +1 -1
  12. package/dist/commands/loop-phases-coder.js +1 -1
  13. package/dist/commands/loop-phases-coder.js.map +1 -1
  14. package/dist/commands/loop-phases-reviewer-follow-ups.d.ts +7 -0
  15. package/dist/commands/loop-phases-reviewer-follow-ups.d.ts.map +1 -0
  16. package/dist/commands/loop-phases-reviewer-follow-ups.js +44 -0
  17. package/dist/commands/loop-phases-reviewer-follow-ups.js.map +1 -0
  18. package/dist/commands/loop-phases-reviewer.d.ts.map +1 -1
  19. package/dist/commands/loop-phases-reviewer.js +6 -45
  20. package/dist/commands/loop-phases-reviewer.js.map +1 -1
  21. package/dist/config/loader.d.ts +4 -0
  22. package/dist/config/loader.d.ts.map +1 -1
  23. package/dist/config/loader.js +27 -0
  24. package/dist/config/loader.js.map +1 -1
  25. package/dist/config/schema.d.ts.map +1 -1
  26. package/dist/config/schema.js +111 -0
  27. package/dist/config/schema.js.map +1 -1
  28. package/dist/config/validator.d.ts.map +1 -1
  29. package/dist/config/validator.js +74 -0
  30. package/dist/config/validator.js.map +1 -1
  31. package/dist/database/intake-queries.d.ts +51 -0
  32. package/dist/database/intake-queries.d.ts.map +1 -0
  33. package/dist/database/intake-queries.js +250 -0
  34. package/dist/database/intake-queries.js.map +1 -0
  35. package/dist/database/queries.d.ts +4 -0
  36. package/dist/database/queries.d.ts.map +1 -1
  37. package/dist/database/queries.js.map +1 -1
  38. package/dist/database/schema.d.ts +2 -2
  39. package/dist/database/schema.d.ts.map +1 -1
  40. package/dist/database/schema.js +49 -0
  41. package/dist/database/schema.js.map +1 -1
  42. package/dist/git/section-pr.d.ts +3 -2
  43. package/dist/git/section-pr.d.ts.map +1 -1
  44. package/dist/git/section-pr.js +95 -15
  45. package/dist/git/section-pr.js.map +1 -1
  46. package/dist/hooks/events.d.ts +10 -1
  47. package/dist/hooks/events.d.ts.map +1 -1
  48. package/dist/hooks/events.js +23 -1
  49. package/dist/hooks/events.js.map +1 -1
  50. package/dist/hooks/integration.d.ts +22 -0
  51. package/dist/hooks/integration.d.ts.map +1 -1
  52. package/dist/hooks/integration.js +52 -0
  53. package/dist/hooks/integration.js.map +1 -1
  54. package/dist/hooks/payload-factories.d.ts +17 -0
  55. package/dist/hooks/payload-factories.d.ts.map +1 -0
  56. package/dist/hooks/payload-factories.js +69 -0
  57. package/dist/hooks/payload-factories.js.map +1 -0
  58. package/dist/hooks/payload-types.d.ts +169 -0
  59. package/dist/hooks/payload-types.d.ts.map +1 -0
  60. package/dist/hooks/payload-types.js +3 -0
  61. package/dist/hooks/payload-types.js.map +1 -0
  62. package/dist/hooks/payload-validation.d.ts +6 -0
  63. package/dist/hooks/payload-validation.d.ts.map +1 -0
  64. package/dist/hooks/payload-validation.js +186 -0
  65. package/dist/hooks/payload-validation.js.map +1 -0
  66. package/dist/hooks/payload.d.ts +6 -334
  67. package/dist/hooks/payload.d.ts.map +1 -1
  68. package/dist/hooks/payload.js +21 -307
  69. package/dist/hooks/payload.js.map +1 -1
  70. package/dist/hooks/template-resolvers.d.ts +9 -0
  71. package/dist/hooks/template-resolvers.d.ts.map +1 -0
  72. package/dist/hooks/template-resolvers.js +133 -0
  73. package/dist/hooks/template-resolvers.js.map +1 -0
  74. package/dist/hooks/templates.d.ts +13 -0
  75. package/dist/hooks/templates.d.ts.map +1 -1
  76. package/dist/hooks/templates.js +40 -118
  77. package/dist/hooks/templates.js.map +1 -1
  78. package/dist/index.js +2 -0
  79. package/dist/index.js.map +1 -1
  80. package/dist/intake/github-gate-api.d.ts +37 -0
  81. package/dist/intake/github-gate-api.d.ts.map +1 -0
  82. package/dist/intake/github-gate-api.js +210 -0
  83. package/dist/intake/github-gate-api.js.map +1 -0
  84. package/dist/intake/github-gate.d.ts +22 -0
  85. package/dist/intake/github-gate.d.ts.map +1 -0
  86. package/dist/intake/github-gate.js +261 -0
  87. package/dist/intake/github-gate.js.map +1 -0
  88. package/dist/intake/github-issues-connector.d.ts +26 -0
  89. package/dist/intake/github-issues-connector.d.ts.map +1 -0
  90. package/dist/intake/github-issues-connector.js +350 -0
  91. package/dist/intake/github-issues-connector.js.map +1 -0
  92. package/dist/intake/index.d.ts +10 -0
  93. package/dist/intake/index.d.ts.map +1 -0
  94. package/dist/intake/index.js +26 -0
  95. package/dist/intake/index.js.map +1 -0
  96. package/dist/intake/pipeline-glue.d.ts +61 -0
  97. package/dist/intake/pipeline-glue.d.ts.map +1 -0
  98. package/dist/intake/pipeline-glue.js +137 -0
  99. package/dist/intake/pipeline-glue.js.map +1 -0
  100. package/dist/intake/poller.d.ts +28 -0
  101. package/dist/intake/poller.d.ts.map +1 -0
  102. package/dist/intake/poller.js +186 -0
  103. package/dist/intake/poller.js.map +1 -0
  104. package/dist/intake/post-pr.d.ts +17 -0
  105. package/dist/intake/post-pr.d.ts.map +1 -0
  106. package/dist/intake/post-pr.js +78 -0
  107. package/dist/intake/post-pr.js.map +1 -0
  108. package/dist/intake/registry.d.ts +17 -0
  109. package/dist/intake/registry.d.ts.map +1 -0
  110. package/dist/intake/registry.js +57 -0
  111. package/dist/intake/registry.js.map +1 -0
  112. package/dist/intake/reviewer-approval.d.ts +10 -0
  113. package/dist/intake/reviewer-approval.d.ts.map +1 -0
  114. package/dist/intake/reviewer-approval.js +70 -0
  115. package/dist/intake/reviewer-approval.js.map +1 -0
  116. package/dist/intake/task-reference.d.ts +10 -0
  117. package/dist/intake/task-reference.d.ts.map +1 -0
  118. package/dist/intake/task-reference.js +23 -0
  119. package/dist/intake/task-reference.js.map +1 -0
  120. package/dist/intake/task-templates.d.ts +25 -0
  121. package/dist/intake/task-templates.d.ts.map +1 -0
  122. package/dist/intake/task-templates.js +98 -0
  123. package/dist/intake/task-templates.js.map +1 -0
  124. package/dist/intake/types.d.ts +103 -0
  125. package/dist/intake/types.d.ts.map +1 -0
  126. package/dist/intake/types.js +3 -0
  127. package/dist/intake/types.js.map +1 -0
  128. package/dist/orchestrator/coder.d.ts +2 -0
  129. package/dist/orchestrator/coder.d.ts.map +1 -1
  130. package/dist/orchestrator/coder.js +25 -2
  131. package/dist/orchestrator/coder.js.map +1 -1
  132. package/dist/providers/gemini.d.ts.map +1 -1
  133. package/dist/providers/gemini.js +7 -14
  134. package/dist/providers/gemini.js.map +1 -1
  135. package/dist/providers/interface.d.ts.map +1 -1
  136. package/dist/providers/interface.js +50 -0
  137. package/dist/providers/interface.js.map +1 -1
  138. package/dist/runners/orchestrator-loop.d.ts.map +1 -1
  139. package/dist/runners/orchestrator-loop.js +7 -0
  140. package/dist/runners/orchestrator-loop.js.map +1 -1
  141. package/dist/runners/system-pressure.d.ts +26 -0
  142. package/dist/runners/system-pressure.d.ts.map +1 -0
  143. package/dist/runners/system-pressure.js +128 -0
  144. package/dist/runners/system-pressure.js.map +1 -0
  145. package/dist/runners/wakeup-checks.d.ts.map +1 -1
  146. package/dist/runners/wakeup-checks.js +16 -1
  147. package/dist/runners/wakeup-checks.js.map +1 -1
  148. package/dist/runners/wakeup-global-cleanup.d.ts +20 -0
  149. package/dist/runners/wakeup-global-cleanup.d.ts.map +1 -0
  150. package/dist/runners/wakeup-global-cleanup.js +317 -0
  151. package/dist/runners/wakeup-global-cleanup.js.map +1 -0
  152. package/dist/runners/wakeup-needed.d.ts +5 -0
  153. package/dist/runners/wakeup-needed.d.ts.map +1 -0
  154. package/dist/runners/wakeup-needed.js +42 -0
  155. package/dist/runners/wakeup-needed.js.map +1 -0
  156. package/dist/runners/wakeup-project-parallel.d.ts +13 -0
  157. package/dist/runners/wakeup-project-parallel.d.ts.map +1 -0
  158. package/dist/runners/wakeup-project-parallel.js +238 -0
  159. package/dist/runners/wakeup-project-parallel.js.map +1 -0
  160. package/dist/runners/wakeup-project.d.ts +10 -0
  161. package/dist/runners/wakeup-project.d.ts.map +1 -0
  162. package/dist/runners/wakeup-project.js +209 -0
  163. package/dist/runners/wakeup-project.js.map +1 -0
  164. package/dist/runners/wakeup-registration.d.ts +2 -0
  165. package/dist/runners/wakeup-registration.d.ts.map +1 -0
  166. package/dist/runners/wakeup-registration.js +42 -0
  167. package/dist/runners/wakeup-registration.js.map +1 -0
  168. package/dist/runners/wakeup-types.d.ts +25 -0
  169. package/dist/runners/wakeup-types.d.ts.map +1 -0
  170. package/dist/runners/wakeup-types.js +3 -0
  171. package/dist/runners/wakeup-types.js.map +1 -0
  172. package/dist/runners/wakeup.d.ts +4 -24
  173. package/dist/runners/wakeup.d.ts.map +1 -1
  174. package/dist/runners/wakeup.js +25 -601
  175. package/dist/runners/wakeup.js.map +1 -1
  176. package/dist/workspace/pool.d.ts.map +1 -1
  177. package/dist/workspace/pool.js +37 -4
  178. package/dist/workspace/pool.js.map +1 -1
  179. package/migrations/025_add_intake_tables.sql +51 -0
  180. package/migrations/026_add_section_coder_and_pr_fields.sql +11 -0
  181. package/migrations/manifest.json +18 -2
  182. package/package.json +1 -1
@@ -3,62 +3,22 @@
3
3
  * Cron wake-up command for restarting stale/dead runners
4
4
  */
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.hasActiveParallelSessionForProject = exports.hasActiveRunnerForProject = exports.getLastWakeupTime = void 0;
6
+ exports.checkWakeupNeeded = exports.hasActiveParallelSessionForProject = exports.hasActiveRunnerForProject = exports.getLastWakeupTime = void 0;
7
7
  exports.wakeup = wakeup;
8
- exports.checkWakeupNeeded = checkWakeupNeeded;
9
- const node_fs_1 = require("node:fs");
10
- const lock_js_1 = require("./lock.js");
11
8
  const global_db_js_1 = require("./global-db.js");
12
- const heartbeat_js_1 = require("./heartbeat.js");
13
9
  const projects_js_1 = require("./projects.js");
14
- const connection_js_1 = require("../database/connection.js");
15
- const loader_js_1 = require("../config/loader.js");
16
- const stuck_task_recovery_js_1 = require("../health/stuck-task-recovery.js");
17
- const invocation_logs_js_1 = require("../cleanup/invocation-logs.js");
18
- // Import from helper files
19
- const wakeup_sanitise_js_1 = require("./wakeup-sanitise.js");
20
- const wakeup_reconcile_js_1 = require("./wakeup-reconcile.js");
21
10
  const wakeup_checks_js_1 = require("./wakeup-checks.js");
22
11
  Object.defineProperty(exports, "hasActiveRunnerForProject", { enumerable: true, get: function () { return wakeup_checks_js_1.hasActiveRunnerForProject; } });
23
12
  Object.defineProperty(exports, "hasActiveParallelSessionForProject", { enumerable: true, get: function () { return wakeup_checks_js_1.hasActiveParallelSessionForProject; } });
24
- const wakeup_runner_js_1 = require("./wakeup-runner.js");
25
13
  const wakeup_timing_js_1 = require("./wakeup-timing.js");
26
14
  Object.defineProperty(exports, "getLastWakeupTime", { enumerable: true, get: function () { return wakeup_timing_js_1.getLastWakeupTime; } });
15
+ const wakeup_global_cleanup_js_1 = require("./wakeup-global-cleanup.js");
16
+ const wakeup_project_js_1 = require("./wakeup-project.js");
17
+ const wakeup_needed_js_1 = require("./wakeup-needed.js");
18
+ Object.defineProperty(exports, "checkWakeupNeeded", { enumerable: true, get: function () { return wakeup_needed_js_1.checkWakeupNeeded; } });
19
+ const system_pressure_js_1 = require("./system-pressure.js");
27
20
  // In-memory mutex to prevent concurrent wakeup cycles in the same process
28
21
  let isWakeupRunning = false;
29
- async function waitForRunnerRegistration(globalDb, projectPath, parallelMode, timeoutMs = 8000) {
30
- const deadline = Date.now() + timeoutMs;
31
- while (Date.now() < deadline) {
32
- if (parallelMode) {
33
- const parallelRunner = globalDb
34
- .prepare(`SELECT 1
35
- FROM runners r
36
- JOIN parallel_sessions ps ON ps.id = r.parallel_session_id
37
- WHERE ps.project_path = ?
38
- AND r.status != 'stopped'
39
- AND r.heartbeat_at > datetime('now', '-5 minutes')
40
- LIMIT 1`)
41
- .get(projectPath);
42
- if (parallelRunner !== undefined)
43
- return true;
44
- }
45
- else {
46
- const standaloneRunner = globalDb
47
- .prepare(`SELECT 1
48
- FROM runners
49
- WHERE project_path = ?
50
- AND parallel_session_id IS NULL
51
- AND status != 'stopped'
52
- AND heartbeat_at > datetime('now', '-5 minutes')
53
- LIMIT 1`)
54
- .get(projectPath);
55
- if (standaloneRunner !== undefined)
56
- return true;
57
- }
58
- await new Promise((resolve) => setTimeout(resolve, 250));
59
- }
60
- return false;
61
- }
62
22
  /**
63
23
  * Main wake-up function
64
24
  * Called by cron every minute to ensure runners are healthy
@@ -86,107 +46,13 @@ async function wakeup(options = {}) {
86
46
  if (!dryRun) {
87
47
  (0, wakeup_timing_js_1.recordWakeupTime)();
88
48
  }
89
- // Step 1: Clean up stale runners first
90
49
  return (0, global_db_js_1.withGlobalDatabase)(async (globalDb) => {
91
- const global = { db: globalDb };
92
- try {
93
- const staleRunners = (0, heartbeat_js_1.findStaleRunners)(global.db);
94
- if (staleRunners.length > 0) {
95
- log(`Found ${staleRunners.length} stale runner(s), cleaning up...`);
96
- if (!dryRun) {
97
- for (const runner of staleRunners) {
98
- // Clean up in-flight task state before removing runner row.
99
- // Process is definitively dead (stale heartbeat), so the lock is deleted
100
- // immediately — unlike the SIGTERM case where the lock is left in place
101
- // to prevent double-execution from a still-alive process.
102
- //
103
- // project_path is the canonical project path for both standalone and
104
- // parallel workstream runners (findStaleRunners JOIN-resolves clone paths
105
- // via parallel_sessions.project_path).
106
- if (runner.current_task_id && runner.project_path) {
107
- try {
108
- const { db: projectDb, close: closeProjectDb } = (0, connection_js_1.openDatabase)(runner.project_path);
109
- try {
110
- const nowMs = Date.now();
111
- projectDb.prepare(`UPDATE task_invocations
112
- SET status = 'failed', success = 0, timed_out = 0, exit_code = 1,
113
- completed_at_ms = ?, duration_ms = ?,
114
- error = COALESCE(error, 'Runner process died (stale heartbeat).')
115
- WHERE task_id = ? AND status = 'running'`).run(nowMs, 0, runner.current_task_id);
116
- // Reset coder-phase tasks (in_progress → pending).
117
- // Leave review-status tasks at 'review' — task selection includes
118
- // review tasks, so the next runner will re-run the reviewer directly
119
- // without unnecessarily re-executing the coder phase.
120
- projectDb.prepare(`UPDATE tasks SET status = 'pending', updated_at = datetime('now')
121
- WHERE id = ? AND status = 'in_progress'`).run(runner.current_task_id);
122
- projectDb.prepare(`DELETE FROM task_locks WHERE task_id = ?`).run(runner.current_task_id);
123
- }
124
- finally {
125
- closeProjectDb();
126
- }
127
- }
128
- catch {
129
- // Project DB errors must not block runner row cleanup.
130
- }
131
- }
132
- if (runner.pid) {
133
- (0, wakeup_runner_js_1.killProcess)(runner.pid);
134
- }
135
- global.db.prepare(`UPDATE workstreams
136
- SET runner_id = NULL,
137
- lease_expires_at = datetime('now')
138
- WHERE runner_id = ?`).run(runner.id);
139
- global.db.prepare('DELETE FROM runners WHERE id = ?').run(runner.id);
140
- }
141
- }
142
- results.push({
143
- action: 'cleaned',
144
- reason: `Cleaned ${staleRunners.length} stale runner(s)`,
145
- staleRunners: staleRunners.length,
146
- });
147
- }
148
- }
149
- catch {
150
- // ignore global DB issues; wakeup will still attempt per-project checks
151
- }
152
- try {
153
- const releasedLeases = global.db.prepare(`UPDATE workstreams
154
- SET runner_id = NULL,
155
- lease_expires_at = NULL
156
- WHERE status = 'running'
157
- AND lease_expires_at IS NOT NULL
158
- AND lease_expires_at <= datetime('now')`).run().changes;
159
- if (releasedLeases > 0) {
160
- log(`Released ${releasedLeases} expired workstream lease(s)`);
161
- }
162
- }
163
- catch {
164
- // ignore lease cleanup issues in wakeup
165
- }
166
- // Step 1b: Reconcile stale workspace pool slots and merge locks
167
- try {
168
- const { reconcileStaleWorkspaces } = await import('../workspace/reconcile.js');
169
- const reconcileResult = reconcileStaleWorkspaces(global.db);
170
- if (reconcileResult.resetSlots > 0 || reconcileResult.deletedLocks > 0) {
171
- log(`Workspace pool reconciliation: reset ${reconcileResult.resetSlots} slot(s), ` +
172
- `deleted ${reconcileResult.deletedLocks} stale lock(s)`);
173
- // Return associated tasks to pending in their project DBs
174
- // (deferred — the next loop iteration will pick them up as pending)
175
- }
176
- }
177
- catch {
178
- // ignore workspace pool reconciliation issues in wakeup
179
- }
180
- // Step 2: Clean zombie lock if present
181
- const lockStatus = (0, lock_js_1.checkLockStatus)();
182
- if (lockStatus.isZombie && lockStatus.pid) {
183
- log(`Found zombie lock (PID: ${lockStatus.pid}), cleaning...`);
184
- if (!dryRun) {
185
- (0, lock_js_1.removeLock)();
186
- }
187
- }
188
- // Step 3: Get all registered projects from global registry
189
- const registeredProjects = (0, projects_js_1.getRegisteredProjects)(false); // enabled only
50
+ results.push(...(await (0, wakeup_global_cleanup_js_1.performWakeupGlobalMaintenance)({
51
+ globalDb,
52
+ dryRun,
53
+ log,
54
+ })));
55
+ const registeredProjects = (0, projects_js_1.getRegisteredProjects)(false);
190
56
  if (registeredProjects.length === 0) {
191
57
  log('No registered projects found');
192
58
  log('Run "steroids projects add <path>" to register a project');
@@ -198,427 +64,21 @@ async function wakeup(options = {}) {
198
64
  return results;
199
65
  }
200
66
  log(`Checking ${registeredProjects.length} registered project(s)...`);
201
- // Step 4: Check each project and start runners as needed
67
+ // Skip spawning new runners if system is under memory/disk pressure
68
+ const pressure = (0, system_pressure_js_1.checkSystemPressure)();
69
+ if (!pressure.ok) {
70
+ log(`System pressure: ${pressure.reason} — skipping runner spawns`);
71
+ results.push({ action: 'skipped', reason: `System pressure: ${pressure.reason}` });
72
+ return results;
73
+ }
202
74
  for (const project of registeredProjects) {
203
- // Skip if project directory doesn't exist
204
- if (!(0, node_fs_1.existsSync)(project.path)) {
205
- log(`Skipping ${project.path}: directory not found`);
206
- results.push({
207
- action: 'none',
208
- reason: 'Directory not found',
209
- projectPath: project.path,
210
- });
211
- continue;
212
- }
213
- // Phase 6 (live monitoring): best-effort retention cleanup of invocation activity logs.
214
- // This is safe to run even if the project has no pending tasks.
215
- let deletedInvocationLogs = 0;
216
- try {
217
- const cleanup = (0, invocation_logs_js_1.cleanupInvocationLogs)(project.path, { retentionDays: 7, dryRun });
218
- deletedInvocationLogs = cleanup.deletedFiles;
219
- if (cleanup.deletedFiles > 0 && !quiet) {
220
- log(`Cleaned ${cleanup.deletedFiles} old invocation log(s) in ${project.path}`);
221
- }
222
- }
223
- catch {
224
- // Ignore cleanup errors; wakeup must remain robust.
225
- }
226
- let recoveredActions = 0;
227
- let skippedRecoveryDueToSafetyLimit = false;
228
- let sanitisedActions = 0;
229
- try {
230
- const { db: projectDb, close: closeProjectDb } = (0, connection_js_1.openDatabase)(project.path);
231
- try {
232
- const sanitiseSummary = (0, wakeup_sanitise_js_1.runPeriodicSanitiseForProject)(global.db, projectDb, project.path, dryRun);
233
- sanitisedActions = (0, wakeup_sanitise_js_1.sanitisedActionCount)(sanitiseSummary);
234
- if (sanitisedActions > 0 && !quiet) {
235
- log(`Sanitised ${sanitisedActions} stale item(s) in ${project.path}`);
236
- }
237
- // Step 4a: Recover stuck tasks (best-effort) before deciding whether to (re)start a runner.
238
- // This is what unblocks orphaned/infinite-hang scenarios without manual intervention.
239
- const config = (0, loader_js_1.loadConfig)(project.path);
240
- const recovery = await (0, stuck_task_recovery_js_1.recoverStuckTasks)({
241
- projectPath: project.path,
242
- projectDb,
243
- globalDb: global.db,
244
- config,
245
- dryRun,
246
- });
247
- recoveredActions = recovery.actions.length;
248
- skippedRecoveryDueToSafetyLimit = recovery.skippedDueToSafetyLimit;
249
- if (recoveredActions > 0 && !quiet) {
250
- log(`Recovered ${recoveredActions} stuck item(s) in ${project.path}`);
251
- }
252
- if (skippedRecoveryDueToSafetyLimit && !quiet) {
253
- log(`Skipping auto-recovery in ${project.path}: safety limit hit (maxIncidentsPerHour)`);
254
- }
255
- }
256
- finally {
257
- closeProjectDb();
258
- }
259
- }
260
- catch {
261
- // If sanitise/recovery can't run (DB missing/corrupt), we still proceed with runner checks.
262
- }
263
- // Check for pending work after sanitise/recovery
264
- const hasWork = await (0, wakeup_checks_js_1.projectHasPendingWork)(project.path);
265
- if (!hasWork) {
266
- const noWorkReason = sanitisedActions > 0
267
- ? `No pending tasks after sanitise (${sanitisedActions} action(s))`
268
- : 'No pending tasks';
269
- log(`Skipping ${project.path}: ${noWorkReason.toLowerCase()}`);
270
- results.push({
271
- action: 'none',
272
- reason: noWorkReason,
273
- projectPath: project.path,
274
- recoveredActions,
275
- skippedRecoveryDueToSafetyLimit,
276
- deletedInvocationLogs,
277
- sanitisedActions,
278
- });
279
- continue;
280
- }
281
- const projectConfig = (0, loader_js_1.loadConfig)(project.path);
282
- // Check global provider backoffs
283
- const coderProvider = projectConfig.ai?.coder?.provider;
284
- const reviewerProvider = projectConfig.ai?.reviewer?.provider;
285
- const providersToCheck = [coderProvider, reviewerProvider].filter(Boolean);
286
- let isBackedOff = false;
287
- let backedOffProvider = '';
288
- let remainingMs = 0;
289
- for (const provider of providersToCheck) {
290
- const ms = (0, global_db_js_1.getProviderBackoffRemainingMs)(provider);
291
- if (ms > 0) {
292
- isBackedOff = true;
293
- backedOffProvider = provider;
294
- remainingMs = ms;
295
- break;
296
- }
297
- }
298
- if (isBackedOff) {
299
- const remainingMinutes = Math.ceil(remainingMs / 60000);
300
- log(`Skipping ${project.path}: Provider '${backedOffProvider}' is in backoff for ${remainingMinutes}m`);
301
- results.push({
302
- action: 'skipped',
303
- reason: `Provider '${backedOffProvider}' backed off for ${remainingMinutes}m`,
304
- projectPath: project.path,
305
- });
306
- continue;
307
- }
308
- const parallelEnabled = projectConfig.runners?.parallel?.enabled === true;
309
- const configuredMaxClonesRaw = Number(projectConfig.runners?.parallel?.maxClones);
310
- const configuredMaxClones = Number.isFinite(configuredMaxClonesRaw) && configuredMaxClonesRaw > 0
311
- ? configuredMaxClonesRaw
312
- : 3;
313
- // Skip projects currently executing a parallel session before attempting recovery/startup.
314
- // This prevents parallel runners from being interfered with by a cron-managed runner.
315
- if ((0, wakeup_checks_js_1.hasActiveParallelSessionForProject)(project.path)) {
316
- let retrySummary = '';
317
- let skipForParallelSession = true;
318
- let scaledDown = 0;
319
- let resumed = 0;
320
- let wouldScaleDown = 0;
321
- let wouldResume = 0;
322
- const activeSessions = global.db
323
- .prepare(`SELECT id
324
- FROM parallel_sessions
325
- WHERE project_path = ?
326
- AND status NOT IN ('completed', 'failed', 'aborted', 'blocked_validation', 'blocked_recovery')`)
327
- .all(project.path);
328
- // Config-aware mode reconciliation (parallel -> single):
329
- // if parallel is disabled, convert only when the active parallel runners
330
- // are idle to avoid interrupting in-flight tasks.
331
- if (!parallelEnabled && activeSessions.length > 0) {
332
- const sessionRunners = activeSessions.flatMap((session) => global.db
333
- .prepare(`SELECT id, pid, status, current_task_id
334
- FROM runners
335
- WHERE parallel_session_id = ?
336
- AND status != 'stopped'
337
- AND heartbeat_at > datetime('now', '-5 minutes')`)
338
- .all(session.id));
339
- const hasBusyRunner = sessionRunners.some((runner) => (runner.status ?? '').toLowerCase() !== 'idle' || !!runner.current_task_id);
340
- if (hasBusyRunner) {
341
- const reason = 'Parallel->single mode switch pending (active workstream runner busy)';
342
- log(`Skipping ${project.path}: ${reason.toLowerCase()}`);
343
- results.push({
344
- action: dryRun ? 'would_start' : 'none',
345
- reason,
346
- projectPath: project.path,
347
- deletedInvocationLogs,
348
- });
349
- continue;
350
- }
351
- if (dryRun) {
352
- const reason = 'Would recycle idle parallel session to apply single-runner mode';
353
- log(`Would reconcile ${project.path}: ${reason.toLowerCase()}`);
354
- results.push({
355
- action: 'would_start',
356
- reason,
357
- projectPath: project.path,
358
- deletedInvocationLogs,
359
- });
360
- continue;
361
- }
362
- for (const runner of sessionRunners) {
363
- if (runner.pid)
364
- (0, wakeup_runner_js_1.killProcess)(runner.pid);
365
- global.db.prepare('DELETE FROM runners WHERE id = ?').run(runner.id);
366
- }
367
- for (const session of activeSessions) {
368
- global.db.prepare(`UPDATE workstreams
369
- SET status = 'aborted',
370
- runner_id = NULL,
371
- lease_expires_at = NULL,
372
- next_retry_at = NULL,
373
- last_reconcile_action = 'mode_switch_to_single',
374
- last_reconciled_at = datetime('now'),
375
- completed_at = COALESCE(completed_at, datetime('now'))
376
- WHERE session_id = ?
377
- AND status NOT IN ('completed', 'failed', 'aborted')`).run(session.id);
378
- global.db.prepare(`UPDATE parallel_sessions
379
- SET status = 'aborted',
380
- completed_at = COALESCE(completed_at, datetime('now'))
381
- WHERE id = ?`).run(session.id);
382
- }
383
- skipForParallelSession = false;
384
- retrySummary = ', recycled idle parallel session to apply single-runner mode';
385
- }
386
- for (const session of activeSessions) {
387
- const sessionRunners = global.db
388
- .prepare(`SELECT id, pid, status, current_task_id
389
- FROM runners
390
- WHERE parallel_session_id = ?
391
- AND status != 'stopped'
392
- AND heartbeat_at > datetime('now', '-5 minutes')
393
- ORDER BY started_at DESC, heartbeat_at DESC`)
394
- .all(session.id);
395
- if (sessionRunners.length > configuredMaxClones) {
396
- const idleCandidate = sessionRunners.find((r) => (r.status ?? '').toLowerCase() === 'idle' && !r.current_task_id);
397
- if (idleCandidate) {
398
- if (dryRun) {
399
- wouldScaleDown += 1;
400
- }
401
- else {
402
- if (idleCandidate.pid) {
403
- (0, wakeup_runner_js_1.killProcess)(idleCandidate.pid);
404
- }
405
- global.db.prepare(`UPDATE workstreams
406
- SET runner_id = NULL,
407
- lease_expires_at = datetime('now', '+5 minutes'),
408
- next_retry_at = datetime('now', '+5 minutes'),
409
- last_reconcile_action = 'concurrency_throttle',
410
- last_reconciled_at = datetime('now')
411
- WHERE session_id = ?
412
- AND runner_id = ?`).run(session.id, idleCandidate.id);
413
- global.db.prepare('DELETE FROM runners WHERE id = ?').run(idleCandidate.id);
414
- scaledDown += 1;
415
- }
416
- }
417
- }
418
- else if (sessionRunners.length < configuredMaxClones) {
419
- const throttled = global.db
420
- .prepare(`SELECT id
421
- FROM workstreams
422
- WHERE session_id = ?
423
- AND status = 'running'
424
- AND runner_id IS NULL
425
- AND next_retry_at > datetime('now')
426
- AND last_reconcile_action = 'concurrency_throttle'
427
- ORDER BY last_reconciled_at ASC
428
- LIMIT 1`)
429
- .get(session.id);
430
- if (throttled) {
431
- if (dryRun) {
432
- wouldResume += 1;
433
- }
434
- else {
435
- global.db.prepare(`UPDATE workstreams
436
- SET lease_expires_at = datetime('now'),
437
- next_retry_at = datetime('now'),
438
- last_reconcile_action = 'concurrency_resume',
439
- last_reconciled_at = datetime('now')
440
- WHERE id = ?`).run(throttled.id);
441
- resumed += 1;
442
- }
443
- }
444
- }
445
- }
446
- if (!dryRun) {
447
- const recovery = (0, wakeup_reconcile_js_1.reconcileParallelSessionRecovery)(global.db, project.path);
448
- if (recovery.workstreamsToRestart.length > 0) {
449
- for (const ws of recovery.workstreamsToRestart) {
450
- (0, wakeup_runner_js_1.restartWorkstreamRunner)(ws);
451
- }
452
- retrySummary += `, restarted ${recovery.workstreamsToRestart.length} workstream runner(s)`;
453
- }
454
- if (recovery.blockedWorkstreams > 0) {
455
- retrySummary += `, blocked ${recovery.blockedWorkstreams} workstream(s)`;
456
- }
457
- if (scaledDown > 0) {
458
- retrySummary += `, scaled down ${scaledDown} idle runner(s) to maxClones=${configuredMaxClones}`;
459
- }
460
- if (resumed > 0) {
461
- retrySummary += `, resumed ${resumed} throttled workstream(s)`;
462
- }
463
- // Re-check activity after recovery. If reconciliation cleared stale
464
- // session state for this project, continue to normal startup logic.
465
- if (!(0, wakeup_checks_js_1.hasActiveParallelSessionForProject)(project.path)) {
466
- skipForParallelSession = false;
467
- if (retrySummary.length > 0) {
468
- retrySummary += ', session state reconciled';
469
- }
470
- else {
471
- retrySummary = ', session state reconciled';
472
- }
473
- }
474
- }
475
- else {
476
- if (wouldScaleDown > 0) {
477
- retrySummary += `, would scale down ${wouldScaleDown} idle runner(s) to maxClones=${configuredMaxClones}`;
478
- }
479
- if (wouldResume > 0) {
480
- retrySummary += `, would resume ${wouldResume} throttled workstream(s)`;
481
- }
482
- }
483
- if (!skipForParallelSession) {
484
- log(`Reconciled stale parallel session for ${project.path}; proceeding with startup`);
485
- }
486
- else {
487
- log(`Skipping ${project.path}: active parallel session in progress${retrySummary}`);
488
- results.push({
489
- action: 'none',
490
- reason: `Parallel session already running${retrySummary}`,
491
- projectPath: project.path,
492
- deletedInvocationLogs,
493
- });
494
- continue;
495
- }
496
- }
497
- // Config-aware mode reconciliation:
498
- // if parallel is enabled but an idle standalone runner is active, recycle it
499
- // so wakeup applies current parallel settings without manual restart.
500
- const activeStandaloneRunner = global.db
501
- .prepare(`SELECT id, pid, status, current_task_id
502
- FROM runners
503
- WHERE project_path = ?
504
- AND parallel_session_id IS NULL
505
- AND status != 'stopped'
506
- AND heartbeat_at > datetime('now', '-5 minutes')
507
- ORDER BY heartbeat_at DESC
508
- LIMIT 1`)
509
- .get(project.path);
510
- if (activeStandaloneRunner && parallelEnabled) {
511
- const isIdle = (activeStandaloneRunner.status ?? '').toLowerCase() === 'idle' && !activeStandaloneRunner.current_task_id;
512
- if (isIdle) {
513
- if (dryRun) {
514
- log(`Would recycle idle standalone runner for ${project.path} to apply parallel mode`);
515
- results.push({
516
- action: 'would_start',
517
- reason: 'Would restart idle runner to apply parallel mode',
518
- projectPath: project.path,
519
- deletedInvocationLogs,
520
- });
521
- continue;
522
- }
523
- if (activeStandaloneRunner.pid) {
524
- (0, wakeup_runner_js_1.killProcess)(activeStandaloneRunner.pid);
525
- }
526
- global.db.prepare('DELETE FROM runners WHERE id = ?').run(activeStandaloneRunner.id);
527
- const restartResult = (0, wakeup_runner_js_1.startRunner)(project.path);
528
- if (restartResult) {
529
- results.push({
530
- action: 'restarted',
531
- reason: 'Restarted idle runner to apply parallel mode',
532
- pid: restartResult.pid,
533
- projectPath: project.path,
534
- deletedInvocationLogs,
535
- });
536
- }
537
- else {
538
- results.push({
539
- action: 'none',
540
- reason: 'Failed to restart idle runner for parallel mode',
541
- projectPath: project.path,
542
- deletedInvocationLogs,
543
- });
544
- }
545
- continue;
546
- }
547
- }
548
- // Skip if project already has an active runner (after recovery, which may have killed/removed it).
549
- if ((0, wakeup_checks_js_1.hasActiveRunnerForProject)(project.path)) {
550
- log(`Skipping ${project.path}: runner already active`);
551
- results.push({
552
- action: 'none',
553
- reason: recoveredActions > 0
554
- ? `Runner already active (recovered ${recoveredActions} stuck item(s))`
555
- : 'Runner already active',
556
- projectPath: project.path,
557
- recoveredActions,
558
- skippedRecoveryDueToSafetyLimit,
559
- deletedInvocationLogs,
560
- sanitisedActions,
561
- });
562
- continue;
563
- }
564
- // Start runner for this project
565
- const willParallel = parallelEnabled;
566
- log(`Starting ${willParallel ? 'parallel session' : 'runner'} for: ${project.path}`);
567
- if (dryRun) {
568
- results.push({
569
- action: 'would_start',
570
- reason: recoveredActions > 0 ? `Recovered ${recoveredActions} stuck item(s); would start runner (dry-run)` : `Would start runner (dry-run)`,
571
- projectPath: project.path,
572
- recoveredActions,
573
- skippedRecoveryDueToSafetyLimit,
574
- deletedInvocationLogs,
575
- sanitisedActions,
576
- });
577
- continue;
578
- }
579
- const startResult = (0, wakeup_runner_js_1.startRunner)(project.path);
580
- if (startResult) {
581
- const mode = startResult.parallel ? 'parallel session' : 'runner';
582
- const registered = await waitForRunnerRegistration(global.db, project.path, startResult.parallel === true);
583
- if (registered) {
584
- results.push({
585
- action: 'started',
586
- reason: recoveredActions > 0 ? `Recovered ${recoveredActions} stuck item(s); started ${mode}` : `Started ${mode}`,
587
- pid: startResult.pid,
588
- projectPath: project.path,
589
- recoveredActions,
590
- skippedRecoveryDueToSafetyLimit,
591
- deletedInvocationLogs,
592
- sanitisedActions,
593
- });
594
- }
595
- else {
596
- results.push({
597
- action: 'none',
598
- reason: recoveredActions > 0
599
- ? `Recovered ${recoveredActions} stuck item(s); ${mode} failed to register`
600
- : `${mode} failed to register`,
601
- projectPath: project.path,
602
- recoveredActions,
603
- skippedRecoveryDueToSafetyLimit,
604
- deletedInvocationLogs,
605
- sanitisedActions,
606
- });
607
- }
608
- }
609
- else {
610
- results.push({
611
- action: 'none',
612
- reason: recoveredActions > 0 ? `Recovered ${recoveredActions} stuck item(s); failed to start runner` : 'Failed to start runner',
613
- projectPath: project.path,
614
- recoveredActions,
615
- skippedRecoveryDueToSafetyLimit,
616
- deletedInvocationLogs,
617
- sanitisedActions,
618
- });
619
- }
75
+ results.push(await (0, wakeup_project_js_1.processWakeupProject)({
76
+ globalDb,
77
+ projectPath: project.path,
78
+ dryRun,
79
+ log,
80
+ }));
620
81
  }
621
- // If no specific results, add a summary
622
82
  if (results.length === 0) {
623
83
  results.push({
624
84
  action: 'none',
@@ -632,40 +92,4 @@ async function wakeup(options = {}) {
632
92
  isWakeupRunning = false;
633
93
  }
634
94
  }
635
- /**
636
- * Check if wake-up is needed without taking action
637
- */
638
- async function checkWakeupNeeded() {
639
- const lockStatus = (0, lock_js_1.checkLockStatus)();
640
- if (lockStatus.locked && lockStatus.pid) {
641
- return (0, global_db_js_1.withGlobalDatabase)(async (db) => {
642
- const staleRunners = (0, heartbeat_js_1.findStaleRunners)(db);
643
- if (staleRunners.length > 0) {
644
- return {
645
- needed: true,
646
- reason: `${staleRunners.length} stale runner(s) need cleanup`,
647
- };
648
- }
649
- return { needed: false, reason: 'Runner is healthy' };
650
- });
651
- }
652
- if (lockStatus.isZombie) {
653
- return { needed: true, reason: 'Zombie lock needs cleanup' };
654
- }
655
- // Check registered projects
656
- const registeredProjects = (0, projects_js_1.getRegisteredProjects)(false);
657
- let projectsWithWork = 0;
658
- for (const project of registeredProjects) {
659
- if ((0, node_fs_1.existsSync)(project.path) && (await (0, wakeup_checks_js_1.projectHasPendingWork)(project.path))) {
660
- projectsWithWork++;
661
- }
662
- }
663
- if (projectsWithWork > 0) {
664
- return {
665
- needed: true,
666
- reason: `${projectsWithWork} project(s) have pending tasks`,
667
- };
668
- }
669
- return { needed: false, reason: 'No pending tasks' };
670
- }
671
95
  //# sourceMappingURL=wakeup.js.map