tide-commander 1.174.0 → 1.176.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/assets/{BossLogsModal-CsjhF9hQ.js → BossLogsModal-qzLlhqPZ.js} +1 -1
- package/dist/assets/{BossSpawnModal-DRgrZpYr.js → BossSpawnModal-Bo5btfZs.js} +1 -1
- package/dist/assets/{ControlsModal-zYwhMaaL.js → ControlsModal-CCP1SS-b.js} +1 -1
- package/dist/assets/{DockerLogsModal-D7bewBqJ.js → DockerLogsModal-d2Isd330.js} +1 -1
- package/dist/assets/{EmbeddedEditor-IxYwI5JV.js → EmbeddedEditor-C2whRCAd.js} +1 -1
- package/dist/assets/{GmailOAuthSetup-CfOuUZnu.js → GmailOAuthSetup-BM4xBt67.js} +1 -1
- package/dist/assets/{GoogleOAuthSetup-Cw1tGSti.js → GoogleOAuthSetup-BMvbdHEK.js} +1 -1
- package/dist/assets/{IframeModal-Bi8MNGZY.js → IframeModal-Diz8yWFu.js} +1 -1
- package/dist/assets/{IntegrationsPanel-01iVXM-l.js → IntegrationsPanel-W_8l0DxS.js} +2 -2
- package/dist/assets/{LogViewerModal-BXhxgz7Z.js → LogViewerModal-DU64WFDr.js} +1 -1
- package/dist/assets/{MonitoringModal-Gf0AkCEl.js → MonitoringModal-BoG0Hr7X.js} +1 -1
- package/dist/assets/{PM2LogsModal-BfoL_mEK.js → PM2LogsModal-ZemBmh8q.js} +1 -1
- package/dist/assets/{RestoreArchivedAreaModal-Ctss__a9.js → RestoreArchivedAreaModal-CEkL1jB3.js} +1 -1
- package/dist/assets/{Scene2DCanvas-BuCW0Zaw.js → Scene2DCanvas-BpX0JWYQ.js} +1 -1
- package/dist/assets/{SceneManager-BFnKKU1I.js → SceneManager-CdCkKNKF.js} +2 -2
- package/dist/assets/{SkillsPanel-D7kJE7eT.js → SkillsPanel-CcJJoxVd.js} +1 -1
- package/dist/assets/{SlackMultiInstanceSetup-Hp7r3YK0.js → SlackMultiInstanceSetup-D0QFzYL8.js} +1 -1
- package/dist/assets/{SpawnModal-Dq9P-FdI.js → SpawnModal-C8EMz9dV.js} +1 -1
- package/dist/assets/{StatisticsModal-BQOkmrmY.js → StatisticsModal-DBWeBd6u.js} +1 -1
- package/dist/assets/{SubordinateAssignmentModal-D2-IpRLn.js → SubordinateAssignmentModal-g2VK-d4p.js} +1 -1
- package/dist/assets/{TriggerManagerPanel-BCZhLuoZ.js → TriggerManagerPanel-CybQfvUd.js} +1 -1
- package/dist/assets/{WorkflowEditorPanel-BEjublaK.js → WorkflowEditorPanel-Doo1fthh.js} +1 -1
- package/dist/assets/{abnfDiagram-VRR7QNED-jjWEEISc.js → abnfDiagram-VRR7QNED-DIzWJ00e.js} +1 -1
- package/dist/assets/{arc-xqT5-bwQ.js → arc-DYnuDU5M.js} +1 -1
- package/dist/assets/{architectureDiagram-ZJ3FMSHR-BXEbOurL.js → architectureDiagram-ZJ3FMSHR-DcQkZge3.js} +1 -1
- package/dist/assets/{area-logos-BH-HM2a-.js → area-logos-CvKL6axv.js} +1 -1
- package/dist/assets/{blockDiagram-677ZJIJ3-D-0HVLDv.js → blockDiagram-677ZJIJ3-P9qfYIyd.js} +1 -1
- package/dist/assets/{c4Diagram-LMCZKHZV-BK39PXfT.js → c4Diagram-LMCZKHZV-Brvvf8IR.js} +1 -1
- package/dist/assets/channel-BjM2BONO.js +1 -0
- package/dist/assets/{chunk-2Q5K7J3B-CuW4NYbO.js → chunk-2Q5K7J3B-huNYyvxh.js} +1 -1
- package/dist/assets/{chunk-32BRIVSS-eJLo7W-b.js → chunk-32BRIVSS-BPwUeNlq.js} +1 -1
- package/dist/assets/{chunk-5VM5RSS4-E2Yaai0b.js → chunk-5VM5RSS4-ClvwfT9j.js} +1 -1
- package/dist/assets/{chunk-EX3LRPZG-9uAbQnMb.js → chunk-EX3LRPZG-r7lw8w07.js} +1 -1
- package/dist/assets/{chunk-JWPE2WC7-kKtXyMkv.js → chunk-JWPE2WC7-Cg-NNKhK.js} +1 -1
- package/dist/assets/{chunk-MOJQB5TN-DPN5ReJ7.js → chunk-MOJQB5TN-D1jRsGiz.js} +1 -1
- package/dist/assets/{chunk-RYQCIY6F-d3yFo4Hu.js → chunk-RYQCIY6F-DzLe8Mhn.js} +1 -1
- package/dist/assets/{chunk-V7JOEXUC--3A8oVa7.js → chunk-V7JOEXUC-CP3S5ECO.js} +1 -1
- package/dist/assets/{chunk-VR4S4FIN-KpHuZy9_.js → chunk-VR4S4FIN-CJsUxduu.js} +1 -1
- package/dist/assets/{chunk-XXDRQBXY-hpU30gZQ.js → chunk-XXDRQBXY-CG5BEMYm.js} +1 -1
- package/dist/assets/classDiagram-OUVF2IWQ-Bo96wh-7.js +1 -0
- package/dist/assets/classDiagram-v2-EOCWNBFH-Bo96wh-7.js +1 -0
- package/dist/assets/{cose-bilkent-JH36ORCC-DOhYmYxe.js → cose-bilkent-JH36ORCC-ByxdBNNQ.js} +1 -1
- package/dist/assets/{cynefin-VYW2F7L2-UDH0NYTi.js → cynefin-VYW2F7L2-DUAFhLA3.js} +1 -1
- package/dist/assets/{cynefinDiagram-TSTJHNR4-DnGUiXaD.js → cynefinDiagram-TSTJHNR4-B2Kk-MGz.js} +1 -1
- package/dist/assets/{dagre-VKFMJZFB-BQ6C_GWY.js → dagre-VKFMJZFB-DFCRJf_G.js} +1 -1
- package/dist/assets/{diagram-FQU43EPY-CREz1035.js → diagram-FQU43EPY-wYMGeAdu.js} +1 -1
- package/dist/assets/{diagram-G47NLZAW-9Jr7GbsE.js → diagram-G47NLZAW-rgjw8pQP.js} +1 -1
- package/dist/assets/{diagram-NH7WQ7WH-DtN5x74X.js → diagram-NH7WQ7WH-BpZASdpx.js} +1 -1
- package/dist/assets/{diagram-OA4YK3LP-UuOi8FAP.js → diagram-OA4YK3LP-CakEHPj1.js} +1 -1
- package/dist/assets/{diagram-WEI45ONY-CWiUGHfM.js → diagram-WEI45ONY-BgF2sKn7.js} +1 -1
- package/dist/assets/{ebnfDiagram-CCIWWBDH-3eMfQx1G.js → ebnfDiagram-CCIWWBDH-D0Kf9aJj.js} +1 -1
- package/dist/assets/{erDiagram-Q63AITRT-DYzSgp8n.js → erDiagram-Q63AITRT-sKvL1I4l.js} +1 -1
- package/dist/assets/{flowDiagram-23GEKE2U-DBLDGx8O.js → flowDiagram-23GEKE2U-DgS4-v6l.js} +1 -1
- package/dist/assets/{ganttDiagram-NO4QXBWP-Dv7Uyp2U.js → ganttDiagram-NO4QXBWP-DxkYU6kL.js} +1 -1
- package/dist/assets/{gitGraphDiagram-IHSO6WYX-leeHrOrC.js → gitGraphDiagram-IHSO6WYX-BP9sXgF1.js} +1 -1
- package/dist/assets/{index-lHF2zw41.js → index-B5rQe41U.js} +1 -1
- package/dist/assets/index-BLo9z-jE.js +1 -0
- package/dist/assets/{index-CFiAyn9P.js → index-CcSdiUnJ.js} +3 -3
- package/dist/assets/index-Cf8OqaQe.js +1 -0
- package/dist/assets/{index-Bnk8mX62.js → index-Ct8OBbU_.js} +1 -1
- package/dist/assets/{index-fP8UTiVP.js → index-D1S-PYLr.js} +2 -2
- package/dist/assets/{index-DY2LUL1D.js → index-DCRtV-sQ.js} +1 -1
- package/dist/assets/{index-CtGcfchE.js → index-DSC1km47.js} +1 -1
- package/dist/assets/{index-Cpf2LUXa.js → index-MzszL_qp.js} +4 -4
- package/dist/assets/{infoDiagram-FWYZ7A6U-B23FL64X.js → infoDiagram-FWYZ7A6U-ODz77wW8.js} +1 -1
- package/dist/assets/{ishikawaDiagram-FXEZZL3T-BPsjNXWV.js → ishikawaDiagram-FXEZZL3T-B-zkHET8.js} +1 -1
- package/dist/assets/{journeyDiagram-5HDEW3XC-3jqJYemY.js → journeyDiagram-5HDEW3XC-3-0ZxNaY.js} +1 -1
- package/dist/assets/{kanban-definition-HUTT4EX6-rHfIfWqi.js → kanban-definition-HUTT4EX6-6TXFrj9B.js} +1 -1
- package/dist/assets/{linear-CvofFTBe.js → linear-D4INeZan.js} +1 -1
- package/dist/assets/main-BNsRZCIt.css +1 -0
- package/dist/assets/{main-ts-uUN3p.js → main-awRasOMk.js} +142 -139
- package/dist/assets/{mermaid.core-BuuQCwr7.js → mermaid.core-VfI8B8Nu.js} +4 -4
- package/dist/assets/{mindmap-definition-LN4V7U3C-CdMbsouS.js → mindmap-definition-LN4V7U3C-BtnLuWah.js} +1 -1
- package/dist/assets/{pegDiagram-2B236MQR-Cl7drnf-.js → pegDiagram-2B236MQR-C7u_WQ8m.js} +1 -1
- package/dist/assets/{pieDiagram-ENE6RG2P-CrfvJw6b.js → pieDiagram-ENE6RG2P-FP2oK2aq.js} +1 -1
- package/dist/assets/{quadrantDiagram-ABIIQ3AL-D6Z7VqX6.js → quadrantDiagram-ABIIQ3AL-C7R49DqX.js} +1 -1
- package/dist/assets/{railroadDiagram-RFXS5EU6-DBhqTpwY.js → railroadDiagram-RFXS5EU6-8Y-09cJG.js} +1 -1
- package/dist/assets/{requirementDiagram-TGXJPOKE-DjYygiNl.js → requirementDiagram-TGXJPOKE-CuLFvAIT.js} +1 -1
- package/dist/assets/{sankeyDiagram-HTMAVEWB-Cb8su5H6.js → sankeyDiagram-HTMAVEWB-C83wAget.js} +1 -1
- package/dist/assets/{sequenceDiagram-DBY2YBRQ-DH326uFa.js → sequenceDiagram-DBY2YBRQ-2bX54qCB.js} +1 -1
- package/dist/assets/{sizeCapture-X5ZJPWSS-BA-8zei3.js → sizeCapture-X5ZJPWSS-CwcxMidm.js} +1 -1
- package/dist/assets/{stateDiagram-2N3HPSRC-DM412fD5.js → stateDiagram-2N3HPSRC-BRLu0Nh3.js} +1 -1
- package/dist/assets/stateDiagram-v2-6OUMAXLB-HDFSwLIz.js +1 -0
- package/dist/assets/{swimlanes-5IMT3BWC-BouPQ02z.js → swimlanes-5IMT3BWC-0ZHVD_6W.js} +2 -2
- package/dist/assets/swimlanesDiagram-G3AALYLV-DIR86oWK.js +8 -0
- package/dist/assets/{timeline-definition-FHXFAJF6-CHbeQn6a.js → timeline-definition-FHXFAJF6-mZxmuno3.js} +1 -1
- package/dist/assets/{vennDiagram-L72KCM5P-HSzYjTi7.js → vennDiagram-L72KCM5P-529a4wgg.js} +1 -1
- package/dist/assets/{wardleyDiagram-EHGQE667-DvrdmzxK.js → wardleyDiagram-EHGQE667-DsYcsK_D.js} +1 -1
- package/dist/assets/{web-BvtZvOK9.js → web-CUfujGZR.js} +1 -1
- package/dist/assets/{web-DT7q-yw_.js → web-CzSim3jK.js} +1 -1
- package/dist/assets/{web-DwD7eqxR.js → web-VJlfgT9G.js} +1 -1
- package/dist/assets/{xychartDiagram-FW5EYKEG-DgSgkHvg.js → xychartDiagram-FW5EYKEG-DImLqIu4.js} +1 -1
- package/dist/build-info.json +1 -1
- package/dist/index.html +2 -2
- package/dist/locales/de/terminal.json +2 -1
- package/dist/locales/en/terminal.json +3 -1
- package/dist/locales/es/terminal.json +3 -1
- package/dist/locales/fr/terminal.json +2 -1
- package/dist/locales/hi/terminal.json +2 -1
- package/dist/locales/it/terminal.json +2 -1
- package/dist/locales/ja/terminal.json +2 -1
- package/dist/locales/pt/terminal.json +2 -1
- package/dist/locales/ru/terminal.json +2 -1
- package/dist/locales/zh-CN/terminal.json +2 -1
- package/dist/src/packages/server/claude/backend.js +26 -2
- package/dist/src/packages/server/claude/interactive/interactive-backend-args.js +1 -1
- package/dist/src/packages/server/claude/interactive/interactive-jsonl-translator.js +20 -0
- package/dist/src/packages/server/claude/interactive/interactive-jsonl-watcher.js +9 -1
- package/dist/src/packages/server/claude/runner/process-lifecycle.js +3 -1
- package/dist/src/packages/server/claude/runner/restart-policy.js +65 -2
- package/dist/src/packages/server/claude/runner/stdout-pipeline.js +10 -1
- package/dist/src/packages/server/claude/runner/watchdog.js +29 -0
- package/dist/src/packages/server/claude/session-loader.js +641 -45
- package/dist/src/packages/server/claude/types.js +10 -1
- package/dist/src/packages/server/codex/app-server/app-server-event-adapter.js +27 -11
- package/dist/src/packages/server/grok/session-watcher.js +60 -0
- package/dist/src/packages/server/routes/agents.js +26 -0
- package/dist/src/packages/server/routes/sessions.js +9 -4
- package/dist/src/packages/server/routes/system.js +16 -0
- package/dist/src/packages/server/services/agent-service.js +10 -0
- package/dist/src/packages/server/services/codex-usage-service.js +141 -2
- package/dist/src/packages/server/services/runtime-service.js +22 -0
- package/dist/src/packages/server/websocket/listeners/runtime-listeners.js +27 -0
- package/dist/src/packages/shared/model-fallback.js +203 -0
- package/dist/src/packages/shared/types.js +1 -0
- package/package.json +1 -1
- package/dist/assets/channel-BgBFpxsT.js +0 -1
- package/dist/assets/classDiagram-OUVF2IWQ-CXHgnAgF.js +0 -1
- package/dist/assets/classDiagram-v2-EOCWNBFH-CXHgnAgF.js +0 -1
- package/dist/assets/index-BrN1fSea.js +0 -1
- package/dist/assets/index-jlRDILHc.js +0 -1
- package/dist/assets/main-CZA5Uyu6.css +0 -1
- package/dist/assets/stateDiagram-v2-6OUMAXLB-BZJwjeVk.js +0 -1
- package/dist/assets/swimlanesDiagram-G3AALYLV-BRQaNmpd.js +0 -8
|
@@ -9,14 +9,22 @@
|
|
|
9
9
|
import * as fs from 'fs';
|
|
10
10
|
import { createFileTailer } from '../runner/tmux-helper.js';
|
|
11
11
|
import { InteractiveJsonlTranslator } from './interactive-jsonl-translator.js';
|
|
12
|
+
import { translateCliModel } from './interactive-backend-args.js';
|
|
13
|
+
import { getAgent } from '../../services/agent-service.js';
|
|
12
14
|
import { createLogger } from '../../utils/logger.js';
|
|
13
15
|
const log = createLogger('Interactive');
|
|
14
16
|
export class InteractiveJsonlWatcher {
|
|
15
17
|
opts;
|
|
16
18
|
tailer = null;
|
|
17
|
-
translator
|
|
19
|
+
translator;
|
|
18
20
|
constructor(opts) {
|
|
19
21
|
this.opts = opts;
|
|
22
|
+
// Baseline + last-known state for silent-fallback detection. The transcript
|
|
23
|
+
// has no `system/init` to read the session model from, so it comes from the
|
|
24
|
+
// agent record — which also carries any fallback still in effect from before
|
|
25
|
+
// a commander restart.
|
|
26
|
+
const agent = getAgent(opts.agentId);
|
|
27
|
+
this.translator = new InteractiveJsonlTranslator(agent?.model ? translateCliModel(agent.model) : null, agent?.modelFallback?.servedModel ?? null);
|
|
20
28
|
}
|
|
21
29
|
start() {
|
|
22
30
|
if (this.tailer)
|
|
@@ -125,12 +125,13 @@ export class RunnerProcessLifecycle {
|
|
|
125
125
|
initialStdin = this.backend.formatStdinInput(prompt);
|
|
126
126
|
log.log(`📤 [TMUX-STDIN] Passing initial prompt (${initialStdin.length} chars) via shell pipe for agent ${agentId}`);
|
|
127
127
|
}
|
|
128
|
+
const closesStdinAfterPrompt = this.backend.shouldCloseStdinAfterPrompt?.() ?? false;
|
|
128
129
|
const tmuxResult = spawnInTmux(executable, args, {
|
|
129
130
|
agentId,
|
|
130
131
|
cwd: workingDir,
|
|
131
132
|
env,
|
|
132
133
|
initialStdin,
|
|
133
|
-
closeStdinAfterPrompt:
|
|
134
|
+
closeStdinAfterPrompt: closesStdinAfterPrompt,
|
|
134
135
|
});
|
|
135
136
|
const activeProcess = {
|
|
136
137
|
agentId,
|
|
@@ -143,6 +144,7 @@ export class RunnerProcessLifecycle {
|
|
|
143
144
|
tmuxSession: tmuxResult.sessionName,
|
|
144
145
|
tmuxLogFile: tmuxResult.logFile,
|
|
145
146
|
tmuxExpectedCommand: path.basename(executable),
|
|
147
|
+
closesStdinAfterPrompt,
|
|
146
148
|
};
|
|
147
149
|
this.activeProcesses.set(agentId, activeProcess);
|
|
148
150
|
this.attachGrokSideChannel(activeProcess, agentId, workingDir, grokWatcherSessionId);
|
|
@@ -1,8 +1,37 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
1
2
|
import { createLogger } from '../../utils/logger.js';
|
|
3
|
+
import * as agentService from '../../services/agent-service.js';
|
|
2
4
|
const log = createLogger('Runner');
|
|
3
5
|
const MAX_RESTART_ATTEMPTS = 3;
|
|
4
6
|
const RESTART_COOLDOWN_MS = 60000;
|
|
5
7
|
const MIN_RUNTIME_FOR_RESTART_MS = 5000;
|
|
8
|
+
/**
|
|
9
|
+
* Did the CLI itself actually write anything to stdout during this spawn?
|
|
10
|
+
*
|
|
11
|
+
* `lastActivityTime` is NOT usable for this. Side channels update it without
|
|
12
|
+
* the CLI producing a byte — the grok session watcher emits a usage_snapshot
|
|
13
|
+
* the moment it attaches, reading signals.json left over from the PREVIOUS
|
|
14
|
+
* turn. That happens ~300ms after spawn, so a stone-dead hung resume looked
|
|
15
|
+
* "active", the restart counter reset to zero every cycle, and the capped
|
|
16
|
+
* restart loop ran forever.
|
|
17
|
+
*
|
|
18
|
+
* The stdout log is truncated on every spawn (tmux-helper writes '' before
|
|
19
|
+
* launching), so a non-empty file means THIS process emitted something.
|
|
20
|
+
*/
|
|
21
|
+
function producedStdout(activeProcess) {
|
|
22
|
+
const file = activeProcess.tmuxLogFile ?? activeProcess.outputFile;
|
|
23
|
+
if (!file) {
|
|
24
|
+
// Pipe mode without a log file: fall back to activity time.
|
|
25
|
+
return activeProcess.lastActivityTime != null
|
|
26
|
+
&& activeProcess.lastActivityTime > activeProcess.startTime;
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
return fs.statSync(file).size > 0;
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
6
35
|
export class RunnerRestartPolicy {
|
|
7
36
|
callbacks;
|
|
8
37
|
activeProcesses;
|
|
@@ -49,10 +78,44 @@ export class RunnerRestartPolicy {
|
|
|
49
78
|
const restartCount = activeProcess.restartCount || 0;
|
|
50
79
|
const lastRestartTime = activeProcess.lastRestartTime || 0;
|
|
51
80
|
const timeSinceLastRestart = Date.now() - lastRestartTime;
|
|
52
|
-
|
|
81
|
+
// Elapsed time alone is NOT proof that a restart was healthy — the CLI must
|
|
82
|
+
// actually have written to stdout. The idle watchdog only kills a wedged CLI
|
|
83
|
+
// after IDLE_RESPAWN_MS (180s), which always exceeds RESTART_COOLDOWN_MS
|
|
84
|
+
// (60s), so a purely time-based reset put every hang→kill→restart cycle back
|
|
85
|
+
// at "attempt 1/3" and the supposedly capped loop ran forever. That is
|
|
86
|
+
// exactly the case the cap exists to stop: grok deadlocks replaying a
|
|
87
|
+
// session that was killed mid-turn, hanging before it opens a single socket
|
|
88
|
+
// and emitting zero bytes, forever.
|
|
89
|
+
const producedOutput = producedStdout(activeProcess);
|
|
90
|
+
const effectiveRestartCount = producedOutput && timeSinceLastRestart > RESTART_COOLDOWN_MS
|
|
91
|
+
? 0
|
|
92
|
+
: restartCount;
|
|
53
93
|
if (effectiveRestartCount >= MAX_RESTART_ATTEMPTS) {
|
|
54
94
|
log.error(`🔄 [AUTO-RESTART] Max restart attempts (${MAX_RESTART_ATTEMPTS}) reached for ${agentId}`);
|
|
55
|
-
|
|
95
|
+
if (!producedOutput && activeProcess.sessionId) {
|
|
96
|
+
// Every attempt replayed the same session and the CLI never wrote a
|
|
97
|
+
// byte — the session itself is unresumable (grok deadlocks loading a
|
|
98
|
+
// session that was killed mid-turn, before it even opens a socket).
|
|
99
|
+
// Retrying it again can only fail the same way, and leaving the id in
|
|
100
|
+
// place strands the agent: every later message resumes the same corpse,
|
|
101
|
+
// so the user "can't send messages" until someone manually clears the
|
|
102
|
+
// context. Drop the id here so the next message starts a fresh session.
|
|
103
|
+
// Deliberately narrow: only when stdout was completely empty.
|
|
104
|
+
log.error(`🔄 [AUTO-RESTART] Session ${activeProcess.sessionId} is unresumable for ${agentId} — clearing it so the next message starts fresh`);
|
|
105
|
+
if (activeProcess.lastRequest)
|
|
106
|
+
activeProcess.lastRequest.sessionId = undefined;
|
|
107
|
+
activeProcess.sessionId = undefined;
|
|
108
|
+
try {
|
|
109
|
+
agentService.updateAgent(agentId, { sessionId: undefined });
|
|
110
|
+
}
|
|
111
|
+
catch (err) {
|
|
112
|
+
log.error(`🔄 [AUTO-RESTART] Failed to clear session for ${agentId}:`, err);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
this.callbacks.onError(agentId, producedOutput
|
|
116
|
+
? `Process keeps crashing - auto-restart disabled after ${MAX_RESTART_ATTEMPTS} attempts. Manual intervention required.`
|
|
117
|
+
: `Session was unresumable - ${MAX_RESTART_ATTEMPTS} restarts produced no output at all. `
|
|
118
|
+
+ `It has been cleared; your next message will start a fresh session (previous conversation is not recoverable).`);
|
|
56
119
|
return;
|
|
57
120
|
}
|
|
58
121
|
log.log(`🔄 [AUTO-RESTART] Restarting ${agentId} (attempt ${effectiveRestartCount + 1}/${MAX_RESTART_ATTEMPTS})...`);
|
|
@@ -140,7 +140,8 @@ export class RunnerStdoutPipeline {
|
|
|
140
140
|
// usage_snapshot (context tracking), error (always important).
|
|
141
141
|
if (this.notificationSent.has(agentId)) {
|
|
142
142
|
const passthrough = event.type === 'init' || event.type === 'step_complete'
|
|
143
|
-
|| event.type === 'usage_snapshot' || event.type === 'error' || event.type === 'compacting'
|
|
143
|
+
|| event.type === 'usage_snapshot' || event.type === 'error' || event.type === 'compacting'
|
|
144
|
+
|| event.type === 'model_fallback';
|
|
144
145
|
if (!passthrough) {
|
|
145
146
|
return;
|
|
146
147
|
}
|
|
@@ -328,6 +329,14 @@ export class RunnerStdoutPipeline {
|
|
|
328
329
|
// Emit as output so runtime-listeners can broadcast it to clients
|
|
329
330
|
this.callbacks.onOutput(agentId, '[System] Compacting context...', false, undefined, event.uuid);
|
|
330
331
|
break;
|
|
332
|
+
case 'model_fallback':
|
|
333
|
+
// The API swapped models on us without asking. Make it a visible row in
|
|
334
|
+
// the terminal — two of the three fallback lanes are silent by design,
|
|
335
|
+
// so this line is the only place the user ever learns about it.
|
|
336
|
+
this.callbacks.onOutput(agentId, event.fallbackRestored
|
|
337
|
+
? `✅ [System] Model restored: back on ${event.text}`
|
|
338
|
+
: `⚠ [System] Model fallback: the API answered with ${event.text} — not the model this agent is configured to run`, false, undefined, event.uuid);
|
|
339
|
+
break;
|
|
331
340
|
default:
|
|
332
341
|
break;
|
|
333
342
|
}
|
|
@@ -18,6 +18,11 @@ const IDLE_RESPAWN_MS = Number.parseInt(process.env.TIDE_IDLE_RESPAWN_MS ?? '',
|
|
|
18
18
|
// count as activity, and message sends flip turnState to 'processing', so neither
|
|
19
19
|
// can trip this.
|
|
20
20
|
const STUCK_WORKING_RECONCILE_MS = Number.parseInt(process.env.TIDE_STUCK_WORKING_RECONCILE_MS ?? '', 10) || 180_000;
|
|
21
|
+
// A one-prompt-per-process CLI (grok/codex/opencode) whose turn already ended
|
|
22
|
+
// should exit on its own within moments. Anything still alive this long after
|
|
23
|
+
// its turn is a corpse squatting on the tmux session — reap it so the next
|
|
24
|
+
// message can spawn and the queue drains.
|
|
25
|
+
const LINGER_AFTER_TURN_MS = Number.parseInt(process.env.TIDE_LINGER_AFTER_TURN_MS ?? '', 10) || 30_000;
|
|
21
26
|
export class RunnerWatchdog {
|
|
22
27
|
activeProcesses;
|
|
23
28
|
lastStderr;
|
|
@@ -124,6 +129,30 @@ export class RunnerWatchdog {
|
|
|
124
129
|
continue;
|
|
125
130
|
}
|
|
126
131
|
}
|
|
132
|
+
// Lingering-after-turn reap: the turn is over (waiting_for_input) but a
|
|
133
|
+
// one-prompt-per-process CLI never exited. Grok does this when the turn
|
|
134
|
+
// spawned a background process (a dev server) that outlives it, or for
|
|
135
|
+
// no visible reason at all. It matters because sendCommand ALWAYS queues
|
|
136
|
+
// for stdin-closed tmux backends and relies on the respawn path to
|
|
137
|
+
// deliver — so while the corpse holds the session, every new message
|
|
138
|
+
// sits in the queue undelivered and the user sees "working" forever.
|
|
139
|
+
// Killing the session routes through watchdog_missing_process →
|
|
140
|
+
// restartPolicy, which skips restart on waiting_for_input and lets the
|
|
141
|
+
// next message spawn cleanly. Only for closesStdinAfterPrompt backends:
|
|
142
|
+
// claude deliberately stays alive between turns to reuse stdin.
|
|
143
|
+
if (activeProcess.closesStdinAfterPrompt
|
|
144
|
+
&& activeProcess.turnState === 'waiting_for_input') {
|
|
145
|
+
const lingeredFor = now - (activeProcess.lastActivityTime ?? activeProcess.startTime);
|
|
146
|
+
if (lingeredFor > LINGER_AFTER_TURN_MS) {
|
|
147
|
+
log.log(`🐕 [WATCHDOG] Agent ${agentId}: turn ended but CLI '${expected}' still alive `
|
|
148
|
+
+ `${Math.round(lingeredFor / 1000)}s later (threshold=${LINGER_AFTER_TURN_MS / 1000}s) `
|
|
149
|
+
+ `— killing tmux session ${activeProcess.tmuxSession} so queued messages can be delivered`);
|
|
150
|
+
activeProcess.tmuxTailer?.drain();
|
|
151
|
+
activeProcess.tmuxTailer?.stop();
|
|
152
|
+
killTmuxSession(agentId);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
127
156
|
// Idle-timeout cleanup: if the owning agent is idle (not actively
|
|
128
157
|
// working) and there's been no activity (output, send, spawn) for
|
|
129
158
|
// longer than the configured threshold, kill the tmux session to
|