termdock 1.4.325 → 1.4.327

package/dist/server/utils/tmuxRecoveryOperations.js

@@ -4,10 +4,27 @@ import { randomUUID } from 'node:crypto';
 export function summarizeRecoveryOperation(operation) {
     const counts = operation.items.reduce((total, item) => {
         total.total += 1;
-        total[item.state] += 1;
+        if (item.state === 'resource_wait') {
+            total.resource_wait += 1;
+            total.paused_count += 1;
+        }
+        else if (item.state === 'paused') {
+            total.paused_count += 1;
+            if (item.pauseReason === 'resources')
+                total.resource_wait += 1;
+        }
+        else if (item.state === 'queued' && operation.paused) {
+            total.paused_count += 1;
+        }
+        else {
+            total[item.state] += 1;
+        }
         return total;
-    }, { total: 0, queued: 0, resource_wait: 0, running: 0, succeeded: 0, failed: 0 });
-    return { ...structuredClone(operation), ...counts };
+    }, { total: 0, queued: 0, resource_wait: 0, running: 0, succeeded: 0, failed: 0, paused_count: 0 });
+    return { ...structuredClone(operation), ...counts, counts: {
+            total: counts.total, queued: counts.queued, running: counts.running,
+            succeeded: counts.succeeded, failed: counts.failed, paused: counts.paused_count,
+        } };
 }
 export class TmuxRecoveryOperationError extends Error {
     code;
@@ -78,7 +95,8 @@ export class TmuxRecoveryOperationStore {
                 if (item.state === 'running') {
                     // A killed worker may have submitted the native resume before it died.
                     // The runner always performs an exact-owner reconciliation before it launches.
-                    item.state = 'queued';
+                    item.state = operation.paused ? 'paused' : 'queued';
+                    item.pauseReason = operation.paused ? 'manual' : undefined;
                     item.phase = 'restart_reconciliation';
                     item.errorCode = 'RECOVERY_WORKER_RESTARTED';
                     item.error = 'Service restarted during recovery; exact runtime ownership will be reconciled before retry.';
@@ -86,7 +104,7 @@ export class TmuxRecoveryOperationStore {
                 }
             }
             if (!terminalState(operation.state) && !operation.paused)
-                operation.state = 'queued';
+                operation.state = operation.items.some(item => item.state === 'paused' && item.pauseReason === 'resources') ? 'paused' : 'queued';
             operation.updatedAt = recovered ? now : operation.updatedAt;
         }
         if (recovered)
@@ -109,7 +127,7 @@ export class TmuxRecoveryOperationStore {
             ? { ...input.scope, sessionIds: [...new Set(input.scope.sessionIds ?? [])].sort() }
             : structuredClone(input.scope);
         const fingerprint = requestHash({ incidentId: input.incidentId, sessionIds, scope, concurrency });
-        const existingKey = this.state.operations.find(operation => operation.idempotencyKey === input.idempotencyKey);
+        const existingKey = this.state.operations.find(operation => operation.idempotencyKey === input.idempotencyKey || operation.idempotencyKeyAliases?.includes(input.idempotencyKey));
         if (existingKey) {
             if (existingKey.requestHash !== fingerprint)
                 throw new TmuxRecoveryOperationError('IDEMPOTENCY_CONFLICT', 'The idempotency key was already used for a different recovery request.');
@@ -118,9 +136,19 @@ export class TmuxRecoveryOperationStore {
         if (this.state.operations.length >= 2000)
             throw new TmuxRecoveryOperationError('RECOVERY_OPERATION_LIMIT', 'Recovery operation history is full; archive diagnostics before creating another operation.');
         const inProgress = this.state.operations.find(operation => !terminalState(operation.state)
-            && operation.items.some(item => sessionIds.includes(item.sessionId) && (item.state === 'queued' || item.state === 'resource_wait' || item.state === 'running')));
-        if (inProgress)
-            throw new TmuxRecoveryOperationError('RECOVERY_SESSION_IN_PROGRESS', 'At least one selected session is already in a recovery operation.', inProgress.id);
+            && operation.requestHash === fingerprint
+            && operation.items.some(item => sessionIds.includes(item.sessionId) && (item.state === 'queued' || item.state === 'resource_wait' || item.state === 'paused' || item.state === 'running')));
+        if (inProgress) {
+            inProgress.idempotencyKeyAliases = [...new Set([...(inProgress.idempotencyKeyAliases ?? []), input.idempotencyKey])];
+            inProgress.updatedAt = Date.now();
+            await this.persist();
+            this.emit();
+            return structuredClone(inProgress);
+        }
+        const overlapping = this.state.operations.find(operation => !terminalState(operation.state)
+            && operation.items.some(item => sessionIds.includes(item.sessionId) && (item.state === 'queued' || item.state === 'resource_wait' || item.state === 'paused' || item.state === 'running')));
+        if (overlapping)
+            throw new TmuxRecoveryOperationError('RECOVERY_SESSION_IN_PROGRESS', 'At least one selected session is already in a recovery operation.', overlapping.id);
         const now = Date.now();
         const operation = {
             id: randomUUID(), incidentId: input.incidentId, idempotencyKey: input.idempotencyKey,
@@ -157,6 +185,13 @@ export class TmuxRecoveryOperationStore {
             return null;
         if (!terminalState(operation.state)) {
             operation.paused = true;
+            operation.pauseReason = 'manual';
+            for (const item of operation.items) {
+                if (item.state === 'queued' || item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')) {
+                    item.state = 'paused';
+                    item.pauseReason = 'manual';
+                }
+            }
             operation.state = 'paused';
             operation.updatedAt = Date.now();
             await this.persist();
@@ -170,7 +205,16 @@ export class TmuxRecoveryOperationStore {
             return null;
         if (!terminalState(operation.state)) {
             operation.paused = false;
-            operation.state = 'queued';
+            operation.pauseReason = undefined;
+            for (const item of operation.items) {
+                if (item.state === 'paused' && item.pauseReason === 'manual') {
+                    item.state = item.resourceWaitReason ? 'paused' : 'queued';
+                    item.pauseReason = item.resourceWaitReason ? 'resources' : undefined;
+                }
+            }
+            const waitingForResources = operation.items.some(item => item.state === 'paused' && item.pauseReason === 'resources');
+            operation.state = waitingForResources ? 'paused' : 'queued';
+            operation.pauseReason = waitingForResources ? 'resources' : undefined;
             operation.updatedAt = Date.now();
             await this.persist();
             this.emit();
@@ -184,7 +228,8 @@ export class TmuxRecoveryOperationStore {
         const lastCompleted = [...items].filter(item => item.completedAt !== null).sort((a, b) => (b.completedAt ?? 0) - (a.completedAt ?? 0))[0];
         return {
             queued: items.filter(item => item.state === 'queued').length,
-            resource_wait: items.filter(item => item.state === 'resource_wait').length,
+            paused: items.filter(item => item.state === 'paused' || item.state === 'resource_wait').length,
+            resource_wait: items.filter(item => item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')).length,
             running: items.filter(item => item.state === 'running').length,
             activeLaunches: this.active,
             lastCompletedAt: lastCompleted?.completedAt ?? null,
@@ -213,8 +258,8 @@ export class TmuxRecoveryOperationStore {
             while (this.active < this.concurrencyLimit) {
                 const candidate = this.state.operations.find(operation => !operation.paused && !terminalState(operation.state)
                     && operation.items.filter(item => item.state === 'running').length < operation.concurrency
-                    && operation.items.some(item => (item.state === 'queued' || item.state === 'resource_wait') && !deferred.has(`${operation.id}:${item.sessionId}`)));
-                const item = candidate?.items.find(entry => (entry.state === 'queued' || entry.state === 'resource_wait') && !deferred.has(`${candidate.id}:${entry.sessionId}`));
+                    && operation.items.some(item => (item.state === 'queued' || item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')) && !deferred.has(`${operation.id}:${item.sessionId}`)));
+                const item = candidate?.items.find(entry => (entry.state === 'queued' || entry.state === 'resource_wait' || (entry.state === 'paused' && entry.pauseReason === 'resources')) && !deferred.has(`${candidate.id}:${entry.sessionId}`));
                 if (!candidate || !item)
                     break;
                 if (this.admission) {
@@ -228,10 +273,12 @@ export class TmuxRecoveryOperationStore {
                     item.memAvailableBytes = admission.memAvailableBytes;
                     item.estimatedRssBytes = admission.estimatedRssBytes ?? item.estimatedRssBytes;
                     if (!admission.admitted) {
-                        item.state = 'resource_wait';
+                        item.state = 'paused';
+                        item.pauseReason = 'resources';
                         item.phase = 'resource_wait';
                         item.resourceWaitReason = admission.reason ?? 'Resource budget is not currently available.';
-                        candidate.state = 'resource_wait';
+                        candidate.pauseReason = 'resources';
+                        candidate.state = candidate.items.some(candidateItem => candidateItem.state === 'running') ? 'running' : 'paused';
                         candidate.updatedAt = Date.now();
                         deferred.add(`${candidate.id}:${item.sessionId}`);
                         await this.persist();
@@ -239,6 +286,8 @@ export class TmuxRecoveryOperationStore {
                         continue;
                     }
                     item.resourceWaitReason = undefined;
+                    item.pauseReason = undefined;
+                    candidate.pauseReason = candidate.items.some(candidateItem => candidateItem.state === 'paused' && candidateItem.pauseReason === 'resources' && candidateItem !== item) ? 'resources' : undefined;
                 }
                 if (candidate.paused)
                     break;
@@ -255,7 +304,7 @@ export class TmuxRecoveryOperationStore {
                 void this.execute(candidate, item);
             }
             if (this.state.operations.some(operation => !operation.paused && !terminalState(operation.state)
-                && operation.items.some(item => item.state === 'resource_wait'))) {
+                && operation.items.some(item => item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')))) {
                 const retry = setTimeout(() => void this.pump(), 5_000);
                 retry.unref?.();
             }
@@ -299,15 +348,20 @@ export class TmuxRecoveryOperationStore {
         finally {
             item.completedAt = Date.now();
             operation.updatedAt = item.completedAt;
-            const hasPending = operation.items.some(candidate => candidate.state === 'queued' || candidate.state === 'resource_wait' || candidate.state === 'running');
+            const hasPending = operation.items.some(candidate => candidate.state === 'queued' || candidate.state === 'resource_wait' || candidate.state === 'paused' || candidate.state === 'running');
             if (!hasPending) {
                 operation.paused = false;
+                operation.pauseReason = undefined;
                 operation.state = operation.items.some(candidate => candidate.state === 'failed') ? 'partial_failure' : 'succeeded';
             }
             else {
                 operation.state = operation.paused ? 'paused'
                     : operation.items.some(candidate => candidate.state === 'running') ? 'running'
-                        : operation.items.some(candidate => candidate.state === 'queued') ? 'queued' : 'resource_wait';
+                        : operation.items.some(candidate => candidate.state === 'queued') ? 'queued' : 'paused';
+                if (operation.paused)
+                    operation.pauseReason = 'manual';
+                else if (!operation.items.some(candidate => candidate.state === 'running' || candidate.state === 'queued'))
+                    operation.pauseReason = 'resources';
             }
             this.active = Math.max(0, this.active - 1);
             try {

package/docs/systemd.md

@@ -0,0 +1,35 @@
+# Running Termdock under systemd
+
+When a systemd unit starts `td --supervise`, set `KillMode=process` if its
+Termdock-managed tmux sessions and Agents must survive a Termdock service or
+supervisor restart. systemd's default `KillMode=control-group` sends the stop
+signal to the entire unit cgroup; that includes tmux servers and Agent
+processes started by Termdock. It can therefore turn a service restart into a
+simultaneous shutdown of every managed Agent.
+
+The repository ships
+[`10-preserve-tmux.conf`](../packaging/systemd/termdock.service.d/10-preserve-tmux.conf)
+as a drop-in fragment. Install it under the drop-in directory for the unit
+that runs Termdock (the example path assumes `termdock.service`):
+
+```sh
+sudo install -D -m 0644 packaging/systemd/termdock.service.d/10-preserve-tmux.conf \
+  /etc/systemd/system/termdock.service.d/10-preserve-tmux.conf
+sudo systemctl daemon-reload
+systemctl show termdock.service -p KillMode
+```
+
+The final command should report `KillMode=process`. If the unit has a different
+name, use its matching `/etc/systemd/system/<unit>.service.d/` directory and
+verify that unit.
+
+This setting intentionally leaves tmux workloads alive when systemd stops the
+Termdock main process. Before permanently removing the service or host, close
+or release managed sessions through Termdock's session lifecycle and verify
+that the runtime inventory is empty. Do not use `systemctl kill` or restore
+`KillMode=control-group` as a way to clean up Agent sessions; those actions can
+terminate unrelated work in the unit.
+
+This is a process-manager boundary, separate from recovery queue admission.
+Termdock will not terminate non-Termdock processes to satisfy its memory or
+resident-Agent budget.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "termdock",
-  "version": "1.4.325",
+  "version": "1.4.327",
   "description": "A complete web-based terminal application with modern UI",
   "type": "module",
   "main": ".desktop-dist/main.js",
@@ -16,6 +16,8 @@
     "dist",
     "runtime-manifest.json",
     "README.md",
+    "docs/systemd.md",
+    "packaging/systemd/termdock.service.d/10-preserve-tmux.conf",
     "LICENSE",
     "scripts",
     "docs/integrations.md"

package/packaging/systemd/termdock.service.d/10-preserve-tmux.conf

@@ -0,0 +1,5 @@
+# Install as a drop-in for the unit that runs `td --supervise`.
+# Termdock-managed tmux sessions are independent runtime workloads and must
+# survive a supervisor/service process restart.
+[Service]
+KillMode=process

package/runtime-manifest.json

@@ -1,13 +1,13 @@
 {
   "schemaVersion": 1,
   "packageName": "termdock",
-  "version": "1.4.325",
+  "version": "1.4.327",
   "runtimeProtocolVersion": 1,
   "minimumDesktopVersion": "1.4.46",
   "nodeMajor": 22,
   "dependencyHash": "sha256-oaGYi/P5ob8sd6VnNBOwshKozjhdV+hYiCmLAsn5x00=",
-  "serverBundleHash": "sha256-lFOekSTHpHZrRuNcWm20UDGwKSjiviUQKjaT/zSYBvc=",
-  "clientBundleHash": "sha256-RVhKCGWk1bExA0aa9FHB8dutqDr/aCkALgScfT+IS80=",
+  "serverBundleHash": "sha256-8rbCJEWANeXtZp/cvJ7UT9tGunnkMMxL7mcvARckvLQ=",
+  "clientBundleHash": "sha256-c+kY8sLmG3zeMRdXsCgqDWaIqC7vboMYVHsXMgZcbHU=",
   "entrypoint": "dist/server/cli.js",
   "clientEntrypoint": "dist/client/index.html"
 }