termdock 1.4.324 → 1.4.326

package/dist/server/utils/tmuxRecoveryOperations.js

@@ -0,0 +1,402 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+export function summarizeRecoveryOperation(operation) {
+    const counts = operation.items.reduce((total, item) => {
+        total.total += 1;
+        if (item.state === 'resource_wait') {
+            total.resource_wait += 1;
+            total.paused_count += 1;
+        }
+        else if (item.state === 'paused') {
+            total.paused_count += 1;
+            if (item.pauseReason === 'resources')
+                total.resource_wait += 1;
+        }
+        else if (item.state === 'queued' && operation.paused) {
+            total.paused_count += 1;
+        }
+        else {
+            total[item.state] += 1;
+        }
+        return total;
+    }, { total: 0, queued: 0, resource_wait: 0, running: 0, succeeded: 0, failed: 0, paused_count: 0 });
+    return { ...structuredClone(operation), ...counts, counts: {
+            total: counts.total, queued: counts.queued, running: counts.running,
+            succeeded: counts.succeeded, failed: counts.failed, paused: counts.paused_count,
+        } };
+}
+export class TmuxRecoveryOperationError extends Error {
+    code;
+    operationId;
+    constructor(code, message, operationId) {
+        super(message);
+        this.code = code;
+        this.operationId = operationId;
+        this.name = 'TmuxRecoveryOperationError';
+    }
+}
+function requestHash(input) {
+    return JSON.stringify(input);
+}
+function terminalState(state) {
+    return state === 'succeeded' || state === 'partial_failure';
+}
+/** Durable, bounded-concurrency queue for explicit tmux recovery operations. */
+export class TmuxRecoveryOperationStore {
+    filePath;
+    concurrencyLimit;
+    onError;
+    state = { version: 1, operations: [] };
+    writeChain = Promise.resolve();
+    active = 0;
+    pumping = false;
+    pumpRequested = false;
+    runner;
+    admission;
+    listeners = new Set();
+    loaded = false;
+    constructor(filePath, concurrencyLimit, onError = () => { }) {
+        this.filePath = filePath;
+        this.concurrencyLimit = concurrencyLimit;
+        this.onError = onError;
+        if (!Number.isInteger(concurrencyLimit) || concurrencyLimit < 1 || concurrencyLimit > 16) {
+            throw new Error('tmux recovery concurrency must be between 1 and 16');
+        }
+    }
+    subscribe(listener) {
+        this.listeners.add(listener);
+        return () => this.listeners.delete(listener);
+    }
+    setRunner(runner) {
+        this.runner = runner;
+    }
+    setAdmission(admission) {
+        this.admission = admission;
+    }
+    async load() {
+        if (this.loaded)
+            return;
+        this.loaded = true;
+        try {
+            const parsed = JSON.parse(await fs.readFile(this.filePath, 'utf8'));
+            if (parsed.version !== 1 || !Array.isArray(parsed.operations))
+                throw new Error('Invalid tmux recovery operation state');
+            this.state = parsed;
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT')
+                throw error;
+        }
+        const now = Date.now();
+        let recovered = false;
+        for (const operation of this.state.operations) {
+            for (const item of operation.items) {
+                if (item.state === 'running') {
+                    // A killed worker may have submitted the native resume before it died.
+                    // The runner always performs an exact-owner reconciliation before it launches.
+                    item.state = operation.paused ? 'paused' : 'queued';
+                    item.pauseReason = operation.paused ? 'manual' : undefined;
+                    item.phase = 'restart_reconciliation';
+                    item.errorCode = 'RECOVERY_WORKER_RESTARTED';
+                    item.error = 'Service restarted during recovery; exact runtime ownership will be reconciled before retry.';
+                    recovered = true;
+                }
+            }
+            if (!terminalState(operation.state) && !operation.paused)
+                operation.state = operation.items.some(item => item.state === 'paused' && item.pauseReason === 'resources') ? 'paused' : 'queued';
+            operation.updatedAt = recovered ? now : operation.updatedAt;
+        }
+        if (recovered)
+            await this.persist();
+    }
+    async create(input) {
+        if (!this.loaded)
+            await this.load();
+        if (!input.idempotencyKey || input.idempotencyKey.length > 200) {
+            throw new TmuxRecoveryOperationError('INVALID_IDEMPOTENCY_KEY', 'A non-empty idempotency key of at most 200 characters is required.');
+        }
+        const sessionIds = [...new Set(input.sessionIds)].sort();
+        if (sessionIds.length === 0)
+            throw new TmuxRecoveryOperationError('RECOVERY_SCOPE_EMPTY', 'The selected recovery scope contains no recoverable sessions.');
+        const concurrency = input.concurrency ?? this.concurrencyLimit;
+        if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > this.concurrencyLimit) {
+            throw new TmuxRecoveryOperationError('INVALID_RECOVERY_CONCURRENCY', `Concurrency must be between 1 and ${this.concurrencyLimit}.`);
+        }
+        const scope = input.scope.kind === 'sessions'
+            ? { ...input.scope, sessionIds: [...new Set(input.scope.sessionIds ?? [])].sort() }
+            : structuredClone(input.scope);
+        const fingerprint = requestHash({ incidentId: input.incidentId, sessionIds, scope, concurrency });
+        const existingKey = this.state.operations.find(operation => operation.idempotencyKey === input.idempotencyKey || operation.idempotencyKeyAliases?.includes(input.idempotencyKey));
+        if (existingKey) {
+            if (existingKey.requestHash !== fingerprint)
+                throw new TmuxRecoveryOperationError('IDEMPOTENCY_CONFLICT', 'The idempotency key was already used for a different recovery request.');
+            return structuredClone(existingKey);
+        }
+        if (this.state.operations.length >= 2000)
+            throw new TmuxRecoveryOperationError('RECOVERY_OPERATION_LIMIT', 'Recovery operation history is full; archive diagnostics before creating another operation.');
+        const inProgress = this.state.operations.find(operation => !terminalState(operation.state)
+            && operation.requestHash === fingerprint
+            && operation.items.some(item => sessionIds.includes(item.sessionId) && (item.state === 'queued' || item.state === 'resource_wait' || item.state === 'paused' || item.state === 'running')));
+        if (inProgress) {
+            inProgress.idempotencyKeyAliases = [...new Set([...(inProgress.idempotencyKeyAliases ?? []), input.idempotencyKey])];
+            inProgress.updatedAt = Date.now();
+            await this.persist();
+            this.emit();
+            return structuredClone(inProgress);
+        }
+        const overlapping = this.state.operations.find(operation => !terminalState(operation.state)
+            && operation.items.some(item => sessionIds.includes(item.sessionId) && (item.state === 'queued' || item.state === 'resource_wait' || item.state === 'paused' || item.state === 'running')));
+        if (overlapping)
+            throw new TmuxRecoveryOperationError('RECOVERY_SESSION_IN_PROGRESS', 'At least one selected session is already in a recovery operation.', overlapping.id);
+        const now = Date.now();
+        const operation = {
+            id: randomUUID(), incidentId: input.incidentId, idempotencyKey: input.idempotencyKey,
+            requestHash: fingerprint, scope, state: 'queued', paused: false,
+            concurrency, createdAt: now, updatedAt: now,
+            items: sessionIds.map(sessionId => ({ sessionId, state: 'queued', attempt: 0, phase: 'queued', startedAt: null, completedAt: null, estimatedRssBytes: 512 * 1024 ** 2 })),
+        };
+        this.state.operations.push(operation);
+        try {
+            await this.persist();
+        }
+        catch (error) {
+            this.state.operations = this.state.operations.filter(candidate => candidate.id !== operation.id);
+            throw error;
+        }
+        this.emit();
+        void this.pump();
+        return structuredClone(operation);
+    }
+    get(id) {
+        const operation = this.state.operations.find(candidate => candidate.id === id);
+        return operation ? summarizeRecoveryOperation(operation) : null;
+    }
+    latest() {
+        const operation = this.state.operations.at(-1);
+        return operation ? summarizeRecoveryOperation(operation) : null;
+    }
+    all() {
+        return this.state.operations.map(summarizeRecoveryOperation);
+    }
+    async pause(id) {
+        const operation = this.state.operations.find(candidate => candidate.id === id);
+        if (!operation)
+            return null;
+        if (!terminalState(operation.state)) {
+            operation.paused = true;
+            operation.pauseReason = 'manual';
+            for (const item of operation.items) {
+                if (item.state === 'queued' || item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')) {
+                    item.state = 'paused';
+                    item.pauseReason = 'manual';
+                }
+            }
+            operation.state = 'paused';
+            operation.updatedAt = Date.now();
+            await this.persist();
+            this.emit();
+        }
+        return this.get(id);
+    }
+    async resume(id) {
+        const operation = this.state.operations.find(candidate => candidate.id === id);
+        if (!operation)
+            return null;
+        if (!terminalState(operation.state)) {
+            operation.paused = false;
+            operation.pauseReason = undefined;
+            for (const item of operation.items) {
+                if (item.state === 'paused' && item.pauseReason === 'manual') {
+                    item.state = item.resourceWaitReason ? 'paused' : 'queued';
+                    item.pauseReason = item.resourceWaitReason ? 'resources' : undefined;
+                }
+            }
+            const waitingForResources = operation.items.some(item => item.state === 'paused' && item.pauseReason === 'resources');
+            operation.state = waitingForResources ? 'paused' : 'queued';
+            operation.pauseReason = waitingForResources ? 'resources' : undefined;
+            operation.updatedAt = Date.now();
+            await this.persist();
+            this.emit();
+            void this.pump();
+        }
+        return this.get(id);
+    }
+    stats() {
+        const operations = this.state.operations;
+        const items = operations.flatMap(operation => operation.items);
+        const lastCompleted = [...items].filter(item => item.completedAt !== null).sort((a, b) => (b.completedAt ?? 0) - (a.completedAt ?? 0))[0];
+        return {
+            queued: items.filter(item => item.state === 'queued').length,
+            paused: items.filter(item => item.state === 'paused' || item.state === 'resource_wait').length,
+            resource_wait: items.filter(item => item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')).length,
+            running: items.filter(item => item.state === 'running').length,
+            activeLaunches: this.active,
+            lastCompletedAt: lastCompleted?.completedAt ?? null,
+            lastCompletedSessionId: lastCompleted?.sessionId ?? null,
+        };
+    }
+    async resumePending() {
+        if (!this.loaded)
+            await this.load();
+        void this.pump();
+    }
+    notifyResourcesChanged() {
+        if (this.pumping) {
+            this.pumpRequested = true;
+            return;
+        }
+        void this.pump();
+    }
+    async flush() { await this.writeChain; }
+    async pump() {
+        if (this.pumping || !this.runner || !this.loaded)
+            return;
+        this.pumping = true;
+        try {
+            const deferred = new Set();
+            while (this.active < this.concurrencyLimit) {
+                const candidate = this.state.operations.find(operation => !operation.paused && !terminalState(operation.state)
+                    && operation.items.filter(item => item.state === 'running').length < operation.concurrency
+                    && operation.items.some(item => (item.state === 'queued' || item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')) && !deferred.has(`${operation.id}:${item.sessionId}`)));
+                const item = candidate?.items.find(entry => (entry.state === 'queued' || entry.state === 'resource_wait' || (entry.state === 'paused' && entry.pauseReason === 'resources')) && !deferred.has(`${candidate.id}:${entry.sessionId}`));
+                if (!candidate || !item)
+                    break;
+                if (this.admission) {
+                    let admission;
+                    try {
+                        admission = await this.admission(candidate.id, item.sessionId);
+                    }
+                    catch (error) {
+                        admission = { admitted: false, reason: error instanceof Error ? error.message : 'RESOURCE_SNAPSHOT_UNAVAILABLE' };
+                    }
+                    item.memAvailableBytes = admission.memAvailableBytes;
+                    item.estimatedRssBytes = admission.estimatedRssBytes ?? item.estimatedRssBytes;
+                    if (!admission.admitted) {
+                        item.state = 'paused';
+                        item.pauseReason = 'resources';
+                        item.phase = 'resource_wait';
+                        item.resourceWaitReason = admission.reason ?? 'Resource budget is not currently available.';
+                        candidate.pauseReason = 'resources';
+                        candidate.state = candidate.items.some(candidateItem => candidateItem.state === 'running') ? 'running' : 'paused';
+                        candidate.updatedAt = Date.now();
+                        deferred.add(`${candidate.id}:${item.sessionId}`);
+                        await this.persist();
+                        this.emit();
+                        continue;
+                    }
+                    item.resourceWaitReason = undefined;
+                    item.pauseReason = undefined;
+                    candidate.pauseReason = candidate.items.some(candidateItem => candidateItem.state === 'paused' && candidateItem.pauseReason === 'resources' && candidateItem !== item) ? 'resources' : undefined;
+                }
+                if (candidate.paused)
+                    break;
+                candidate.state = 'running';
+                item.state = 'running';
+                item.phase = item.phase === 'restart_reconciliation' ? item.phase : 'restoring';
+                item.attempt += 1;
+                item.startedAt = Date.now();
+                item.completedAt = null;
+                candidate.updatedAt = Date.now();
+                this.active += 1;
+                await this.persist();
+                this.emit();
+                void this.execute(candidate, item);
+            }
+            if (this.state.operations.some(operation => !operation.paused && !terminalState(operation.state)
+                && operation.items.some(item => item.state === 'resource_wait' || (item.state === 'paused' && item.pauseReason === 'resources')))) {
+                const retry = setTimeout(() => void this.pump(), 5_000);
+                retry.unref?.();
+            }
+        }
+        catch (error) {
+            this.onError(error);
+        }
+        finally {
+            this.pumping = false;
+            if (this.pumpRequested) {
+                this.pumpRequested = false;
+                queueMicrotask(() => void this.pump());
+            }
+        }
+    }
+    async execute(operation, item) {
+        try {
+            const result = await this.runner(operation.id, item.sessionId, phase => {
+                item.phase = phase.slice(0, 80);
+                operation.updatedAt = Date.now();
+                void this.persist().then(() => this.emit()).catch(error => this.onError(error));
+            });
+            if (result?.actualRssBytes !== undefined)
+                item.actualRssBytes = result.actualRssBytes;
+            item.state = 'succeeded';
+            item.phase = 'completed';
+            item.error = undefined;
+            item.errorCode = undefined;
+        }
+        catch (error) {
+            const value = error && typeof error === 'object' ? error : {};
+            const message = error instanceof Error ? error.message : String(error);
+            const code = typeof value.code === 'string' ? value.code
+                : message.includes('TERMINAL_INPUT_TARGET_CHANGED') ? 'TERMINAL_INPUT_TARGET_CHANGED' : 'TMUX_RECOVERY_FAILED';
+            item.state = 'failed';
+            item.phase = code === 'TERMINAL_INPUT_TARGET_CHANGED' ? 'target_changed' : 'failed';
+            item.errorCode = code;
+            item.error = message.slice(0, 1000);
+            this.onError({ operationId: operation.id, sessionId: item.sessionId, code, message });
+        }
+        finally {
+            item.completedAt = Date.now();
+            operation.updatedAt = item.completedAt;
+            const hasPending = operation.items.some(candidate => candidate.state === 'queued' || candidate.state === 'resource_wait' || candidate.state === 'paused' || candidate.state === 'running');
+            if (!hasPending) {
+                operation.paused = false;
+                operation.pauseReason = undefined;
+                operation.state = operation.items.some(candidate => candidate.state === 'failed') ? 'partial_failure' : 'succeeded';
+            }
+            else {
+                operation.state = operation.paused ? 'paused'
+                    : operation.items.some(candidate => candidate.state === 'running') ? 'running'
+                        : operation.items.some(candidate => candidate.state === 'queued') ? 'queued' : 'paused';
+                if (operation.paused)
+                    operation.pauseReason = 'manual';
+                else if (!operation.items.some(candidate => candidate.state === 'running' || candidate.state === 'queued'))
+                    operation.pauseReason = 'resources';
+            }
+            this.active = Math.max(0, this.active - 1);
+            try {
+                await this.persist();
+            }
+            catch (error) {
+                this.onError(error);
+            }
+            this.emit();
+            void this.pump();
+        }
+    }
+    emit() { for (const listener of this.listeners) {
+        try {
+            listener();
+        }
+        catch { /* observer failure is isolated */ }
+    } }
+    persist() {
+        const snapshot = JSON.stringify(this.state, null, 2);
+        const write = this.writeChain.then(async () => {
+            await fs.mkdir(path.dirname(this.filePath), { recursive: true });
+            const tempPath = `${this.filePath}.${process.pid}.${randomUUID()}.tmp`;
+            await fs.writeFile(tempPath, snapshot, { mode: 0o600 });
+            await fs.rename(tempPath, this.filePath);
+        });
+        this.writeChain = write.catch(error => { this.onError(error); });
+        return write;
+    }
+}
+export function configuredTmuxRecoveryConcurrency(value = process.env.TD_TMUX_RECOVERY_CONCURRENCY) {
+    if (value === undefined || value.trim() === '')
+        return 3;
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < 1 || parsed > 16)
+        throw new Error('Recovery concurrency must be an integer from 1 to 16');
+    return parsed;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "termdock",
-  "version": "1.4.324",
+  "version": "1.4.326",
   "description": "A complete web-based terminal application with modern UI",
   "type": "module",
   "main": ".desktop-dist/main.js",

package/runtime-manifest.json

@@ -1,13 +1,13 @@
 {
   "schemaVersion": 1,
   "packageName": "termdock",
-  "version": "1.4.324",
+  "version": "1.4.326",
   "runtimeProtocolVersion": 1,
   "minimumDesktopVersion": "1.4.46",
   "nodeMajor": 22,
   "dependencyHash": "sha256-oaGYi/P5ob8sd6VnNBOwshKozjhdV+hYiCmLAsn5x00=",
-  "serverBundleHash": "sha256-yjPt7bAXNp3KrOLjsqzi7sVLnPj/UD3Qz7jSJwgX9gU=",
-  "clientBundleHash": "sha256-AoWtEiTNIyzQQDP3vrVfbjymEXr6cyRy/8C++LTIBtc=",
+  "serverBundleHash": "sha256-8rbCJEWANeXtZp/cvJ7UT9tGunnkMMxL7mcvARckvLQ=",
+  "clientBundleHash": "sha256-9MzxeyAxwRf4rg374l2f6wUzrybLPL+H5boh1/uodkQ=",
   "entrypoint": "dist/server/cli.js",
   "clientEntrypoint": "dist/client/index.html"
 }