termdock 1.4.383 → 1.4.384

package/dist/server/agent/collaborationStore.js

@@ -6,6 +6,7 @@ import path from 'node:path';
 import crypto from 'node:crypto';
 import { directLocalId, directRemoteAddress } from './collaborationDirectProtocol.js';
 import { hasUnconfirmedTerminalWrite } from './collaborationProtocol.js';
+import { JsonStorageBudget } from '../utils/jsonStorageBudget.js';
 import { chatPresentation, validChatUIValues } from '../shared/chatPresentation.js';
 import { COLLAB_NAME_FORBIDDEN, COLLAB_LIMITS, CollaborationError, STATUS_RANK, canonicalShortId, drawCollaborationId, newCollaborationId, resolveIdPrefix, validateExtras } from './collaborationProtocol.js';
 function newerRevision(a, b) {
@@ -37,7 +38,7 @@ function pruneRoles(group, sessionIds) {
 }
 /** Public collaboration facts never expose legacy read guesses. */
 export function terminalMessage(message, raw = false) {
-    const { readAt: _readAt, readSource: _readSource, ...body } = message;
+    const { readAt: _readAt, readSource: _readSource, retainUntil: _retainUntil, ...body } = message;
     const legacyConsumption = message.deliverySource === 'consumer_read';
     return { ...body, status: legacyConsumption ? 'pending' : message.status === 'read' ? 'delivered' : message.status,
         deliveredAt: legacyConsumption ? null : message.deliveredAt,
@@ -87,12 +88,20 @@ export class CollaborationStore {
     groupCreationAvailable = true;
     groupRenamesAvailable = true;
     groupArchivesAvailable = true;
+    storage;
+    committedMessageIds = new Set();
+    committedPendingIds = new Set();
+    historyBytes;
     /** `drawId` is the random source behind newCollaborationId. It is a
      *  constructor seam so tests can force a collision and prove the guard
      *  below actually skips it; production always uses the CSPRNG default. */
-    constructor(filePath, drawId = drawCollaborationId) {
+    constructor(filePath, drawId = drawCollaborationId, options = {}) {
         this.filePath = filePath;
         this.drawId = drawId;
+        this.storage = new JsonStorageBudget(filePath, 'collaboration-messages', 'TERMDOCK_MESSAGE_STORAGE_MAX_BYTES', 512 * 1024 * 1024, () => new CollaborationError('MESSAGE_STORAGE_FULL', '消息新增内容达到容量预算；待投递消息、投递确认和消费水位保留，请消费已有消息或调整 TERMDOCK_MESSAGE_STORAGE_MAX_BYTES', 503), options);
+        this.historyBytes = options.historyBytes ?? Number(process.env.TERMDOCK_MESSAGE_HISTORY_BYTES ?? 64 * 1024 * 1024);
+        if (!Number.isSafeInteger(this.historyBytes) || this.historyBytes < 1)
+            throw new CollaborationError('INVALID_STORAGE_CONFIG', 'TERMDOCK_MESSAGE_HISTORY_BYTES must be a positive safe integer');
         let initializeChatAttention = false;
         try {
             const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
@@ -183,11 +192,14 @@ export class CollaborationStore {
                 && message.toPrincipalId && !chatPresentation(message.content, message.metadata).ui?.streaming).map(message => message.id);
         }
         this.persistedDocument = JSON.stringify(this.document);
+        this.committedMessageIds = new Set(this.document.messages.map(message => message.id));
+        this.committedPendingIds = new Set(this.document.messages.filter(message => message.status === 'pending').map(message => message.id));
         // Persist before serving any indices so an idle restart cannot move the
         // baseline forward and swallow replies received since the first migration.
         if (initializeChatAttention)
             this.persist(true);
     }
+    storageStatus() { return { ...this.storage.status(), history_target_bytes: this.historyBytes }; }
     list() {
         return this.document.groups.filter((group) => !group.deleted && !group.sessionChatSessionId).sort((a, b) => b.updatedAt - a.updatedAt);
     }
@@ -282,7 +294,7 @@ export class CollaborationStore {
                 throw new CollaborationError('GROUP_CREATE_TARGET_UNAVAILABLE', 'Original local group was deleted or changed; this key cannot create a replacement', 409);
             // A previous directory-fsync failure may have committed the key without
             // confirming durability. Reconfirm the current group+key before ACK.
-            this.persist(true);
+            this.persistLocalGroup();
             return structuredClone(previous.group);
         }
         if (Object.keys(this.document.localGroupRequests ?? {}).length >= 2000)
@@ -294,9 +306,43 @@ export class CollaborationStore {
             } };
         // persist restores the last committed document on failure. A post-rename
         // fsync failure retains the committed request so retries cannot duplicate.
-        this.persist(true);
+        this.persistLocalGroup();
         return structuredClone(group);
     }
+    persistLocalGroup() {
+        const proposed = this.document;
+        try {
+            if (!this.storage.status().commit_unconfirmed) {
+                this.persist(true);
+                return;
+            }
+            const serialized = JSON.stringify(proposed);
+            this.storage.reconfirm(serialized);
+            this.persistedDocument = serialized;
+            this.committedMessageIds = new Set(proposed.messages.map(message => message.id));
+            this.committedPendingIds = new Set(proposed.messages.filter(message => message.status === 'pending').map(message => message.id));
+            for (const listener of this.changeListeners) {
+                try {
+                    listener();
+                }
+                catch { /* A durable receipt can be replayed again. */ }
+            }
+        }
+        catch (error) {
+            // Keep a post-rename request visible for idempotent retry, but never
+            // report success or admit another write before its durability is confirmed.
+            this.document = JSON.parse(this.persistedDocument);
+            try {
+                const serialized = JSON.stringify(proposed);
+                if (fs.readFileSync(this.filePath, 'utf8') === serialized) {
+                    this.document = proposed;
+                    this.persistedDocument = serialized;
+                }
+            }
+            catch { /* Retain the normal rollback when the replacement is not proven. */ }
+            throw error;
+        }
+    }
     /** Exact local lookup; no prefix/name resolution or private group configuration. */
     getLocalGroupMetadata(id) {
         if (typeof id !== 'string' || !/^[a-zA-Z0-9._-]{1,80}$/.test(id))
@@ -335,7 +381,7 @@ export class CollaborationStore {
             if (!target || target.deleted || target.federated)
                 throw new CollaborationError('GROUP_ARCHIVE_TARGET_UNAVAILABLE', 'Original local archive target is unavailable; no replacement is created', 409);
             const group = this.getLocalGroupMetadata(id);
-            this.persist(true);
+            this.persistLocalGroup();
             return { group, applied: { ...previous.applied }, replayed: true };
         }
         const current = this.getLocalGroupMetadata(id);
@@ -354,7 +400,7 @@ export class CollaborationStore {
         this.document = { ...this.document, groups: this.document.groups.map(group => group.id === id
                 ? { ...group, updatedAt, ...(archivedAt !== undefined ? { archivedAt } : { archivedAt: undefined }) } : group),
             localGroupArchives: { ...this.document.localGroupArchives, [requestKey]: { hash, applied } } };
-        this.persist(true);
+        this.persistLocalGroup();
         return { group: { ...applied }, applied: { ...applied }, replayed: false };
     }
     /** Display metadata and its original receipt share a durable commit. */
@@ -378,7 +424,7 @@ export class CollaborationStore {
             if (!target || target.deleted || target.federated)
                 throw new CollaborationError('GROUP_RENAME_TARGET_UNAVAILABLE', 'Original local rename target is unavailable; no replacement is created', 409);
             const group = this.getLocalGroupMetadata(id);
-            this.persist(true);
+            this.persistLocalGroup();
             return { group, applied: { ...previous.applied }, replayed: true };
         }
         const current = this.getLocalGroupMetadata(id);
@@ -393,7 +439,7 @@ export class CollaborationStore {
         const applied = { id, name, updatedAt, ...(target.archivedAt !== undefined ? { archivedAt: target.archivedAt } : {}) };
         this.document = { ...this.document, groups: this.document.groups.map(group => group.id === id ? { ...group, name, updatedAt } : group),
             localGroupRenames: { ...this.document.localGroupRenames, [requestKey]: { hash, applied } } };
-        this.persist(true);
+        this.persistLocalGroup();
         return { group: { ...applied }, applied: { ...applied }, replayed: false };
     }
     setMemberLaunchProfile(groupId, sessionId, profile) {
@@ -799,7 +845,7 @@ export class CollaborationStore {
         reply.metadata = { ...reply.metadata, ...extras.metadata, openui: { ...extras.metadata?.openui, version: 2, patch: false, revision: Number(revision) + 1 } };
         reply.responseKind = next.streaming ? 'progress' : 'result';
         (this.document.idempotency ??= {})[key] = { hash, ids: [reply.id], expiresAt: Date.now() + COLLAB_LIMITS.idempotency_retention_ms };
-        this.persist(true);
+        this.persist(true, true);
         return reply;
     }
     /** References only: callers can select a bounded page before projecting bodies. */
@@ -921,7 +967,7 @@ export class CollaborationStore {
         const diagnostic = this.diagnostic(messageId);
         if (diagnostic)
             this.recordTransport(messageId, { ...diagnostic, next_retry_at: null });
-        this.persist();
+        this.persist(true);
     }
     markDelivered(messageIds) {
         return this.updateStatus(messageIds, 'delivered');
@@ -948,7 +994,7 @@ export class CollaborationStore {
             return updated;
         });
         if (changed.length > 0)
-            this.persist();
+            this.persist(true);
         return changed;
     }
     federationSnapshot() {
@@ -1259,17 +1305,22 @@ export class CollaborationStore {
     commitCursor(sessionId, cursor, consumer) {
         if (!consumer || consumer.length > 128)
             throw new CollaborationError('INVALID_CONSUMER', 'consumer must contain 1–128 characters');
+        let key, sequence;
         try {
             const decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString());
             if (JSON.parse(decoded.scope)[0] !== sessionId || !Number.isSafeInteger(decoded.sequence) || decoded.sequence < 0 || decoded.sequence > (this.document.sequence ?? 0))
                 throw new Error();
-            const key = JSON.stringify([decoded.scope, consumer]);
-            (this.document.consumers ??= {})[key] = Math.max(this.document.consumers?.[key] ?? 0, decoded.sequence);
-            this.persist();
+            key = JSON.stringify([decoded.scope, consumer]);
+            sequence = decoded.sequence;
         }
         catch {
             throw new CollaborationError('INVALID_CURSOR', 'Invalid consumption cursor');
         }
+        const previous = this.document.consumers?.[key];
+        if (previous !== undefined && sequence <= previous)
+            return;
+        (this.document.consumers ??= {})[key] = sequence;
+        this.persist(true);
     }
     acceptFragment(fragment) {
         const group = this.getGroup(fragment.group_id);
@@ -1309,36 +1360,55 @@ export class CollaborationStore {
         this.persist();
         return { message_id: fragment.message_id, received, total: state.total, complete: received === state.total };
     }
-    persist(durable = false) {
+    persist(durable = false, contentGrowth = false) {
         const now = Date.now();
-        for (const [key, record] of Object.entries(this.document.idempotency ?? {}))
-            if (record.expiresAt <= now)
-                delete this.document.idempotency[key];
-        const protectedIds = new Set(Object.values(this.document.idempotency ?? {}).flatMap((record) => record.ids));
-        const history = this.document.messages.filter((message) => message.status !== 'pending' && !protectedIds.has(message.id)).slice(-MAX_MESSAGES);
-        const retained = new Set(history.map((message) => message.id));
-        this.document.messages = this.document.messages.filter((message) => {
-            if (message.status === 'pending' || protectedIds.has(message.id) || retained.has(message.id))
-                return true;
-            (this.document.prunedThrough ??= {})[message.toSessionId] = Math.max(this.document.prunedThrough?.[message.toSessionId] ?? 0, message.sequence ?? 0);
-            if (message.fromSessionId)
-                (this.document.prunedSenders ??= {})[message.fromSessionId] = true;
-            delete this.document.transport?.[message.id];
-            return false;
-        });
-        const messageIds = new Set(this.document.messages.map(message => message.id));
-        this.document.sessionChatAttentionBaseline = this.document.sessionChatAttentionBaseline?.filter(id => messageIds.has(id));
-        const chatIds = new Set(this.document.groups.filter(group => !group.deleted && group.sessionChatSessionId).map(group => group.id));
-        for (const [id, ids] of Object.entries(this.document.sessionChatViewed ?? {})) {
-            if (!chatIds.has(id))
-                delete this.document.sessionChatViewed[id];
-            else
-                this.document.sessionChatViewed[id] = ids.filter(messageId => messageIds.has(messageId));
-        }
-        const temporaryPath = `${this.filePath}.${process.pid}.tmp`;
         try {
+            for (const [key, record] of Object.entries(this.document.idempotency ?? {}))
+                if (record.expiresAt <= now)
+                    delete this.document.idempotency[key];
+            const protectedIds = new Set(Object.values(this.document.idempotency ?? {}).flatMap((record) => record.ids));
+            for (const message of this.document.messages) {
+                if (message.status !== 'pending' && this.committedPendingIds.has(message.id))
+                    message.retainUntil = now + 5 * 60_000;
+                if ((message.retainUntil ?? 0) > now)
+                    protectedIds.add(message.id);
+            }
+            const history = this.document.messages.filter((message) => message.status !== 'pending' && !protectedIds.has(message.id)).slice(-MAX_MESSAGES);
+            const retained = new Set(history.map((message) => message.id));
+            const sizes = new Map(this.document.messages.map(message => [message.id, Buffer.byteLength(JSON.stringify(message)) + 1]));
+            let messageBytes = this.document.messages.reduce((sum, message) => sum + (message.status === 'pending' || protectedIds.has(message.id) || retained.has(message.id) ? sizes.get(message.id) : 0), 0);
+            // Existing count retention also needs a byte target. Pending messages and
+            // unexpired deduplication receipts are never sacrificed to meet it.
+            for (const message of history)
+                if (messageBytes > this.historyBytes) {
+                    retained.delete(message.id);
+                    messageBytes -= sizes.get(message.id);
+                }
+            this.document.messages = this.document.messages.filter((message) => {
+                if (message.status === 'pending' || protectedIds.has(message.id) || retained.has(message.id))
+                    return true;
+                (this.document.prunedThrough ??= {})[message.toSessionId] = Math.max(this.document.prunedThrough?.[message.toSessionId] ?? 0, message.sequence ?? 0);
+                if (message.fromSessionId)
+                    (this.document.prunedSenders ??= {})[message.fromSessionId] = true;
+                delete this.document.transport?.[message.id];
+                return false;
+            });
+            const messageIds = new Set(this.document.messages.map(message => message.id));
+            this.document.sessionChatAttentionBaseline = this.document.sessionChatAttentionBaseline?.filter(id => messageIds.has(id));
+            const chatIds = new Set(this.document.groups.filter(group => !group.deleted && group.sessionChatSessionId).map(group => group.id));
+            for (const [id, ids] of Object.entries(this.document.sessionChatViewed ?? {})) {
+                if (!chatIds.has(id))
+                    delete this.document.sessionChatViewed[id];
+                else
+                    this.document.sessionChatViewed[id] = ids.filter(messageId => messageIds.has(messageId));
+            }
+            const temporaryPath = `${this.filePath}.${process.pid}.tmp`;
             fs.mkdirSync(path.dirname(this.filePath), { recursive: true });
             const serialized = JSON.stringify(this.document);
+            const newMessage = this.document.messages.some(message => !this.committedMessageIds.has(message.id));
+            this.storage.admit(serialized, !newMessage && !contentGrowth);
+            // Enqueue success must survive a crash before a delivery worker runs.
+            durable ||= newMessage || contentGrowth;
             const descriptor = fs.openSync(temporaryPath, 'w', 0o600);
             try {
                 fs.writeFileSync(descriptor, serialized);
@@ -1349,7 +1419,6 @@ export class CollaborationStore {
                 fs.closeSync(descriptor);
             }
             fs.renameSync(temporaryPath, this.filePath);
-            this.persistedDocument = serialized;
             if (durable) {
                 const directory = fs.openSync(path.dirname(this.filePath), 'r');
                 try {
@@ -1359,6 +1428,10 @@ export class CollaborationStore {
                     fs.closeSync(directory);
                 }
             }
+            this.persistedDocument = serialized;
+            this.storage.committed(serialized);
+            this.committedMessageIds = new Set(this.document.messages.map(message => message.id));
+            this.committedPendingIds = new Set(this.document.messages.filter(message => message.status === 'pending').map(message => message.id));
             for (const listener of this.changeListeners) {
                 try {
                     listener();
@@ -1367,6 +1440,7 @@ export class CollaborationStore {
             }
         }
         catch (error) {
+            this.storage.failed();
             this.document = JSON.parse(this.persistedDocument);
             throw error;
         }

package/dist/server/agent/collaborationTaskStore.js

@@ -1,7 +1,7 @@
 import { GOAL_EXPERIENCE_GUIDE } from './collaborationTaskGuidance.js';
 import { randomBytes, createHash } from 'node:crypto';
-import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs';
-import { dirname } from 'node:path';
+import { readFileSync } from 'node:fs';
+import { JsonStorageBudget } from '../utils/jsonStorageBudget.js';
 import { collaborationResultPresentation } from './collaborationResultPresentation.js';
 import { taskExecution, taskPurpose } from './collaborationTaskPurpose.js';
 import { CollaborationError } from './collaborationProtocol.js';
@@ -121,13 +121,13 @@ function validateSnapshot(task) {
 }
 /** Atomic document writes keep decisions and their outgoing messages in one transaction. */
 export class CollaborationTaskStore {
-    file;
     listeners = new Set();
     subscribe(listener) { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; }
     snapshot() { return structuredClone(this.document.tasks); }
     document = { version: 1, tasks: [], outbox: [], requests: {} };
-    constructor(file) {
-        this.file = file;
+    storage;
+    constructor(file, options = {}) {
+        this.storage = new JsonStorageBudget(file, 'collaboration-tasks', 'TERMDOCK_TASK_STORAGE_MAX_BYTES', 64 * 1024 * 1024, () => new CollaborationError('TASK_STORAGE_FULL', '任务新增内容达到容量预算；已有任务仍可确认、收尾或关闭，管理员可配置 TERMDOCK_TASK_STORAGE_MAX_BYTES', 409), options);
         try {
             const doc = JSON.parse(readFileSync(file, 'utf8'));
             if (doc.version !== 1 || !Array.isArray(doc.tasks) || !Array.isArray(doc.outbox) || !doc.requests)
@@ -139,7 +139,8 @@ export class CollaborationTaskStore {
                 throw error;
         }
     }
-    transaction(mutate, origin) {
+    storageStatus() { return this.storage.status(); }
+    transaction(mutate, origin, control = false) {
         const next = structuredClone(this.document);
         const result = mutate(next);
         if (origin) {
@@ -154,12 +155,7 @@ export class CollaborationTaskStore {
         if (next.tasks.some(task => Buffer.byteLength(JSON.stringify(task)) > 512_000))
             throw new CollaborationError('TASK_HISTORY_LIMIT', '此任务记录达到 500 KiB，请建立后续任务并保留本任务为历史', 409);
         const serialized = JSON.stringify(next);
-        if (Buffer.byteLength(serialized) > 64 * 1024 * 1024)
-            throw new CollaborationError('TASK_STORAGE_FULL', '任务记录已达到保存上限，请导出历史后整理', 409);
-        mkdirSync(dirname(this.file), { recursive: true, mode: 0o700 });
-        const temporary = `${this.file}.${process.pid}.tmp`;
-        writeFileSync(temporary, serialized, { mode: 0o600 });
-        renameSync(temporary, this.file);
+        this.storage.write(serialized, control);
         this.document = next;
         for (const listener of this.listeners) {
             try {
@@ -679,7 +675,8 @@ export class CollaborationTaskStore {
                 delete task.automationIssue;
             doc.requests[request.key] = { hash: request.hash, taskId, eventId: task.events.at(-1)?.id };
             return task;
-        }, origin);
+        }, origin, ['close', 'pause', 'accept', 'review'].includes(input.kind)
+            || input.kind === 'report' && ['ack', 'complete', 'failed', 'blocked'].includes(input.status ?? ''));
     }
     touchParent(doc, task, content) {
         const root = task.workflow?.rootTaskId ? doc.tasks.find(t => t.id === task.workflow.rootTaskId) : undefined;
@@ -895,7 +892,7 @@ export class CollaborationTaskStore {
                 && !doc.outbox.some(o => o.taskId === task.id && o.lastError)
                 && !task.deliveries.some(d => d.attemptId === task.activeAttemptId && ['failed', 'expired'].includes(d.status)))
                 delete task.automationIssue;
-        });
+        }, undefined, true);
     }
     failed(outboxId, error) {
         const current = this.document.outbox.find(o => o.id === outboxId);
@@ -925,6 +922,6 @@ export class CollaborationTaskStore {
                     this.touchParent(doc, task, task.automationIssue);
                 }
             }
-        });
+        }, undefined, true);
     }
 }

package/dist/server/agent/integrationCli.js

@@ -8,6 +8,12 @@ import { COLLAB_HELP, executeCollaborationCommand, duration } from './collaborat
 import { getTermdockVersion } from '../utils/version.js';
 export const INTEGRATION_HELP = `td integration — local scoped integration administration
   groups | list
+  storage (read event capacity, warnings and metadata usage; local administrator only)
+  consumers (read confirmation watermarks, lag and retirement state)
+  consumer retire --principal <id> --consumer <name> --expected-sequence <N> --idempotency-key <key>
+    Explicitly ends a consumer's retention protection; preserves its watermark.
+    Its old name can no longer subscribe/ACK. Reconcile before using a new name.
+    Never retire a slow/idle consumer merely because its last ACK is old.
   group get <exact-group-id>
   group rename <exact-group-id> --name <name> --expected-updated-at <N> --idempotency-key <key>
     Administrator only; minimum CLI/running service 1.4.319;
@@ -154,6 +160,28 @@ export async function runIntegrationAdmin(argv, port, adminToken, write) {
             return 0;
         }
         const socketPath = integrationSocketPath(port);
+        if (action === 'storage' || action === 'consumers') {
+            if (argv.length !== 1)
+                throw new Error('Usage: td integration storage | consumers');
+            write(JSON.stringify(parsed(await request(socketPath, 'GET', `/admin/${action}`, undefined, headers(undefined, adminToken)))));
+            return 0;
+        }
+        if (action === 'consumer') {
+            const args = {};
+            if (argv[1] !== 'retire')
+                throw new Error('Usage: td integration consumer retire --principal <id> --consumer <name> --expected-sequence <N> --idempotency-key <key>');
+            for (let index = 2; index < argv.length; index += 2) {
+                if (!['--principal', '--consumer', '--expected-sequence', '--idempotency-key'].includes(argv[index]) || !argv[index + 1] || Object.hasOwn(args, argv[index]))
+                    throw new Error('Invalid consumer retirement arguments');
+                args[argv[index]] = argv[index + 1];
+            }
+            if (Object.keys(args).length !== 4 || !/^(0|[1-9][0-9]*)$/.test(args['--expected-sequence']))
+                throw new Error('Consumer retirement requires all four arguments and an integer sequence');
+            write(JSON.stringify(parsed(await request(socketPath, 'POST', '/admin/consumers/retire', {
+                principal_id: args['--principal'], consumer: args['--consumer'], expected_sequence: Number(args['--expected-sequence']), idempotency_key: args['--idempotency-key'],
+            }, headers(undefined, adminToken)))));
+            return 0;
+        }
         if (action === 'group' && ['get', 'rename', 'archive', 'unarchive'].includes(argv[1])) {
             const operation = argv[1], id = argv[2], args = {};
             const usage = 'Usage: td integration group get <exact-id> | group rename <exact-id> --name <name> --expected-updated-at <N> --idempotency-key <key> | group archive|unarchive <exact-id> --expected-updated-at <N> --idempotency-key <key>';