switchroom 0.19.30 → 0.19.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1121 -584
- package/dist/host-control/main.js +151 -73
- package/package.json +1 -1
- package/profiles/_base/cron-session.sh.hbs +5 -1
- package/profiles/_base/start.sh.hbs +15 -1
- package/telegram-plugin/dist/gateway/gateway.js +1435 -551
- package/telegram-plugin/edit-flood-fuse.ts +70 -20
- package/telegram-plugin/gateway/boot-beacon.ts +364 -0
- package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
- package/telegram-plugin/gateway/gateway.ts +87 -88
- package/telegram-plugin/gateway/inbound-spool.ts +39 -0
- package/telegram-plugin/gateway/narrative-lane.ts +12 -0
- package/telegram-plugin/gateway/obligation-store.ts +28 -0
- package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
- package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
- package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
- package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
- package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
- package/telegram-plugin/gateway/status-pin-store.ts +33 -11
- package/telegram-plugin/registry/turns-schema.ts +21 -1
- package/telegram-plugin/retry-api-call.ts +46 -21
- package/telegram-plugin/shared/bot-runtime.ts +61 -17
- package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
- package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
- package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
- package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
- package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
- package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
- package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
- package/telegram-plugin/tests/obligation-store.test.ts +67 -1
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
- package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
- package/telegram-plugin/tool-activity-summary.ts +104 -38
- package/telegram-plugin/worker-activity-feed.ts +33 -16
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
- package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
statSync, renameSync, realpathSync, chmodSync, openSync, closeSync,
|
|
20
20
|
existsSync, unlinkSync, appendFileSync,
|
|
21
21
|
} from 'fs'
|
|
22
|
+
import { fsyncPathSync } from '../../src/util/atomic.js'
|
|
22
23
|
import { homedir } from 'os'
|
|
23
24
|
import { join, sep, basename } from 'path'
|
|
24
25
|
|
|
@@ -642,12 +643,9 @@ import { runStatusPinReconcile, type StatusPinClaim } from './status-pin-retarge
|
|
|
642
643
|
import { createPeriodicSweepGuard } from './periodic-sweep-guard.js'
|
|
643
644
|
import { withDeadline } from './with-deadline.js'
|
|
644
645
|
import { createStatusPinApi, type PinCapableBot, type RobustApiSeam } from './status-pin-api.js'
|
|
645
|
-
import {
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
unexpiredStoreRepinIds,
|
|
649
|
-
type DmPinSweeper,
|
|
650
|
-
} from './dm-pin-sweep.js'
|
|
646
|
+
import { collectSweepTargets, type StalePinSweeper, type SweepTarget } from './stale-pin-sweep.js'
|
|
647
|
+
import { createGatewayStalePinSweeper } from './stale-pin-sweep-wiring.js'
|
|
648
|
+
import { atomicWriteFileSync } from '../../src/util/atomic.js'
|
|
651
649
|
import { startWebhookIngestServer } from './webhook-ingest-server.js'
|
|
652
650
|
import { recordWebhookEvent } from '../../src/web/webhook-gateway-record.js'
|
|
653
651
|
|
|
@@ -849,11 +847,10 @@ import { listRecords as listWorktreeRecords, touchHeartbeat as touchWorktreeHear
|
|
|
849
847
|
import { makeWorktreeWatchProvider } from '../worktree-watch-cwds.js'
|
|
850
848
|
import {
|
|
851
849
|
startBootCard,
|
|
852
|
-
resolvePersonaName,
|
|
853
|
-
type BootCardHandle,
|
|
850
|
+
resolvePersonaName, shouldSkipDuplicateBootCard,
|
|
851
|
+
type BootCardHandle, type RestartReason,
|
|
854
852
|
} from './boot-card.js'
|
|
855
853
|
import { determineRestartReason } from './boot-reason.js'
|
|
856
|
-
import { shouldSkipDuplicateBootCard, type RestartReason } from './boot-card.js'
|
|
857
854
|
import { maybeRenderUpdateAnnouncement } from './update-announce.js'
|
|
858
855
|
import { createIssuesCardHandle, type IssuesCardHandle } from '../issues-card.js'
|
|
859
856
|
import { startIssuesWatcher, type IssuesWatcherHandle } from '../issues-watcher.js'
|
|
@@ -914,6 +911,7 @@ import {
|
|
|
914
911
|
TURN_ACTIVE_IDLE_SWEEP_MS,
|
|
915
912
|
} from './turn-active-marker.js'
|
|
916
913
|
import { startGatewayHeartbeat } from './gateway-heartbeat.js'
|
|
914
|
+
import { attachBootBeacon } from './boot-beacon.js'
|
|
917
915
|
import { startOutboxSweep } from './outbox-sweep.js'
|
|
918
916
|
import {
|
|
919
917
|
VERSION,
|
|
@@ -2345,7 +2343,7 @@ function checkApprovals(): void {
|
|
|
2345
2343
|
)
|
|
2346
2344
|
}
|
|
2347
2345
|
}
|
|
2348
|
-
if (isGatewayMain && !STATIC) setInterval(checkApprovals, 5000).unref()
|
|
2346
|
+
if (isGatewayMain && !STATIC) setInterval(attachBootBeacon(STATE_DIR, checkApprovals), 5000).unref() // beacon rides this EXISTING tick — boot-beacon.ts
|
|
2349
2347
|
// Gateway liveness heartbeat — touches `<STATE_DIR>/gateway-heartbeat` while
|
|
2350
2348
|
// the gateway lives so the silent-end Stop hook's single-writer election can
|
|
2351
2349
|
// confirm the gateway is alive (and WILL run its turn_end delivery) before
|
|
@@ -2838,6 +2836,7 @@ const obligationStoreFs = {
|
|
|
2838
2836
|
writeFileSync: (p: string, d: string) => writeFileSync(p, d),
|
|
2839
2837
|
renameSync: (a: string, b: string) => renameSync(a, b),
|
|
2840
2838
|
existsSync: (p: string) => existsSync(p),
|
|
2839
|
+
fsyncFileSync: fsyncPathSync, fsyncDirSync: fsyncPathSync,
|
|
2841
2840
|
}
|
|
2842
2841
|
const obligationLedger = new ObligationLedger(OBLIGATION_REPRESENT_MAX, {
|
|
2843
2842
|
onChange:
|
|
@@ -9102,6 +9101,10 @@ async function reconcileStatusPin(
|
|
|
9102
9101
|
pinKey: string,
|
|
9103
9102
|
chatId: string,
|
|
9104
9103
|
desired: DesiredPin,
|
|
9104
|
+
/** Forum topic the pinned message lives in, when the caller knows it. Keyed
|
|
9105
|
+
* into the claim + the durable row so a post-crash sweep can aim
|
|
9106
|
+
* `unpinAllForumTopicMessages` at the right topic. */
|
|
9107
|
+
threadId?: number,
|
|
9105
9108
|
): Promise<void> {
|
|
9106
9109
|
// Fire-and-forget hard boundary. Most callers invoke this as
|
|
9107
9110
|
// `void reconcileStatusPin(...)` (auto status-pin is best-effort — it must
|
|
@@ -9119,7 +9122,7 @@ async function reconcileStatusPin(
|
|
|
9119
9122
|
// Serialize per pinKey (F2): reconcileStatusPinInner reads `prev` from the
|
|
9120
9123
|
// in-memory claim map at its top, so overlapping same-key reconciles must
|
|
9121
9124
|
// run one-at-a-time or a stale `prev` clears the disk row under a live pin.
|
|
9122
|
-
await withPinReconcileLock(pinKey, () => reconcileStatusPinInner(pinKey, chatId, desired))
|
|
9125
|
+
await withPinReconcileLock(pinKey, () => reconcileStatusPinInner(pinKey, chatId, desired, threadId))
|
|
9123
9126
|
} catch (err) {
|
|
9124
9127
|
const msg = err instanceof Error ? err.message : String(err)
|
|
9125
9128
|
process.stderr.write(
|
|
@@ -9133,6 +9136,7 @@ async function reconcileStatusPinInner(
|
|
|
9133
9136
|
pinKey: string,
|
|
9134
9137
|
chatId: string,
|
|
9135
9138
|
desired: DesiredPin,
|
|
9139
|
+
threadId?: number,
|
|
9136
9140
|
): Promise<void> {
|
|
9137
9141
|
if (!PIN_STATUS_WHILE_WORKING) return
|
|
9138
9142
|
if (chatId.length === 0) return
|
|
@@ -9172,6 +9176,7 @@ async function reconcileStatusPinInner(
|
|
|
9172
9176
|
await runStatusPinReconcile({
|
|
9173
9177
|
pinKey,
|
|
9174
9178
|
chatId,
|
|
9179
|
+
threadId,
|
|
9175
9180
|
prev,
|
|
9176
9181
|
desired,
|
|
9177
9182
|
// persist:null ⇒ no durable row (STATIC / feature-off).
|
|
@@ -9203,81 +9208,62 @@ async function unpinAllStatusPins(): Promise<void> {
|
|
|
9203
9208
|
}
|
|
9204
9209
|
}
|
|
9205
9210
|
|
|
9206
|
-
// ───
|
|
9207
|
-
//
|
|
9208
|
-
//
|
|
9209
|
-
//
|
|
9210
|
-
//
|
|
9211
|
-
//
|
|
9212
|
-
//
|
|
9213
|
-
//
|
|
9211
|
+
// ─── Stale-pin stack sweep ──────────────────────────────────────────────────
|
|
9212
|
+
// A Telegram chat holds a STACK of pins; `getChat().pinned_message` exposes only
|
|
9213
|
+
// the TOP and there is no list-pins method, so a probe-based cleanup never sees
|
|
9214
|
+
// the rest. Worse (verified live 2026-07-29): a bare `unpinChatMessage` on a
|
|
9215
|
+
// stale entry is a SILENT NO-OP that returns ok:true, so the gateway logged a
|
|
9216
|
+
// clean sweep on every boot while the stack grew unbounded — 37 orphans across
|
|
9217
|
+
// 5 agents, 21 in one chat. The drain, the rate gates and the durable
|
|
9218
|
+
// obligation cursor live in stale-pin-sweep.ts; this is just the binding.
|
|
9214
9219
|
//
|
|
9215
9220
|
// Eligibility mirrors statusPinBootCleanup's mutex gate: set true ONLY after
|
|
9216
9221
|
// this gateway wins the startup lock, so a losing double-boot never clears the
|
|
9217
9222
|
// live holder's pins.
|
|
9218
|
-
let
|
|
9219
|
-
|
|
9220
|
-
|
|
9221
|
-
|
|
9222
|
-
|
|
9223
|
-
|
|
9224
|
-
|
|
9225
|
-
|
|
9226
|
-
|
|
9227
|
-
|
|
9228
|
-
|
|
9229
|
-
|
|
9230
|
-
|
|
9231
|
-
|
|
9232
|
-
|
|
9233
|
-
|
|
9234
|
-
|
|
9235
|
-
|
|
9236
|
-
|
|
9237
|
-
|
|
9238
|
-
// both the boot sweep (in-memory maps still empty then) and a later
|
|
9239
|
-
// first-inbound sweep see them. Best-effort: a store read failure
|
|
9240
|
-
// degrades to in-memory-only. The sweeper dedupes.
|
|
9241
|
-
liveTrackedMessageIds: (chatId) => {
|
|
9242
|
-
const ids: number[] = []
|
|
9243
|
-
for (const claim of statusPinClaims.values()) {
|
|
9244
|
-
if (claim.chatId === chatId) ids.push(claim.messageId)
|
|
9245
|
-
}
|
|
9246
|
-
if (statusPinPersistEnabled || bannerPinPersistEnabled || toolPinPersistEnabled) {
|
|
9247
|
-
try {
|
|
9248
|
-
ids.push(
|
|
9249
|
-
...unexpiredStoreRepinIds(
|
|
9250
|
-
loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs),
|
|
9251
|
-
chatId,
|
|
9252
|
-
Date.now(),
|
|
9253
|
-
),
|
|
9254
|
-
)
|
|
9255
|
-
} catch (err) {
|
|
9256
|
-
process.stderr.write(
|
|
9257
|
-
`telegram gateway: dm-pin-sweep: store repin scan failed ` +
|
|
9258
|
-
`(chat=${chatId}): ${(err as Error).message}\n`,
|
|
9259
|
-
)
|
|
9260
|
-
}
|
|
9261
|
-
}
|
|
9262
|
-
return ids
|
|
9223
|
+
let stalePinSweepEligible = false
|
|
9224
|
+
// Durable ledger: fsync'd tempfile + atomic rename, so a power cut leaves either
|
|
9225
|
+
// the previous complete ledger or the new one — never a truncated file that
|
|
9226
|
+
// forgets an in-flight drain.
|
|
9227
|
+
const STALE_PIN_SWEEP_STORE_PATH = join(STATE_DIR, 'stale-pin-sweep.json')
|
|
9228
|
+
const stalePinSweeper: StalePinSweeper = createGatewayStalePinSweeper({
|
|
9229
|
+
telegram: { handle: () => lockedBot, call: robustApiCall },
|
|
9230
|
+
claims: () => statusPinClaims.values(),
|
|
9231
|
+
loadPinRows: () =>
|
|
9232
|
+
statusPinPersistEnabled || bannerPinPersistEnabled || toolPinPersistEnabled
|
|
9233
|
+
? loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs)
|
|
9234
|
+
: [],
|
|
9235
|
+
eligible: () => stalePinSweepEligible,
|
|
9236
|
+
store: {
|
|
9237
|
+
path: STALE_PIN_SWEEP_STORE_PATH,
|
|
9238
|
+
fs: {
|
|
9239
|
+
readFileSync: (p) => readFileSync(p, 'utf-8'),
|
|
9240
|
+
writeFileSync: (p, data) => atomicWriteFileSync(p, data, 0o600),
|
|
9241
|
+
existsSync: (p) => existsSync(p),
|
|
9242
|
+
},
|
|
9263
9243
|
},
|
|
9264
|
-
|
|
9265
|
-
|
|
9244
|
+
// Per-deployment override only. UNSET (the normal case) means "take the
|
|
9245
|
+
// standing policy" — UNPIN_ALL_FORUM_TOPIC_ENABLED in stale-pin-sweep.ts,
|
|
9246
|
+
// i.e. the WHOLESALE topic drain stays off because it also removes pins this
|
|
9247
|
+
// gateway never placed. Set the env var to 1 to opt a deployment in.
|
|
9248
|
+
allowUnpinAllForumTopic:
|
|
9249
|
+
process.env.SWITCHROOM_PIN_SWEEP_UNPIN_ALL_TOPIC == null
|
|
9250
|
+
? undefined
|
|
9251
|
+
: process.env.SWITCHROOM_PIN_SWEEP_UNPIN_ALL_TOPIC === '1',
|
|
9266
9252
|
})
|
|
9267
9253
|
|
|
9268
9254
|
/**
|
|
9269
|
-
* Boot-time pin cleanup +
|
|
9270
|
-
* prior-session pin record BEFORE the store reapers empty the
|
|
9271
|
-
* three boot reapers, marks the
|
|
9272
|
-
* recorded
|
|
9255
|
+
* Boot-time pin cleanup + stale-pin stack sweep. Collects the `(chat, thread)`
|
|
9256
|
+
* targets with a prior-session pin record BEFORE the store reapers empty the
|
|
9257
|
+
* stores, runs the three boot reapers, marks the sweep eligible, then drains
|
|
9258
|
+
* each recorded target. Ordering and per-step isolation live in
|
|
9273
9259
|
* `runBootPinSweepSteps` (boot-sweep-gate.ts) — a throwing reaper must not
|
|
9274
|
-
* strand `
|
|
9260
|
+
* strand `enableSweep`. Dispatched only via `bootPinSweepGate` below (mutex
|
|
9275
9261
|
* won AND `lockedBot` constructed); fire-and-forget — never blocks boot.
|
|
9276
9262
|
*/
|
|
9277
|
-
function
|
|
9263
|
+
function runBootPinCleanupAndStalePinSweep(): Promise<void> {
|
|
9278
9264
|
return runBootPinSweepSteps({
|
|
9279
|
-
|
|
9280
|
-
|
|
9265
|
+
scanSweepTargets: () =>
|
|
9266
|
+
collectSweepTargets({
|
|
9281
9267
|
statusPins:
|
|
9282
9268
|
statusPinPersistEnabled || bannerPinPersistEnabled || toolPinPersistEnabled
|
|
9283
9269
|
? loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs)
|
|
@@ -9292,19 +9278,19 @@ function runBootPinCleanupAndDmSweep(): Promise<void> {
|
|
|
9292
9278
|
statusPinCleanup: statusPinBootCleanup,
|
|
9293
9279
|
activityCardReaper: activityCardBootReaper,
|
|
9294
9280
|
queuedCardReaper: queuedCardBootReaper,
|
|
9295
|
-
// This gateway owns the shared pin state — enable the
|
|
9296
|
-
//
|
|
9297
|
-
|
|
9298
|
-
|
|
9281
|
+
// This gateway owns the shared pin state — enable the drain (this sweep
|
|
9282
|
+
// AND lazy first-inbound sweeps).
|
|
9283
|
+
enableSweep: () => {
|
|
9284
|
+
stalePinSweepEligible = true
|
|
9299
9285
|
},
|
|
9300
|
-
|
|
9286
|
+
sweepTarget: (t: SweepTarget) => stalePinSweeper.sweepTarget(t),
|
|
9301
9287
|
log: (line) => process.stderr.write(line),
|
|
9302
9288
|
})
|
|
9303
9289
|
}
|
|
9304
9290
|
|
|
9305
9291
|
// #3664: needs BOTH the mutex (arm) and a constructed `lockedBot` (botReady,
|
|
9306
9292
|
// end of initGatewayBot) before it may run — see boot-sweep-gate.ts.
|
|
9307
|
-
const bootPinSweepGate = createBootSweepGate({ run:
|
|
9293
|
+
const bootPinSweepGate = createBootSweepGate({ run: runBootPinCleanupAndStalePinSweep, onError: (err) => process.stderr.write(`telegram gateway: boot pin cleanup / stale-pin sweep failed: ${(err as Error).message}\n`) })
|
|
9308
9294
|
|
|
9309
9295
|
// Activity feed. The gateway streams a live "what it's doing" tool-activity
|
|
9310
9296
|
// feed for every turn. The PreToolUse sidecar emits a `tool_label` per tool
|
|
@@ -9926,6 +9912,7 @@ if (isGatewayMain) inboundSpool = STATIC
|
|
|
9926
9912
|
renameSync: (a, b) => renameSync(a, b),
|
|
9927
9913
|
existsSync: (p) => existsSync(p),
|
|
9928
9914
|
statSizeSync: (p) => statSync(p).size,
|
|
9915
|
+
fsyncFileSync: fsyncPathSync, fsyncDirSync: fsyncPathSync,
|
|
9929
9916
|
},
|
|
9930
9917
|
// #2789 B: durability degradation must be visible, not silent. When
|
|
9931
9918
|
// spool appends start failing we're back to in-memory-only — a
|
|
@@ -14750,14 +14737,26 @@ export async function handleInbound(
|
|
|
14750
14737
|
// #2862 — operator is back; re-offer any approvals that timed out meanwhile.
|
|
14751
14738
|
maybePostMissedApprovalDigest('operator inbound')
|
|
14752
14739
|
|
|
14753
|
-
//
|
|
14754
|
-
// the boot getChat() probe can't see (DMs skip the probe
|
|
14755
|
-
// exposes the newest pin).
|
|
14756
|
-
//
|
|
14757
|
-
//
|
|
14740
|
+
// First inbound after boot: drain any stale STACKED pins in this exact
|
|
14741
|
+
// (chat, thread) that the boot getChat() probe can't see (DMs skip the probe
|
|
14742
|
+
// AND getChat exposes only the newest pin). The durable obligation cursor
|
|
14743
|
+
// dedups with the boot sweep and with a prior drain of the same target; the
|
|
14744
|
+
// rights precheck and rate gates apply as usual, and a non-forum supergroup
|
|
14745
|
+
// is skipped. No-op until the startup mutex is won. Fire-and-forget.
|
|
14746
|
+
//
|
|
14747
|
+
// This runs on EVERY inbound; `sweepTarget` is at-most-once per target per
|
|
14748
|
+
// process (and coalesces concurrent callers), so a busy chat cannot
|
|
14749
|
+
// re-attempt a partial drain once per message. That guard lives in the
|
|
14750
|
+
// sweeper, not here, so the boot caller inherits it too.
|
|
14758
14751
|
{
|
|
14759
14752
|
const inboundChatId = ctx.chat?.id
|
|
14760
|
-
if (inboundChatId != null)
|
|
14753
|
+
if (inboundChatId != null) {
|
|
14754
|
+
void stalePinSweeper.sweepTarget({
|
|
14755
|
+
chatId: String(inboundChatId),
|
|
14756
|
+
threadId: ctx.message?.message_thread_id,
|
|
14757
|
+
isForum: ctx.chat?.is_forum === true,
|
|
14758
|
+
})
|
|
14759
|
+
}
|
|
14761
14760
|
// Outbox envelope-less routing (F2) is scoped to each record's OWN stamped
|
|
14762
14761
|
// per-session origin chat (captured at Stop), not a gateway-global stamp.
|
|
14763
14762
|
}
|
|
@@ -24049,11 +24048,11 @@ async function startGateway(): Promise<void> { // #2996 P0c: the boot IIFE, now
|
|
|
24049
24048
|
// (its claim still names that id), leaving live work unpinned. The
|
|
24050
24049
|
// feed drives this: pin `wk:group:<feedKey>` when the group first
|
|
24051
24050
|
// paints, unpin only when the group empties (messageId === null).
|
|
24052
|
-
reconcilePin: ({ feedKey, chatId, messageId }) => {
|
|
24051
|
+
reconcilePin: ({ feedKey, chatId, threadId, messageId }) => {
|
|
24053
24052
|
if (!PIN_STATUS_WHILE_WORKING) return
|
|
24054
24053
|
const key = `wk:group:${feedKey}`
|
|
24055
24054
|
if (messageId != null) {
|
|
24056
|
-
void reconcileStatusPin(key, chatId, { pinned: true, messageId })
|
|
24055
|
+
void reconcileStatusPin(key, chatId, { pinned: true, messageId }, threadId)
|
|
24057
24056
|
} else {
|
|
24058
24057
|
const unpinChat = chatId || statusPinClaims.get(key)?.chatId
|
|
24059
24058
|
if (unpinChat != null && unpinChat.length > 0) {
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
* gateway (mirrors the #1544/#1546/#1549 pure-seam idiom).
|
|
42
42
|
*/
|
|
43
43
|
|
|
44
|
+
import { dirname } from 'node:path'
|
|
44
45
|
import type { InboundMessage } from './ipc-protocol.js'
|
|
45
46
|
|
|
46
47
|
/** Stable dedup id for an inbound. Real Telegram messages have a
|
|
@@ -161,6 +162,17 @@ export interface InboundSpoolFsSeam {
|
|
|
161
162
|
renameSync: (from: string, to: string) => void
|
|
162
163
|
existsSync: (path: string) => boolean
|
|
163
164
|
statSizeSync: (path: string) => number
|
|
165
|
+
/** fsync the FILE at `path` — flush its bytes to stable storage.
|
|
166
|
+
* `appendFileSync` returning only means the page cache took the write;
|
|
167
|
+
* without this a host power cut loses the record even though every
|
|
168
|
+
* syscall succeeded. */
|
|
169
|
+
fsyncFileSync: (path: string) => void
|
|
170
|
+
/** fsync the DIRECTORY at `path` — makes a completed `renameSync`
|
|
171
|
+
* durable. rename(2) is atomic (never a torn file) but that is an
|
|
172
|
+
* ordering guarantee, not a durability one: the new directory entry
|
|
173
|
+
* lives in the parent's own cached metadata and can vanish in a power
|
|
174
|
+
* cut. Call AFTER the rename. */
|
|
175
|
+
fsyncDirSync: (path: string) => void
|
|
164
176
|
}
|
|
165
177
|
|
|
166
178
|
export interface InboundSpoolOptions {
|
|
@@ -345,6 +357,13 @@ export function createInboundSpool(opts: InboundSpoolOptions): InboundSpool {
|
|
|
345
357
|
function appendRecord(rec: SpoolRecord): void {
|
|
346
358
|
try {
|
|
347
359
|
fs.appendFileSync(path, JSON.stringify(rec) + '\n')
|
|
360
|
+
// Durability barrier. The whole point of this file is that the
|
|
361
|
+
// record survives the process dying, and `appendFileSync` alone
|
|
362
|
+
// only guarantees the page cache holds it — enough for a SIGKILL,
|
|
363
|
+
// not for a host power cut. fsync before we report success, so a
|
|
364
|
+
// failure here is treated as the durability loss it is (latched
|
|
365
|
+
// `degraded`) rather than a silent downgrade to in-memory.
|
|
366
|
+
fs.fsyncFileSync(path)
|
|
348
367
|
// #2789 B: a successful append after a failure run means the spool
|
|
349
368
|
// is durable again — un-latch degraded state and surface recovery
|
|
350
369
|
// so the health signal clears.
|
|
@@ -435,12 +454,32 @@ export function createInboundSpool(opts: InboundSpoolOptions): InboundSpool {
|
|
|
435
454
|
const tmp = path + '.compact.tmp'
|
|
436
455
|
try {
|
|
437
456
|
fs.writeFileSync(tmp, lines.length ? lines.join('\n') + '\n' : '')
|
|
457
|
+
// fsync the tmp file BEFORE publishing it: rename orders the
|
|
458
|
+
// metadata, it does not put the DATA on the platter. Renaming an
|
|
459
|
+
// unsynced tmp over the log can leave the spool naming a file whose
|
|
460
|
+
// contents were never written — worse than not compacting at all.
|
|
461
|
+
fs.fsyncFileSync(tmp)
|
|
438
462
|
fs.renameSync(tmp, path)
|
|
439
463
|
log(`inbound-spool: compacted path=${path} live=${live.size}\n`)
|
|
440
464
|
} catch (err) {
|
|
441
465
|
// Compaction is opportunistic — a failure keeps the (larger but
|
|
442
466
|
// correct) append-only log; never lose data trying to shrink it.
|
|
443
467
|
log(`inbound-spool: compact FAILED path=${path}: ${(err as Error).message}\n`)
|
|
468
|
+
return
|
|
469
|
+
}
|
|
470
|
+
// ...then fsync the containing DIRECTORY, which is what makes the rename
|
|
471
|
+
// itself survive a power cut. Separate try: the rename has already
|
|
472
|
+
// landed, so this failing is a weaker durability claim, NOT a failed
|
|
473
|
+
// compaction — logging it as one would send an operator hunting a
|
|
474
|
+
// rewrite that actually worked.
|
|
475
|
+
try {
|
|
476
|
+
fs.fsyncDirSync(dirname(path))
|
|
477
|
+
} catch (err) {
|
|
478
|
+
log(
|
|
479
|
+
`inbound-spool: compact directory fsync FAILED path=${path}: ` +
|
|
480
|
+
`${(err as Error).message} — compacted log written but the rename ` +
|
|
481
|
+
`may not survive a power cut\n`,
|
|
482
|
+
)
|
|
444
483
|
}
|
|
445
484
|
}
|
|
446
485
|
|
|
@@ -133,6 +133,15 @@ export function createNarrativeLane(deps: NarrativeLaneDeps) {
|
|
|
133
133
|
// The parent's OWN running token total → `· N tok` on the metrics line.
|
|
134
134
|
// 0 → tokenSegment omits it (clean, same as the worker feed).
|
|
135
135
|
totalTokens: turn.totalTokens,
|
|
136
|
+
// Terminal `✅ <summary>` footer, rendered through the SAME
|
|
137
|
+
// `deriveCardResult` the 🛠 worker card uses. The agent's analogue of the
|
|
138
|
+
// worker's `latestSummary` is the answer it actually delivered this turn
|
|
139
|
+
// (`lastReplyText`, stamped on the reply/stream_reply tool_use just
|
|
140
|
+
// before the first-reply `clearActivitySummary` hand-off). Only supplied
|
|
141
|
+
// on a FINAL render; empty (a turn that finalised before any reply — the
|
|
142
|
+
// foreground handoff-clear path — or a genuinely silent turn) → the
|
|
143
|
+
// shared derivation omits the block, exactly as the worker card does.
|
|
144
|
+
resultText: final ? turn.lastReplyText : undefined,
|
|
136
145
|
}
|
|
137
146
|
return renderActivityFeedWithNested(turn.mirrorLines, childLines, final, liveSuffix, stepCount, header)
|
|
138
147
|
}
|
|
@@ -892,6 +901,9 @@ export function createNarrativeLane(deps: NarrativeLaneDeps) {
|
|
|
892
901
|
const livenessHeader: SessionActivityHeader = {
|
|
893
902
|
label: 'Agent', elapsedMs: livenessElapsed, toolCount: turn.labeledToolCount, state: 'done',
|
|
894
903
|
model: turn.currentModel,
|
|
904
|
+
// Same terminal `✅ <summary>` footer as the normal final render —
|
|
905
|
+
// this liveness-only card is still a FINAL agent card.
|
|
906
|
+
resultText: turn.lastReplyText,
|
|
895
907
|
}
|
|
896
908
|
finalHtml = renderActivityFeedWithNested(['Working…'], [], true, '', undefined, livenessHeader)
|
|
897
909
|
}
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
* unbounded. PURE w.r.t. the injected fs seam ⇒ unit-testable.
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
+
import { dirname } from 'node:path'
|
|
28
29
|
import type { Obligation } from './obligation-ledger.js'
|
|
29
30
|
|
|
30
31
|
export interface ObligationStoreFsSeam {
|
|
@@ -34,6 +35,14 @@ export interface ObligationStoreFsSeam {
|
|
|
34
35
|
* the snapshot. */
|
|
35
36
|
renameSync: (from: string, to: string) => void
|
|
36
37
|
existsSync: (path: string) => boolean
|
|
38
|
+
/** fsync the FILE at `path`. Atomicity ≠ durability: rename orders the
|
|
39
|
+
* metadata, it does not put the tmp file's DATA on stable storage. Call
|
|
40
|
+
* on the tmp file BEFORE the rename. */
|
|
41
|
+
fsyncFileSync: (path: string) => void
|
|
42
|
+
/** fsync the DIRECTORY at `path`, AFTER the rename — the new directory
|
|
43
|
+
* entry lives in the parent's own cached metadata and is otherwise lost
|
|
44
|
+
* in a power cut. */
|
|
45
|
+
fsyncDirSync: (path: string) => void
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
interface SnapshotEnvelope {
|
|
@@ -133,11 +142,30 @@ export function persistObligations(
|
|
|
133
142
|
const tmp = path + '.tmp'
|
|
134
143
|
try {
|
|
135
144
|
fs.writeFileSync(tmp, JSON.stringify(env))
|
|
145
|
+
// Order matters: fsync the tmp file's DATA, THEN publish it by rename.
|
|
146
|
+
// Renaming first would let a power cut leave the snapshot path pointing
|
|
147
|
+
// at an inode whose bytes never reached the platter — a zero-length or
|
|
148
|
+
// stale snapshot that reads as "no open obligations", which is exactly
|
|
149
|
+
// the silent-drop this store exists to prevent.
|
|
150
|
+
fs.fsyncFileSync(tmp)
|
|
136
151
|
fs.renameSync(tmp, path)
|
|
137
152
|
} catch (err) {
|
|
138
153
|
log(
|
|
139
154
|
`obligation-store: persist FAILED path=${path}: ${(err as Error).message} — ` +
|
|
140
155
|
`durability degraded to in-memory\n`,
|
|
141
156
|
)
|
|
157
|
+
return
|
|
158
|
+
}
|
|
159
|
+
// The rename landed, so the snapshot is already correct for a process
|
|
160
|
+
// crash; this last barrier only upgrades it to survive a power cut. Kept
|
|
161
|
+
// in its own try so a directory-fsync failure isn't mislabelled as a
|
|
162
|
+
// failed persist — the data IS written, just not provably on the platter.
|
|
163
|
+
try {
|
|
164
|
+
fs.fsyncDirSync(dirname(path))
|
|
165
|
+
} catch (err) {
|
|
166
|
+
log(
|
|
167
|
+
`obligation-store: directory fsync FAILED path=${path}: ${(err as Error).message} — ` +
|
|
168
|
+
`snapshot written but the rename may not survive a power cut\n`,
|
|
169
|
+
)
|
|
142
170
|
}
|
|
143
171
|
}
|