sintbot 1.4.0 → 1.4.2

package/src/c2updater.js

@@ -0,0 +1,361 @@
+'use strict'
+
+// Software updates for the multi-group service. Checking is read-only. Only a
+// launchd-managed installation upgrades from the console, and only while every
+// group is idle and settled: the exact stable version is installed side by
+// side under the releases directory, then a detached switch process replaces
+// the service entry with the same steps as the deploy scripts and rolls back
+// on any failure. Foreground and development installations get instructions.
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+const { execFile, execFileSync, spawn } = require('node:child_process')
+const { promisify } = require('node:util')
+const { createHash } = require('node:crypto')
+const { stableVersion, compareVersion, fetchVersion, CHECK_INTERVAL_MS } = require('./updater')
+const { atomicWriteJSON } = require('./core/store')
+
+const PACKAGE = 'sintbot'
+const REGISTRIES = Object.freeze([
+  { registry: 'https://registry.npmjs.org', source: 'npm' },
+  { registry: 'https://registry.npmmirror.com', source: 'mirror' },
+])
+const QUEUE_TIMEOUT_MS = 24 * 60 * 60 * 1000
+const ACTIVE = Object.freeze(['queued', 'installing', 'switching'])
+const LABEL = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
+const execFileAsync = promisify(execFile)
+
+function readJSON(file) {
+  try {
+    const value = JSON.parse(fs.readFileSync(file, 'utf8'))
+    return value && typeof value === 'object' ? value : {}
+  } catch { return {} }
+}
+
+function readPlist(file, { exec = execFileSync } = {}) {
+  return JSON.parse(exec('/usr/bin/plutil', ['-convert', 'json', '-o', '-', file],
+    { encoding: 'utf8', timeout: 10_000 }))
+}
+
+const real = file => { try { return fs.realpathSync(file) } catch { return null } }
+
+// launchd names its job in XPC_SERVICE_NAME. The job counts as managing this
+// process only when its plist starts this exact entry and launchd reports this
+// exact PID; anything else is a foreground run.
+function detectRunMode({ privateBuild, entry, env = process.env, pid = process.pid,
+  uid = process.getuid?.(), home = os.homedir(), exec = execFileSync } = {}) {
+  if (privateBuild === true) return Object.freeze({ mode: 'dev' })
+  const label = env.XPC_SERVICE_NAME
+  if (typeof label !== 'string' || !LABEL.test(label) || label === '0') {
+    return Object.freeze({ mode: 'foreground' })
+  }
+  const plist = path.join(home, 'Library', 'LaunchAgents', `${label}.plist`)
+  try {
+    const spec = readPlist(plist, { exec })
+    const args = spec.ProgramArguments
+    if (spec.Label !== label || !Array.isArray(args) || args.length < 3 ||
+        !args.every(arg => typeof arg === 'string') ||
+        real(args[1]) === null || real(args[1]) !== real(entry)) {
+      return Object.freeze({ mode: 'foreground', reason: 'launchd entry differs' })
+    }
+    const printed = exec('/bin/launchctl', ['print', `gui/${uid}/${label}`],
+      { encoding: 'utf8', timeout: 10_000 })
+    if (!new RegExp(`^\\s*pid = ${pid}\\s*$`, 'm').test(printed)) {
+      return Object.freeze({ mode: 'foreground', reason: 'launchd pid differs' })
+    }
+    return Object.freeze({ mode: 'managed', label, plist, entry: args[1],
+      logFile: typeof spec.StandardErrorPath === 'string' ? spec.StandardErrorPath : null })
+  } catch {
+    return Object.freeze({ mode: 'foreground', reason: 'launchd job unreadable' })
+  }
+}
+
+function registriesFromEnv(env = process.env) {
+  const custom = env.SINTBOT_UPDATE_REGISTRY
+  if (custom === undefined || custom === '') return REGISTRIES
+  const url = new URL(custom)
+  if (!['http:', 'https:'].includes(url.protocol)) throw new Error('SINTBOT_UPDATE_REGISTRY must be an http(s) URL')
+  return Object.freeze([{ registry: url.href.replace(/\/$/, ''), source: 'custom' }])
+}
+
+function defaultReleasesRoot(env = process.env, home = os.homedir()) {
+  const dir = env.SINTBOT_RELEASES_DIR
+  if (dir !== undefined && dir !== '') {
+    if (!path.isAbsolute(dir) || path.resolve(dir) !== dir) throw new Error('SINTBOT_RELEASES_DIR must be a normalized absolute path')
+    return dir
+  }
+  return path.join(home, '.local', 'share', 'sintbot', 'releases')
+}
+
+const entryOf = prefix => path.join(prefix, 'lib', 'node_modules', PACKAGE, 'bin', 'sintbot-v2.js')
+
+function npmBin() {
+  const sibling = path.join(path.dirname(process.execPath), 'npm')
+  return fs.existsSync(sibling) ? sibling : 'npm'
+}
+
+async function defaultInstallVersion({ version, prefix, registry }) {
+  const env = {}
+  for (const key of ['PATH', 'HOME', 'USER', 'TMPDIR', 'HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY',
+    'http_proxy', 'https_proxy', 'no_proxy', 'LANG', 'LC_ALL']) {
+    if (process.env[key] !== undefined) env[key] = process.env[key]
+  }
+  try {
+    await execFileAsync(npmBin(), ['install', '--global', '--prefix', prefix, `--registry=${registry}`,
+      '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', `${PACKAGE}@${version}`],
+    { timeout: 10 * 60_000, maxBuffer: 4 * 1024 * 1024, env })
+  } catch (error) {
+    throw new Error(`安装 v${version} 失败（退出码 ${error?.code ?? '未知'}）；没有提权，也没有改用其他版本`)
+  }
+}
+
+// The same verification as the public installer, in a child process so the
+// service never loads modules of another version.
+function defaultVerifyInstall({ version, prefix }) {
+  const root = path.join(prefix, 'lib', 'node_modules', PACKAGE)
+  // Only literal requires in this file: the package builder rejects computed ones.
+  const script = `
+const fs = require('node:fs');
+const path = require('node:path');
+const { pathToFileURL } = require('node:url');
+(async () => {
+  const root = process.argv[1];
+  const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
+  if (pkg.name !== 'sintbot' || pkg.version !== process.argv[2] || pkg.private === true ||
+      pkg.bin?.sintbot !== 'bin/sintbot-v2.js') throw new Error('package identity differs');
+  await import(pathToFileURL(path.join(root, 'src/agent/privatecore.js')).href);
+  await import(pathToFileURL(path.join(root, 'src/platform/feishu.js')).href);
+})().catch(() => process.exit(1));`
+  try {
+    execFileSync(process.execPath, ['-e', script, root, version], { timeout: 60_000, stdio: 'ignore', cwd: '/' })
+    const reported = execFileSync(process.execPath, [entryOf(prefix), 'version'],
+      { encoding: 'utf8', timeout: 30_000, cwd: '/' }).trim()
+    if (reported !== version) throw new Error('version differs')
+  } catch {
+    throw new Error(`v${version} 安装后的核对没有通过；新目录保留在 ${prefix}，服务没有切换`)
+  }
+}
+
+function defaultSpawnSwitch({ entry, stateFile }) {
+  const child = spawn(process.execPath, [entry, 'upgrade-switch', '--state', stateFile],
+    { detached: true, stdio: 'ignore', cwd: '/', env: process.env })
+  child.unref()
+  return child.pid
+}
+
+function journalHashes(dataDir) {
+  const out = {}
+  for (const name of fs.readdirSync(dataDir).filter(name => /^c2-v2.*\.log$/.test(name)).sort()) {
+    out[name] = createHash('sha256').update(fs.readFileSync(path.join(dataDir, name))).digest('hex')
+  }
+  return out
+}
+
+function alive(pid) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return false
+  try { process.kill(pid, 0); return true } catch (error) { return error.code === 'EPERM' }
+}
+
+function createC2Updater({ currentVersion, runMode, stateFile, dataDir, configPath,
+  gate, beginMaintenance = () => {}, endMaintenance = () => {}, expectedGroups = () => 0,
+  registries = REGISTRIES, fetchImpl = globalThis.fetch, releasesRoot = defaultReleasesRoot(),
+  installVersion = defaultInstallVersion, verifyInstall = defaultVerifyInstall,
+  spawnSwitch = defaultSpawnSwitch, hashJournals = journalHashes, isAlive = alive,
+  now = Date.now, waitMs = 5_000, queueTimeoutMs = QUEUE_TIMEOUT_MS,
+  checkIntervalMs = CHECK_INTERVAL_MS, autoCheck = true, log = () => {} } = {}) {
+  if (typeof currentVersion !== 'string' || !runMode || typeof runMode.mode !== 'string' ||
+      typeof stateFile !== 'string' || !path.isAbsolute(stateFile) || typeof gate !== 'function') {
+    throw new TypeError('C2 updater needs the version, run mode, state file and idle gate')
+  }
+  const current = stableVersion(currentVersion)
+  let data = readJSON(stateFile)
+  let timer = null
+  let checking = null
+  let closed = false
+
+  function persist() {
+    try { atomicWriteJSON(stateFile, data, { mode: 0o600 }) } catch (error) { log(`update state: ${error.message}`) }
+  }
+  function setJob(patch, message) {
+    data.job = { ...data.job, ...patch, ...(message ? { message } : {}), updatedAt: now() }
+    persist()
+    return data.job
+  }
+  function fail(message) {
+    setJob({ state: 'failed' }, String(message).slice(0, 300))
+    return status()
+  }
+
+  // A switch that this process did not finish is settled from the evidence:
+  // the new version reached here, the switch process is still working, or it
+  // stopped without writing a result.
+  if (data.job?.state === 'switching' && data.job.to !== current && !isAlive(data.job.helperPid)) {
+    fail('切换过程中断，现场已保留；请检查后重新发起')
+  } else if (data.job?.state === 'installing') {
+    fail('安装被中断，可重新发起升级')
+  }
+
+  // The switch process writes this file too; always act on its latest bytes.
+  function refresh() { data = readJSON(stateFile) }
+  function status() {
+    refresh()
+    const latest = stableVersion(data.latestVersion)
+    const upgradeSupported = runMode.mode === 'managed'
+    return {
+      currentVersion, latestVersion: latest, checkedAt: Number(data.checkedAt) || null,
+      source: data.source || '', ignoredVersion: data.ignoredVersion || '',
+      available: !!current && !!latest && compareVersion(latest, current) > 0 && data.ignoredVersion !== latest,
+      mode: runMode.mode, upgradeSupported, announceSupported: false,
+      job: data.job ? { ...data.job } : null,
+    }
+  }
+
+  async function check() {
+    if (closed) return status()
+    if (checking) return checking
+    checking = (async () => {
+      let lastError = null
+      for (const item of registries) {
+        try {
+          const version = await fetchVersion({ fetchImpl, url: `${item.registry}/${PACKAGE}/latest` })
+          refresh()
+          data.latestVersion = version
+          data.checkedAt = now()
+          data.source = item.source
+          data.registry = item.registry
+          if (data.ignoredVersion && data.ignoredVersion !== version) data.ignoredVersion = ''
+          persist()
+          return status()
+        } catch (error) { lastError = error }
+      }
+      if (lastError) log(`update check unavailable: ${lastError.message || 'network error'}`)
+      return status()
+    })().finally(() => { checking = null })
+    return checking
+  }
+
+  function ignore(version) {
+    refresh()
+    const v = stableVersion(version)
+    if (!v || v !== stableVersion(data.latestVersion)) throw new Error('只能忽略当前检测到的版本')
+    data.ignoredVersion = v
+    persist()
+    return status()
+  }
+
+  function startUpgrade({ version } = {}) {
+    refresh()
+    if (runMode.mode !== 'managed') {
+      throw new Error(runMode.mode === 'dev' ? '开发版请用部署脚本升级' : '前台运行请按卡片上的步骤升级')
+    }
+    const v = stableVersion(version)
+    if (!current || !v || v !== stableVersion(data.latestVersion) || compareVersion(v, current) <= 0) {
+      throw new Error('版本信息已变化，请先重新检查')
+    }
+    if (data.job && ACTIVE.includes(data.job.state)) throw new Error('已有升级正在进行')
+    if (gate().includes('held')) throw new Error('有暂停的群，先处理后再升级')
+    data.job = { id: `${now()}-${Math.random().toString(36).slice(2, 8)}`, state: 'queued',
+      from: current, to: v, registry: data.registry || registries[0].registry,
+      createdAt: now(), updatedAt: now(), message: '等所有群空闲、记录结清后升级' }
+    persist()
+    schedule(0)
+    return status()
+  }
+
+  function schedule(ms = waitMs) {
+    if (closed || timer || data.job?.state !== 'queued') return
+    timer = setTimeout(() => { timer = null; void processQueue() }, ms)
+    timer.unref?.()
+  }
+
+  async function processQueue() {
+    const job = data.job
+    if (closed || job?.state !== 'queued') return status()
+    if (now() - job.createdAt > queueTimeoutMs) return fail('等待群空闲超时；可在空闲后重新发起')
+    const blocked = gate()
+    if (blocked.includes('held')) return fail('有暂停的群，升级已取消；先处理暂停的群')
+    if (blocked.length) { schedule(); return status() }
+    const prefix = path.join(releasesRoot, job.to)
+    if (!job.installed) {
+      setJob({ state: 'installing', prefix }, `正在安装 v${job.to}`)
+      try {
+        if (fs.existsSync(prefix)) {
+          // A directory left by an earlier attempt or a manual install is reused
+          // only when it verifies as exactly this version.
+          await verifyInstall({ version: job.to, prefix })
+        } else {
+          fs.mkdirSync(releasesRoot, { recursive: true, mode: 0o700 })
+          await installVersion({ version: job.to, prefix, registry: job.registry })
+          await verifyInstall({ version: job.to, prefix })
+        }
+      } catch (error) { return fail(error.message) }
+      if (closed) return status()
+      setJob({ state: 'queued', installed: true }, `v${job.to} 已安装，等所有群空闲后切换`)
+    }
+    // Close admission first, then prove idleness once more: nothing can start
+    // between this check and the switch process stopping the service.
+    if (gate().length) { schedule(); return status() }
+    beginMaintenance()
+    const settled = gate()
+    if (settled.length) { endMaintenance(); schedule(); return status() }
+    let hashes
+    try { hashes = hashJournals(dataDir) } catch (error) { endMaintenance(); return fail(`无法读取群记录：${error.message}`) }
+    setJob({ state: 'switching', oldEntry: runMode.entry, newEntry: entryOf(prefix), label: runMode.label,
+      plist: runMode.plist, logFile: runMode.logFile, dataDir, configPath,
+      expectedGroups: expectedGroups(), journals: hashes, servicePid: process.pid,
+      backupDir: path.join(dataDir, 'upgrades', new Date(now()).toISOString().replace(/[:.]/g, '-')),
+      steps: [] }, `正在切换到 v${job.to}；期间的消息会提示稍后重发`)
+    try {
+      const pid = spawnSwitch({ entry: runMode.entry, stateFile })
+      setJob({ helperPid: pid })
+    } catch (error) {
+      endMaintenance()
+      return fail(`切换进程没有启动：${error.message}`)
+    }
+    return status()
+  }
+
+  // The new version marks the switch done only after every group connected.
+  function confirmStarted() {
+    refresh()
+    if (data.job?.state === 'switching' && data.job.to === current) {
+      setJob({ state: 'done', finishedAt: now() }, `已升级到 v${current}`)
+    }
+    return status()
+  }
+
+  if (data.job?.state === 'queued') schedule(0)
+  const checkTimer = autoCheck ? setInterval(() => { void check() }, checkIntervalMs) : null
+  checkTimer?.unref?.()
+  return { status, check, ignore, startUpgrade, processQueue, confirmStarted,
+    start() { return autoCheck ? check() : Promise.resolve(status()) },
+    close() { closed = true; if (checkTimer) clearInterval(checkTimer); if (timer) clearTimeout(timer); timer = null } }
+}
+
+// What keeps a console upgrade from starting: 'not-serving', 'held', 'busy'
+// or 'unsettled'. The updater's own maintenance flag never blocks it, so the
+// final check after admission closes can pass.
+function createC2UpgradeGate({ isRunning, service, capacity, assertJournalScope }) {
+  return () => {
+    const reasons = new Set()
+    const live = service()
+    if (!isRunning() || !live) return ['not-serving']
+    if (live.held.size) reasons.add('held')
+    const shared = capacity.snapshot()
+    if (shared.closed) reasons.add('not-serving')
+    if (shared.active || shared.waiting) reasons.add('busy')
+    for (const [chatId, binding] of live.bindings) {
+      if (binding.gate.blocked) reasons.add('not-serving')
+      for (const rt of binding.engine.rts.values()) {
+        if (rt.v2NormalRouteActive || rt.v2AdmissionPending || rt.v2Controller ||
+            rt.v2QueueReceiptIds?.length) reasons.add('busy')
+      }
+      try { assertJournalScope(binding.journal, chatId, { requireSettled: true }) }
+      catch { reasons.add('unsettled') }
+    }
+    return [...reasons]
+  }
+}
+
+module.exports = { createC2Updater, createC2UpgradeGate, detectRunMode, registriesFromEnv, defaultReleasesRoot,
+  entryOf, journalHashes, readPlist, defaultVerifyInstall, REGISTRIES, PACKAGE }

package/src/c2upgradeswitch.js

@@ -0,0 +1,169 @@
+'use strict'
+
+// Detached switch process started by the console upgrade. It runs outside the
+// service's process group, so stopping the launchd job does not stop it. The
+// steps are those of the deploy scripts: stop the job, wait for the writer
+// lock, prove the journals unchanged, back up data and plist, preflight the
+// new entry with the job's environment, replace only the entry in the plist,
+// start the job and wait for every group to connect. Any failure restores the
+// original plist bytes and restarts the previous version.
+const fs = require('node:fs')
+const path = require('node:path')
+const { execFileSync, spawnSync } = require('node:child_process')
+const { isDeepStrictEqual } = require('node:util')
+const { atomicWriteJSON } = require('./core/store')
+const { journalHashes, readPlist } = require('./c2updater')
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
+
+function defaultSystem() {
+  const uid = process.getuid()
+  const run = (file, args, options = {}) => spawnSync(file, args, { encoding: 'utf8', timeout: 60_000, ...options })
+  return {
+    bootout: label => run('/bin/launchctl', ['bootout', `gui/${uid}/${label}`]),
+    bootstrap: plist => run('/bin/launchctl', ['bootstrap', `gui/${uid}`, plist]),
+    jobPid: label => {
+      const out = run('/bin/launchctl', ['print', `gui/${uid}/${label}`])
+      if (out.status !== 0) return null
+      const match = /^\s*pid = (\d+)\s*$/m.exec(out.stdout || '')
+      return match ? Number(match[1]) : 0
+    },
+    alive: pid => { try { process.kill(pid, 0); return true } catch (error) { return error.code === 'EPERM' } },
+    readPlist: file => readPlist(file),
+    writePlist: (file, value) => {
+      const json = `${file}.upgrade.json`, tmp = `${file}.upgrade.tmp`
+      fs.writeFileSync(json, JSON.stringify(value), { mode: 0o600 })
+      try {
+        execFileSync('/usr/bin/plutil', ['-convert', 'xml1', '-o', tmp, json], { timeout: 10_000 })
+        fs.chmodSync(tmp, 0o600)
+        execFileSync('/usr/bin/plutil', ['-lint', tmp], { timeout: 10_000, stdio: 'ignore' })
+        fs.renameSync(tmp, file)
+      } finally { fs.rmSync(json, { force: true }); fs.rmSync(tmp, { force: true }) }
+    },
+    backupData: (dataDir, file) => {
+      const base = path.basename(dataDir)
+      const out = run('/usr/bin/tar', ['-czf', file, '--exclude', `${base}/upgrades`, '-C', path.dirname(dataDir), base])
+      if (out.status !== 0) throw new Error(`data backup failed: ${(out.stderr || '').trim().slice(0, 200)}`)
+      fs.chmodSync(file, 0o600)
+    },
+    preflight: (spec, entry) => {
+      const env = { HOME: process.env.HOME, USER: process.env.USER, ...(spec.EnvironmentVariables || {}) }
+      const args = [entry, ...spec.ProgramArguments.slice(2), '--preflight']
+      return run(spec.ProgramArguments[0], args, { env, cwd: '/', timeout: 180_000 })
+    },
+    logSize: file => { try { return fs.statSync(file).size } catch { return 0 } },
+    logSince: (file, offset) => {
+      try {
+        const fd = fs.openSync(file, 'r')
+        try {
+          const size = fs.fstatSync(fd).size
+          if (size <= offset) return ''
+          const buffer = Buffer.alloc(Math.min(size - offset, 4 * 1024 * 1024))
+          fs.readSync(fd, buffer, 0, buffer.length, offset)
+          return buffer.toString('utf8')
+        } finally { fs.closeSync(fd) }
+      } catch { return '' }
+    },
+  }
+}
+
+async function runUpgradeSwitch({ stateFile, system = defaultSystem(), hashJournals = journalHashes,
+  now = Date.now, wait = sleep, stopTimeoutMs = 60_000, connectTimeoutMs = 90_000, pollMs = 500 } = {}) {
+  const read = () => JSON.parse(fs.readFileSync(stateFile, 'utf8'))
+  const state = read()
+  const job = state.job
+  if (job?.state !== 'switching' || typeof job.label !== 'string' || typeof job.plist !== 'string' ||
+      typeof job.oldEntry !== 'string' || typeof job.newEntry !== 'string' ||
+      typeof job.dataDir !== 'string' || typeof job.backupDir !== 'string') {
+    throw new Error('No switching upgrade job to run')
+  }
+  const steps = []
+  const write = (patch, message) => {
+    const latest = read()
+    if (latest.job?.id !== job.id) throw new Error('The upgrade job changed during the switch')
+    latest.job = { ...latest.job, ...patch, ...(message ? { message } : {}), steps: [...steps], updatedAt: now() }
+    atomicWriteJSON(stateFile, latest, { mode: 0o600 })
+    return latest.job
+  }
+  const step = (name, ok = true, detail = '') => {
+    steps.push({ at: now(), step: name, ok, ...(detail ? { detail: String(detail).slice(0, 300) } : {}) })
+    try { write({}) } catch {}
+  }
+  const lockFile = path.join(job.dataDir, 'daemon.lock')
+  const until = async (predicate, ms) => {
+    const end = now() + ms
+    while (now() < end) { if (await predicate()) return true; await wait(pollMs) }
+    return !!(await predicate())
+  }
+  const stopped = () => until(() => {
+    const pid = system.jobPid(job.label)
+    return (pid === null || pid === 0) && !system.alive(job.servicePid) && !fs.existsSync(lockFile)
+  }, stopTimeoutMs)
+  const connected = (offset, expectDone) => until(() => {
+    const text = system.logSince(job.logFile, offset)
+    if (/ held \(|multi-group failure|startup cleanup was not confirmed/.test(text)) return false
+    if (job.expectedGroups > 0 &&
+        text.includes(`SintBot Feishu ingress connected for ${job.expectedGroups} groups`)) return true
+    try { return expectDone && read().job?.state === 'done' } catch { return false }
+  }, connectTimeoutMs)
+
+  fs.mkdirSync(job.backupDir, { recursive: true, mode: 0o700 })
+  const original = fs.readFileSync(job.plist)
+  fs.writeFileSync(path.join(job.backupDir, 'service.plist'), original, { mode: 0o600 })
+  const spec = system.readPlist(job.plist)
+  if (spec.Label !== job.label || spec.ProgramArguments?.[1] !== job.oldEntry) {
+    write({ state: 'failed' }, '服务配置和发起升级时不一致，没有切换')
+    return 'failed'
+  }
+
+  // Restore the exact original plist and start the previous version.
+  async function rollback(reason) {
+    step('rollback', true, reason)
+    system.bootout(job.label)
+    await stopped()
+    fs.writeFileSync(job.plist, original, { mode: 0o600 })
+    const offset = system.logSize(job.logFile)
+    const started = system.bootstrap(job.plist)
+    const back = started.status === 0 && await connected(offset, false)
+    step('previous version started', back)
+    write({ state: back ? 'rolled-back' : 'failed', finishedAt: now() }, back
+      ? `没有切换到 v${job.to}：${reason}；已换回 v${job.from}`
+      : `没有切换到 v${job.to}：${reason}；换回 v${job.from} 后服务没有连上，请手动检查`)
+    return back ? 'rolled-back' : 'failed'
+  }
+
+  const out = system.bootout(job.label)
+  step('stop service', out.status === 0, out.stderr)
+  if (!await stopped()) return rollback('旧服务没有在规定时间内停止')
+  if (!isDeepStrictEqual(hashJournals(job.dataDir), job.journals)) {
+    // A journal changed after the idle check: never switch on top of it.
+    return rollback('停止过程中群记录发生了变化')
+  }
+  step('journals unchanged')
+  try { system.backupData(job.dataDir, path.join(job.backupDir, 'data.tgz')); step('data backed up') }
+  catch (error) { return rollback(error.message) }
+  const check = system.preflight(spec, job.newEntry)
+  step('new version preflight', check.status === 0, (check.stderr || '').trim().split('\n').pop())
+  if (check.status !== 0) return rollback('新版本预检没有通过')
+  const next = { ...spec, ProgramArguments: [...spec.ProgramArguments] }
+  next.ProgramArguments[1] = job.newEntry
+  try {
+    system.writePlist(job.plist, next)
+    const written = system.readPlist(job.plist)
+    const expected = { ...spec, ProgramArguments: next.ProgramArguments }
+    if (!isDeepStrictEqual(written, expected)) throw new Error('rewritten service configuration differs')
+    step('service entry replaced')
+  } catch (error) { return rollback(error.message) }
+  const offset = system.logSize(job.logFile)
+  const started = system.bootstrap(job.plist)
+  step('start new version', started.status === 0, started.stderr)
+  if (started.status !== 0 || !await connected(offset, true)) {
+    return rollback(`新版本启动后没有在 ${Math.round(connectTimeoutMs / 1000)} 秒内连上全部群`)
+  }
+  const latest = read().job
+  if (latest?.state !== 'done') write({ state: 'done', finishedAt: now() }, `已升级到 v${job.to}`)
+  else write({})
+  return 'done'
+}
+
+module.exports = { runUpgradeSwitch, defaultSystem }

package/src/core/c2sourcequarantine.js

@@ -29,14 +29,21 @@ function createC2V2SourceQuarantine(value, { required = false } = {}) {
 
 // Every sealed incident under <data>/incidents/*/incident.json keeps its
 // unsettled source messages out of admission for that group, on every start.
-// A malformed seal stops startup: an unreadable fence cannot be skipped.
+// A seal written by `sintbot incident seal` also lists the receipts its
+// journal had already settled, so the group's new journal never runs one of
+// those messages again. A malformed seal stops startup: an unreadable fence
+// cannot be skipped.
 const CHAT_ID = /^oc_[A-Za-z0-9_-]{1,124}$/
+const RECEIPT_ID = /^[0-9a-f]{32}$/
 function loadC2IncidentQuarantine(stateDir, { fs = require('node:fs'), path = require('node:path') } = {}) {
   if (typeof stateDir !== 'string' || !path.isAbsolute(stateDir)) {
     throw new TypeError('C2 incident quarantine needs the installation directory')
   }
   const root = path.join(stateDir, 'incidents')
+  // Loaded here so the source quarantine above stays a standalone module.
+  const { taskId } = require('./enginecheckpoint')
   const byChat = new Map()
+  const settled = new Map()
   const fail = name => { throw new Error(`Sealed incident record is invalid: ${name}; preserve it and review`) }
   let entries = []
   try { entries = fs.readdirSync(root, { withFileTypes: true }) }
@@ -53,14 +60,26 @@ function loadC2IncidentQuarantine(stateDir, { fs = require('node:fs'), path = re
     try { record = JSON.parse(fs.readFileSync(file, 'utf8')) } catch { fail(entry.name) }
     if (record?.schema !== 'sbot-c2-incident-seal-v1' || !CHAT_ID.test(record.chatId || '') ||
         !Array.isArray(record.pending) ||
-        record.pending.some(item => !FEISHU_MESSAGE_ID.test(item?.sourceId || ''))) fail(entry.name)
+        record.pending.some(item => !FEISHU_MESSAGE_ID.test(item?.sourceId || '')) ||
+        record.completedReceiptIds !== undefined && (!Array.isArray(record.completedReceiptIds) ||
+          record.completedReceiptIds.some(id => typeof id !== 'string' || !RECEIPT_ID.test(id)))) fail(entry.name)
     const ids = byChat.get(record.chatId) || new Set()
     for (const item of record.pending) ids.add(item.sourceId)
     byChat.set(record.chatId, ids)
+    if (record.completedReceiptIds?.length) {
+      const done = settled.get(record.chatId) || new Set()
+      for (const id of record.completedReceiptIds) done.add(id)
+      settled.set(record.chatId, done)
+    }
   }
   const sourceIds = Object.freeze([...byChat.values()].flatMap(ids => [...ids]))
+  const settledBefore = message => {
+    const done = settled.get(message?.chatId)
+    if (!done || typeof message.msgId !== 'string' || !message.msgId) return false
+    return done.has(taskId({ source: 'message', chatId: message.chatId, sourceId: message.msgId }))
+  }
   return Object.freeze({ sourceIds,
-    rejects: message => !!byChat.get(message?.chatId)?.has(message?.msgId) })
+    rejects: message => !!byChat.get(message?.chatId)?.has(message?.msgId) || settledBefore(message) })
 }
 
 module.exports = { createC2V2SourceQuarantine, loadC2IncidentQuarantine }

package/src/core/v2engine.js

@@ -596,6 +596,7 @@ async runDurableV2Message(msg, platform) {
     rt.v2NormalRouteActive = true
     const execute = async () => {
       let release = null
+      let failed = false
       try {
         let receiptId = accepted.receiptId
         if (this.durableV2AcquireCapacity) {
@@ -628,12 +629,13 @@ async runDurableV2Message(msg, platform) {
         // Seal shared grants before releasing this turn's permit. Waiting
         // siblings must not dispatch in the microtask before the service sees
         // this rejection and begins its process-wide stop.
+        failed = true
         this.durableV2SealCapacity?.(error)
         throw error
       } finally {
         rt.v2AdmissionPending = false
         rt.v2AdmissionController = null
-        release?.()
+        if (release) this._releaseDurableV2Capacity(rt, release, failed)
       }
     }
     const execution = execute()
@@ -649,6 +651,24 @@ async runDurableV2Message(msg, platform) {
     }
   }
 
+// A failed route returns its shared model permit only once its runner is
+// proven closed or proven never started. Otherwise the permit stays taken
+// until the process exits, so a sibling group never runs beside a child that
+// may still be alive.
+_releaseDurableV2Capacity(rt, release, failed) {
+    const closed = () => !rt.v2Controller && (rt.v2RunnerStopVerified === true ||
+      rt.v2RunnerNotStarted === true)
+    const stop = rt.v2RunnerStopPromise
+    if (!failed || !rt.v2Controller && (!stop || closed())) return release()
+    const kept = () => this.log(`group ${rt.chatId}: model permit kept until restart; runner close unproven`)
+    if (!stop) return kept()
+    stop.then(() => {
+      if (!closed()) return kept()
+      // A failed binary release closes the shared capacity itself.
+      try { release() } catch (error) { this.log(`group ${rt.chatId}: model permit release failed: ${error.message}`) }
+    }, kept)
+  }
+
 async _runDurableV2Steer(rt, receiptId) {
     const authority = this.durableV2Runtime
     const chatId = rt.chatId
@@ -1323,6 +1343,7 @@ async startDurableV2Runner({ chatId, receiptId, turnToken, effectId } = {}) {
       rt.v2Controller = controller
       rt.v2RunnerStopPromise = null
       rt.v2RunnerStopVerified = false
+      rt.v2RunnerNotStarted = false
       let streamChain = Promise.resolve()
       let streamFailure = null
       const enqueueFrame = event => {
@@ -1508,6 +1529,7 @@ async startDurableV2Runner({ chatId, receiptId, turnToken, effectId } = {}) {
         rt.v2RunnerGeneration = null
         rt.v2RunningJobs = null
         rt.v2DispatchEffect = null
+        rt.v2RunnerNotStarted = true
         return { effectId, accepted: false, startFailureRecovered: true }
       }
       if (capability && !accepted) {