switchroom 0.5.0 → 0.7.8

package/telegram-plugin/tests/boot-probes.test.ts

@@ -13,8 +13,17 @@ import { join } from 'path'
 
 import {
   probeAgentProcess,
+  probeScheduler,
+  probeBroker,
+  probeKernel,
+  probeSkills,
   probeQuota,
   watchAgentProcess,
+  findAgentProcessInContainer,
+  uptimeMsForStarttime,
+  type ProcFsImpl,
+  type SchedulerFsImpl,
+  type SkillsFsImpl,
 } from '../gateway/boot-probes.js'
 import { readQuotaCache, RATE_LIMIT_TTL_MS } from '../gateway/quota-cache.js'
 
@@ -448,4 +457,553 @@ describe('watchAgentProcess — #296: re-poll after window expiry', () => {
     expect(yields[0].status).toBe('ok')
     expect(extraCalls).toBe(1) // only the initial probe; no follow-up
   })
+
+// ── docker mode: skip systemctl, use /proc walk ───────────────────────────
+
+describe('probeAgentProcess — docker mode skips systemctl', () => {
+  it('probeAgentProcess(dockerMode) returns the injected /proc result without execing', async () => {
+    let execFileCalls = 0
+    const execFileImpl: ExecFileFn = async () => {
+      execFileCalls++
+      throw new Error('systemctl should never be called under dockerMode')
+    }
+    const dockerProbeImpl = () => ({
+      status: 'ok' as const,
+      label: 'Agent',
+      detail: 'PID 42 · up 3.0s · 128 MB',
+    })
+    const result = await probeAgentProcess('clerk', {
+      dockerMode: true,
+      dockerProbeImpl,
+      execFileImpl: execFileImpl as never,
+      sleepImpl: noopSleep,
+      retryIntervalMs: 1,
+      retryMaxMs: 5,
+    })
+    expect(result.status).toBe('ok')
+    expect(result.label).toBe('Agent')
+    expect(result.detail).toBe('PID 42 · up 3.0s · 128 MB')
+    expect(execFileCalls).toBe(0)
+  })
+
+  it('probeAgentProcess(dockerMode) surfaces fail when no claude process found', async () => {
+    const dockerProbeImpl = () => ({
+      status: 'fail' as const,
+      label: 'Agent',
+      detail: 'claude process not found',
+    })
+    const result = await probeAgentProcess('clerk', {
+      dockerMode: true,
+      dockerProbeImpl,
+    })
+    expect(result.status).toBe('fail')
+    expect(result.detail).toBe('claude process not found')
+  })
+
+  it('watchAgentProcess(dockerMode) yields the /proc result once and exits', async () => {
+    let execFileCalls = 0
+    const execFileImpl: ExecFileFn = async () => {
+      execFileCalls++
+      return { stdout: '', stderr: '' }
+    }
+    const dockerProbeImpl = () => ({
+      status: 'ok' as const,
+      label: 'Agent',
+      detail: 'PID 42 · up 5.0s · 200 MB',
+    })
+    const yields: Array<{ status: string; detail: string }> = []
+    const gen = watchAgentProcess('clerk', {
+      dockerMode: true,
+      dockerProbeImpl,
+      execFileImpl: execFileImpl as never,
+      sleepImpl: noopSleep,
+      liveWindowMs: 1000,
+      pollIntervalMs: 10,
+      followupRepollMs: 0,
+    })
+    for await (const r of gen) yields.push({ status: r.status, detail: r.detail })
+    expect(yields).toHaveLength(1)
+    expect(yields[0].status).toBe('ok')
+    expect(execFileCalls).toBe(0)
+  })
+
+})
+
+// ── probeScheduler — in-container agent-scheduler (Phase 4 cron-fold-in) ──
+
+function makeSchedulerFs(files: Record<string, { content?: string; mtimeMs?: number }>): SchedulerFsImpl {
+  return {
+    readFile: (p) => {
+      const f = files[p]
+      if (!f || f.content === undefined) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' })
+      return f.content
+    },
+    mtimeMs: (p) => {
+      const f = files[p]
+      if (!f || f.mtimeMs === undefined) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' })
+      return f.mtimeMs
+    },
+    exists: (p) => p in files,
+  }
+}
+
+describe('probeScheduler', () => {
+  it('returns ok n/a when not in dockerMode (Phase 4 deleted host-side scheduler)', async () => {
+    const result = await probeScheduler('clerk', { dockerMode: false })
+    expect(result.status).toBe('ok')
+    expect(result.label).toBe('Scheduler')
+    expect(result.detail).toContain('non-docker')
+  })
+
+  it('fails when lockfile is missing (sidecar never started or supervisor gave up)', async () => {
+    const fs = makeSchedulerFs({})
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: () => true,
+      // Disable settle softening so the verdict is the hard fail —
+      // the "boot still settling" case is covered by the dedicated
+      // test below.
+      containerBootTimeMs: null,
+    })
+    expect(result.status).toBe('fail')
+    expect(result.detail).toContain('no lockfile')
+  })
+
+  it('softens fail to degraded when container booted recently (still-settling window)', async () => {
+    // No lockfile, but container PID 1 started 5 s ago — the supervisor
+    // is still bringing the scheduler up. /status hit at this instant
+    // should NOT show 🔴 for a non-issue.
+    const now = 1_700_000_000_000
+    const fs = makeSchedulerFs({})
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: () => true,
+      now: () => now,
+      containerBootTimeMs: now - 5_000,
+    })
+    expect(result.status).toBe('degraded')
+    expect(result.detail).toContain('still settling')
+  })
+
+  it('does NOT soften when container booted long ago (real wedge, not a flap)', async () => {
+    // Container has been up for an hour; missing lockfile is a real
+    // problem, not a settle window. Hard fail expected.
+    const now = 1_700_000_000_000
+    const fs = makeSchedulerFs({})
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: () => true,
+      now: () => now,
+      containerBootTimeMs: now - 60 * 60_000,
+    })
+    expect(result.status).toBe('fail')
+    expect(result.detail).not.toContain('still settling')
+  })
+
+  it('honors SWITCHROOM_AGENT_SCHEDULER_LOCK env override', async () => {
+    const customLock = '/custom/sched.lock'
+    const fs = makeSchedulerFs({
+      [customLock]: { content: '1234' },
+    })
+    const oldEnv = process.env.SWITCHROOM_AGENT_SCHEDULER_LOCK
+    process.env.SWITCHROOM_AGENT_SCHEDULER_LOCK = customLock
+    try {
+      const result = await probeScheduler('clerk', {
+        dockerMode: true,
+        fs,
+        isAlive: (pid) => pid === 1234,
+        containerBootTimeMs: null,
+      })
+      expect(result.status).toBe('ok')
+      expect(result.detail).toContain('pid 1234')
+    } finally {
+      if (oldEnv === undefined) delete process.env.SWITCHROOM_AGENT_SCHEDULER_LOCK
+      else process.env.SWITCHROOM_AGENT_SCHEDULER_LOCK = oldEnv
+    }
+  })
+
+  it('honors SWITCHROOM_AGENT_SCHEDULER_JSONL env override for freshness hint', async () => {
+    const lockPath = '/state/agent/scheduler.lock'
+    const customJsonl = '/custom/audit.jsonl'
+    const now = 1_700_000_000_000
+    const fs = makeSchedulerFs({
+      [lockPath]: { content: '5555' },
+      [customJsonl]: { content: '{}\n', mtimeMs: now - 90_000 },
+    })
+    const oldEnv = process.env.SWITCHROOM_AGENT_SCHEDULER_JSONL
+    process.env.SWITCHROOM_AGENT_SCHEDULER_JSONL = customJsonl
+    try {
+      const result = await probeScheduler('clerk', {
+        dockerMode: true,
+        fs,
+        isAlive: () => true,
+        now: () => now,
+        containerBootTimeMs: null,
+      })
+      expect(result.status).toBe('ok')
+      expect(result.detail).toContain('last fire')
+    } finally {
+      if (oldEnv === undefined) delete process.env.SWITCHROOM_AGENT_SCHEDULER_JSONL
+      else process.env.SWITCHROOM_AGENT_SCHEDULER_JSONL = oldEnv
+    }
+  })
+
+  it('returns ok with last-fire age when lock is held by a live PID', async () => {
+    const lockPath = '/state/agent/scheduler.lock'
+    const jsonlPath = '/state/agent/scheduler.jsonl'
+    const now = 1_700_000_000_000
+    const fs = makeSchedulerFs({
+      [lockPath]: { content: '4242\n' },
+      [jsonlPath]: { content: '{}\n', mtimeMs: now - 5 * 60_000 },
+    })
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: (pid) => pid === 4242,
+      now: () => now,
+    })
+    expect(result.status).toBe('ok')
+    expect(result.detail).toContain('pid 4242')
+    expect(result.detail).toContain('last fire')
+  })
+
+  it('degraded when lockfile holder PID is not alive (mid-restart)', async () => {
+    const lockPath = '/state/agent/scheduler.lock'
+    const fs = makeSchedulerFs({
+      [lockPath]: { content: '99\n' },
+    })
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: () => false,
+    })
+    expect(result.status).toBe('degraded')
+    expect(result.detail).toContain('pid 99 not alive')
+  })
+
+  it('degraded when lockfile contents are not a valid PID', async () => {
+    const lockPath = '/state/agent/scheduler.lock'
+    const fs = makeSchedulerFs({
+      [lockPath]: { content: 'garbage' },
+    })
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: () => true,
+    })
+    expect(result.status).toBe('degraded')
+    expect(result.detail).toContain('invalid')
+  })
+
+  it('returns ok without freshness hint when scheduler.jsonl has never been written', async () => {
+    const lockPath = '/state/agent/scheduler.lock'
+    const fs = makeSchedulerFs({
+      [lockPath]: { content: '5555' },
+    })
+    const result = await probeScheduler('clerk', {
+      dockerMode: true,
+      fs,
+      isAlive: () => true,
+    })
+    expect(result.status).toBe('ok')
+    expect(result.detail).toContain('pid 5555')
+    expect(result.detail).not.toContain('last fire')
+  })
+})
+
+// ── probeBroker / probeKernel — UDS reachability ─────────────────────────
+
+describe('probeBroker / probeKernel', () => {
+  it('probeBroker returns ok n/a when not in dockerMode', async () => {
+    const result = await probeBroker('/some/path', { dockerMode: false })
+    expect(result.status).toBe('ok')
+    expect(result.detail).toContain('non-docker')
+  })
+
+  it('probeBroker fails when no socket path is configured', async () => {
+    const oldEnv = process.env.SWITCHROOM_BROKER_SOCKET
+    delete process.env.SWITCHROOM_BROKER_SOCKET
+    try {
+      const result = await probeBroker(undefined, { dockerMode: true })
+      expect(result.status).toBe('fail')
+      expect(result.detail).toContain('not configured')
+    } finally {
+      if (oldEnv !== undefined) process.env.SWITCHROOM_BROKER_SOCKET = oldEnv
+    }
+  })
+
+  it('probeBroker reports ok when connect resolves', async () => {
+    const result = await probeBroker('/run/switchroom/broker/clerk/sock', {
+      dockerMode: true,
+      connectImpl: async () => { /* connect ok */ },
+    })
+    expect(result.status).toBe('ok')
+    expect(result.label).toBe('Broker')
+    expect(result.detail).toBe('reachable')
+  })
+
+  it('probeBroker reports fail with ENOENT detail when socket missing', async () => {
+    const result = await probeBroker('/run/switchroom/broker/clerk/sock', {
+      dockerMode: true,
+      connectImpl: async () => {
+        const err = new Error('ENOENT') as NodeJS.ErrnoException
+        err.code = 'ENOENT'
+        throw err
+      },
+    })
+    expect(result.status).toBe('fail')
+    expect(result.detail).toContain('socket missing')
+  })
+
+  it('probeBroker reports fail with ECONNREFUSED detail when daemon down', async () => {
+    const result = await probeBroker('/run/switchroom/broker/clerk/sock', {
+      dockerMode: true,
+      connectImpl: async () => {
+        const err = new Error('ECONNREFUSED') as NodeJS.ErrnoException
+        err.code = 'ECONNREFUSED'
+        throw err
+      },
+    })
+    expect(result.status).toBe('fail')
+    expect(result.detail).toContain('connection refused')
+  })
+
+  it('probeKernel mirrors probeBroker shape with Kernel label', async () => {
+    const result = await probeKernel('/run/switchroom/kernel/clerk/sock', {
+      dockerMode: true,
+      connectImpl: async () => { /* connect ok */ },
+    })
+    expect(result.status).toBe('ok')
+    expect(result.label).toBe('Kernel')
+    expect(result.detail).toBe('reachable')
+  })
+})
+
+// ── probeSkills — symlink validity ───────────────────────────────────────
+
+function makeSkillsFs(entries: Record<string, string[]>, files: Set<string>): SkillsFsImpl {
+  return {
+    readdir: (p) => {
+      if (!(p in entries)) throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' })
+      return entries[p]
+    },
+    exists: (p) => files.has(p) || p in entries,
+  }
+}
+
+describe('probeSkills', () => {
+  const agentDir = '/state/agent'
+  const skillsDir = '/state/agent/.claude/skills'
+
+  it('returns ok when no skills dir exists (no skills configured is normal)', async () => {
+    const fs = makeSkillsFs({}, new Set())
+    const result = await probeSkills(agentDir, { fs })
+    expect(result.status).toBe('ok')
+    expect(result.detail).toContain('no skills dir')
+  })
+
+  it('returns ok with count when every skill resolves', async () => {
+    const fs = makeSkillsFs(
+      { [skillsDir]: ['simplify', 'review'] },
+      new Set([
+        `${skillsDir}/simplify`, `${skillsDir}/simplify/SKILL.md`,
+        `${skillsDir}/review`, `${skillsDir}/review/SKILL.md`,
+      ]),
+    )
+    const result = await probeSkills(agentDir, { fs })
+    expect(result.status).toBe('ok')
+    expect(result.detail).toContain('2 resolved')
+  })
+
+  it('degraded when at least one symlink dangles, names them up to cap', async () => {
+    const fs = makeSkillsFs(
+      { [skillsDir]: ['simplify', 'gone-skill', 'also-gone', 'and-this', 'review'] },
+      new Set([
+        `${skillsDir}/simplify`, `${skillsDir}/simplify/SKILL.md`,
+        `${skillsDir}/review`, `${skillsDir}/review/SKILL.md`,
+        // gone-skill, also-gone, and-this NOT in files set → dangling
+      ]),
+    )
+    const result = await probeSkills(agentDir, { fs, maxNamesShown: 2 })
+    expect(result.status).toBe('degraded')
+    expect(result.detail).toContain('3/5 dangling')
+    expect(result.detail).toContain('gone-skill')
+    expect(result.detail).toContain('also-gone')
+    expect(result.detail).toContain('+1 more')
+    expect(result.detail).not.toContain('and-this')
+  })
+
+  it('returns ok when entries dir is empty', async () => {
+    const fs = makeSkillsFs({ [skillsDir]: [] }, new Set([skillsDir]))
+    // Make exists() report true for the dir itself
+    const wrappedFs: SkillsFsImpl = {
+      readdir: fs.readdir,
+      exists: (p) => p === skillsDir || fs.exists(p),
+    }
+    const result = await probeSkills(agentDir, { fs: wrappedFs })
+    expect(result.status).toBe('ok')
+    expect(result.detail).toBe('0 skills')
+  })
+})
+
+// ── /proc parser unit tests (synthetic fs) ────────────────────────────────
+
+/** Build a /proc/<pid>/stat string for tests. */
+function makeStat(pid: number, comm: string, starttime: number): string {
+  // Layout: pid (comm) state ppid pgrp session tty_nr tpgid flags
+  //         minflt cminflt majflt cmajflt utime stime cutime cstime
+  //         priority nice num_threads itrealvalue starttime ...
+  // We need starttime at field 22 (1-indexed). Pad fields 4..21 with zeros.
+  const middle = new Array(18).fill('0').join(' ')
+  return `${pid} (${comm}) S ${middle} ${starttime} 0 0 0 0\n`
+}
+
+function makeProcFs(
+  procs: Array<{ pid: number; comm: string; rssKb: number; starttime: number }>,
+  extraFiles: Record<string, string> = {},
+): ProcFsImpl {
+  const files: Record<string, string> = { ...extraFiles }
+  const dirEntries = ['1', '2', 'self', 'uptime', 'meminfo']
+  for (const p of procs) {
+    files[`/proc/${p.pid}/comm`] = `${p.comm}\n`
+    files[`/proc/${p.pid}/status`] = `Name:\t${p.comm}\nVmRSS:\t  ${p.rssKb} kB\n`
+    files[`/proc/${p.pid}/stat`] = makeStat(p.pid, p.comm, p.starttime)
+    if (!dirEntries.includes(String(p.pid))) dirEntries.push(String(p.pid))
+  }
+  return {
+    readdir: (path: string) => {
+      if (path === '/proc') return dirEntries
+      throw new Error(`unexpected readdir: ${path}`)
+    },
+    readFile: (path: string) => {
+      if (path in files) return files[path]
+      throw new Error(`ENOENT: ${path}`)
+    },
+  }
+}
+
+describe('findAgentProcessInContainer — /proc parser', () => {
+  it('picks heaviest claude process across multiple candidates', () => {
+    const fs = makeProcFs([
+      { pid: 100, comm: 'claude', rssKb: 50_000, starttime: 1000 },
+      { pid: 101, comm: 'claude', rssKb: 200_000, starttime: 1100 },
+      { pid: 102, comm: 'bash',   rssKb: 5_000,   starttime: 900  },
+      { pid: 103, comm: 'tmux',   rssKb: 2_000,   starttime: 800  },
+    ])
+    const found = findAgentProcessInContainer(fs)
+    expect(found).not.toBeNull()
+    expect(found!.pid).toBe(101)
+    expect(found!.rssKb).toBe(200_000)
+    expect(found!.starttime).toBe(1100)
+  })
+
+  it('falls back to heaviest non-wrapper node when no claude exists', () => {
+    const fs = makeProcFs([
+      { pid: 200, comm: 'node', rssKb: 80_000, starttime: 500 },
+      { pid: 201, comm: 'node', rssKb: 30_000, starttime: 600 },
+      { pid: 202, comm: 'bash', rssKb: 5_000,  starttime: 400 },
+    ])
+    const found = findAgentProcessInContainer(fs)
+    expect(found!.pid).toBe(200)
+  })
+
+  it('returns null when only wrappers exist', () => {
+    const fs = makeProcFs([
+      { pid: 1,  comm: 'tini', rssKb: 500,  starttime: 100 },
+      { pid: 50, comm: 'bash', rssKb: 1000, starttime: 200 },
+      { pid: 51, comm: 'tmux', rssKb: 800,  starttime: 250 },
+    ])
+    const found = findAgentProcessInContainer(fs)
+    expect(found).toBeNull()
+  })
+
+  it('handles a comm containing parens via lastIndexOf(\")\")', () => {
+    // Tests are the only protection against off-by-one when comm is funky.
+    const procs = [{ pid: 300, comm: '(weird)comm', rssKb: 90_000, starttime: 1234 }]
+    const fs = makeProcFs(procs)
+    // Override stat with a path comm has parens — comm content goes inside (...)
+    // Note: the kernel actually wraps comm in single parens in /proc/<pid>/stat,
+    // so a comm of `(weird)comm` lands as `300 ((weird)comm) S ...` — making
+    // lastIndexOf the only safe parse anchor.
+    const fsWithFunky: ProcFsImpl = {
+      readdir: () => ['300', 'uptime'],
+      readFile: (path: string) => {
+        if (path === '/proc/300/comm') return '(weird)comm\n'
+        if (path === '/proc/300/status') return 'VmRSS:\t90000 kB\n'
+        if (path === '/proc/300/stat') return `300 ((weird)comm) S ${new Array(18).fill('0').join(' ')} 1234 0 0 0\n`
+        throw new Error(`ENOENT: ${path}`)
+      },
+    }
+    // Funky comm is neither 'claude' nor 'node', so it's filtered out.
+    expect(findAgentProcessInContainer(fsWithFunky)).toBeNull()
+    void fs; void procs
+  })
+
+  it('handles a claude comm with trailing parens correctly', () => {
+    const fs: ProcFsImpl = {
+      readdir: () => ['400', 'uptime'],
+      readFile: (path: string) => {
+        if (path === '/proc/400/comm') return 'claude\n'
+        if (path === '/proc/400/status') return 'VmRSS:\t  150000 kB\n'
+        // Use a `claude` comm followed by 18 zero pad fields then starttime=9999.
+        if (path === '/proc/400/stat') return `400 (claude) S ${new Array(18).fill('0').join(' ')} 9999 0 0 0\n`
+        throw new Error(`ENOENT: ${path}`)
+      },
+    }
+    const found = findAgentProcessInContainer(fs)
+    expect(found).not.toBeNull()
+    expect(found!.pid).toBe(400)
+    expect(found!.starttime).toBe(9999)
+    expect(found!.rssKb).toBe(150_000)
+  })
+
+  it('skips entries that fail to read', () => {
+    const fs: ProcFsImpl = {
+      readdir: () => ['1', '500'],
+      readFile: (path: string) => {
+        if (path === '/proc/500/comm') return 'claude\n'
+        if (path === '/proc/500/status') return 'VmRSS:\t10000 kB\n'
+        if (path === '/proc/500/stat') return `500 (claude) S ${new Array(18).fill('0').join(' ')} 7000 0 0 0\n`
+        throw new Error(`ENOENT: ${path}`) // PID 1 reads fail → skipped
+      },
+    }
+    const found = findAgentProcessInContainer(fs)
+    expect(found!.pid).toBe(500)
+  })
+})
+
+describe('uptimeMsForStarttime', () => {
+  it('computes uptime in ms from starttime ticks and /proc/uptime', () => {
+    const fs: ProcFsImpl = {
+      readdir: () => [],
+      readFile: (path: string) => {
+        if (path === '/proc/uptime') return '1234.56 1000.00\n'
+        throw new Error(`unexpected: ${path}`)
+      },
+    }
+    // boot uptime = 1234.56 s, starttime = 1000 ticks → 10 s in.
+    // Process uptime = 1234.56 - 10 = 1224.56 s = 1_224_560 ms.
+    expect(uptimeMsForStarttime(1000, fs)).toBe(1_224_560)
+  })
+
+  it('returns null if /proc/uptime is unreadable', () => {
+    const fs: ProcFsImpl = {
+      readdir: () => [],
+      readFile: () => { throw new Error('ENOENT') },
+    }
+    expect(uptimeMsForStarttime(1000, fs)).toBeNull()
+  })
+
+  it('returns null when computed uptime is negative', () => {
+    // starttime in the future relative to boot uptime → invalid.
+    const fs: ProcFsImpl = {
+      readdir: () => [],
+      readFile: () => '5.0 0.0\n',
+    }
+    expect(uptimeMsForStarttime(99999999, fs)).toBeNull()
+  })
+})
 })