spexcode 0.6.2 → 0.6.6

package/spec-cli/src/ranker.ts

@@ -1,191 +0,0 @@
-// tier multipliers, name > desc > body. A doc's NAME is the strongest signal, its one-line DESC a curated
-// summary (next), its BODY the weakest per-hit — but the body carries BM25 term-frequency so a doc that
-// genuinely concentrates a rare word still climbs. IDF scales all three. The spread only ORDERS the tiers;
-// the discriminating magnitude comes from rarity (IDF) and body-density (BM25), not these constants — which
-// is why they sit in flat plateaus rather than being fitted to any case.
-const W_NAME_PREFIX = 8
-const W_NAME_SUBSTR = 5
-const W_DESC = 2.2
-const W_BODY = 1
-
-// a tiny stoplist of question scaffolding + length-1 tokens, dropped so "how does the … is it …" can't drown
-// the content words. Deliberately small and general — NOT tuned to any benchmark; just the function words a
-// natural-language query carries that match nothing meaningful. Quantifiers (many, several, same, too…) are
-// NOT stopped: in this corpus they are load-bearing ("too many owners" IS the multi-ownership concept —
-// dropping them measurably breaks that reach).
-const STOP = new Set([
-  'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'is', 'it', 'its', 'as', 'at', 'by', 'for',
-  'how', 'does', 'do', 'what', 'which', 'that', 'this', 'these', 'those', 'with', 'from', 'into', 'are',
-  'be', 'can', 'just', 'them', 'they', 'their', 'so', 'if', 'not', 'no', 'but', 'vs', 'us', 'we', 'you',
-])
-
-// The corpus is mostly English but not only — some nodes carry CJK prose (the root spexcode node is a whole
-// Chinese paragraph), and the dashboard palette ([[shared-ranker]]) ranks session/issue titles that are
-// frequently Chinese. CJK has no spaces, so a whitespace/`[^a-z0-9]` split silently discards ALL of it. We
-// tokenize the SAME way on both sides: an ASCII alphanumeric run is one token; each CJK character is its own
-// token (a unigram). Unigrams — not bigrams — keep the shared prefix-match/IDF/BM25 machinery untouched (a
-// single-char query still matches, no bigram edge cases) and stay BLUNT & ROBUST, the floor's whole stance.
-// Han (incl. Ext-A + compat) plus Japanese kana; enough to cover the CJK a spec body or a session title carries.
-const CJK = '\\u3400-\\u4dbf\\u4e00-\\u9fff\\uf900-\\ufaff\\u3040-\\u30ff' // Ext-A · Unified · Compat · kana
-const TOKEN_RE = new RegExp(`[a-z0-9]+|[${CJK}]`, 'g')
-const CJK_RE = new RegExp(`[${CJK}]`)
-function isCjk(t: string): boolean { return CJK_RE.test(t) }
-function tokenize(text: string): string[] {
-  return text.toLowerCase().match(TOKEN_RE) ?? []
-}
-
-// tokenize, lowercase, drop stopwords + length-1 ASCII tokens (a length-1 CJK token is a real word — keep it), de-dup.
-export function terms(query: string): string[] {
-  const seen = new Set<string>()
-  for (const w of tokenize(query)) {
-    if (isCjk(w) || (w.length > 1 && !STOP.has(w))) seen.add(w)
-  }
-  return [...seen]
-}
-
-// the words of a field, lowercased — used for word-boundary (prefix-of-a-word) matching, which kills
-// short-token pollution (`main` must not match inside `domain`); CJK chars are single-char words.
-function words(text: string): string[] {
-  return tokenize(text)
-}
-
-// light query-side stem for prefix matching: drop a trailing plural 's' (len≥4, not 'ss') then a mute 'e'
-// (len≥5) — so `sessions` prefix-reaches `session`, `merge`→`merg` reaches `merging`, `declare`→`declar`
-// reaches `declaration`. Without the e-drop the spec's promised merge↔merging reach silently never worked
-// (`'merging'.startsWith('merge')` is false). Query-side only; IDF self-neutralises the extra reach (a
-// looser term matches more docs → bigger df → smaller idf), so no flood.
-function stem(t: string): string {
-  let s = t
-  if (s.length >= 4 && s.endsWith('s') && !s.endsWith('ss')) s = s.slice(0, -1)
-  if (s.length >= 5 && s.endsWith('e')) s = s.slice(0, -1)
-  return s
-}
-
-// name match is forward-only (a chosen short field — reverse would let a stray short word swallow it);
-// desc/body match bidirectionally so a longer doc word still reaches a shorter query term, reverse gated to
-// words ≥3 chars so a stray short word can't swallow a longer term (IDF neutralises the generic words it
-// pulls in).
-function nameMatch(term: string, w: string): boolean { return w.startsWith(stem(term)) }
-function textMatch(term: string, w: string): boolean { return w.startsWith(stem(term)) || (w.length >= 3 && term.startsWith(w)) }
-
-// how strongly a query term's name-prefix hit counts, over a doc's name words: the FRACTION of the matched
-// name word the query term actually spans. A full word (`port`→`port`, `api`→`api`) is 1.0 — the strong
-// signal a name tier is for; a short query term that only PREFIXES a longer, unrelated word (`port`→`portable`,
-// `governs`→`governed`) covers less of it and is weaker evidence, so it earns proportionally less than the
-// full name weight. Floored at 0.5 (a prefix is still evidence, not noise). Returns 0 when no name word is
-// prefixed. This is what stops a corpus-growth collision — a sibling whose NAME merely starts with a query
-// word (portable-layout for "port", governed-related for "governs") — from outranking the node that owns the
-// concept; the floor sat at an implicit 1.0 before, which let those partial prefixes score a full name hit.
-const NAME_COVER_FLOOR = 0.5
-function nameCover<T>(term: string, n: Fields<T>): number {
-  const st = stem(term)
-  let best = 0
-  for (const w of n.nameWords) {
-    if (w.startsWith(st)) { const r = st.length / w.length; if (r > best) best = r }
-  }
-  return best > 0 ? Math.max(NAME_COVER_FLOOR, best) : 0
-}
-
-// classic BM25 tf: frequency with saturation (K1 sets how fast it saturates) and length-normalisation (B),
-// both in a wide insensitive plateau. tf=0 → 0.
-const K1 = 1.2
-const B = 0.4
-function bm25tf(tf: number, len: number, avgLen: number): number {
-  if (tf <= 0) return 0
-  return (tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * len) / (avgLen || 1)))
-}
-
-// the precomputed searchable fields of one doc (built once, reused for df, scoring, and snippet).
-type Fields<T> = { ref: T; name: string; nameWords: string[]; desc: string; descWords: string[]; bodyWords: string[]; snippetText: string }
-
-// the pre-IDF weight a term earns against one doc, picking its single best tier (three fields): a name
-// word-prefix beats a name substring beats a desc hit beats a body hit. Name is a short, chosen field →
-// near-binary, scaled only by prefix COVERAGE (a full word counts full; a query term that is a mere prefix
-// of a longer name word counts proportionally less — see nameCover). Desc is presence too (a curated
-// one-liner — repetition there is stuffing, not evidence)
-// but LENGTH-NORMALISED: it was flat-binary until descs drifted long and a bloated desc became a cheat code
-// (one 60-word desc catches every query term a curated one-liner can't). bm25tf(1, avgLen, avgLen) = 1, so
-// a hit in an average-length desc scores exactly the old binary W_DESC — the normalisation only bites
-// outliers. The body keeps the full BM25-saturated term-frequency that discriminates the long ties.
-function tierWeight<T>(term: string, n: Fields<T>, avgBodyLen: number, avgDescLen: number): number {
-  const cover = nameCover(term, n)
-  if (cover > 0) return W_NAME_PREFIX * cover
-  if (n.name.includes(term)) return W_NAME_SUBSTR
-  if (n.descWords.some((w) => textMatch(term, w))) return W_DESC * bm25tf(1, n.descWords.length, avgDescLen)
-  const tf = n.bodyWords.reduce((c, w) => c + (textMatch(term, w) ? 1 : 0), 0)
-  return W_BODY * bm25tf(tf, n.bodyWords.length, avgBodyLen)
-}
-
-// a short single-line window of prose around the FIRST matched term, so a reader sees WHY it matched. Falls
-// back to the desc (then the text head) when only the name matched. Collapsed to one line, ~window chars.
-function snippetFor(text: string, desc: string, qterms: string[], window = 140): string {
-  const flat = text.replace(/\s+/g, ' ').trim()
-  const lower = flat.toLowerCase()
-  let at = -1
-  for (const t of qterms) {
-    // ASCII terms locate at a word boundary (so `main` doesn't hit inside `domain`); a CJK term has no
-    // `\b` around it (JS `\b` is ASCII-only), so locate it by plain substring.
-    const i = isCjk(t) ? lower.indexOf(t) : lower.search(new RegExp('\\b' + t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')))
-    if (i >= 0 && (at < 0 || i < at)) at = i
-  }
-  if (at < 0) {
-    const fb = (desc || flat).replace(/\s+/g, ' ').trim()
-    return fb.length > window ? fb.slice(0, window).trimEnd() + '…' : fb
-  }
-  const start = Math.max(0, at - Math.floor(window / 3))
-  let s = flat.slice(start, start + window).trim()
-  if (start > 0) s = '…' + s
-  if (start + window < flat.length) s = s + '…'
-  return s
-}
-
-// what a caller hands in per doc: the original item (`ref`, returned verbatim) + its three text fields.
-export type RankInput<T> = { ref: T; name: string; desc: string; body: string }
-export type Ranked<T> = { ref: T; score: number; snippet: string }
-
-// the shared entrypoint: sum each query term's best-tier weight × IDF, keep docs hitting ≥1 term, sort by
-// score desc (stable — equal scores keep the caller's pre-sorted input order), cap to `limit` (default 10).
-export function rankDocs<T>(query: string, inputs: RankInput<T>[], opts: { limit?: number } = {}): Ranked<T>[] {
-  const limit = opts.limit ?? 10
-  const qterms = terms(query)
-  if (!qterms.length) return []
-
-  const docs: Fields<T>[] = inputs.map((d) => {
-    const name = d.name.toLowerCase()
-    return {
-      ref: d.ref, name, nameWords: words(name),
-      desc: d.desc.toLowerCase(), descWords: words(d.desc),
-      bodyWords: words(d.body),
-      snippetText: `${d.desc}\n${d.body}`,
-    }
-  })
-
-  // IDF per query term: df = docs containing it (any field), idf = ln(N/df) — a term in every doc scores 0,
-  // a rare one carries the rank. Read from the corpus, not hand-set.
-  const N = docs.length
-  const avgBodyLen = docs.reduce((a, n) => a + n.bodyWords.length, 0) / (N || 1)
-  const avgDescLen = docs.reduce((a, n) => a + n.descWords.length, 0) / (N || 1)
-  const idf: Record<string, number> = {}
-  for (const t of qterms) {
-    let df = 0
-    for (const n of docs) {
-      if (n.nameWords.some((w) => nameMatch(t, w)) || n.descWords.some((w) => textMatch(t, w)) || n.bodyWords.some((w) => textMatch(t, w))) df++
-    }
-    idf[t] = df > 0 ? Math.log(N / df) : 0
-  }
-
-  const scored: Ranked<T>[] = []
-  for (const n of docs) {
-    let score = 0
-    // cap EACH term's contribution at W_NAME_PREFIX: idf and BM25 order the tiers WITHIN this ceiling, but no
-    // single query term may out-score a full name hit. A many-word question should be answered by the node
-    // that matches it BROADLY, not by one that spikes on a single rare word it happens to carry in its NAME
-    // (a `spex search`-named node swallowing "…searches specs…", an injected-* node swallowing "injected…") —
-    // the corpus-growth failure where one uncapped idf×name term buried a node matching more of the query.
-    for (const t of qterms) score += Math.min(tierWeight(t, n, avgBodyLen, avgDescLen) * idf[t], W_NAME_PREFIX)
-    if (score <= 0) continue
-    scored.push({ ref: n.ref, score: Math.round(score * 100) / 100, snippet: snippetFor(n.snippetText, n.desc, qterms) })
-  }
-  scored.sort((a, b) => b.score - a.score)   // stable: equal scores keep the caller's pre-sorted input order
-  return scored.slice(0, limit)
-}

package/spec-cli/src/reaper.ts

@@ -1,125 +0,0 @@
-// @@@ socket-level connection reaper ([[spec-cli]]) - the ONE mechanism that reaps abandoned sockets (the
-// 135-conn starvation that wedged the public port and triggered the mass-restore cascade), and the SINGLE
-// OWNER of the deadlines it enforces. Node's own `headersTimeout`/`keepAliveTimeout` cover the same phases
-// (pre-request, idle-between-requests) and so are a second mechanism racing this one: MEASURED (eval
-// server-reaps-abandoned-connections, issue #65) a `headersTimeout: 20000` set beside the reaper won the
-// race on every reap at default config and silently capped SPEXCODE_REAP_HEADER_MS above 20s — the knob
-// went dead while the close still looked timely (Node's 408, not the reaper's destroy). So install()
-// DISABLES those overlapping Node timeouts on the server it guards; this helper is an explicit per-socket
-// deadline at the server boundary, with no platform machinery shadowing it.
-//
-// It keys on "no request has completed yet / idle between requests" — never on response DURATION — so a
-// long-lived ESTABLISHED stream (the /api/graph/stream SSE, a terminal WebSocket upgrade) is exempt for as
-// long as it streams. The lifecycle per socket:
-//   - on socket birth: arm the HEADER deadline — the socket must produce a fully-parsed request within it,
-//     else it is a slow-loris and gets destroyed.
-//   - on 'request' (headers complete → the request is in flight): disarm. An active request/response is
-//     never reaped, however long its body/response takes (a slow board build or a streaming SSE response).
-//   - when the response finishes/closes and no request is left in flight: re-arm the IDLE deadline (the
-//     keep-alive window). Another request disarms it again; silence past it reaps the idle keep-alive socket.
-//   - on 'upgrade' (WebSocket): mark exempt permanently — a persistent bidirectional stream, not a request
-//     that ever "completes".
-//
-// WHICH socket the deadline lives on is load-bearing. On a TLS server (the public gateway) 'connection'
-// fires with the RAW TCP socket, but every 'request'/'upgrade' reports the wrapping TLSSocket — a deadline
-// armed on the raw socket is unreachable from request handling, never disarms, and destroys every TLS
-// connection at the header deadline no matter how alive it is (MEASURED, eval
-// stream-survives-public-gateway: the dashboard's actively-pinging SSE + terminal WS through the https
-// gateway died every ~30s — the "reconnecting…" storm). So per-socket state is keyed on the socket requests
-// actually report: the TLSSocket (born at 'secureConnection') on a TLS server, the plain socket elsewhere.
-// The raw pre-handshake phase gets its own header-deadline guard (a TCP connect that never completes the TLS
-// handshake is the same slow-loris one layer down), linked raw→TLS by the connection's addr:port 4-tuple —
-// public API only, no reliance on Node's private TLSSocket internals.
-//
-// The child's reaped close propagates back through the supervisor's raw-TCP proxy (which pairs socket
-// halves), so a reaped upstream frees its public-side socket too — no separate raw timeout on the proxy,
-// which would blind it to a legitimately-idle WS/SSE.
-import type { Server as HttpServer } from 'node:http'
-import type { Socket } from 'node:net'
-import { Server as TlsServer } from 'node:tls'
-
-export interface ReaperOptions {
-  // ms a freshly-connected (or post-response idle) socket has to START a new request before it is reaped as a
-  // slow-loris. Env: SPEXCODE_REAP_HEADER_MS. Default 30s.
-  headerMs?: number
-  // ms a keep-alive socket may sit idle BETWEEN requests before reaping. Env: SPEXCODE_REAP_IDLE_MS. Default 15s.
-  idleMs?: number
-}
-
-interface SocketState { timer?: NodeJS.Timeout; active: number; upgraded: boolean; arm(ms: number): void; disarm(): void }
-const STATE = Symbol('spexcode.reaper')
-
-function resolveMs(explicit: number | undefined, env: string | undefined, fallback: number): number {
-  if (typeof explicit === 'number' && explicit > 0) return explicit
-  const n = Number(env)
-  return Number.isFinite(n) && n > 0 ? n : fallback
-}
-
-// Attach the reaper to a Node http/https server. Call it right after the server is created (before or just
-// after listen); it hooks the socket-birth event ('connection', or 'secureConnection' on TLS servers),
-// 'request' and 'upgrade', and needs no changes to the request handlers.
-export function installConnectionReaper(server: HttpServer, opts: ReaperOptions = {}): void {
-  const headerMs = resolveMs(opts.headerMs, process.env.SPEXCODE_REAP_HEADER_MS, 30000)
-  const idleMs = resolveMs(opts.idleMs, process.env.SPEXCODE_REAP_IDLE_MS, 15000)
-
-  // claim single ownership of the phases the reaper covers (see header comment): Node's overlapping
-  // timeouts would otherwise race these deadlines and silently cap the env knobs. `requestTimeout` is
-  // deliberately LEFT at Node's default (~5 min): it bounds the in-flight request-body phase the reaper
-  // exempts (a silently-abandoned mid-body upload has no other reaper), and 5 min shadows no sane knob.
-  server.headersTimeout = 0
-  server.keepAliveTimeout = 0
-
-  // per-socket tracking, on the SAME socket object 'request'/'upgrade' will report (see header comment).
-  const track = (socket: Socket) => {
-    const state: SocketState = {
-      timer: undefined, active: 0, upgraded: false,
-      disarm() { if (state.timer) { clearTimeout(state.timer); state.timer = undefined } },
-      arm(ms: number) { state.disarm(); state.timer = setTimeout(() => socket.destroy(), ms); state.timer.unref?.() },
-    }
-    ;(socket as unknown as Record<symbol, SocketState>)[STATE] = state
-    state.arm(headerMs)                 // slow-loris guard: first request's headers must complete within headerMs
-    socket.once('close', state.disarm)  // socket gone → drop its pending timer
-  }
-
-  if (server instanceof TlsServer) {
-    // TLS: requests report the TLSSocket, so that is where the deadline must live. The raw phase before the
-    // handshake completes still needs a guard of its own; the 4-tuple key hands it off to the TLSSocket.
-    const pendingHandshake = new Map<string, () => void>()
-    server.on('connection', (raw: Socket) => {
-      const key = `${raw.remoteAddress}:${raw.remotePort}`
-      const timer = setTimeout(() => raw.destroy(), headerMs)
-      timer.unref?.()
-      const done = () => { clearTimeout(timer); pendingHandshake.delete(key) }
-      pendingHandshake.set(key, done)
-      raw.once('close', done)
-    })
-    server.on('secureConnection', (tlsSocket) => {
-      pendingHandshake.get(`${tlsSocket.remoteAddress}:${tlsSocket.remotePort}`)?.()
-      track(tlsSocket as unknown as Socket)
-    })
-  } else {
-    server.on('connection', track)
-  }
-
-  server.on('request', (req, res) => {
-    const s = (req.socket as unknown as Record<symbol, SocketState>)[STATE]
-    if (!s) return
-    s.active++
-    s.disarm()                    // a request is in flight — never reap an active request/response
-    let ended = false
-    const done = () => {
-      if (ended) return
-      ended = true
-      s.active--
-      // response over and nothing else in flight → this is now an idle keep-alive socket; re-arm.
-      if (s.active === 0 && !s.upgraded && !req.socket.destroyed) s.arm(idleMs)
-    }
-    res.once('finish', done)      // response fully sent
-    res.once('close', done)       // response aborted / connection dropped
-  })
-
-  server.on('upgrade', (req) => {
-    const s = (req.socket as unknown as Record<symbol, SocketState>)[STATE]
-    if (s) { s.upgraded = true; s.disarm() }   // persistent stream — exempt for its lifetime
-  })
-}

package/spec-cli/src/resilience.ts

@@ -1,41 +0,0 @@
-import { existsSync } from 'node:fs'
-
-function describe(e: unknown): string {
-  return e instanceof Error ? e.message : String(e)
-}
-
-// run a per-worktree DETAIL read; on a throw, branch on whether the worktree DIRECTORY still exists.
-//   dir gone   → the worktree was genuinely removed mid-read → return null so the caller omits it.
-//   dir present → a flaky detail read (ENOENT race on a sibling file, or a git index/ref lock under a
-//                 concurrent merge) → return the caller's `degraded` row so an EXISTING worktree is NEVER
-//                 dropped from the board. read-failure != non-existence.
-// `dir` is the worktree path (the existence fact). Accepts a sync OR async fn — resolveLayout's overlay
-// read is async, listSessions' row read is sync; `degraded` is a sync raw-facts fallback.
-export async function guardWorktree<T>(dir: string, fn: () => T | Promise<T>, degraded: () => T): Promise<T | null> {
-  try {
-    return await fn()
-  } catch (e) {
-    if (!existsSync(dir)) {
-      console.warn(`spec-cli: worktree ${dir} gone from disk (removed mid-read), omitting: ${describe(e)}`)
-      return null
-    }
-    console.warn(`spec-cli: worktree ${dir} detail-read failed (transient/lock); serving degraded row: ${describe(e)}`)
-    return degraded()
-  }
-}
-
-// the last-resort process net: log an otherwise-fatal async throw and KEEP SERVING. Registering these
-// handlers is itself what overrides Node's default "print the stack and exit" — so the backend rides out
-// a transient race it didn't anticipate instead of dropping the public port. Idempotent (guarded) so a
-// double-install in one process can't stack duplicate handlers.
-let guardsInstalled = false
-export function installProcessGuards(): void {
-  if (guardsInstalled) return
-  guardsInstalled = true
-  process.on('unhandledRejection', (reason) => {
-    console.error(`spec-cli: unhandledRejection kept the server alive (investigate): ${reason instanceof Error ? reason.stack : describe(reason)}`)
-  })
-  process.on('uncaughtException', (err) => {
-    console.error(`spec-cli: uncaughtException kept the server alive (investigate): ${err?.stack || describe(err)}`)
-  })
-}

package/spec-cli/src/reviewSnapshot.ts

@@ -1,28 +0,0 @@
-export type ReviewEvalNode = {
-  id: string
-  hue?: number
-  scenarios: any[]
-  evals: any[]
-  readings: any[]
-}
-
-export type ReviewSnapshot = {
-  issues: any[]
-  evalNodes: ReviewEvalNode[]
-  forgeRevision: number
-}
-
-let current: ReviewSnapshot | null = null
-
-export function publishReviewSnapshot(snapshot: ReviewSnapshot): void {
-  current = snapshot
-}
-
-export function readReviewSnapshot(): ReviewSnapshot {
-  if (!current) throw new Error('review snapshot is unavailable before the first successful graph build')
-  return current
-}
-
-export function hasReviewSnapshot(): boolean {
-  return current !== null
-}