sweet-search 2.5.2 → 2.5.3

package/core/incremental-indexing/infrastructure/hcgs-invalidation.mjs

@@ -0,0 +1,182 @@
+/**
+ * HCGS summary invalidation.
+ *
+ * Plan § 7.7. HCGS summaries are not one of the five first-stage indices,
+ * but stale summaries can still poison reader trust. The reconcile path
+ * therefore owns HCGS invalidation in v1:
+ *
+ *   1. Each summary records `(source_entity_ids, source_chunk_struct_ids,
+ *      source_hashes, epoch_written, epoch_retired)`.
+ *   2. When graph/vector deltas retire or replace any source entity/chunk,
+ *      mark the dependent summary retired in the same manifest epoch.
+ *   3. Search and MCP must never serve a summary whose source epoch is not
+ *      visible in the pinned manifest. If a fresh summary is missing,
+ *      omit it or trigger existing on-demand regeneration; do not serve
+ *      the stale text.
+ *   4. Regeneration is low-priority CPU / existing provider policy and
+ *      happens outside the reconcile tick. Invalidation is the
+ *      correctness requirement; eager LLM regeneration is not.
+ *
+ * The current HCGS implementation (`core/graph/hcgs-generator.js`) stores
+ * `summary` and `summary_embedding` directly on the `entities` table. We
+ * add a sidecar table `hcgs_summary_metadata` that the reconciler updates
+ * in lock-step with the entity/vector deltas. The HCGS query path consults
+ * the sidecar to decide whether the live summary is visible at the pinned
+ * manifest epoch.
+ *
+ * Schema:
+ *   CREATE TABLE hcgs_summary_metadata (
+ *     entity_id          TEXT PRIMARY KEY,
+ *     source_entity_ids  TEXT NOT NULL,    -- JSON array
+ *     source_chunk_struct_ids TEXT NOT NULL, -- JSON array
+ *     source_hashes      TEXT NOT NULL,    -- JSON object {entity_id -> hash}
+ *     epoch_written      INTEGER NOT NULL DEFAULT 0,
+ *     epoch_retired      INTEGER
+ *   ) WITHOUT ROWID;
+ *
+ * The reconciler writes one row per summary it observes; the maintenance
+ * worker prunes rows whose `epoch_retired` is older than `minLiveEpoch`.
+ */
+
+/**
+ * Ensure the HCGS sidecar schema exists.
+ *
+ * @param {import('better-sqlite3').Database} db
+ */
+export function ensureHcgsSidecarSchema(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS hcgs_summary_metadata (
+      entity_id TEXT PRIMARY KEY,
+      source_entity_ids TEXT NOT NULL,
+      source_chunk_struct_ids TEXT NOT NULL,
+      source_hashes TEXT NOT NULL,
+      epoch_written INTEGER NOT NULL DEFAULT 0,
+      epoch_retired INTEGER
+    ) WITHOUT ROWID
+  `);
+  db.exec(`
+    CREATE INDEX IF NOT EXISTS idx_hcgs_meta_epoch_written
+      ON hcgs_summary_metadata (epoch_written);
+  `);
+  db.exec(`
+    CREATE INDEX IF NOT EXISTS idx_hcgs_meta_epoch_retired
+      ON hcgs_summary_metadata (epoch_retired)
+      WHERE epoch_retired IS NOT NULL;
+  `);
+}
+
+/**
+ * Record / refresh a summary's source-dependency snapshot.
+ *
+ * Plan § 7.7 step 1. Caller commits within the per-file transaction so the
+ * sidecar can't be in disagreement with the entity row.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {string} entityId
+ * @param {{sourceEntityIds:string[], sourceChunkStructIds:string[], sourceHashes:object, epoch:number}} payload
+ */
+export function recordSummary(db, entityId, payload) {
+  db.prepare(`
+    INSERT INTO hcgs_summary_metadata (
+      entity_id, source_entity_ids, source_chunk_struct_ids, source_hashes,
+      epoch_written, epoch_retired
+    ) VALUES (?, ?, ?, ?, ?, NULL)
+    ON CONFLICT(entity_id) DO UPDATE SET
+      source_entity_ids = excluded.source_entity_ids,
+      source_chunk_struct_ids = excluded.source_chunk_struct_ids,
+      source_hashes = excluded.source_hashes,
+      epoch_written = excluded.epoch_written,
+      epoch_retired = NULL
+  `).run(
+    entityId,
+    JSON.stringify(payload.sourceEntityIds ?? []),
+    JSON.stringify(payload.sourceChunkStructIds ?? []),
+    JSON.stringify(payload.sourceHashes ?? {}),
+    Number(payload.epoch),
+  );
+}
+
+/**
+ * Retire summaries whose source entities or chunks changed in this tick.
+ *
+ * Caller passes `{ retiredEntityIds, retiredChunkStructIds }` collected
+ * from the graph + vector deltas. Plan § 7.7 step 2 fires the
+ * retirement in the same manifest epoch.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {{retiredEntityIds:Set<string>|Array<string>, retiredChunkStructIds:Set<string>|Array<string>}} sources
+ * @param {number} epoch
+ * @returns {number}  How many summaries were retired.
+ */
+export function retireDependentSummaries(db, sources, epoch) {
+  const entityIds = Array.from(sources.retiredEntityIds ?? []);
+  const chunkIds = Array.from(sources.retiredChunkStructIds ?? []);
+  if (entityIds.length === 0 && chunkIds.length === 0) return 0;
+
+  const rows = db.prepare(`
+    SELECT entity_id, source_entity_ids, source_chunk_struct_ids
+    FROM hcgs_summary_metadata
+    WHERE epoch_retired IS NULL
+  `).all();
+  const entitySet = new Set(entityIds);
+  const chunkSet = new Set(chunkIds);
+  const stmt = db.prepare(`
+    UPDATE hcgs_summary_metadata
+       SET epoch_retired = ?
+     WHERE entity_id = ? AND epoch_retired IS NULL
+  `);
+  let count = 0;
+  for (const row of rows) {
+    const srcEnts = JSON.parse(row.source_entity_ids || '[]');
+    const srcChunks = JSON.parse(row.source_chunk_struct_ids || '[]');
+    let hit = false;
+    for (const e of srcEnts) if (entitySet.has(e)) { hit = true; break; }
+    if (!hit) {
+      for (const c of srcChunks) if (chunkSet.has(c)) { hit = true; break; }
+    }
+    if (hit) {
+      stmt.run(epoch, row.entity_id);
+      count += 1;
+    }
+  }
+  return count;
+}
+
+/**
+ * SQL fragment that filters HCGS summary metadata rows visible at a given
+ * manifest epoch. Plan § 7.7 step 3 — readers MUST add this predicate to
+ * any join against the sidecar before returning a summary.
+ *
+ * Returns `epoch_written <= :manifestEpoch
+ *           AND (epoch_retired IS NULL OR epoch_retired > :manifestEpoch)`.
+ *
+ * @param {string} [alias]
+ * @returns {string}
+ */
+export function summaryVisibilityPredicate(alias = '') {
+  const normalizedAlias = String(alias || '').endsWith('.') ? String(alias).slice(0, -1) : String(alias || '');
+  const a = normalizedAlias.length > 0 ? `${normalizedAlias}.` : '';
+  return (
+    `${a}epoch_written <= :manifestEpoch ` +
+    `AND (${a}epoch_retired IS NULL OR ${a}epoch_retired > :manifestEpoch)`
+  );
+}
+
+/**
+ * Drop retired rows older than the prune frontier. Plan § 8.1.1 step 4.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @param {number} pruneFrontier
+ * @returns {number}
+ */
+export function pruneRetiredSummaries(db, pruneFrontier) {
+  if (!Number.isInteger(pruneFrontier)) {
+    throw new Error('pruneRetiredSummaries: pruneFrontier must be an integer');
+  }
+  const res = db.prepare(`
+    DELETE FROM hcgs_summary_metadata
+     WHERE epoch_retired IS NOT NULL
+       AND epoch_retired <= ?
+  `).run(pruneFrontier);
+  return res.changes ?? 0;
+}

package/core/incremental-indexing/infrastructure/li-segment-merge.mjs

@@ -0,0 +1,278 @@
+/**
+ * LI segment batch merge / compaction.
+ *
+ * The reconcile write path (`application/production-li-delta.mjs`) seals one
+ * new SSLX segment per tick that has any add op, so segment count grows
+ * ~1/tick forever. The per-segment `li_segment` handler only drops
+ * tombstoned docs *within* a segment — it never reduces the segment count.
+ * This module is the missing segment-count bound: it merges many small
+ * live segments into `ceil(liveDocs / SEGMENT_SIZE)` segments, dropping
+ * tombstoned docs, while leaving already-full sealed segments untouched
+ * (those are handled by the per-segment stale-ratio path).
+ *
+ * Publish safety (no mixed-epoch / no ENOENT for in-flight readers):
+ *   1. Sweep crash orphans (segment files not referenced by the manifest
+ *      and not already quarantined) — makes a crashed prior merge idempotent.
+ *   2. Sweep the quarantine journal — physically delete consumed segment
+ *      files whose grace window has elapsed.
+ *   3. Write the merged segment files under unique new names via
+ *      `*.compacting.tmp` + rename (new names never collide with live ones,
+ *      so a reader on the OLD manifest never sees a half-written file).
+ *   4. Atomically publish `manifest.json` (tmp + rename) listing the kept
+ *      full segments + the new merged segments only.
+ *   5. Quarantine the consumed small-segment files (+ stale sidecars) — they
+ *      stay readable for any reader still mid-`_loadSegmented` on the old
+ *      manifest; physical deletion is deferred to a later pass once the
+ *      grace window has elapsed.
+ *
+ * Crash recovery:
+ *   - crash before (4): old manifest still references the old segments; the
+ *     new files are orphans cleaned by step 1 on the next run.
+ *   - crash after (4) before (5): new manifest is live; old files become
+ *     orphans cleaned by step 1 on the next run (>= 1 tick later, well past
+ *     any reader load window).
+ *   - crash after (5): the quarantine sweep (step 2) finishes the deletion.
+ *
+ * Segment naming is collision-proof via a monotonic `nextSeq` persisted in
+ * the manifest (see `nextSegmentSeq`); the write path uses the same counter.
+ */
+
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { LateInteractionIndex } from '../../ranking/late-interaction-index.js';
+import { STALE_SIDECAR_EXT, nextSegmentSeq, LI_SEGMENT_SIZE } from './li-segment-state.mjs';
+
+export { nextSegmentSeq, LI_SEGMENT_SIZE };
+
+/** Default quarantine grace before a consumed segment file is unlinked. */
+export const LI_MERGE_GRACE_MS = 60_000;
+const QUARANTINE_FILE = 'pending-delete.jsonl';
+
+function readJson(filePath, fallback = null) {
+  try { return JSON.parse(fs.readFileSync(filePath, 'utf-8')); } catch { return fallback; }
+}
+
+function safeUnlink(filePath) {
+  try { fs.unlinkSync(filePath); return true; } catch { return false; }
+}
+
+async function writeJsonAtomic(filePath, payload) {
+  const tmp = `${filePath}.tmp.${process.pid}`;
+  await fsp.writeFile(tmp, JSON.stringify(payload, null, 2));
+  await fsp.rename(tmp, filePath);
+}
+
+/**
+ * Resolve the segmented LI layout from the stub. Returns null for legacy /
+ * missing / corrupt indices (callers skip — nothing to merge).
+ */
+export function resolveSegmentedLayout(stateDir) {
+  const stubPath = path.join(stateDir, 'codebase-late-interaction.db');
+  if (!fs.existsSync(stubPath)) return null;
+  const stub = readJson(stubPath);
+  if (!stub || stub.format !== 'segmented' || !stub.segmentDir) return null;
+  const segmentDir = path.resolve(stateDir, stub.segmentDir);
+  const manifestPath = path.join(segmentDir, 'manifest.json');
+  const manifest = readJson(manifestPath);
+  if (!manifest || !Array.isArray(manifest.segments)) return null;
+  return { stubPath, segmentDir, manifestPath, manifest };
+}
+
+function quarantinePath(segmentDir) {
+  return path.join(segmentDir, QUARANTINE_FILE);
+}
+
+function readQuarantine(segmentDir) {
+  const p = quarantinePath(segmentDir);
+  if (!fs.existsSync(p)) return [];
+  const out = [];
+  for (const line of fs.readFileSync(p, 'utf-8').split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      const entry = JSON.parse(trimmed);
+      if (entry && Array.isArray(entry.paths)) out.push(entry);
+    } catch { /* skip torn line */ }
+  }
+  return out;
+}
+
+function writeQuarantine(segmentDir, entries) {
+  const p = quarantinePath(segmentDir);
+  if (entries.length === 0) { safeUnlink(p); return; }
+  fs.writeFileSync(p, entries.map((e) => JSON.stringify(e)).join('\n') + '\n');
+}
+
+/**
+ * Physically delete quarantined segment files whose grace window elapsed.
+ * Entries still inside the grace window are retained verbatim.
+ *
+ * @returns {number} files unlinked
+ */
+export function sweepQuarantine(segmentDir, graceMs = LI_MERGE_GRACE_MS, now = Date.now()) {
+  const entries = readQuarantine(segmentDir);
+  if (entries.length === 0) return 0;
+  const survivors = [];
+  let deleted = 0;
+  for (const entry of entries) {
+    if (now - (entry.retiredAtMs || 0) > graceMs) {
+      for (const rel of entry.paths) {
+        if (safeUnlink(path.join(segmentDir, rel))) deleted += 1;
+      }
+    } else {
+      survivors.push(entry);
+    }
+  }
+  writeQuarantine(segmentDir, survivors);
+  return deleted;
+}
+
+/**
+ * Delete crash-orphan `*.bin` segment files (and orphan stale sidecars) that
+ * the manifest does not reference and that are not currently quarantined.
+ * Safe because no reader ever resolves a path the manifest does not list,
+ * and the daemon is the single writer.
+ *
+ * @returns {number} files unlinked
+ */
+export function sweepOrphanSegments(segmentDir, manifest) {
+  let names;
+  try { names = fs.readdirSync(segmentDir); } catch { return 0; }
+  const referenced = new Set((manifest.segments || []).map((s) => s.path));
+  const quarantined = new Set();
+  for (const entry of readQuarantine(segmentDir)) {
+    for (const rel of entry.paths) quarantined.add(rel);
+  }
+  const isProtected = (segName) => referenced.has(segName) || quarantined.has(segName);
+  let deleted = 0;
+  for (const name of names) {
+    if (name.endsWith(STALE_SIDECAR_EXT)) {
+      const base = name.slice(0, -STALE_SIDECAR_EXT.length);
+      if (!isProtected(base)) { if (safeUnlink(path.join(segmentDir, name))) deleted += 1; }
+      continue;
+    }
+    if (!name.endsWith('.bin')) continue;
+    if (!isProtected(name)) { if (safeUnlink(path.join(segmentDir, name))) deleted += 1; }
+  }
+  return deleted;
+}
+
+/**
+ * Merge small live LI segments into fewer larger segments.
+ *
+ * Pass `sweepOnly: true` to run ONLY the cheap housekeeping sweeps (orphan +
+ * quarantine) without loading or rewriting the index. The watermark uses this
+ * for the `pending_delete` re-fire so a large index is not reloaded every tick
+ * just to drain a few quarantined files.
+ *
+ * @param {string} stateDir
+ * @param {{segmentSize?:number, graceMs?:number, minSmallSegments?:number, sweepOnly?:boolean}} [opts]
+ * @returns {Promise<object>} summary, never throws on missing/legacy index
+ */
+export async function mergeLiSegments(stateDir, opts = {}) {
+  const segmentSize = Number.isInteger(opts.segmentSize) && opts.segmentSize > 0 ? opts.segmentSize : LI_SEGMENT_SIZE;
+  const graceMs = Number.isFinite(opts.graceMs) && opts.graceMs >= 0 ? opts.graceMs : LI_MERGE_GRACE_MS;
+  const minSmall = Number.isInteger(opts.minSmallSegments) && opts.minSmallSegments > 0 ? opts.minSmallSegments : 2;
+
+  const layout = resolveSegmentedLayout(stateDir);
+  if (!layout) return { skipped: 'no-segmented-index' };
+  const { stubPath, segmentDir, manifestPath, manifest } = layout;
+
+  // Step 1 + 2: housekeeping that must run regardless of whether we merge.
+  const orphansDeleted = sweepOrphanSegments(segmentDir, manifest);
+  const quarantineDeleted = sweepQuarantine(segmentDir, graceMs);
+
+  if (opts.sweepOnly) {
+    return { swept: true, orphansDeleted, quarantineDeleted, segmentCount: manifest.segments.length };
+  }
+
+  const small = manifest.segments.filter((s) => Number.isFinite(s.count) && s.count < segmentSize);
+  if (small.length < minSmall) {
+    return { skipped: 'too-few-small-segments', orphansDeleted, quarantineDeleted, segmentCount: manifest.segments.length };
+  }
+  const keptFull = manifest.segments.filter((s) => !(Number.isFinite(s.count) && s.count < segmentSize));
+  const smallNames = new Set(small.map((s) => s.path));
+  const smallAbs = new Set(small.map((s) => path.join(segmentDir, s.path)));
+
+  // Load the live docs (the loader drops tombstoned docs per the stale bitmap).
+  const index = new LateInteractionIndex({ indexPath: stubPath, loadExisting: true, modelId: manifest.modelId || null });
+  await index.init();
+
+  const collected = [];
+  for (const [docId, doc] of index.documents.entries()) {
+    const pos = index._docSegmentPositions?.get(docId);
+    if (!pos || !smallAbs.has(pos.segmentPath)) continue;
+    collected.push({ segmentPath: pos.segmentPath, docIndex: pos.docIndex, docId, doc });
+  }
+  collected.sort((a, b) => (a.segmentPath < b.segmentPath ? -1 : a.segmentPath > b.segmentPath ? 1
+    : a.docIndex - b.docIndex));
+
+  // Writer used purely as the SSLX serializer (model/quant params copied verbatim).
+  const writer = new LateInteractionIndex({
+    indexPath: stubPath,
+    loadExisting: false,
+    tokenDim: index.tokenDim,
+    maxTokens: index.maxTokens,
+    useInt8: index.useInt8,
+    quantBits: index.quantBits,
+    modelId: index.modelId,
+    poolFactor: index.poolFactor,
+    whtSeed: index.whtSeed,
+    whtOrdering: index.whtOrdering,
+    matryoshkaDim: index.matryoshkaDim,
+  });
+  await writer.init();
+
+  let seq = nextSegmentSeq(manifest);
+  const newSegments = [];
+  const writtenFinals = [];
+  for (let i = 0; i < collected.length; i += segmentSize) {
+    const batch = new Map();
+    for (const { docId, doc } of collected.slice(i, i + segmentSize)) batch.set(docId, doc);
+    const segName = `segment-${String(seq).padStart(4, '0')}.bin`;
+    seq += 1;
+    const finalPath = path.join(segmentDir, segName);
+    const tmpPath = finalPath + '.compacting.tmp';
+    await writer._writeSegmentFile(tmpPath, batch);
+    fs.renameSync(tmpPath, finalPath);
+    writtenFinals.push(finalPath);
+    newSegments.push({ path: segName, count: batch.size });
+  }
+
+  // Step 4: atomic manifest publish (kept full segments + new merged segments).
+  const nextManifest = {
+    ...manifest,
+    segments: [...keptFull, ...newSegments],
+    nextSeq: seq,
+  };
+  nextManifest.totalDocuments = nextManifest.segments.reduce((sum, s) => sum + (s?.count || 0), 0);
+  await writeJsonAtomic(manifestPath, nextManifest);
+
+  // Step 5: quarantine the consumed small-segment files (+ stale sidecars).
+  const consumedPaths = [];
+  for (const name of smallNames) {
+    consumedPaths.push(name);
+    if (fs.existsSync(path.join(segmentDir, name + STALE_SIDECAR_EXT))) {
+      consumedPaths.push(name + STALE_SIDECAR_EXT);
+    }
+  }
+  if (consumedPaths.length > 0) {
+    const journal = readQuarantine(segmentDir);
+    journal.push({ retiredAtMs: Date.now(), paths: consumedPaths });
+    writeQuarantine(segmentDir, journal);
+  }
+
+  return {
+    tier: 'li_segments',
+    mergedFrom: small.length,
+    mergedInto: newSegments.length,
+    keptFull: keptFull.length,
+    keptDocs: collected.length,
+    segmentCountBefore: manifest.segments.length,
+    segmentCountAfter: nextManifest.segments.length,
+    orphansDeleted,
+    quarantineDeleted,
+    quarantined: consumedPaths.length,
+  };
+}

package/core/incremental-indexing/infrastructure/li-segment-state.mjs

@@ -0,0 +1,173 @@
+/**
+ * LI segment state and per-segment tombstone tracking.
+ *
+ * Plan § 7.5. Sweet-search's late-interaction index ships as SSLX v3
+ * segments of ≤ 10 K docs each. The newest segment is the "growing"
+ * write target; sealed segments are immutable. Edits to docs in a sealed
+ * segment produce two writes:
+ *
+ *   - the old doc's bit is set in the sealed segment's `*.stale.bin`,
+ *   - the new doc is appended to the growing segment.
+ *
+ * This module tracks per-segment stale counts and decides which segments
+ * cross the per-segment watermark (`stale_doc_ratio > 0.20` per plan
+ * § 7.5 step 3 / `domain/watermark-scheduler.mjs`).
+ *
+ * The stale bitmap for each segment lives at `<segmentPath>.stale.bin`
+ * and uses the layout in `infrastructure/tombstone-bitmap.mjs`.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {
+  createBitmap, loadBitmap, saveBitmap,
+  resizeBitmap, setBit, isSet, filterLive,
+} from './tombstone-bitmap.mjs';
+
+export const STALE_SIDECAR_EXT = '.stale.bin';
+/** SSLX sealed-segment capacity (docs). Mirrors LI_SEGMENT_SIZE in
+ *  core/ranking/late-interaction-index.js; a segment with fewer docs is
+ *  "small" and a candidate for batch merge. */
+export const LI_SEGMENT_SIZE = 10_000;
+
+function staleSidecarPath(segmentPath) {
+  return segmentPath + STALE_SIDECAR_EXT;
+}
+
+/**
+ * Next monotonic segment sequence number for a segment manifest. Persisted
+ * as `manifest.nextSeq`; bootstrapped from the max existing `segment-<n>`
+ * index so segment naming stays collision-proof even after the batch merge
+ * removes segments out of order. Both the reconcile write path and the
+ * merge handler bump this counter.
+ *
+ * @param {object} manifest
+ * @returns {number}
+ */
+export function nextSegmentSeq(manifest) {
+  if (Number.isInteger(manifest?.nextSeq)) return manifest.nextSeq;
+  let maxIdx = -1;
+  for (const seg of manifest?.segments || []) {
+    const m = typeof seg?.path === 'string' && seg.path.match(/segment-(\d+)\.bin$/);
+    if (m) maxIdx = Math.max(maxIdx, Number(m[1]));
+  }
+  return maxIdx + 1;
+}
+
+/**
+ * Per-segment state object that the LI tier uses while applying deltas.
+ *
+ * @param {string} segmentPath
+ * @param {number} docCount
+ */
+export function openSegmentState(segmentPath, docCount) {
+  let bitmap = loadBitmap(staleSidecarPath(segmentPath));
+  if (!bitmap) {
+    bitmap = createBitmap(Math.max(1, docCount));
+  } else if (bitmap.capacity < docCount) {
+    bitmap = resizeBitmap(bitmap, docCount);
+  }
+  return {
+    segmentPath,
+    docCount,
+    bitmap,
+  };
+}
+
+/**
+ * Mark a doc tombstoned. Persists the bitmap to disk so a daemon crash
+ * before manifest publish does not lose the tombstone state.
+ *
+ * @param {object} segmentState
+ * @param {number} docIndex
+ */
+export function tombstoneDoc(segmentState, docIndex) {
+  setBit(segmentState.bitmap, docIndex);
+}
+
+/**
+ * Persist the stale bitmap to disk via atomic temp+rename.
+ *
+ * @param {object} segmentState
+ */
+export function persistSegmentState(segmentState) {
+  saveBitmap(staleSidecarPath(segmentState.segmentPath), segmentState.bitmap);
+}
+
+/**
+ * Compute the stale_doc_ratio for the watermark check.
+ *
+ * @param {object} segmentState
+ * @returns {number}
+ */
+export function staleDocRatio(segmentState) {
+  if (segmentState.docCount === 0) return 0;
+  let tombstoned = 0;
+  for (let i = 0; i < segmentState.docCount; i += 1) {
+    if (isSet(segmentState.bitmap, i)) tombstoned += 1;
+  }
+  return tombstoned / segmentState.docCount;
+}
+
+/**
+ * Enumerate segment paths in a segments directory. Returns paths sorted
+ * lexicographically (caller is responsible for any ordering they need
+ * beyond that).
+ *
+ * @param {string} segmentsDir
+ * @returns {string[]}
+ */
+export function listSegments(segmentsDir) {
+  if (!fs.existsSync(segmentsDir)) return [];
+  return fs
+    .readdirSync(segmentsDir)
+    .filter((n) => n.endsWith('.bin') && !n.endsWith(STALE_SIDECAR_EXT))
+    .sort()
+    .map((n) => path.join(segmentsDir, n));
+}
+
+/**
+ * For each segment in the directory, evaluate its current stale ratio.
+ * The watermark scheduler (`domain/watermark-scheduler.mjs`) consumes
+ * the returned array via `liSegments` input.
+ *
+ * @param {string} segmentsDir
+ * @param {Map<string, number>} [docCountsBySegment]   Optional override.
+ * @returns {Array<{segmentId:string, staleDocRatio:number}>}
+ */
+export function evaluateSegmentRatios(segmentsDir, docCountsBySegment = new Map()) {
+  const out = [];
+  for (const segmentPath of listSegments(segmentsDir)) {
+    const docCount = docCountsBySegment.get(segmentPath) ?? 0;
+    if (docCount === 0) continue;
+    const state = openSegmentState(segmentPath, docCount);
+    out.push({
+      segmentId: path.basename(segmentPath),
+      staleDocRatio: staleDocRatio(state),
+    });
+  }
+  return out;
+}
+
+/**
+ * Filter a candidate doc-index list by the segment's stale bitmap. Used
+ * by the LI scorer (plan § 7.5 step 5).
+ *
+ * @param {object} segmentState
+ * @param {number[]} candidates
+ * @returns {number[]}
+ */
+export function filterLiveDocs(segmentState, candidates) {
+  return filterLive(segmentState.bitmap, candidates);
+}
+
+/**
+ * Check whether a doc index is alive (false) or tombstoned (true).
+ *
+ * @param {object} segmentState
+ * @param {number} docIndex
+ * @returns {boolean}
+ */
+export function isDocTombstoned(segmentState, docIndex) {
+  return isSet(segmentState.bitmap, docIndex);
+}