pi-mega-compact 0.9.2 → 0.10.1

package/dist/src/store/turns/sqlite-store.test.js

@@ -0,0 +1,144 @@
+/**
+ * sqlite-store.test.ts — SqliteTurnStore compliance + SQLite-specific tests.
+ *
+ * Compliance: shared suite run in-memory (fast).
+ * SQLite-specific: file-backed tests verifying:
+ *   - Separate turns.db (not sqlite.db)
+ *   - WAL mode active
+ *   - Connection caching (same path → same connection)
+ *   - dbSizeBytes reports file size
+ *   - File persistence across store instances
+ */
+import { describe, it, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, statSync, rmSync, mkdtempSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { SqliteTurnStore } from "./sqlite-store.js";
+import { runComplianceSuite } from "./contract-compliance.test.js";
+import { closeAllTurnDbs } from "./connection.js";
+// ── Compliance suite (in-memory, fast) ──────────────────────────
+runComplianceSuite("SqliteTurnStore", (options) => new SqliteTurnStore(options), { stateDir: join(tmpdir(), "turns-compliance-sqlite"), inMemory: true });
+// ── SQLite-specific file-backed tests ───────────────────────────
+describe("SqliteTurnStore — file-backed", () => {
+    let stateDir;
+    let store;
+    beforeEach(() => {
+        closeAllTurnDbs(); // clear any cached connections
+        stateDir = mkdtempSync(join(tmpdir(), "turns-file-"));
+        store = new SqliteTurnStore({ stateDir });
+    });
+    afterEach(() => {
+        try {
+            store.close();
+        }
+        catch {
+            // best-effort
+        }
+        closeAllTurnDbs();
+        try {
+            rmSync(stateDir, { recursive: true, force: true });
+        }
+        catch {
+            // best-effort
+        }
+    });
+    it("creates turns.db separate from sqlite.db", () => {
+        const turnsDbPath = join(stateDir, "turns.db");
+        assert.ok(existsSync(turnsDbPath), "turns.db should exist");
+        // sqlite.db should NOT be created by the turns store
+        const sqliteDbPath = join(stateDir, "sqlite.db");
+        assert.ok(!existsSync(sqliteDbPath), "sqlite.db should NOT be created");
+    });
+    it("uses WAL journal mode", () => {
+        const row = store
+            .db.prepare("PRAGMA journal_mode")
+            .get();
+        assert.equal(row.journal_mode, "wal");
+    });
+    it("caches connections for the same path", () => {
+        // Creating a second store with the same stateDir should reuse
+        // the cached connection (not create a new file lock conflict)
+        const store2 = new SqliteTurnStore({ stateDir });
+        try {
+            // Both should be able to write without locking errors
+            const entry = {
+                conversationId: "conv_cache",
+                sessionId: "sess_cache",
+                turnIndex: 0,
+                role: "assistant",
+                endedAt: Date.now(),
+            };
+            store.asWriter().appendTurn(entry);
+            store2.asWriter().appendTurn({
+                ...entry,
+                turnIndex: 1,
+            });
+            assert.equal(store.asReader().countTurns("conv_cache"), 2);
+        }
+        finally {
+            store2.close();
+        }
+    });
+    it("reports dbSizeBytes for file-backed store", () => {
+        const entry = {
+            conversationId: "conv_size",
+            sessionId: "sess_size",
+            turnIndex: 0,
+            role: "assistant",
+            endedAt: Date.now(),
+        };
+        store.asWriter().appendTurn(entry);
+        // prune returns a PruneReport with freedBytes; we can also
+        // verify the file exists and has content
+        const turnsDbPath = join(stateDir, "turns.db");
+        const size = statSync(turnsDbPath).size;
+        assert.ok(size > 0, "turns.db should have non-zero size after writes");
+    });
+    it("persists data across store instances", () => {
+        const entry = {
+            conversationId: "conv_persist",
+            sessionId: "sess_persist",
+            turnIndex: 0,
+            role: "assistant",
+            endedAt: Date.now(),
+            model: "test-model",
+        };
+        store.asWriter().appendTurn(entry);
+        store.close();
+        closeAllTurnDbs();
+        // Re-open the same stateDir
+        const store2 = new SqliteTurnStore({ stateDir });
+        try {
+            const turns = store2.asReader().query({ conversationId: "conv_persist" });
+            assert.equal(turns.length, 1);
+            assert.equal(turns[0].model, "test-model");
+        }
+        finally {
+            store2.close();
+            closeAllTurnDbs();
+        }
+    });
+    it("supports custom dbPath", () => {
+        const customPath = join(stateDir, "custom-turns.db");
+        const customStore = new SqliteTurnStore({
+            stateDir,
+            dbPath: customPath,
+        });
+        try {
+            assert.ok(existsSync(customPath), "custom db path should be created");
+            customStore.asWriter().appendTurn({
+                conversationId: "conv_custom",
+                sessionId: "sess_custom",
+                turnIndex: 0,
+                role: "user",
+                endedAt: Date.now(),
+            });
+            assert.equal(customStore.asReader().countTurns("conv_custom"), 1);
+        }
+        finally {
+            customStore.close();
+            closeAllTurnDbs();
+        }
+    });
+});

package/dist/src/store/turns/types.js

@@ -0,0 +1,16 @@
+/**
+ * types.ts — S49 contract module (the source of truth).
+ *
+ * Every implementation must satisfy these interfaces. Hosts import only this
+ * file + the factory from index.ts. SQL schemas are private to implementations.
+ *
+ * Design principles (from docs/specs/s49-rev1-architecture-upgrade.md):
+ *   1. Contract-first — the interface IS the spec.
+ *   2. Append-only — TurnWriter.append* are the only write methods; no UPDATE.
+ *   3. Capability-gated — asReader/asWriter/asAdmin return subset views.
+ *   4. Ledger protocol — host PUSHES facts, PULLS views. Store never initiates.
+ *   5. StoreSnapshot — checkpoint/restore for backup, migration, test seeding.
+ *
+ * PREVENT-PI-004: zero network. PREVENT-002: no SQL here (private to impls).
+ */
+export {};

package/dist/src/topics/cluster.js

@@ -0,0 +1,141 @@
+/**
+ * cluster.ts — S51A k-means topic clustering over real stored embeddings.
+ *
+ * Host-agnostic (no pi imports), pure local math (PREVENT-PI-004): clusters the
+ * real `context_chunks.embedding_blob` vectors with k-means++ (reused from
+ * src/dedup/raptor/kmeans.ts), picks k by a real criterion (elbow on WCSS, or
+ * silhouette when the corpus is large enough), and labels each cluster with the
+ * top TF-IDF terms of its real member text. Pure math only — no model calls,
+ * no network, no remote service (PREVENT-PI-004).
+ *
+ * The topic view is derived + read-only over `context_chunks` — building a model
+ * never mutates any memory (PREVENT-PI-001/002). Parameterized SQL (PREVENT-002).
+ */
+import { cosineSimilarity } from "../embedder.js";
+import { decodeEmbedding } from "../store/sqlite/utils.js";
+import { tfidfScores, labelFromScores, membershipConfidence, } from "./labels.js";
+import { selectK } from "./kselection.js";
+export const DEFAULT_CLUSTER_CONFIG = {
+    kRange: [3, 15],
+    labelTopTerms: 5,
+    restarts: 5,
+    seed: 0x9e3779b9,
+};
+/**
+ * Load chunks that have a usable embedding. Text is the best-available surface
+ * (normalized_text → summary → topic_summary). Skips rows with no embedding or
+ * no text. Returns in stable (session_id, id) order for determinism.
+ */
+export function loadEmbeddings(mainDb) {
+    const rows = mainDb
+        .prepare(`SELECT id, session_id, embedding_blob,
+              COALESCE(normalized_text, summary, topic_summary) AS text
+       FROM context_chunks
+       WHERE embedding_blob IS NOT NULL
+       ORDER BY session_id ASC, id ASC`)
+        .all();
+    const out = [];
+    for (const r of rows) {
+        const vec = decodeEmbedding(r.embedding_blob);
+        if (vec.length === 0)
+            continue;
+        const text = (r.text ?? "").trim();
+        if (text.length === 0)
+            continue;
+        out.push({ chunkId: r.id, sessionId: r.session_id, vec, text });
+    }
+    return out;
+}
+/**
+ * Build a topic model from the real stored chunks. Picks k by silhouette when
+ * computable, else elbow; degenerate/small corpus → single `general` cluster.
+ * Never throws on degenerate input — returns the safe single-cluster model.
+ */
+export function buildTopicModel(mainDb, cfg = DEFAULT_CLUSTER_CONFIG) {
+    const builtAt = Date.now();
+    const chunks = loadEmbeddings(mainDb);
+    const totalChunks = chunks.length;
+    // Too small to cluster meaningfully → single general cluster.
+    if (totalChunks < Math.max(2, cfg.kRange[0])) {
+        return singleClusterModel(chunks, builtAt, cfg);
+    }
+    const points = chunks.map((c) => c.vec);
+    const [minK, maxK] = cfg.kRange;
+    const hi = Math.min(maxK, Math.max(minK, Math.floor(totalChunks / 2)));
+    const ks = [];
+    for (let k = minK; k <= hi; k++)
+        ks.push(k);
+    if (ks.length === 0)
+        ks.push(minK);
+    const { candidate: chosen, criterion } = selectK(points, ks, cfg);
+    const meanSil = chosen.silhouette;
+    return assembleModel(chunks, chosen, criterion, meanSil, builtAt, cfg);
+}
+/** Single `general` cluster holding every chunk (degenerate/small corpus). */
+function singleClusterModel(chunks, builtAt, cfg) {
+    const topic = {
+        id: "topic_0",
+        label: labelFromScores(tfidfScores(chunks, chunks), cfg.labelTopTerms),
+        termScores: tfidfScores(chunks, chunks),
+        memoryCount: chunks.length,
+        lastUpdated: builtAt,
+    };
+    const assignments = chunks.map((c) => ({
+        memoryId: c.chunkId,
+        sessionId: c.sessionId,
+        topicId: "topic_0",
+        confidence: 1,
+        assignedAt: builtAt,
+        method: "kmeans+tfidf",
+    }));
+    return {
+        topics: chunks.length > 0 ? [topic] : [],
+        assignments,
+        k: chunks.length > 0 ? 1 : 0,
+        criterion: "elbow",
+        silhouetteScore: null,
+        totalChunks: chunks.length,
+        builtAt,
+    };
+}
+/** Assemble topics + assignments from a chosen clustering. */
+function assembleModel(chunks, chosen, criterion, meanSil, builtAt, cfg) {
+    const topics = [];
+    const assignments = [];
+    for (let c = 0; c < chosen.k; c++) {
+        const members = chunks.filter((_, i) => chosen.assignments[i] === c);
+        if (members.length === 0)
+            continue;
+        const id = `topic_${topics.length}`;
+        const scores = tfidfScores(members, chunks);
+        topics.push({
+            id,
+            label: labelFromScores(scores, cfg.labelTopTerms),
+            termScores: scores,
+            memoryCount: members.length,
+            lastUpdated: builtAt,
+        });
+    }
+    for (let i = 0; i < chunks.length; i++) {
+        const cluster = chosen.assignments[i];
+        const centroid = chosen.centroids[cluster];
+        const sim = centroid ? cosineSimilarity(chunks[i].vec, centroid) : 0;
+        assignments.push({
+            memoryId: chunks[i].chunkId,
+            sessionId: chunks[i].sessionId,
+            topicId: `topic_${cluster}`,
+            confidence: membershipConfidence(sim),
+            assignedAt: builtAt,
+            method: "kmeans+tfidf",
+        });
+    }
+    return {
+        topics,
+        assignments,
+        k: topics.length,
+        criterion,
+        silhouetteScore: meanSil,
+        totalChunks: chunks.length,
+        builtAt,
+    };
+}

package/dist/src/topics/cluster.test.js

@@ -0,0 +1,150 @@
+/**
+ * cluster.test.ts — S51A clustering + labeling tests. No network, temp dirs.
+ * Includes the honest-boundary grep-asserts (no ollama/llm/fetch, no fabricated
+ * keyword literals) required by the s47/s51 acceptance criteria.
+ */
+import { test, beforeEach, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, readFileSync, readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { openStore } from "../store/sqlite/utils.js";
+import { upsertCheckpoint } from "../store/sqlite/checkpoints.js";
+import { loadEmbeddings, buildTopicModel, DEFAULT_CLUSTER_CONFIG } from "./cluster.js";
+import { tfidfScores, membershipConfidence } from "./labels.js";
+let tmpDir;
+let counter = 0;
+beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "mc-topics-"));
+});
+afterEach(() => {
+    rmSync(tmpDir, { recursive: true, force: true });
+});
+function stateDir() {
+    return join(tmpDir, `run-${counter++}`);
+}
+/** Unit vector along axis `axis` (dim = total dims). */
+function axisVec(axis, dim) {
+    const v = new Array(dim).fill(0);
+    v[axis] = 1;
+    return v;
+}
+function cp(id, text, embedding, session = "sess_t") {
+    return {
+        checkpointId: id,
+        sessionId: session,
+        summary: text,
+        normalizedText: text,
+        keyDecisions: [],
+        nextSteps: [],
+        filesModified: [],
+        tokenEstimate: 0,
+        regionHash: "r",
+        embedding,
+        timestamp: 1,
+    };
+}
+/** Seed N clusters of M chunks each: cluster i sits near axis i with distinct terms. */
+function seedClusters(dir, clusters, perCluster, dim) {
+    openStore(dir);
+    const terms = ["sqlite", "wal", "checkpoint", "recovery", "backup", "replay", "fork", "merge"];
+    for (let c = 0; c < clusters; c++) {
+        for (let m = 0; m < perCluster; m++) {
+            // Near-axis vector with slight per-member jitter (stays closest to its axis).
+            const v = axisVec(c, dim).map((x, i) => x + (i === c ? 0 : (m + 1) * 1e-4 * (i + 1)));
+            const term = terms[(c * 2) % terms.length];
+            const term2 = terms[(c * 2 + 1) % terms.length];
+            upsertCheckpoint(cp(`chkpt_c${c}_${m}`, `${term} ${term2} work item ${m} details`, v), dir);
+        }
+    }
+}
+test("loadEmbeddings returns only chunks with embeddings + text", () => {
+    const dir = stateDir();
+    openStore(dir);
+    upsertCheckpoint(cp("a", "hello world", axisVec(0, 4)), dir);
+    upsertCheckpoint(cp("b", "", axisVec(1, 4)), dir); // empty text → skipped
+    upsertCheckpoint(cp("c", "no embedding here", [], "sess_t"), dir); // empty embedding → skipped
+    const chunks = loadEmbeddings(openStore(dir));
+    assert.equal(chunks.length, 1);
+    assert.equal(chunks[0].chunkId, "a");
+    assert.equal(chunks[0].text, "hello world");
+});
+test("buildTopicModel finds 3 well-separated clusters (silhouette picks k=3)", () => {
+    const dir = stateDir();
+    seedClusters(dir, 3, 6, 8); // 18 chunks, 3 tight clusters
+    const model = buildTopicModel(openStore(dir), { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
+    assert.equal(model.totalChunks, 18);
+    assert.equal(model.k, 3);
+    assert.equal(model.topics.length, 3);
+    // Every chunk assigned; assignment count == chunk count.
+    assert.equal(model.assignments.length, 18);
+    // Labels reflect the seeded discriminative terms (sqlite/wal, checkpoint/recovery, backup/replay).
+    const labels = model.topics.map((t) => t.label).join(" | ");
+    assert.ok(/sqlite|checkpoint|backup/.test(labels), `labels should carry seeded terms: ${labels}`);
+});
+test("corpus too small → single general cluster, no crash", () => {
+    const dir = stateDir();
+    openStore(dir);
+    upsertCheckpoint(cp("only", "lone chunk about sqlite", axisVec(0, 4)), dir);
+    const model = buildTopicModel(openStore(dir), DEFAULT_CLUSTER_CONFIG); // kRange[0]=3 > 1 chunk
+    assert.equal(model.k, 1);
+    assert.equal(model.topics.length, 1);
+    assert.equal(model.topics[0].memoryCount, 1);
+    assert.equal(model.silhouetteScore, null);
+});
+test("empty corpus → zero topics, no crash", () => {
+    const dir = stateDir();
+    openStore(dir);
+    const model = buildTopicModel(openStore(dir), DEFAULT_CLUSTER_CONFIG);
+    assert.equal(model.k, 0);
+    assert.equal(model.topics.length, 0);
+    assert.equal(model.assignments.length, 0);
+});
+test("tfidfScores surfaces discriminative terms, down-weights common ones", () => {
+    const mk = (text) => ({
+        chunkId: text, sessionId: "s", vec: [1], text,
+    });
+    const corpus = [
+        mk("sqlite wal sqlite pragma"),
+        mk("sqlite wal journal"),
+        mk("react component render"),
+        mk("react hooks state"),
+    ];
+    const members = corpus.slice(0, 2); // the sqlite cluster
+    const scores = tfidfScores(members, corpus);
+    const top = scores[0]?.term;
+    assert.ok(top === "sqlite" || top === "wal", `top term should be discriminative, got ${top}`);
+    // 'react' never appears in members → not scored for this cluster.
+    assert.ok(!scores.some((s) => s.term === "react"));
+});
+test("membershipConfidence maps cosine [-1,1] → [0,1]", () => {
+    assert.equal(membershipConfidence(1), 1);
+    assert.equal(membershipConfidence(-1), 0);
+    assert.equal(membershipConfidence(0), 0.5);
+});
+test("buildTopicModel is deterministic for a fixed seed", () => {
+    const dir = stateDir();
+    seedClusters(dir, 3, 5, 8);
+    const db = openStore(dir);
+    const a = buildTopicModel(db, { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
+    const b = buildTopicModel(db, { ...DEFAULT_CLUSTER_CONFIG, kRange: [2, 5] });
+    assert.equal(a.k, b.k);
+    assert.deepEqual(a.topics.map((t) => t.label).sort(), b.topics.map((t) => t.label).sort());
+});
+test("honest boundary: no LLM/Ollama/network/keyword-literals in src/topics", () => {
+    const here = dirname(fileURLToPath(import.meta.url));
+    const srcTopics = join(here, "..", "..", "..", "src", "topics"); // dist/src/topics → src/topics
+    const files = readdirSync(srcTopics).filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts"));
+    const banned = [/ollama/i, /\bllm\b/i, /\bfetch\s*\(/, /\bhybrid\b/i];
+    const fabricated = ["graphql", "docker", "stacktrace", "bottleneck", "trade-off"];
+    for (const f of files) {
+        const text = readFileSync(join(srcTopics, f), "utf8");
+        for (const re of banned) {
+            assert.ok(!re.test(text), `${f} must not contain ${re} (PREVENT-PI-004 / no-LLM boundary)`);
+        }
+        for (const word of fabricated) {
+            assert.ok(!text.toLowerCase().includes(`"${word}"`), `${f} must not hardcode fabricated taxonomy term "${word}"`);
+        }
+    }
+});

package/dist/src/topics/index.js

@@ -0,0 +1,6 @@
+/**
+ * index.ts — S51 topics barrel. Re-export only; no logic.
+ */
+export { loadEmbeddings, buildTopicModel, DEFAULT_CLUSTER_CONFIG, } from "./cluster.js";
+export { createTopicStore, getWikiCompactCounter, bumpWikiCompactCounter } from "./store.js";
+export { tokenize, tfidfScores, labelFromScores, membershipConfidence } from "./labels.js";

package/dist/src/topics/kselection.js

@@ -0,0 +1,142 @@
+/**
+ * kselection.ts — S51A k-selection criterion math for topic clustering.
+ *
+ * Pure local math (PREVENT-PI-004): within-cluster sum of squares (cosine
+ * distance), mean silhouette score, and the elbow heuristic. k-means++ is
+ * reused from src/dedup/raptor/kmeans.ts. No model calls, no network.
+ *
+ * Extracted from cluster.ts so each module stays under the project's 300-line
+ * file-size cap with a single responsibility: this module owns the
+ * "which k is best" criterion; cluster.ts owns load + orchestration.
+ *
+ * Host-agnostic (no pi imports). Parameterized only by Vector arrays.
+ */
+import { cosineDistance, kmeanspp, meanVector, } from "../dedup/raptor/kmeans.js";
+/** Within-cluster sum of squares (cosine distance) for one clustering. */
+export function wcss(points, assignments, centroids) {
+    let s = 0;
+    for (let i = 0; i < points.length; i++) {
+        const d = cosineDistance(points[i], centroids[assignments[i]]);
+        s += d * d;
+    }
+    return s;
+}
+/**
+ * Mean silhouette score across all points (−1..1); null when undefined
+ * (k<=1 or k>=n, or every cluster is a singleton).
+ */
+export function silhouette(points, assignments, k) {
+    const n = points.length;
+    if (k <= 1 || k >= n)
+        return null;
+    // Precompute cluster sizes once.
+    const sizes = new Map();
+    for (const a of assignments)
+        sizes.set(a, (sizes.get(a) ?? 0) + 1);
+    let sum = 0;
+    let counted = 0;
+    for (let i = 0; i < n; i++) {
+        const a = assignments[i];
+        const ownSize = sizes.get(a) ?? 0;
+        if (ownSize <= 1)
+            continue; // singleton cluster → contribution 0 (skip)
+        let aSum = 0;
+        const bSum = new Map();
+        for (let j = 0; j < n; j++) {
+            if (i === j)
+                continue;
+            const d = cosineDistance(points[i], points[j]);
+            if (assignments[j] === a) {
+                aSum += d;
+            }
+            else {
+                bSum.set(assignments[j], (bSum.get(assignments[j]) ?? 0) + d);
+            }
+        }
+        const aMean = aSum / (ownSize - 1); // exclude self
+        let b = Infinity;
+        for (const [cluster, total] of bSum) {
+            const cnt = sizes.get(cluster) ?? 0;
+            if (cnt > 0)
+                b = Math.min(b, total / cnt);
+        }
+        if (!Number.isFinite(b))
+            continue;
+        const denom = Math.max(aMean, b);
+        if (denom > 0) {
+            sum += (b - aMean) / denom;
+            counted++;
+        }
+    }
+    return counted > 0 ? sum / counted : null;
+}
+/** Elbow: index of max curvature on the WCSS-vs-k curve. */
+export function elbowIndex(ks, wcssVals) {
+    if (ks.length <= 2)
+        return 0;
+    // Normalize and find the point farthest from the line joining the endpoints.
+    const x0 = ks[0];
+    const y0 = wcssVals[0];
+    const x1 = ks[ks.length - 1];
+    const y1 = wcssVals[wcssVals.length - 1];
+    const dx = x1 - x0;
+    const dy = y1 - y0;
+    const len = Math.hypot(dx, dy) || 1;
+    let best = 0;
+    let bestDist = -1;
+    for (let i = 0; i < ks.length; i++) {
+        const dist = Math.abs(dy * ks[i] - dx * wcssVals[i] + x1 * y0 - y1 * x0) / len;
+        if (dist > bestDist) {
+            bestDist = dist;
+            best = i;
+        }
+    }
+    return best;
+}
+/** Run k-means with restarts for one k; keep the lowest-WCSS result. */
+export function bestForK(points, k, cfg) {
+    let best = null;
+    for (let r = 0; r < cfg.restarts; r++) {
+        const res = kmeanspp(points, k, { seed: cfg.seed + r });
+        if (res.k === 0)
+            continue;
+        const w = wcss(points, res.assignments, res.centroids);
+        if (!best || w < best.wcss) {
+            best = {
+                k: res.k,
+                assignments: res.assignments,
+                centroids: res.centroids,
+                wcss: w,
+                silhouette: null,
+            };
+        }
+    }
+    // Degenerate: kmeans collapsed everything to < k clusters.
+    if (!best) {
+        const mean = meanVector(points);
+        best = {
+            k: 1,
+            assignments: points.map(() => 0),
+            centroids: [mean],
+            wcss: 0,
+            silhouette: null,
+        };
+    }
+    best.silhouette = silhouette(points, best.assignments, best.k);
+    return best;
+}
+/**
+ * Run k-means across the k search space and pick the best candidate:
+ * prefer silhouette when any candidate produced a computable score, else
+ * the elbow on the WCSS curve. `points` must be non-empty and `ks` non-empty.
+ */
+export function selectK(points, ks, cfg) {
+    const candidates = ks.map((k) => bestForK(points, k, cfg));
+    const withSil = candidates.filter((c) => c.silhouette !== null);
+    if (withSil.length > 0) {
+        const chosen = withSil.reduce((a, b) => (b.silhouette ?? -1) > (a.silhouette ?? -1) ? b : a);
+        return { candidate: chosen, criterion: "silhouette" };
+    }
+    const idx = elbowIndex(candidates.map((c) => c.k), candidates.map((c) => c.wcss));
+    return { candidate: candidates[idx], criterion: "elbow" };
+}

package/dist/src/topics/labels.js

@@ -0,0 +1,56 @@
+/**
+ * labels.ts — S51A TF-IDF cluster labeling + membership confidence.
+ *
+ * Pure local math over real stored text (PREVENT-PI-004). NO fabricated keyword
+ * taxonomy, NO model: a cluster's label is the top TF-IDF terms of its real member
+ * chunks (real TF × real IDF over the whole corpus), so common words are
+ * naturally down-weighted by IDF rather than by a hand-written stopword list.
+ */
+/** Tokenize text: lowercase, split on non-alphanumeric, drop empties/1-char. */
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((t) => t.length > 1);
+}
+/**
+ * Compute TF-IDF term scores for one cluster's member chunks against the whole
+ * corpus. TF = term count across the cluster's members; IDF = log(1 + N/df)
+ * where N = total chunks and df = chunks containing the term. Returns the full
+ * list sorted by score desc.
+ */
+export function tfidfScores(memberChunks, corpus) {
+    const tf = new Map();
+    for (const c of memberChunks) {
+        for (const term of tokenize(c.text))
+            tf.set(term, (tf.get(term) ?? 0) + 1);
+    }
+    if (tf.size === 0)
+        return [];
+    // Document frequency over the whole corpus (one vote per chunk per term).
+    const df = new Map();
+    for (const c of corpus) {
+        for (const term of new Set(tokenize(c.text)))
+            df.set(term, (df.get(term) ?? 0) + 1);
+    }
+    const n = Math.max(1, corpus.length);
+    const out = [];
+    for (const [term, f] of tf) {
+        const idf = Math.log(1 + n / (df.get(term) ?? 1));
+        out.push({ term, score: f * idf });
+    }
+    out.sort((a, b) => b.score - a.score || a.term.localeCompare(b.term));
+    return out;
+}
+/** Join the top `topTerms` TF-IDF terms into a human label. */
+export function labelFromScores(scores, topTerms) {
+    const top = scores.slice(0, Math.max(1, topTerms)).map((s) => s.term);
+    return top.length > 0 ? top.join(" · ") : "general";
+}
+/**
+ * Normalized cosine membership confidence in [0,1]. Cosine is in [-1,1]; map to
+ * [0,1] via (1 + cos)/2. A chunk sitting exactly on its centroid → 1.
+ */
+export function membershipConfidence(cosineSimilarity) {
+    return Math.max(0, Math.min(1, (1 + cosineSimilarity) / 2));
+}