pi-mega-compact 0.13.4 → 0.13.6

package/dist/src/wiki/curation.js

@@ -0,0 +1,147 @@
+/**
+ * curation.ts — W2 user curation of the auto-categorized wiki (factory shell).
+ *
+ * Persist rename / merge / split over the `topic_overrides` +
+ * `topic_evolution` tables (reserved in the W1.1 schema). Every mutation is
+ * atomic (SAVEPOINT via withTx) so a partial write never leaks. The DB
+ * primitives live in curation-helpers.ts; this file owns the three
+ * transactions (renameTopic / mergeTopics / splitTopic) + label resolution.
+ *
+ * Host-agnostic (no pi imports); pure local node:sqlite (PREVENT-PI-004), all
+ * SQL parameterized (PREVENT-002). Overrides are immutable per (topic_id, kind).
+ * `topics`/`memory_topics` are writable; the append-only invariant applies only
+ * to the `turns` table.
+ */
+import { openTurnStore, withTx } from "../store/turns/connection.js";
+import { getStateDir } from "../store.js";
+import { getTopicLabel, recountTopic, customLabelOverride, writeLabelOverride, moveMemory, recordEvolution, createSplitTopic, } from "./curation-helpers.js";
+/** Open the wiki curation store over the shared turns.db connection. */
+export function createWikiCuration(stateDir) {
+    const db = openTurnStore(stateDir ?? getStateDir());
+    const now = () => Date.now();
+    function tryCatchResult(res, fn) {
+        try {
+            return fn();
+        }
+        catch {
+            return { ...res, ok: false };
+        }
+    }
+    function renameTopic(topicId, label) {
+        const base = {
+            ok: false,
+            topicId,
+            label,
+            edited: false,
+            merged: false,
+            split: false,
+        };
+        if (!getTopicLabel(db, topicId))
+            return base;
+        return tryCatchResult(base, () => {
+            withTx(db, () => writeLabelOverride(db, topicId, label, now()));
+            return { ...base, ok: true, edited: label.trim() !== "" };
+        });
+    }
+    function mergeTopics(sourceTopicId, targetTopicId) {
+        const base = {
+            ok: false,
+            topicId: targetTopicId,
+            label: "",
+            edited: false,
+            merged: true,
+            split: false,
+        };
+        if (sourceTopicId === targetTopicId)
+            return base;
+        const sourceLabel = getTopicLabel(db, sourceTopicId);
+        const targetLabel = getTopicLabel(db, targetTopicId);
+        if (!sourceLabel || !targetLabel)
+            return base;
+        return tryCatchResult(base, () => {
+            withTx(db, () => {
+                const t = now();
+                const members = db
+                    .prepare(`SELECT memory_id, session_id FROM memory_topics
+						 WHERE topic_id = ? ORDER BY memory_id ASC`)
+                    .all(sourceTopicId);
+                for (const m of members) {
+                    moveMemory(db, m.memory_id, targetTopicId);
+                    recordEvolution(db, targetTopicId, m.memory_id, m.session_id, t, "merge");
+                }
+                db.prepare(`INSERT OR REPLACE INTO topic_overrides
+					   (topic_id, kind, merged_into, merged_memory_ids, overridden_at)
+					 VALUES (?, 'merge', ?, ?, ?)`).run(sourceTopicId, targetTopicId, JSON.stringify(members.map((m) => m.memory_id)), t);
+                db.prepare("DELETE FROM memory_topics WHERE topic_id = ?").run(sourceTopicId);
+                db.prepare("DELETE FROM topics WHERE id = ?").run(sourceTopicId);
+                recountTopic(db, targetTopicId);
+            });
+            return {
+                ...base,
+                ok: true,
+                label: getTopicLabel(db, targetTopicId) ?? "",
+                edited: customLabelOverride(db, targetTopicId) !== null,
+            };
+        });
+    }
+    function splitTopic(topicId, memoryIds) {
+        const base = {
+            ok: false,
+            topicId,
+            label: "",
+            edited: false,
+            merged: false,
+            split: true,
+        };
+        const sourceLabel = getTopicLabel(db, topicId);
+        if (!sourceLabel || memoryIds.length === 0)
+            return base;
+        return tryCatchResult(base, () => {
+            let newTopicId = "";
+            withTx(db, () => {
+                const t = now();
+                newTopicId = createSplitTopic(db, topicId, sourceLabel, t);
+                for (const memoryId of memoryIds) {
+                    const row = db
+                        .prepare("SELECT session_id FROM memory_topics WHERE memory_id = ? AND topic_id = ?")
+                        .get(memoryId, topicId);
+                    if (!row)
+                        continue;
+                    moveMemory(db, memoryId, newTopicId);
+                    recordEvolution(db, newTopicId, memoryId, row.session_id, t, "split");
+                    recordEvolution(db, topicId, memoryId, row.session_id, t, "split");
+                }
+                db.prepare(`INSERT OR REPLACE INTO topic_overrides
+					   (topic_id, kind, split_from, split_memory_ids, overridden_at)
+					 VALUES (?, 'split', ?, ?, ?)`).run(newTopicId, topicId, JSON.stringify(memoryIds), t);
+                recountTopic(db, topicId);
+                recountTopic(db, newTopicId);
+            });
+            return {
+                ...base,
+                ok: true,
+                topicId: newTopicId,
+                label: getTopicLabel(db, newTopicId) ?? "",
+            };
+        });
+    }
+    function resolveLabel(topicId, autoLabel) {
+        const custom = customLabelOverride(db, topicId);
+        if (custom)
+            return { label: custom, edited: true };
+        return { label: autoLabel, edited: false };
+    }
+    function overrideKinds(topicId) {
+        const rows = db
+            .prepare("SELECT kind FROM topic_overrides WHERE topic_id = ? ORDER BY kind")
+            .all(topicId);
+        return rows.map((r) => r.kind);
+    }
+    return {
+        renameTopic,
+        mergeTopics,
+        splitTopic,
+        resolveLabel,
+        overrideKinds,
+    };
+}

package/dist/src/wiki/incremental.js

@@ -0,0 +1,126 @@
+import { cosineSimilarity } from "../embedder.js";
+import { loadEmbeddings } from "../topics/cluster.js";
+import { silhouette } from "../topics/kselection.js";
+import { openTurnStore, withTx } from "../store/turns/connection.js";
+/** Min cosine to a centroid before a chunk is admitted to a topic. */
+const ASSIGN_SIM_FLOOR = 0.3;
+/** Average of vectors → the centroid for a topic's current members. */
+function meanOf(vectors) {
+    const dim = vectors[0]?.length ?? 0;
+    const centroid = new Array(dim).fill(0);
+    for (const v of vectors) {
+        for (let i = 0; i < dim; i++)
+            centroid[i] += v[i];
+    }
+    return centroid.map((x) => x / Math.max(1, vectors.length));
+}
+/**
+ * Assign new memories (not already in memory_topics) to their nearest existing
+ * topic centroid without a full rebuild. `db` is the main memory store whose
+ * `context_chunks` holds the embeddings; `stateDir` locates the turns.db where
+ * `topics` / `memory_topics` live. Writes are best-effort + non-fatal.
+ */
+export function assignNewMemoriesIncremental(db, stateDir) {
+    const result = {
+        assigned: 0,
+        pending: 0,
+        silhouette: null,
+    };
+    try {
+        const turnDb = openTurnStore(stateDir);
+        const assignedRows = turnDb
+            .prepare("SELECT memory_id, topic_id FROM memory_topics")
+            .all();
+        const chunks = loadEmbeddings(db);
+        if (chunks.length === 0)
+            return result;
+        // Map memory_id → chunk for both centroid build + unassigned scan.
+        const chunkById = new Map();
+        for (const c of chunks)
+            chunkById.set(c.chunkId, c);
+        const assignedIds = new Set(assignedRows.map((r) => r.memory_id));
+        const centroids = new Map();
+        // Accumulate real member vectors per topic from assigned, embedded chunks.
+        const byTopic = new Map();
+        for (const r of assignedRows) {
+            const c = chunkById.get(r.memory_id);
+            if (!c)
+                continue;
+            const list = byTopic.get(r.topic_id) ?? [];
+            list.push(c.vec);
+            byTopic.set(r.topic_id, list);
+        }
+        for (const [topicId, vecs] of byTopic) {
+            if (vecs.length > 0)
+                centroids.set(topicId, meanOf(vecs));
+        }
+        // Assign unassigned chunks to the nearest non-empty centroid.
+        const assignmentsToWrite = new Map();
+        for (const c of chunks) {
+            if (assignedIds.has(c.chunkId))
+                continue;
+            let bestTopic = "";
+            let bestSim = ASSIGN_SIM_FLOOR;
+            for (const [topicId, centroid] of centroids) {
+                const sim = cosineSimilarity(c.vec, centroid);
+                if (sim > bestSim) {
+                    bestSim = sim;
+                    bestTopic = topicId;
+                }
+            }
+            if (bestTopic) {
+                assignmentsToWrite.set(c.chunkId, bestTopic);
+            }
+            else {
+                result.pending++;
+            }
+        }
+        // Persist new assignments atomically + best-effort.
+        if (assignmentsToWrite.size > 0) {
+            try {
+                withTx(turnDb, () => {
+                    const ins = turnDb.prepare(`INSERT OR REPLACE INTO memory_topics
+						   (memory_id, topic_id, confidence, assigned_at, method, session_id)
+						 VALUES (?, ?, ?, ?, 'kmeans+tfidf', ?)`);
+                    const now = Date.now();
+                    for (const [memoryId, topicId] of assignmentsToWrite) {
+                        const cent = centroids.get(topicId);
+                        const c = chunkById.get(memoryId);
+                        const sim = cent && c ? cosineSimilarity(c.vec, cent) : 0;
+                        ins.run(memoryId, topicId, sim, now, c?.sessionId ?? null);
+                    }
+                });
+                result.assigned = assignmentsToWrite.size;
+            }
+            catch {
+                /* non-fatal: assignment write failure never breaks the loop */
+            }
+        }
+        // Compute mean silhouette over current + newly assigned memberships.
+        const allPoints = [];
+        const assignments = [];
+        const topicIndex = new Map();
+        const lookup = new Map();
+        for (const r of assignedRows)
+            lookup.set(r.memory_id, { topic: r.topic_id });
+        for (const c of chunks) {
+            const topic = lookup.has(c.chunkId)
+                ? lookup.get(c.chunkId).topic
+                : assignmentsToWrite.get(c.chunkId);
+            if (!topic)
+                continue;
+            if (!topicIndex.has(topic))
+                topicIndex.set(topic, topicIndex.size);
+            allPoints.push(c.vec);
+            assignments.push(topicIndex.get(topic));
+        }
+        if (allPoints.length > 0) {
+            result.silhouette = silhouette(allPoints, assignments, topicIndex.size);
+        }
+        return result;
+    }
+    catch {
+        /* non-fatal: incremental assignment never breaks compaction */
+        return result;
+    }
+}

package/dist/src/wiki/index.js

@@ -0,0 +1,5 @@
+/**
+ * index.ts — wiki barrel. Re-export only; no logic.
+ */
+export { createWikiCuration, } from "./curation.js";
+export { assignNewMemoriesIncremental, } from "./incremental.js";