presubmit 0.1.0

package/dist/apps/server/src/semantic-history.js

@@ -0,0 +1,186 @@
+import { createHash } from 'node:crypto';
+import { mkdir } from 'node:fs/promises';
+const digest = (value) => createHash('sha256').update(value).digest('hex');
+const columns = ['sourceId', 'sessionId', 'sessionTitle', 'runId', 'kind', 'createdAt', 'text'];
+const quote = (value) => `'${value.replaceAll("'", "''")}'`;
+/** One rebuildable LanceDB index per project; shared by the search dialog and agent RAG. */
+export class SemanticHistory {
+    directory;
+    embeddings;
+    sources;
+    connection;
+    table;
+    opening;
+    pending;
+    timer;
+    stop = new AbortController();
+    count = 0;
+    progress;
+    error;
+    vectors = new Map();
+    searches = new Set();
+    name = '';
+    constructor(directory, embeddings, sources) {
+        this.directory = directory;
+        this.embeddings = embeddings;
+        this.sources = sources;
+    }
+    status() {
+        return {
+            enabled: true,
+            indexing: !!this.pending,
+            indexedChunks: this.count,
+            progress: this.progress && { ...this.progress },
+            model: this.embeddings.model,
+            error: this.error,
+        };
+    }
+    async open() {
+        this.stop.signal.throwIfAborted();
+        if (!this.opening) {
+            this.opening = (async () => {
+                const identity = await this.embeddings.identity(this.stop.signal);
+                this.name = `history_v1_${digest(identity).slice(0, 16)}`;
+                await mkdir(this.directory, { recursive: true, mode: 0o700 });
+                const { connect } = await import('#presubmit/lancedb');
+                this.connection = await connect(this.directory);
+                if ((await this.connection.tableNames()).includes(this.name)) {
+                    this.table = await this.connection.openTable(this.name);
+                    this.count = await this.table.countRows();
+                }
+            })().catch((error) => {
+                this.table?.close();
+                this.connection?.close();
+                this.table = undefined;
+                this.connection = undefined;
+                this.count = 0;
+                this.opening = undefined;
+                throw error;
+            });
+        }
+        await this.opening;
+    }
+    start() {
+        if (this.timer || this.stop.signal.aborted)
+            return;
+        void this.refresh();
+        this.timer = setInterval(() => {
+            void this.refresh();
+        }, 15000);
+        this.timer.unref();
+    }
+    /** Serialize background updates; unchanged chunks never need another embedding. */
+    refresh() {
+        if (this.stop.signal.aborted)
+            return Promise.resolve();
+        if (this.pending)
+            return this.pending;
+        this.progress = undefined;
+        this.pending = this.sync()
+            .then(() => {
+            this.error = undefined;
+        })
+            .catch(() => {
+            if (!this.stop.signal.aborted)
+                this.error =
+                    'Related search is unavailable. Check the local embedding service and model.';
+        })
+            .finally(() => {
+            this.pending = undefined;
+        });
+        return this.pending;
+    }
+    async sync() {
+        await this.open();
+        const docs = this.sources();
+        const existing = new Map();
+        if (this.table)
+            for (const row of await this.table.query().select(['sourceId', 'digest']).toArray())
+                existing.set(row.sourceId, row.digest);
+        const changed = docs
+            .map((doc) => ({ ...doc, digest: digest(JSON.stringify(doc)) }))
+            .filter((doc) => existing.get(doc.sourceId) !== doc.digest);
+        const progress = { completed: docs.length - changed.length, total: docs.length };
+        this.progress = progress;
+        for (let offset = 0; offset < changed.length; offset += 16) {
+            this.stop.signal.throwIfAborted();
+            const batch = changed.slice(offset, offset + 16);
+            const vectors = await this.embeddings.embed(batch.map((d) => d.text), 'document', this.stop.signal);
+            this.stop.signal.throwIfAborted();
+            const rows = batch.map((doc, i) => ({ ...doc, vector: vectors[i] }));
+            if (!this.table)
+                this.table = await this.connection.createTable(this.name, rows);
+            else
+                await this.table
+                    .mergeInsert('sourceId')
+                    .whenMatchedUpdateAll()
+                    .whenNotMatchedInsertAll()
+                    .execute(rows);
+            this.count = await this.table.countRows();
+            progress.completed += batch.length;
+            await new Promise((resolve) => setImmediate(resolve));
+        }
+        const ids = new Set(docs.map((doc) => doc.sourceId));
+        const removed = [...existing.keys()].filter((id) => !ids.has(id));
+        for (let offset = 0; offset < removed.length; offset += 100)
+            await this.table.delete(`sourceId IN (${removed
+                .slice(offset, offset + 100)
+                .map(quote)
+                .join(',')})`);
+        if (this.table)
+            this.count = await this.table.countRows();
+    }
+    search(query, limit = 5, excludeRunId) {
+        const pending = this.query(query, limit, excludeRunId);
+        this.searches.add(pending);
+        void pending.finally(() => this.searches.delete(pending)).catch(() => { });
+        return pending;
+    }
+    async query(query, limit, excludeRunId) {
+        const text = query.trim();
+        if (!text || text.length > 200 || !Number.isInteger(limit) || limit < 1 || limit > 10)
+            throw new Error('Use a query of 1–200 characters and a limit of 1–10.');
+        await this.open();
+        if (!this.table) {
+            if (this.error)
+                throw new Error(this.error);
+            if (this.pending)
+                throw new Error('Conversation history is still being indexed. Try again shortly.');
+            return [];
+        }
+        let vector = this.vectors.get(text);
+        if (!vector) {
+            [vector] = await this.embeddings.embed([text], 'query', this.stop.signal);
+            this.vectors.set(text, vector);
+            if (this.vectors.size > 32)
+                this.vectors.delete(this.vectors.keys().next().value);
+        }
+        let search = this.table
+            .vectorSearch(vector)
+            .distanceType('cosine')
+            .select([...columns, '_distance'])
+            .limit(limit);
+        if (excludeRunId)
+            search = search.where(`runId != ${quote(excludeRunId)}`);
+        return (await search.toArray()).map((row) => ({
+            sourceId: row.sourceId,
+            sessionId: row.sessionId,
+            sessionTitle: row.sessionTitle,
+            runId: row.runId,
+            kind: row.kind,
+            createdAt: row.createdAt,
+            text: row.text,
+            score: Math.max(0, Math.min(1, 1 - row._distance)),
+        }));
+    }
+    async close() {
+        this.stop.abort();
+        if (this.timer)
+            clearInterval(this.timer);
+        await this.pending;
+        await this.opening?.catch(() => { });
+        await Promise.allSettled(this.searches);
+        this.table?.close();
+        this.connection?.close();
+    }
+}