pulseed 0.5.3 → 0.5.4

package/dist/platform/soil/sqlite-repository.js

@@ -2,336 +2,11 @@ import * as fsp from "node:fs/promises";
 import * as path from "node:path";
 import { randomUUID } from "node:crypto";
 import Database from "better-sqlite3";
-import { cosineSimilarity } from "../knowledge/embedding-client.js";
 import { getDefaultSoilSqliteIndexPath, resolveSoilRootDir } from "./config.js";
-import { SoilEmbeddingSchema, SoilMutationSchema, SoilPageMemberSchema, SoilPageSchema, SoilRecordFilterSchema, SoilRecordSchema, SoilSearchRequestSchema, } from "./contracts.js";
-import { SOIL_SCHEMA_SQL } from "./ddl.js";
-function buildSnippet(text, query) {
-    const haystack = text.trim();
-    if (!haystack)
-        return "";
-    const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
-    const lower = haystack.toLowerCase();
-    for (const token of tokens) {
-        const index = lower.indexOf(token);
-        if (index >= 0) {
-            const start = Math.max(0, index - 50);
-            const end = Math.min(haystack.length, index + token.length + 100);
-            return haystack.slice(start, end);
-        }
-    }
-    return haystack.slice(0, 160);
-}
-function serializeJson(value) {
-    return JSON.stringify(value ?? {});
-}
-function parseJsonObject(input) {
-    try {
-        const parsed = JSON.parse(input);
-        return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
-            ? parsed
-            : {};
-    }
-    catch {
-        return {};
-    }
-}
-function parseJsonArray(input) {
-    try {
-        const parsed = JSON.parse(input);
-        return Array.isArray(parsed) ? parsed.filter((item) => typeof item === "string") : [];
-    }
-    catch {
-        return [];
-    }
-}
-function parseReindexRecordIds(input) {
-    try {
-        const payload = JSON.parse(input);
-        if (typeof payload !== "object" || payload === null || Array.isArray(payload)) {
-            return [];
-        }
-        const value = payload.record_ids;
-        return Array.isArray(value)
-            ? value.filter((recordId) => typeof recordId === "string" && recordId.length > 0)
-            : [];
-    }
-    catch {
-        return [];
-    }
-}
-function encodeEmbedding(entry) {
-    if (entry.encoding === "f32le") {
-        const floats = entry.embedding instanceof Uint8Array
-            ? new Float32Array(entry.embedding.buffer.slice(entry.embedding.byteOffset, entry.embedding.byteOffset + entry.embedding.byteLength))
-            : Float32Array.from(entry.embedding);
-        return Buffer.from(floats.buffer.slice(floats.byteOffset, floats.byteOffset + floats.byteLength));
-    }
-    const payload = entry.embedding instanceof Uint8Array
-        ? Array.from(entry.embedding.values())
-        : entry.embedding;
-    return Buffer.from(JSON.stringify(payload), "utf8");
-}
-function decodeEmbedding(row) {
-    if (row.encoding === "f32le") {
-        const copy = row.embedding.buffer.slice(row.embedding.byteOffset, row.embedding.byteOffset + row.embedding.byteLength);
-        return Array.from(new Float32Array(copy));
-    }
-    return JSON.parse(row.embedding.toString("utf8"));
-}
-function toRecord(row) {
-    return SoilRecordSchema.parse({
-        ...row,
-        is_active: Boolean(row.is_active),
-        metadata_json: parseJsonObject(row.metadata_json),
-    });
-}
-function toPage(row) {
-    return SoilPageSchema.parse(row);
-}
-function toPageMember(row) {
-    return SoilPageMemberSchema.parse(row);
-}
-function unique(values) {
-    return [...new Set(values.filter((value) => value.length > 0))];
-}
-function dedupeCandidates(candidates, limit) {
-    const byChunkId = new Map();
-    for (const candidate of candidates) {
-        const current = byChunkId.get(candidate.chunk_id);
-        if (!current ||
-            candidate.score > current.score ||
-            (candidate.score === current.score && current.page_id === null && candidate.page_id !== null)) {
-            byChunkId.set(candidate.chunk_id, candidate);
-        }
-    }
-    return [...byChunkId.values()]
-        .sort((left, right) => right.score - left.score || left.chunk_id.localeCompare(right.chunk_id))
-        .slice(0, limit)
-        .map((candidate, index) => ({ ...candidate, rank: index + 1 }));
-}
-function fuseCandidates(lexical, dense, limit, rrfK = 60) {
-    const byChunkId = new Map();
-    for (const [lane, weight, candidates] of [
-        ["lexical", 1, lexical],
-        ["dense", 0.85, dense],
-    ]) {
-        for (const candidate of candidates) {
-            const prior = byChunkId.get(candidate.chunk_id);
-            const laneScore = weight / (rrfK + candidate.rank);
-            const metadata_json = {
-                ...(prior?.metadata_json ?? candidate.metadata_json),
-                [`${lane}_rank`]: candidate.rank,
-                [`${lane}_score`]: candidate.score,
-            };
-            if (!prior) {
-                byChunkId.set(candidate.chunk_id, {
-                    ...candidate,
-                    lane: "hybrid",
-                    score: laneScore,
-                    metadata_json,
-                });
-                continue;
-            }
-            byChunkId.set(candidate.chunk_id, {
-                ...prior,
-                page_id: prior.page_id ?? candidate.page_id,
-                snippet: prior.snippet ?? candidate.snippet,
-                score: prior.score + laneScore,
-                metadata_json,
-            });
-        }
-    }
-    return [...byChunkId.values()]
-        .sort((left, right) => right.score - left.score || left.chunk_id.localeCompare(right.chunk_id))
-        .slice(0, limit)
-        .map((candidate, index) => ({ ...candidate, rank: index + 1 }));
-}
-function hasExplicitMetadataFilter(request) {
-    const recordFilter = request.record_filter;
-    const pageFilter = request.page_filter;
-    return Boolean(recordFilter.record_ids?.length ||
-        recordFilter.record_keys?.length ||
-        recordFilter.record_types?.length ||
-        recordFilter.statuses?.length ||
-        recordFilter.goal_ids?.length ||
-        recordFilter.task_ids?.length ||
-        recordFilter.source_types?.length ||
-        recordFilter.source_ids?.length ||
-        recordFilter.valid_at ||
-        recordFilter.updated_after ||
-        recordFilter.updated_before ||
-        pageFilter.page_ids?.length ||
-        pageFilter.soil_ids?.length ||
-        pageFilter.routes?.length ||
-        pageFilter.kinds?.length ||
-        pageFilter.page_statuses?.length ||
-        pageFilter.relative_paths?.length);
-}
-function buildRecordFilterSql(request, params) {
-    const clauses = [];
-    const filter = request.record_filter;
-    if (filter.active_only) {
-        clauses.push("r.is_active = 1");
-    }
-    if (filter.record_ids?.length) {
-        clauses.push(`r.record_id IN (${filter.record_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.record_ids);
-    }
-    if (filter.record_keys?.length) {
-        clauses.push(`r.record_key IN (${filter.record_keys.map(() => "?").join(", ")})`);
-        params.push(...filter.record_keys);
-    }
-    if (filter.record_types?.length) {
-        clauses.push(`r.record_type IN (${filter.record_types.map(() => "?").join(", ")})`);
-        params.push(...filter.record_types);
-    }
-    if (filter.statuses?.length) {
-        clauses.push(`r.status IN (${filter.statuses.map(() => "?").join(", ")})`);
-        params.push(...filter.statuses);
-    }
-    if (filter.goal_ids?.length) {
-        clauses.push(`r.goal_id IN (${filter.goal_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.goal_ids);
-    }
-    if (filter.task_ids?.length) {
-        clauses.push(`r.task_id IN (${filter.task_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.task_ids);
-    }
-    if (filter.source_types?.length) {
-        clauses.push(`r.source_type IN (${filter.source_types.map(() => "?").join(", ")})`);
-        params.push(...filter.source_types);
-    }
-    if (filter.source_ids?.length) {
-        clauses.push(`r.source_id IN (${filter.source_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.source_ids);
-    }
-    if (filter.valid_at) {
-        clauses.push("(r.valid_from IS NULL OR r.valid_from <= ?)");
-        clauses.push("(r.valid_to IS NULL OR r.valid_to > ?)");
-        params.push(filter.valid_at, filter.valid_at);
-    }
-    if (filter.updated_after) {
-        clauses.push("r.updated_at >= ?");
-        params.push(filter.updated_after);
-    }
-    if (filter.updated_before) {
-        clauses.push("r.updated_at <= ?");
-        params.push(filter.updated_before);
-    }
-    return clauses;
-}
-function buildPageFilterSql(request, params) {
-    const clauses = [];
-    const filter = request.page_filter;
-    if (filter.page_ids?.length) {
-        clauses.push(`p.page_id IN (${filter.page_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.page_ids);
-    }
-    if (filter.soil_ids?.length) {
-        clauses.push(`p.soil_id IN (${filter.soil_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.soil_ids);
-    }
-    if (filter.routes?.length) {
-        clauses.push(`p.route IN (${filter.routes.map(() => "?").join(", ")})`);
-        params.push(...filter.routes);
-    }
-    if (filter.kinds?.length) {
-        clauses.push(`p.kind IN (${filter.kinds.map(() => "?").join(", ")})`);
-        params.push(...filter.kinds);
-    }
-    if (filter.page_statuses?.length) {
-        clauses.push(`p.status IN (${filter.page_statuses.map(() => "?").join(", ")})`);
-        params.push(...filter.page_statuses);
-    }
-    if (filter.relative_paths?.length) {
-        clauses.push(`p.relative_path IN (${filter.relative_paths.map(() => "?").join(", ")})`);
-        params.push(...filter.relative_paths);
-    }
-    return clauses;
-}
-function buildPageExistsSql(recordIdExpr, request, params) {
-    const filter = request.page_filter;
-    const pagePredicates = [];
-    if (filter.page_ids?.length) {
-        pagePredicates.push(`p.page_id IN (${filter.page_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.page_ids);
-    }
-    if (filter.soil_ids?.length) {
-        pagePredicates.push(`p.soil_id IN (${filter.soil_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.soil_ids);
-    }
-    if (filter.routes?.length) {
-        pagePredicates.push(`p.route IN (${filter.routes.map(() => "?").join(", ")})`);
-        params.push(...filter.routes);
-    }
-    if (filter.kinds?.length) {
-        pagePredicates.push(`p.kind IN (${filter.kinds.map(() => "?").join(", ")})`);
-        params.push(...filter.kinds);
-    }
-    if (filter.page_statuses?.length) {
-        pagePredicates.push(`p.status IN (${filter.page_statuses.map(() => "?").join(", ")})`);
-        params.push(...filter.page_statuses);
-    }
-    if (filter.relative_paths?.length) {
-        pagePredicates.push(`p.relative_path IN (${filter.relative_paths.map(() => "?").join(", ")})`);
-        params.push(...filter.relative_paths);
-    }
-    if (pagePredicates.length === 0) {
-        return [];
-    }
-    return [
-        `EXISTS (
-      SELECT 1
-      FROM soil_page_members spm
-      JOIN soil_pages p ON p.page_id = spm.page_id
-      WHERE spm.record_id = ${recordIdExpr}
-        AND ${pagePredicates.join(" AND ")}
-    )`,
-    ];
-}
-function buildCandidatePageIdSql(recordIdExpr, request, params) {
-    const filter = request.page_filter;
-    const pagePredicates = [];
-    if (filter.page_ids?.length) {
-        pagePredicates.push(`p.page_id IN (${filter.page_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.page_ids);
-    }
-    if (filter.soil_ids?.length) {
-        pagePredicates.push(`p.soil_id IN (${filter.soil_ids.map(() => "?").join(", ")})`);
-        params.push(...filter.soil_ids);
-    }
-    if (filter.routes?.length) {
-        pagePredicates.push(`p.route IN (${filter.routes.map(() => "?").join(", ")})`);
-        params.push(...filter.routes);
-    }
-    if (filter.kinds?.length) {
-        pagePredicates.push(`p.kind IN (${filter.kinds.map(() => "?").join(", ")})`);
-        params.push(...filter.kinds);
-    }
-    if (filter.page_statuses?.length) {
-        pagePredicates.push(`p.status IN (${filter.page_statuses.map(() => "?").join(", ")})`);
-        params.push(...filter.page_statuses);
-    }
-    if (filter.relative_paths?.length) {
-        pagePredicates.push(`p.relative_path IN (${filter.relative_paths.map(() => "?").join(", ")})`);
-        params.push(...filter.relative_paths);
-    }
-    const baseQuery = pagePredicates.length > 0
-        ? `SELECT spm.page_id
-       FROM soil_page_members spm
-       JOIN soil_pages p ON p.page_id = spm.page_id
-       WHERE spm.record_id = ${recordIdExpr}
-         AND ${pagePredicates.join(" AND ")}
-       ORDER BY CASE WHEN spm.role = 'primary' THEN 0 ELSE 1 END, spm.ordinal, p.relative_path, spm.page_id
-       LIMIT 1`
-        : `SELECT spm.page_id
-       FROM soil_page_members spm
-       WHERE spm.record_id = ${recordIdExpr}
-       ORDER BY CASE WHEN spm.role = 'primary' THEN 0 ELSE 1 END, spm.ordinal, spm.page_id
-       LIMIT 1`;
-    return `(${baseQuery})`;
-}
+import { SoilRecordFilterSchema, SoilSearchRequestSchema, } from "./contracts.js";
+import { fuseCandidates, toPage, toPageMember, toRecord, unique } from "./sqlite-repository-helpers.js";
+import { buildRecordFilterSql, lookupDirectCandidates, searchDenseCandidates, searchLexicalCandidates, shouldRunDenseSearch, } from "./sqlite-repository-search.js";
+import { applySoilMutation, initializeReadonlySoilSqlite, initializeSoilSqlite, loadOpenEmbeddingReindexRecordIds, replaceSoilPageMembers, } from "./sqlite-repository-storage.js";
 export class SqliteSoilRepository {
     db;
     dbPath;
@@ -344,195 +19,27 @@ export class SqliteSoilRepository {
         const indexPath = configInput.indexPath ? path.resolve(configInput.indexPath) : getDefaultSoilSqliteIndexPath(rootDir);
         await fsp.mkdir(path.dirname(indexPath), { recursive: true });
         const db = new Database(indexPath);
-        db.pragma("journal_mode = WAL");
-        db.pragma("foreign_keys = ON");
-        db.exec(SOIL_SCHEMA_SQL);
+        initializeSoilSqlite(db);
+        return new SqliteSoilRepository(db, indexPath);
+    }
+    static async openExisting(configInput = {}) {
+        const rootDir = resolveSoilRootDir(configInput.rootDir);
+        const indexPath = configInput.indexPath ? path.resolve(configInput.indexPath) : getDefaultSoilSqliteIndexPath(rootDir);
+        try {
+            await fsp.access(indexPath);
+        }
+        catch {
+            return null;
+        }
+        const db = new Database(indexPath, { readonly: true, fileMustExist: true });
+        initializeReadonlySoilSqlite(db);
         return new SqliteSoilRepository(db, indexPath);
     }
     close() {
         this.db.close();
     }
     async applyMutation(input) {
-        const mutation = SoilMutationSchema.parse(input);
-        const contentMutatedRecordIds = new Set();
-        const embeddedChunkIds = new Set();
-        const embeddedRecordIds = new Set();
-        const tx = this.db.transaction(() => {
-            for (const record of mutation.records) {
-                if (record.is_active) {
-                    this.db
-                        .prepare("UPDATE soil_records SET is_active = 0 WHERE record_key = ? AND record_id != ? AND is_active = 1")
-                        .run(record.record_key, record.record_id);
-                }
-                this.db.prepare(`
-          INSERT INTO soil_records (
-            record_id, record_key, version, record_type, soil_id, title, summary, canonical_text,
-            goal_id, task_id, status, confidence, importance, source_reliability,
-            valid_from, valid_to, supersedes_record_id, is_active, source_type, source_id,
-            metadata_json, created_at, updated_at
-          ) VALUES (
-            @record_id, @record_key, @version, @record_type, @soil_id, @title, @summary, @canonical_text,
-            @goal_id, @task_id, @status, @confidence, @importance, @source_reliability,
-            @valid_from, @valid_to, @supersedes_record_id, @is_active, @source_type, @source_id,
-            @metadata_json, @created_at, @updated_at
-          )
-          ON CONFLICT(record_id) DO UPDATE SET
-            record_key = excluded.record_key,
-            version = excluded.version,
-            record_type = excluded.record_type,
-            soil_id = excluded.soil_id,
-            title = excluded.title,
-            summary = excluded.summary,
-            canonical_text = excluded.canonical_text,
-            goal_id = excluded.goal_id,
-            task_id = excluded.task_id,
-            status = excluded.status,
-            confidence = excluded.confidence,
-            importance = excluded.importance,
-            source_reliability = excluded.source_reliability,
-            valid_from = excluded.valid_from,
-            valid_to = excluded.valid_to,
-            supersedes_record_id = excluded.supersedes_record_id,
-            is_active = excluded.is_active,
-            source_type = excluded.source_type,
-            source_id = excluded.source_id,
-            metadata_json = excluded.metadata_json,
-            created_at = excluded.created_at,
-            updated_at = excluded.updated_at
-        `).run({
-                    ...record,
-                    is_active: record.is_active ? 1 : 0,
-                    metadata_json: serializeJson(record.metadata_json),
-                });
-                contentMutatedRecordIds.add(record.record_id);
-            }
-            for (const chunk of mutation.chunks) {
-                this.db.prepare(`
-          INSERT INTO soil_chunks (
-            chunk_id, record_id, soil_id, chunk_index, chunk_kind, heading_path_json,
-            chunk_text, token_count, checksum, created_at
-          ) VALUES (
-            @chunk_id, @record_id, @soil_id, @chunk_index, @chunk_kind, @heading_path_json,
-            @chunk_text, @token_count, @checksum, @created_at
-          )
-          ON CONFLICT(chunk_id) DO UPDATE SET
-            record_id = excluded.record_id,
-            soil_id = excluded.soil_id,
-            chunk_index = excluded.chunk_index,
-            chunk_kind = excluded.chunk_kind,
-            heading_path_json = excluded.heading_path_json,
-            chunk_text = excluded.chunk_text,
-            token_count = excluded.token_count,
-            checksum = excluded.checksum,
-            created_at = excluded.created_at
-        `).run({
-                    ...chunk,
-                    heading_path_json: serializeJson(chunk.heading_path_json),
-                });
-                contentMutatedRecordIds.add(chunk.record_id);
-            }
-            for (const page of mutation.pages) {
-                this.db.prepare(`
-          INSERT INTO soil_pages (
-            page_id, soil_id, relative_path, route, kind, status, markdown, checksum, projected_at
-          ) VALUES (
-            @page_id, @soil_id, @relative_path, @route, @kind, @status, @markdown, @checksum, @projected_at
-          )
-          ON CONFLICT(page_id) DO UPDATE SET
-            soil_id = excluded.soil_id,
-            relative_path = excluded.relative_path,
-            route = excluded.route,
-            kind = excluded.kind,
-            status = excluded.status,
-            markdown = excluded.markdown,
-            checksum = excluded.checksum,
-            projected_at = excluded.projected_at
-        `).run(page);
-            }
-            for (const member of mutation.page_members) {
-                this.db.prepare(`
-          INSERT INTO soil_page_members (page_id, record_id, ordinal, role, confidence)
-          VALUES (@page_id, @record_id, @ordinal, @role, @confidence)
-          ON CONFLICT(page_id, record_id, role) DO UPDATE SET
-            ordinal = excluded.ordinal,
-            confidence = excluded.confidence
-        `).run(member);
-            }
-            for (const embedding of mutation.embeddings) {
-                const parsed = SoilEmbeddingSchema.parse(embedding);
-                this.db.prepare(`
-          INSERT INTO soil_embeddings (
-            chunk_id, model, embedding_version, encoding, embedding, embedded_at
-          ) VALUES (
-            @chunk_id, @model, @embedding_version, @encoding, @embedding, @embedded_at
-          )
-          ON CONFLICT(chunk_id, model, embedding_version) DO UPDATE SET
-            encoding = excluded.encoding,
-            embedding = excluded.embedding,
-            embedded_at = excluded.embedded_at
-        `).run({
-                    chunk_id: parsed.chunk_id,
-                    model: parsed.model,
-                    embedding_version: parsed.embedding_version,
-                    encoding: parsed.encoding,
-                    embedding: encodeEmbedding(parsed),
-                    embedded_at: parsed.embedded_at,
-                });
-                const chunk = this.db
-                    .prepare("SELECT record_id FROM soil_chunks WHERE chunk_id = ?")
-                    .get(parsed.chunk_id);
-                if (chunk) {
-                    embeddedChunkIds.add(parsed.chunk_id);
-                    embeddedRecordIds.add(chunk.record_id);
-                }
-            }
-            for (const edge of mutation.edges) {
-                this.db.prepare(`
-          INSERT INTO soil_edges (src_record_id, edge_type, dst_record_id, confidence)
-          VALUES (@src_record_id, @edge_type, @dst_record_id, @confidence)
-          ON CONFLICT(src_record_id, edge_type, dst_record_id) DO UPDATE SET
-            confidence = excluded.confidence
-        `).run(edge);
-            }
-            for (const tombstone of mutation.tombstones) {
-                this.db.prepare(`
-          INSERT INTO soil_tombstones (tombstone_id, record_id, record_key, version, reason, deleted_at)
-          VALUES (?, ?, ?, ?, ?, ?)
-        `).run(randomUUID(), tombstone.record_id, tombstone.record_key, tombstone.version, tombstone.reason, tombstone.deleted_at);
-                if (tombstone.record_id) {
-                    this.db
-                        .prepare("UPDATE soil_records SET is_active = 0 WHERE record_id = ?")
-                        .run(tombstone.record_id);
-                    contentMutatedRecordIds.add(tombstone.record_id);
-                }
-                if (tombstone.record_key) {
-                    this.db
-                        .prepare("UPDATE soil_records SET is_active = 0 WHERE record_key = ?")
-                        .run(tombstone.record_key);
-                    const rows = this.db
-                        .prepare("SELECT record_id FROM soil_records WHERE record_key = ?")
-                        .all(tombstone.record_key);
-                    for (const row of rows) {
-                        contentMutatedRecordIds.add(row.record_id);
-                    }
-                }
-            }
-            this.syncFts(unique([...contentMutatedRecordIds]));
-            const fullyEmbeddedRecordIds = new Set(unique([...embeddedRecordIds]).filter((recordId) => this.recordHasCompleteEmbeddingMutation(recordId, embeddedChunkIds)));
-            this.completeOpenEmbeddingJobs(fullyEmbeddedRecordIds);
-            if (contentMutatedRecordIds.size > 0) {
-                const openEmbeddingRecordIds = new Set(this.loadOpenEmbeddingReindexRecordIds());
-                const insertJob = this.db.prepare(`
-          INSERT INTO soil_reindex_jobs (
-            job_id, scope, reason, status, payload_json, created_at
-          ) VALUES (?, ?, ?, ?, ?, ?)
-        `);
-                for (const recordId of unique([...contentMutatedRecordIds]).filter((id) => !fullyEmbeddedRecordIds.has(id) && !openEmbeddingRecordIds.has(id))) {
-                    insertJob.run(randomUUID(), "embedding", "content mutation invalidated embeddings", "pending", JSON.stringify({ record_ids: [recordId] }), new Date().toISOString());
-                }
-            }
-        });
-        tx();
+        applySoilMutation(this.db, input);
     }
     async loadRecords(input = {}) {
         const record_filter = SoilRecordFilterSchema.parse(input);
@@ -560,122 +67,10 @@ export class SqliteSoilRepository {
         await this.applyMutation({ pages });
     }
     async replacePageMembers(pageId, members) {
-        const parsed = members.map((member) => SoilPageMemberSchema.parse({ ...member, page_id: pageId }));
-        const tx = this.db.transaction(() => {
-            this.db.prepare("DELETE FROM soil_page_members WHERE page_id = ?").run(pageId);
-            for (const member of parsed) {
-                this.db.prepare(`
-          INSERT INTO soil_page_members (page_id, record_id, ordinal, role, confidence)
-          VALUES (@page_id, @record_id, @ordinal, @role, @confidence)
-        `).run(member);
-            }
-        });
-        tx();
+        replaceSoilPageMembers(this.db, pageId, members);
     }
     async lookupDirect(input) {
-        const request = SoilSearchRequestSchema.parse(input);
-        if (!request.direct_lookup) {
-            return { request, candidates: [] };
-        }
-        const pageParams = [request.query, request.query, request.query];
-        const pageWhere = ["(soil_id = ? OR relative_path = ? OR page_id = ?)"];
-        pageWhere.push(...buildPageFilterSql(request, pageParams));
-        pageParams.push(request.limit);
-        const pageMatches = this.db.prepare(`
-      SELECT page_id, soil_id
-      FROM soil_pages p
-      WHERE ${pageWhere.join(" AND ")}
-      LIMIT ?
-    `).all(...pageParams);
-        const recordParams = [request.query, request.query, request.query, request.query];
-        const recordWhere = ["(r.record_id = ? OR r.record_key = ? OR r.soil_id = ? OR r.source_id = ?)"];
-        recordWhere.push(...buildRecordFilterSql(request, recordParams));
-        recordParams.push(request.limit);
-        const recordRows = this.db.prepare(`
-      SELECT *
-      FROM soil_records r
-      WHERE ${recordWhere.join(" AND ")}
-      LIMIT ?
-    `).all(...recordParams);
-        const pageIdBySoilId = new Map();
-        for (const page of pageMatches) {
-            if (!pageIdBySoilId.has(page.soil_id)) {
-                pageIdBySoilId.set(page.soil_id, page.page_id);
-            }
-        }
-        const firstChunkByRecordId = new Map();
-        const recordIds = unique(recordRows.map((row) => row.record_id));
-        if (recordIds.length > 0) {
-            const chunkRows = this.db
-                .prepare(`
-          SELECT *
-          FROM soil_chunks
-          WHERE record_id IN (${recordIds.map(() => "?").join(", ")})
-          ORDER BY record_id, chunk_index
-        `)
-                .all(...recordIds);
-            for (const chunk of chunkRows) {
-                if (!firstChunkByRecordId.has(chunk.record_id)) {
-                    firstChunkByRecordId.set(chunk.record_id, chunk);
-                }
-            }
-        }
-        const firstPageMemberByPageId = new Map();
-        const pageIds = unique(pageMatches.map((page) => page.page_id));
-        if (pageIds.length > 0) {
-            const pageMemberRows = this.db
-                .prepare(`
-          SELECT spm.page_id, spm.record_id, sc.chunk_id, sc.chunk_text
-          FROM soil_page_members spm
-          LEFT JOIN soil_chunks sc ON sc.record_id = spm.record_id
-          WHERE spm.page_id IN (${pageIds.map(() => "?").join(", ")})
-          ORDER BY spm.page_id, spm.ordinal, sc.chunk_index
-        `)
-                .all(...pageIds);
-            for (const row of pageMemberRows) {
-                if (!firstPageMemberByPageId.has(row.page_id)) {
-                    firstPageMemberByPageId.set(row.page_id, row);
-                }
-            }
-        }
-        const candidates = [];
-        const representedPageIds = new Set();
-        for (const row of recordRows) {
-            const chunk = firstChunkByRecordId.get(row.record_id);
-            const page_id = pageIdBySoilId.get(row.soil_id) ?? null;
-            if (page_id) {
-                representedPageIds.add(page_id);
-            }
-            candidates.push({
-                chunk_id: chunk?.chunk_id ?? `record:${row.record_id}`,
-                record_id: row.record_id,
-                soil_id: row.soil_id,
-                lane: "direct",
-                rank: candidates.length + 1,
-                score: 1,
-                snippet: chunk ? buildSnippet(chunk.chunk_text, request.query) : row.summary ?? row.title,
-                page_id,
-                metadata_json: parseJsonObject(row.metadata_json),
-            });
-        }
-        for (const page of pageMatches) {
-            if (representedPageIds.has(page.page_id))
-                continue;
-            representedPageIds.add(page.page_id);
-            const member = firstPageMemberByPageId.get(page.page_id);
-            candidates.push({
-                chunk_id: member?.chunk_id ?? `page:${page.page_id}`,
-                record_id: member?.record_id ?? `page:${page.page_id}`,
-                soil_id: page.soil_id,
-                lane: "direct",
-                rank: candidates.length + 1,
-                score: 1,
-                snippet: member?.chunk_text ? buildSnippet(member.chunk_text, request.query) : page.soil_id,
-                page_id: page.page_id,
-                metadata_json: {},
-            });
-        }
-        return { request, candidates: candidates.slice(0, request.limit) };
+        return lookupDirectCandidates(this.db, input);
     }
     async searchHybrid(input) {
         const request = SoilSearchRequestSchema.parse(input);
@@ -690,174 +85,20 @@ export class SqliteSoilRepository {
             return lexical;
         }
         const lexicalRecordIds = unique(lexical.map((candidate) => candidate.record_id));
-        const dense = lexicalRecordIds.length > 0
+        const dense = shouldRunDenseSearch(request, lexicalRecordIds)
             ? await this.searchDense({
                 ...request,
                 direct_lookup: false,
-                dense_candidate_record_ids: lexicalRecordIds,
+                ...(lexicalRecordIds.length > 0 ? { dense_candidate_record_ids: lexicalRecordIds } : {}),
             })
-            : hasExplicitMetadataFilter(request)
-                ? await this.searchDense({ ...request, direct_lookup: false })
-                : [];
+            : [];
         return fuseCandidates(lexical, dense, request.limit);
     }
     async searchLexical(input) {
-        const request = SoilSearchRequestSchema.parse(input);
-        const pageIdParams = [];
-        const pageIdSql = buildCandidatePageIdSql("r.record_id", request, pageIdParams);
-        const whereParams = [];
-        const where = ["soil_chunk_fts MATCH ?"];
-        where.push(...buildRecordFilterSql(request, whereParams));
-        where.push(...buildPageExistsSql("r.record_id", request, whereParams));
-        const params = [...pageIdParams, request.query, ...whereParams, request.lexical_top_k];
-        const rows = this.db.prepare(`
-      SELECT
-        soil_chunk_fts.chunk_id AS chunk_id,
-        soil_chunk_fts.record_id AS record_id,
-        soil_chunk_fts.soil_id AS soil_id,
-        ${pageIdSql} AS page_id,
-        r.title AS title,
-        r.summary AS summary,
-        sc.chunk_text AS chunk_text,
-        bm25(soil_chunk_fts, 8.0, 5.0, 3.0, 1.0) AS score
-      FROM soil_chunk_fts
-      JOIN soil_chunks sc ON sc.chunk_id = soil_chunk_fts.chunk_id
-      JOIN soil_records r ON r.record_id = soil_chunk_fts.record_id
-      WHERE ${where.join(" AND ")}
-      ORDER BY score
-      LIMIT ?
-    `).all(...params);
-        const candidates = rows.map((row, index) => ({
-            chunk_id: row.chunk_id,
-            record_id: row.record_id,
-            soil_id: row.soil_id,
-            page_id: row.page_id,
-            lane: "lexical",
-            rank: index + 1,
-            score: -1 * row.score,
-            snippet: buildSnippet(row.chunk_text, request.query),
-            metadata_json: { title: row.title, summary: row.summary },
-        }));
-        return dedupeCandidates(candidates, request.limit);
+        return searchLexicalCandidates(this.db, input);
     }
     async searchDense(input) {
-        const request = SoilSearchRequestSchema.parse(input);
-        if (!request.query_embedding?.length) {
-            return [];
-        }
-        const pageIdParams = [];
-        const pageIdSql = buildCandidatePageIdSql("r.record_id", request, pageIdParams);
-        const params = [];
-        const where = buildRecordFilterSql(request, params);
-        where.push(...buildPageExistsSql("r.record_id", request, params));
-        const denseCandidateRecordIds = request.dense_candidate_record_ids ? unique(request.dense_candidate_record_ids) : null;
-        if (denseCandidateRecordIds?.length === 0) {
-            return [];
-        }
-        if (denseCandidateRecordIds) {
-            where.push(`r.record_id IN (${denseCandidateRecordIds.map(() => "?").join(", ")})`);
-            params.push(...denseCandidateRecordIds);
-        }
-        const excludedRecordIds = this.loadOpenEmbeddingReindexRecordIds();
-        if (excludedRecordIds.length > 0) {
-            where.push(`r.record_id NOT IN (${excludedRecordIds.map(() => "?").join(", ")})`);
-            params.push(...excludedRecordIds);
-        }
-        if (request.query_embedding_model) {
-            where.push("se.model = ?");
-            params.push(request.query_embedding_model);
-        }
-        where.push(`
-      NOT EXISTS (
-        SELECT 1
-        FROM soil_embeddings newer
-        WHERE newer.chunk_id = se.chunk_id
-          AND newer.model = se.model
-          AND newer.embedding_version > se.embedding_version
-      )
-    `);
-        const queryParams = [...pageIdParams, ...params];
-        const rows = this.db.prepare(`
-      SELECT
-        se.chunk_id,
-        se.model,
-        se.embedding_version,
-        se.encoding,
-        se.embedding,
-        se.embedded_at,
-        sc.record_id,
-        sc.soil_id,
-        sc.chunk_text,
-        r.title,
-        r.summary,
-        ${pageIdSql} AS page_id
-      FROM soil_embeddings se
-      JOIN soil_chunks sc ON sc.chunk_id = se.chunk_id
-      JOIN soil_records r ON r.record_id = sc.record_id
-      ${where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""}
-    `).all(...queryParams);
-        const scored = [];
-        for (const row of rows) {
-            try {
-                scored.push({
-                    row,
-                    similarity: cosineSimilarity(request.query_embedding, decodeEmbedding(row)),
-                });
-            }
-            catch {
-                continue;
-            }
-        }
-        scored.sort((left, right) => right.similarity - left.similarity);
-        const candidates = scored.slice(0, request.dense_top_k).map(({ row, similarity }, index) => ({
-            chunk_id: row.chunk_id,
-            record_id: row.record_id,
-            soil_id: row.soil_id,
-            page_id: row.page_id,
-            lane: "dense",
-            rank: index + 1,
-            score: similarity,
-            snippet: buildSnippet(row.chunk_text, request.query),
-            metadata_json: { model: row.model, embedding_version: row.embedding_version, title: row.title, summary: row.summary },
-        }));
-        return dedupeCandidates(candidates, request.limit);
-    }
-    recordHasCompleteEmbeddingMutation(recordId, embeddedChunkIds) {
-        const chunks = this.db
-            .prepare("SELECT chunk_id FROM soil_chunks WHERE record_id = ?")
-            .all(recordId);
-        return chunks.length > 0 && chunks.every((chunk) => embeddedChunkIds.has(chunk.chunk_id));
-    }
-    completeOpenEmbeddingJobs(recordIds) {
-        if (recordIds.size === 0)
-            return;
-        const jobs = this.db.prepare(`
-      SELECT job_id, payload_json
-      FROM soil_reindex_jobs
-      WHERE scope = 'embedding'
-        AND status IN ('pending', 'running')
-    `).all();
-        const complete = this.db.prepare("UPDATE soil_reindex_jobs SET status = 'completed', completed_at = ? WHERE job_id = ?");
-        const completedAt = new Date().toISOString();
-        for (const job of jobs) {
-            const jobRecordIds = parseReindexRecordIds(job.payload_json);
-            if (jobRecordIds.length > 0 && jobRecordIds.every((recordId) => recordIds.has(recordId))) {
-                complete.run(completedAt, job.job_id);
-            }
-        }
-    }
-    loadOpenEmbeddingReindexRecordIds() {
-        const rows = this.db.prepare(`
-      SELECT payload_json
-      FROM soil_reindex_jobs
-      WHERE scope = 'embedding'
-        AND status IN ('pending', 'running')
-    `).all();
-        const recordIds = [];
-        for (const row of rows) {
-            recordIds.push(...parseReindexRecordIds(row.payload_json));
-        }
-        return unique(recordIds);
+        return searchDenseCandidates(this.db, input, loadOpenEmbeddingReindexRecordIds(this.db));
     }
     async loadPagesForRecords(recordIds) {
         const ids = unique(recordIds);
@@ -890,40 +131,5 @@ export class SqliteSoilRepository {
     `).all(...ids);
         return rows.map((row) => toPageMember(row));
     }
-    syncFts(recordIds) {
-        if (recordIds.length === 0)
-            return;
-        if (recordIds.length > 0) {
-            this.db.prepare(`DELETE FROM soil_chunk_fts WHERE record_id IN (${recordIds.map(() => "?").join(", ")})`).run(...recordIds);
-        }
-        const rows = this.db.prepare(`
-      SELECT
-        sc.chunk_id,
-        sc.record_id,
-        sc.soil_id,
-        (
-          SELECT spm.page_id
-          FROM soil_page_members spm
-          WHERE spm.record_id = r.record_id
-          ORDER BY CASE WHEN spm.role = 'primary' THEN 0 ELSE 1 END, spm.ordinal, spm.page_id
-          LIMIT 1
-        ) AS page_id,
-        r.title AS title_context,
-        COALESCE(r.summary, '') AS summary_context,
-        sc.heading_path_json,
-      sc.chunk_text
-      FROM soil_chunks sc
-      JOIN soil_records r ON r.record_id = sc.record_id
-      WHERE sc.record_id IN (${recordIds.map(() => "?").join(", ")})
-    `).all(...recordIds);
-        const insert = this.db.prepare(`
-      INSERT INTO soil_chunk_fts (
-        chunk_id, record_id, soil_id, page_id, title_context, summary_context, heading_context, chunk_text
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-    `);
-        for (const row of rows) {
-            insert.run(row.chunk_id, row.record_id, row.soil_id, row.page_id, row.title_context, row.summary_context, parseJsonArray(row.heading_path_json).join(" / "), row.chunk_text);
-        }
-    }
 }
 //# sourceMappingURL=sqlite-repository.js.map