renor-engine 0.0.0-stage → 1.0.0

package/dist/mvp/engine.js

@@ -0,0 +1,1065 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BACKGROUND_INDEX_FIRST_DELAY_MS = exports.UNFINISHED_RETRY_MS = exports.UNFINISHED_FILES = exports.TIMESTAMP_TICK_MS = exports.FULL_VAULT_CHECK_MS = exports.INDEX_FILE_CONCURRENCY = exports.ENTITY_LOOKUP_SURFACE = exports.QUERY_CACHE_MAX_BYTES = exports.QUERY_CACHE_MAX_ENTRIES = exports.DEFAULT_EMBED_BUDGETS = void 0;
+exports.resolveEmbedBudgets = resolveEmbedBudgets;
+exports.readEmbeddingTableDims = readEmbeddingTableDims;
+exports.queryEmbeddingCachePath = queryEmbeddingCachePath;
+exports.createEmbeddingProvider = createEmbeddingProvider;
+exports.attachEngine = attachEngine;
+exports.semanticReady = semanticReady;
+exports.keywordIndexReady = keywordIndexReady;
+exports.engineRebuildEmbeddings = engineRebuildEmbeddings;
+exports.runningEmbeddingRebuild = runningEmbeddingRebuild;
+exports.clientPath = clientPath;
+exports.resolveEngineEvidence = resolveEngineEvidence;
+exports.resolveEngineDigests = resolveEngineDigests;
+exports.engineSearchMemory = engineSearchMemory;
+exports.engineSearchText = engineSearchText;
+exports.engineLookupEntity = engineLookupEntity;
+exports.engineStatus = engineStatus;
+exports.engineIndex = engineIndex;
+exports.engineIndexChanged = engineIndexChanged;
+exports.engineFinishIndexInBackground = engineFinishIndexInBackground;
+exports.runningBackgroundIndex = runningBackgroundIndex;
+exports.hasIndexWorkToFinish = hasIndexWorkToFinish;
+exports.engineRememberNote = engineRememberNote;
+const ledgerLiveness_1 = require("../retrieval/ledgerLiveness");
+const invisibleText_1 = require("../memory/invisibleText");
+const fs_1 = __importDefault(require("fs"));
+const crypto_1 = __importDefault(require("crypto"));
+const path_1 = __importDefault(require("path"));
+const readCache_1 = require("../db/readCache");
+const statements_1 = require("../db/statements");
+const vectorIndex_1 = require("../embeddings/vectorIndex");
+const space_1 = require("../embeddings/space");
+const rebuild_1 = require("../embeddings/rebuild");
+const openai_1 = require("../embeddings/openai");
+const local_1 = require("../embeddings/local");
+const vertex_1 = require("../embeddings/vertex");
+const caching_1 = require("../embeddings/caching");
+const interactive_1 = require("../embeddings/interactive");
+const coalescing_1 = require("../embeddings/coalescing");
+const readerLayout_1 = require("../retrieval/readerLayout");
+const orchestrator_1 = require("../retrieval/orchestrator");
+const ledgerEventTime_1 = require("../retrieval/ledgerEventTime");
+const entityLookup_1 = require("../retrieval/entityLookup");
+const writer_1 = require("../memory/writer");
+const memoryMap_1 = require("../memorymap/memoryMap");
+const pipeline_1 = require("../indexer/pipeline");
+const walker_1 = require("../indexer/walker");
+const queryHelpers_1 = require("../retrieval/queryHelpers");
+const keyword_1 = require("../retrieval/keyword");
+const noteIdentity_1 = require("../retrieval/noteIdentity");
+const ledgerPresentation_1 = require("../retrieval/ledgerPresentation");
+const scope_1 = require("./scope");
+const lifecycle_1 = require("../memory/lifecycle");
+const inventory_1 = require("../memory/inventory");
+const readScope_1 = require("./readScope");
+const attach_1 = require("../evidence/attach");
+const options_1 = require("../evidence/options");
+const background_1 = require("../evidence/background");
+const contextRelevance_1 = require("./contextRelevance");
+const sanitize_1 = require("./sanitize");
+const digests_1 = require("../digests/digests");
+const worker_1 = require("../digests/worker");
+exports.DEFAULT_EMBED_BUDGETS = { writeMs: 5_000, readMs: 3_000 };
+/**
+ * `RENOR_EMBED_WRITE_BUDGET_MS` and `RENOR_EMBED_READ_BUDGET_MS` set the two budgets; `0` for either (or both) lets
+ * that kind of call wait as long as the provider's retries take, which is what every call did before they existed.
+ */
+function resolveEmbedBudgets(env = process.env) {
+    const read = (name, fallback) => {
+        const raw = env[name]?.trim();
+        if (raw === undefined || raw === '')
+            return fallback;
+        const value = Number(raw);
+        if (!Number.isInteger(value) || value < 0 || (value > 0 && value < 100) || value > 600_000) {
+            throw new Error(`${name} must be 0 (no limit) or a number of milliseconds between 100 and 600000`);
+        }
+        return value;
+    };
+    const writeMs = read('RENOR_EMBED_WRITE_BUDGET_MS', exports.DEFAULT_EMBED_BUDGETS.writeMs);
+    const readMs = read('RENOR_EMBED_READ_BUDGET_MS', exports.DEFAULT_EMBED_BUDGETS.readMs);
+    return writeMs === 0 && readMs === 0 ? undefined : { writeMs: writeMs || Infinity, readMs: readMs || Infinity };
+}
+/** `provider` as one call that has a budget sees it: every call it makes ends within `budgetMs` (see EmbedOptions). */
+function withBudget(provider, budgetMs) {
+    if (budgetMs === undefined || !Number.isFinite(budgetMs))
+        return provider;
+    return {
+        dimensions: provider.dimensions,
+        identity: provider.identity,
+        embed: texts => provider.embed(texts, { budgetMs }),
+        embedQuery: text => provider.embedQuery ? provider.embedQuery(text, { budgetMs }) : provider.embed([text], { budgetMs }).then(vectors => vectors[0]),
+    };
+}
+/** sqlite-vec table dimension, or null if the table does not exist. */
+function readEmbeddingTableDims(db) {
+    const row = db.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embeddings'`).get();
+    if (!row?.sql)
+        return null;
+    const m = row.sql.match(/FLOAT\[(\d+)\]/i);
+    return m ? Number(m[1]) : null;
+}
+/**
+ * Where a workspace keeps the vectors of the queries it has embedded: its own private state directory, the one
+ * that holds the OAuth client registry (`MVP_AUTH_STATE_PATH`), which config validation keeps out of the memory
+ * and history trees. So export, import and memory_status never see it, and it is one file per workspace.
+ */
+function queryEmbeddingCachePath(config) {
+    return path_1.default.join(path_1.default.dirname(config.authStatePath), 'query-embeddings.db');
+}
+/** Rows and vector bytes the read-path cache may hold: a few thousand distinct queries (3 KiB each at 768 dims). */
+exports.QUERY_CACHE_MAX_ENTRIES = 4096;
+exports.QUERY_CACHE_MAX_BYTES = 32 * 1024 * 1024;
+/**
+ * The vector of a query already asked costs no second provider call. It is a cache: if it cannot be opened, the
+ * file is derived and is removed once and started again, and if it still cannot be opened the provider is
+ * returned as it is.
+ */
+function withQueryCache(provider, config, scope) {
+    const cachePath = queryEmbeddingCachePath(config);
+    const open = () => new caching_1.CachingEmbeddingProvider(provider, {
+        cachePath,
+        serving: true,
+        scope,
+        maxEntries: exports.QUERY_CACHE_MAX_ENTRIES,
+        maxBytes: exports.QUERY_CACHE_MAX_BYTES,
+    });
+    try {
+        return open();
+    }
+    catch {
+        try {
+            for (const suffix of ['', '-wal', '-shm'])
+                fs_1.default.rmSync(cachePath + suffix, { force: true });
+            return open();
+        }
+        catch {
+            return provider;
+        }
+    }
+}
+function createEmbeddingProvider(config, 
+/** A test points the Vertex endpoint at a local stub and shortens its retry waits; production passes nothing. */
+vertexSeam) {
+    if (config.embeddingProvider === 'none')
+        return null;
+    if (config.embeddingProvider === 'local') {
+        return new local_1.LocalEmbeddingProvider(config.embeddingModel);
+    }
+    if (config.embeddingProvider === 'vertex') {
+        // The provider's identity is its kind and model; the project and location it is served from are not in it. The
+        // endpoint has no setting of its own (it is built from the location), so these two name it.
+        const served = crypto_1.default.createHash('sha256').update(`${config.vertexProject ?? ''}\0${config.vertexLocation ?? ''}`).digest('hex').slice(0, 12);
+        // The query cache is outermost: a query already asked is answered from it with the service down, and only a query
+        // that must go to the service meets the interactive budget and the failure memory in front of it.
+        // Document embeddings from concurrent index passes share requests (coalescing.ts): below the budget, so each caller's
+        // budget still ends its own call, and above the service, so a request holds at most the usual four in flight.
+        return withQueryCache(new interactive_1.InteractiveEmbeddingProvider(new coalescing_1.CoalescingEmbeddingProvider(new vertex_1.VertexEmbeddingProvider({
+            keyFile: config.vertexKeyFile,
+            project: config.vertexProject,
+            location: config.vertexLocation,
+            model: config.embeddingModel,
+            dimensions: config.embeddingDims,
+            ...vertexSeam,
+        }))), config, `p-${served}`);
+    }
+    const keys = (config.embeddingApiKey ?? '').split(',').map(k => k.trim()).filter(Boolean);
+    return new openai_1.OpenAIEmbeddingProvider(keys, config.embeddingModel, config.embeddingBaseURL, config.embeddingDims);
+}
+function attachEngine(db, config, override) {
+    const provider = override === undefined ? createEmbeddingProvider(config) : override;
+    if (!provider) {
+        // What this process saves gets no vector: an index that has a model's vectors is marked unfinished first.
+        if ((0, vectorIndex_1.markVectorsUnfinished)(db)) {
+            console.log('[embeddings] no embedding provider is configured: what is saved now gets no vector, so the index is marked unfinished; '
+                + 'the next start with its model embeds those chunks before semantic search reads the index again');
+        }
+        return { provider: null, vectorIndex: null, dims: readEmbeddingTableDims(db) };
+    }
+    const existing = readEmbeddingTableDims(db);
+    if (existing !== null && existing !== provider.dimensions) {
+        throw new Error(`Embedding dimension mismatch: index.db chunk_embeddings is FLOAT[${existing}] but the configured provider is ${provider.dimensions}-d. `
+            + `Do not mix Voyage/e5/other models in one book (ADR-0003). Set MVP_EMBEDDING_DIMS and MVP_EMBEDDING_MODEL to match the index, or use a separate MVP_DB_PATH.`);
+    }
+    const vectorIndex = new vectorIndex_1.VectorIndex(db, provider.dimensions);
+    vectorIndex.initTable();
+    claimEmbeddingSpace(db, vectorIndex, provider);
+    const modelKey = override === undefined
+        ? (0, contextRelevance_1.embeddingModelKey)(config.embeddingProvider, config.embeddingModel, config.embeddingDims)
+        : injectedModelKey(provider);
+    const embedBudgets = resolveEmbedBudgets();
+    return {
+        provider,
+        vectorIndex,
+        dims: provider.dimensions,
+        ...(modelKey ? { modelKey } : {}),
+        ...(embedBudgets ? { embedBudgets } : {}),
+    };
+}
+/**
+ * The model key of an injected provider, from the identity it declares and the size it embeds at: exactly what the
+ * configured path composes, so a caller that brings its own provider (a bench, a test, an in-process harness) keeps
+ * the calibrated relevance bar its model has instead of losing the verdict altogether. An injected provider that
+ * declares no identity names no model, so it has no bar: `undefined`, and the signal stays silent rather than
+ * guessing (the eval-hash stub is exactly this case — lexical vectors, and nothing calibrated to compare them to).
+ */
+function injectedModelKey(provider) {
+    const identity = provider.identity;
+    return identity ? (0, contextRelevance_1.embeddingModelKey)(identity.kind, identity.model, provider.dimensions) : undefined;
+}
+/**
+ * Dimensions alone cannot tell two embedding models apart, so the index records the space its vectors are in
+ * (provider, model, dimensions). A new index takes the configured model's space. An index whose vectors are in
+ * another space, or in none anyone can vouch for (it holds vectors and records none: it was built before spaces
+ * were recorded), is not named and is not read: search would compare numbers from different models. It is rebuilt
+ * instead (`engineRebuildEmbeddings`, which the service starts), and until that finishes semantic search is off
+ * (`semanticReady`). An index that holds chunks and no vector at all was written without an embedding provider (a
+ * workspace that searched by keyword until now): it is named and marked unfinished, so the same background job
+ * embeds every one of its chunks, once, before semantic search reads it. A provider that declares no identity claims
+ * nothing, and nothing is held back for it.
+ */
+function claimEmbeddingSpace(db, vectorIndex, provider) {
+    const wanted = (0, space_1.embeddingSpace)(provider);
+    if (!wanted)
+        return;
+    const stored = vectorIndex.space();
+    if (stored === wanted)
+        return;
+    if (stored === null && !vectorIndex.hasVectors()) {
+        const chunks = db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
+        if (chunks === 0) {
+            vectorIndex.setSpace(wanted); // a new index: nothing to embed, so it is whole
+            return;
+        }
+        vectorIndex.beginRebuild(wanted); // there is no vector to clear: this names the index and marks it unfinished
+        console.log(`[embeddings] index.db holds ${chunks} chunk(s) and no vectors (it was written without an embedding provider): `
+            + `every chunk is embedded with ${wanted} in the background, and until that finishes semantic search is off and keyword search keeps working.`);
+        return;
+    }
+    console.warn(stored === null
+        ? '[embeddings] index.db holds vectors but no record of the model that wrote them (it was built before models were recorded). '
+        : `[embeddings] index.db holds vectors from ${stored}, but the configured embedding is ${wanted}. `, `Search cannot use vectors from another model: a rebuild embeds every chunk again with ${wanted}, and until it finishes `
+        + 'semantic search is off and keyword search keeps working.');
+}
+/**
+ * Whether a semantic search may read this engine's index: every vector in it is in the configured model's space and
+ * none is still to come. An index in another space, in none anyone can vouch for, or part way through a rebuild is
+ * not read: search would compare numbers from different models, or find only part of the index. A provider that
+ * declares no identity makes no claim, so it is never held back.
+ */
+function semanticReady(engine) {
+    if (!engine.provider || !engine.vectorIndex)
+        return false;
+    const wanted = (0, space_1.embeddingSpace)(engine.provider);
+    if (!wanted)
+        return true;
+    return engine.vectorIndex.space() === wanted && !engine.vectorIndex.rebuilding();
+}
+/** What a caller is told while semantic search is off for a rebuild. */
+function rebuildNotice(db, engine) {
+    const chunks = db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
+    const left = engine.vectorIndex.space() === (0, space_1.embeddingSpace)(engine.provider) ? (0, rebuild_1.chunksWithoutVectors)(db) : chunks;
+    return `Semantic search is off while this workspace's embeddings are built with the configured model (${left} of ${chunks} chunks left). Keyword search works meanwhile.`;
+}
+/** Why something failed, on one line a client may be shown (host paths redacted). */
+function failureReason(err, max) {
+    return (0, sanitize_1.sanitizeErrorMessage)(err).replace(/\s+/g, ' ').trim().slice(0, max);
+}
+/**
+ * The last search by meaning that failed, per engine, until one succeeds: memory_status reports it, so a workspace whose
+ * embedding service stopped answering is not reported healthy (it makes no embeddings call of its own).
+ */
+const semanticFailures = new WeakMap();
+/** Whether keyword search can run: its index table exists. A damaged or half-migrated index file has none. */
+function keywordIndexReady(db) {
+    return (0, statements_1.prepareOnce)(db, `SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_fts'`).get() !== undefined;
+}
+/** What a caller is told when a search by meaning failed: why, and what still answers. */
+function semanticFailureNotice(err) {
+    return `Semantic search is unavailable right now (${failureReason(err, 200)}), so nothing here was found by meaning; `
+        + 'the saved facts and keyword search do not need it and still answer. The next request tries semantic search again.';
+}
+const embeddingRebuilds = new WeakMap();
+/**
+ * Rebuild, in the background, every vector this engine's index holds with the configured model, when its index is
+ * not whole in that model's space (see `semanticReady`), and record the space once it is. The clearing and naming
+ * happen before this returns; the embedding happens after, a batch at a time. The index is whole only when every
+ * chunk has a vector. A provider that is down, or a chunk it will not embed, interrupts the rebuild; it is resumed
+ * after a wait that grows (30 s, doubling, up to 5 min), semantic search stays off, and the log says how many chunks
+ * are left and which one was refused. So it ends only when the index is whole or `abort` is called. Returns null
+ * when there is nothing to do; a rebuild already running for this index is returned rather than started twice.
+ */
+function engineRebuildEmbeddings(db, engine, options = {}) {
+    const { provider, vectorIndex } = engine;
+    if (!provider || !vectorIndex)
+        return null;
+    const space = (0, space_1.embeddingSpace)(provider);
+    if (!space || semanticReady(engine))
+        return null;
+    const running = embeddingRebuilds.get(vectorIndex);
+    if (running)
+        return running;
+    const { outageDelayMs = (outage) => Math.min(300_000, 30_000 * 2 ** outage), ...rebuildOptions } = options;
+    const controller = new AbortController();
+    const startedAt = Date.now();
+    let waiting = false;
+    const wait = async (ms) => {
+        waiting = true;
+        try {
+            await (0, rebuild_1.sleep)(ms, controller.signal);
+        }
+        finally {
+            waiting = false;
+        }
+    };
+    const run = async () => {
+        const chunks = () => db.prepare('SELECT COUNT(*) FROM chunks').pluck().get();
+        console.log(`[embeddings] rebuilding the vectors of ${chunks()} chunk(s) with ${space}; semantic search is off until it finishes`);
+        for (let outage = 0; !controller.signal.aborted; outage++) {
+            try {
+                const result = await (0, rebuild_1.rebuildEmbeddings)(db, vectorIndex, provider, space, { ...rebuildOptions, signal: controller.signal });
+                if (result.done) {
+                    console.log(`[embeddings] rebuild finished in ${((Date.now() - startedAt) / 1000).toFixed(1)} s; semantic search is on`);
+                    return;
+                }
+            }
+            catch (err) {
+                const delay = outageDelayMs(outage);
+                console.error(`[embeddings] rebuild interrupted with ${(0, rebuild_1.chunksWithoutVectors)(db)} of ${chunks()} chunk(s) still to embed; semantic search stays off; trying again in ${Math.round(delay / 1000)} s:`, err);
+                await wait(delay);
+            }
+        }
+    };
+    // The clearing and naming run now, in the synchronous start of the first call, so the index is never left
+    // looking whole while it holds another model's vectors.
+    const rebuild = {
+        abort: () => controller.abort(),
+        waiting: () => waiting,
+        done: run().catch((err) => console.error('[embeddings] rebuild failed:', err)).finally(() => embeddingRebuilds.delete(vectorIndex)),
+    };
+    embeddingRebuilds.set(vectorIndex, rebuild);
+    return rebuild;
+}
+/** The rebuild `engineRebuildEmbeddings` started for this engine's index that has not ended yet, or null; starts none. */
+function runningEmbeddingRebuild(engine) {
+    return (engine.vectorIndex && embeddingRebuilds.get(engine.vectorIndex)) ?? null;
+}
+/** A search shows the path of every hit it judges, and the notes it finds are the same ones read after read. */
+const clientPaths = new Map();
+const MAX_CLIENT_PATHS = 4096;
+function clientPath(memoryDir, stored) {
+    // A relative path resolves against the working directory, so it is part of the key then, and only then (as in
+    // path.resolve, which asks for it only when it has to).
+    const key = path_1.default.isAbsolute(stored) && path_1.default.isAbsolute(memoryDir) ? `${memoryDir}\0${stored}` : `${process.cwd()}\0${memoryDir}\0${stored}`;
+    let shown = clientPaths.get(key);
+    if (shown === undefined) {
+        const resolved = path_1.default.resolve(stored);
+        const root = path_1.default.resolve(memoryDir);
+        shown = resolved === root || resolved.startsWith(root + path_1.default.sep) ? path_1.default.relative(root, resolved) || stored : path_1.default.basename(stored);
+        if (clientPaths.size >= MAX_CLIENT_PATHS)
+            clientPaths.clear();
+        clientPaths.set(key, shown);
+    }
+    return shown;
+}
+function resolveEngineEvidence(config, env = process.env, idle = null) {
+    if (!(0, attach_1.searchEvidenceEnabled)(env))
+        return undefined;
+    const options = (0, options_1.resolveEvidenceToolOptions)({ dbPath: config.dbPath }, env);
+    // Bounded-wait evidence (RENOR_EVIDENCE_ASYNC, default off) is read here and nowhere else: a background extraction
+    // needs a long-lived server to finish in, and the CLI, the benchmark's entry point, must always wait for its table.
+    const background = (0, background_1.resolveEvidenceBackground)(env);
+    if (background && idle) {
+        background.hold = flight => {
+            idle.track(() => flight).catch(() => undefined); // its failure is the flight's to report
+        };
+    }
+    return { options: background ? { ...options, background } : options, extractor: (0, options_1.createEvidenceExtractor)(options) };
+}
+function resolveEngineDigests(db, config, engine, idle = null, env = process.env) {
+    const options = (0, digests_1.resolveDigestOptions)({ dbPath: config.dbPath }, env);
+    if (!options)
+        return null;
+    if (!engine.provider) {
+        console.log('[digests] RENOR_DIGESTS is on, but this workspace has no embedding provider, so digests are off for it');
+        return null;
+    }
+    const worker = new worker_1.DigestWorker({
+        db,
+        config: { memoryDir: config.memoryDir, ledgerRelPath: config.ledgerRelPath },
+        options,
+        provider: engine.provider,
+        track: idle ? work => idle.track(work) : undefined,
+    });
+    console.log(`[digests] write-time digests are on (model ${options.model}, at most ${options.maxUsdPerDay} USD a day)`);
+    return { options, worker };
+}
+/**
+ * What search_memory answers on a workspace without semantic search. It searched nothing, and the error code
+ * alone left the agent without a next step, so `note` names the search every workspace has.
+ */
+const SEMANTIC_UNAVAILABLE_NOTE = 'This workspace has no semantic search, so nothing was searched by meaning. Use search_text instead: it matches exact words, so search one distinctive word (a name, a project, a term).';
+function semanticUnavailable() {
+    return { ok: false, error: 'semantic-unavailable', note: SEMANTIC_UNAVAILABLE_NOTE, hits: [] };
+}
+async function engineSearchMemory(db, engine, memoryDir, query, topK, scope, evidence, includeHistory = false, ledgerRelPath = lifecycle_1.DEFAULT_LEDGER_REL, 
+/** Out-param for get_context: the query embedding and the best raw similarity among the returned (scoped, live) semantic hits. */
+probe, 
+/** Write-time digests (RENOR_DIGESTS): when present, the digests of the window's notes lead the hits as one table. No model call. */
+digests, 
+/** Per-read default: get_context passes 0; searches leave this unset for the default cap of 2. */
+noteCapWhenUnset, 
+/** Internal floor input: expose note similarities so the MCP search can filter its completed page. */
+showSimilarity = false, 
+/** search_memory's reader layout (readerLayout.ts) needs each chunk's place in its note: the ids ride on the hits until it has used them. */
+withChunkIds = false, 
+/** search_memory under RENOR_SEARCH_BEST_FIRST: the window best match first (retrieval/orchestrator.ts). */
+bestFirst = false) {
+    // A query with no searchable characters can never truthfully match stored data. Refuse before
+    // any embedding call (eval-hash or provider) so whitespace-only input yields no hits instead
+    // of an arbitrary semantic match, matching search_history's blank-query handling.
+    if ((0, invisibleText_1.isBlankOrInvisible)(query)) {
+        return { ok: true, hits: [] };
+    }
+    if (!engine.provider || !engine.vectorIndex) {
+        return semanticUnavailable();
+    }
+    // Never search a mixed or unfinished space: the caller falls back to keyword search and is told why.
+    if (!semanticReady(engine)) {
+        return { ok: false, error: 'semantic-rebuilding', notice: rebuildNotice(db, engine), hits: [] };
+    }
+    const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath });
+    // The query is embedded within the read budget: past it the search says so (below) and the caller answers from
+    // the facts and keyword search, as it does for any failure of the provider.
+    const provider = withBudget(engine.provider, engine.embedBudgets?.readMs);
+    const vectorIndex = engine.vectorIndex;
+    const asked = probe ?? {};
+    const scopeNorm = (0, lifecycle_1.scopeFilterFor)(scope);
+    const inScope = scopeNorm ? (0, readScope_1.createReadScope)(db, scopeNorm) : null;
+    // The scope is judged before the window is cut, so a scope whose memories rank below other scopes' still gets its
+    // best ones (memory scout, finding 7: judged only after the cut, they were all gone); and again on what comes back.
+    // Which chunks the scope holds is read from every chunk of the index (and the scope of every ledger event), so it is
+    // read once and kept until something is written; reads that follow one another, as an agent's do, share it.
+    const scoped = inScope
+        ? {
+            shows: h => inScope.has(h, clientPath(memoryDir, h.notePath)),
+            chunkIds: () => (0, readCache_1.readUntilWrite)(db, `scopedChunkIds\0${scopeNorm}\0${memoryDir}`, () => inScope.chunkIds(stored => clientPath(memoryDir, stored))),
+        }
+        : undefined;
+    const search = () => (0, orchestrator_1.retrieveContext)(db, vectorIndex, provider, query, topK, [], {}, { includeHistory, isLedgerNote: isLedger, probe: asked, scope: scoped, noteCapWhenUnset, ...(bestFirst ? { bestFirst } : {}) });
+    let result;
+    try {
+        result = await search();
+        // What a save could not embed is embedded now, once the provider answers, and searched with the rest.
+        if (await finishUnfinishedFiles(db, memoryDir, engine, provider, query, asked))
+            result = await search();
+    }
+    catch (err) {
+        // Search by meaning failed: the embedding provider is down, or refuses the key or the quota (or the semantic lane
+        // broke, which the log shows). The facts and keyword search do not need it, so a caller answers with those and
+        // says why nothing was found by meaning, instead of failing whole.
+        console.error('[search] semantic search failed:', err);
+        semanticFailures.set(engine, { at: new Date().toISOString(), reason: failureReason(err, 120) });
+        return { ok: false, error: 'semantic-unavailable', notice: semanticFailureNotice(err), hits: [] };
+    }
+    semanticFailures.delete(engine);
+    // Scope must be enforced before any full-note hydration/evidence extraction.
+    // Otherwise a scoped query could derive evidence from notes outside its scope
+    // even if those ranked hits are filtered from the final response later.
+    const scopedRetrievalHits = inScope
+        ? inScope.hits(result.hits, h => clientPath(memoryDir, h.notePath))
+        : result.hits;
+    if (probe) {
+        let top;
+        for (const h of scopedRetrievalHits)
+            if (typeof h.similarity === 'number' && (top === undefined || h.similarity > top.similarity))
+                top = h;
+        probe.topSimilarity = top ? top.similarity : null;
+        // The same hit as the hits below show it: a ledger event as what it says (presentLedgerEvents judges each hit alone).
+        const shown = top && (0, ledgerPresentation_1.presentLedgerEvents)([top], isLedger)[0];
+        probe.topHit = shown ? { notePath: clientPath(memoryDir, shown.notePath), headingPath: shown.headingPath, text: shown.text } : undefined;
+        // Every returned semantic hit as the memories show it, with its raw cosine: a budget reads which of them are above the bar.
+        probe.similarities = scopedRetrievalHits.flatMap(h => {
+            const asShown = typeof h.similarity === 'number' ? (0, ledgerPresentation_1.presentLedgerEvents)([h], isLedger)[0] : undefined;
+            return asShown ? [{ notePath: clientPath(memoryDir, asShown.notePath), headingPath: asShown.headingPath, text: asShown.text, similarity: h.similarity }] : [];
+        });
+    }
+    const closedEvents = (0, ledgerLiveness_1.closedLedgerEvents)(db);
+    const noteDate = (0, orchestrator_1.createNoteDater)(db);
+    // Without an evidence model no second pass runs, and nothing is said about one: on every search of such a workspace
+    // (every hosted one) a report that nothing was missing told the agent nothing. With a model, every state is reported,
+    // and so is the kill switch ("off").
+    const attached = evidence?.extractor
+        ? await (0, attach_1.attachSearchEvidence)({
+            // The ledger note holds every scope's events, so hydration drops other scopes' before closed ones.
+            transformBody: text => (0, ledgerLiveness_1.resolveLedgerHits)([{ text: inScope ? inScope.body(text) : text, reason: 'evidence-source' }], closedEvents, includeHistory)[0]?.text ?? '',
+            query,
+            // Evidence hydrates whole notes and prints one "Session dated" line above each: the note's date. A hit's own
+            // date is its excerpt's, and one ledger event's would date every other event in the same body as well.
+            hits: scopedRetrievalHits.map(h => ({ ...h, session_date: noteDate(h.notePath) })),
+            vaultPath: memoryDir,
+            options: evidence.options,
+            extractor: evidence.extractor,
+            enabled: true,
+        })
+        : evidence
+            ? null
+            : { hits: scopedRetrievalHits, evidence: { status: 'skipped', reason: 'flag-off' } };
+    const shown = (0, ledgerPresentation_1.presentLedgerEvents)(scopedRetrievalHits, isLedger);
+    // The evidence table, when there is one, leads the hits the agent is shown; they keep their own dates.
+    let led = attached ? (0, attach_1.prependEvidenceHit)(shown, attached.evidence) : shown;
+    // The digest table follows it. Only notes of the window the scope already allowed are read, so a scoped read never sees
+    // another scope's digests, and no model is called: it reads the digest cache and the query embedding the search made.
+    if (digests) {
+        try {
+            const table = await (0, digests_1.digestTable)({
+                db, query, hits: scopedRetrievalHits, options: digests.options, provider, queryVector: asked.queryVector, isLedgerNote: isLedger,
+            });
+            if (table.items.length > 0) {
+                const lead = (0, digests_1.leadingSummaryHits)(led);
+                led = [...led.slice(0, lead), (0, digests_1.digestItemsToHit)(table.items), ...led.slice(lead)];
+            }
+        }
+        catch (err) {
+            // Optional augmentation: a fault here never fails the search.
+            console.error('[digests] reading the digests failed:', failureReason(err, 200));
+        }
+    }
+    const isConversation = withChunkIds ? (0, readerLayout_1.conversationNoteMatcher)(db) : () => false;
+    const hits = led.map(h => ({
+        notePath: clientPath(memoryDir, h.notePath),
+        headingPath: h.headingPath,
+        text: h.text,
+        score: h.score,
+        reason: h.reason,
+        ...(h.session_date ? { session_date: h.session_date } : {}),
+        // Only a conversation's chunks: the layout merges those and no others.
+        ...(withChunkIds && 'chunkId' in h && typeof h.chunkId === 'number' && h.chunkId > 0 && isConversation(h.notePath) ? { chunkId: h.chunkId } : {}),
+        // The author of a fact an identity question read: the reader's view marks it (mvp/factMarkers.ts).
+        ...('author' in h && h.author !== undefined ? { author: h.author } : {}),
+        // Raw cosine before boosts, rounded as in the checked floor arm. A ledger event is no note:
+        // its chunk is event markup, and its cosine says little about what it holds (contextRelevance.ts).
+        ...(showSimilarity && typeof h.similarity === 'number' && !isLedger(h.notePath) ? { similarity: Math.round(h.similarity * 1000) / 1000 } : {}),
+    }));
+    return {
+        ok: true,
+        rankedBy: result.rankedBy,
+        hits,
+        // Present whenever an evidence model is configured, and when the kill switch is off: `state` is how a caller tells
+        // an intended "off" from a pass that failed or did not apply.
+        ...(attached ? { evidence: (0, attach_1.productEvidenceReport)(attached.evidence) } : {}),
+        ...(scopeNorm ? { scope: scopeNorm } : {}),
+    };
+}
+function engineSearchText(db, memoryDir, query, topK, scope, includeHistory = false, ledgerRelPath = lifecycle_1.DEFAULT_LEDGER_REL, ledgerRecords) {
+    const fts = (0, statements_1.prepareOnce)(db, `SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'chunks_fts'`).get();
+    if (!fts)
+        return { ok: false, error: 'fts-unavailable', results: [] };
+    const match = (0, queryHelpers_1.quoteFtsQuery)(query);
+    if (!match)
+        return { ok: true, results: [] };
+    const scopeNorm = (0, lifecycle_1.scopeFilterFor)(scope);
+    const inScope = scopeNorm ? (0, readScope_1.createReadScope)(db, scopeNorm) : null;
+    const isLedger = (0, ledgerPresentation_1.ledgerNoteMatcher)({ memoryDir, ledgerRelPath });
+    // A search may judge thousands of rows of one note (the ledger is one note), so each stored path is made the path a
+    // client is shown once per search, not once per row.
+    const shownPaths = new Map();
+    const shownPath = (stored) => {
+        let shown = shownPaths.get(stored);
+        if (shown === undefined)
+            shownPaths.set(stored, (shown = clientPath(memoryDir, stored)));
+        return shown;
+    };
+    const toResult = (r) => ({
+        notePath: shownPath(r.notePath),
+        headingPath: r.headingPath,
+        text: r.text,
+        score: r.score,
+        reason: r.reason,
+    });
+    const closedEvents = (0, ledgerLiveness_1.closedLedgerEvents)(db);
+    let prose;
+    try {
+        // A row is shown when its event is live (liveness is resolved first), it is not the ledger's own markup, and the
+        // scope read holds it. All three are judged before the page is cut, from a read that goes on until the page is full
+        // or the index has no more rows: a closed event, a record of the ledger or another scope's row never takes a place
+        // from a row that is shown, however many of them rank above it. The first read is the window of the best
+        // `topK * 10` rows (at most 200) this search has always read.
+        prose = (0, queryHelpers_1.shownPage)(n => (0, keyword_1.keywordSearch)(db, match, n), rows => {
+            const live = (0, ledgerPresentation_1.keywordLedgerHits)((0, ledgerLiveness_1.resolveLedgerHits)(rows.map(r => toResult({ ...r, score: Math.abs(r.score), reason: 'fts' })), closedEvents, includeHistory), isLedger, query, ledgerRecords);
+            return inScope ? inScope.hits(live, r => r.notePath) : live;
+        }, topK, Math.min(topK * 10, 200));
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { ok: false, error: 'fts-query', detail: message.slice(0, 200), results: [] };
+    }
+    // Only the prose hits actually returned count as "already have"; scope is applied
+    // before the identity limit.
+    const have = new Set(prose.map(h => h.notePath));
+    // The ledger is a note too (`ledger.md` names it), so its opening chunk obeys the same liveness and scope rules:
+    // judged by the scope of the event it opens with, not by the ledger's path, and before the identity limit.
+    const named = (0, ledgerLiveness_1.resolveLedgerHits)((0, noteIdentity_1.selectNoteIdentityHits)(db, query, noteIdentity_1.NOTE_IDENTITY_EXTRA, new Set(), (p, opening) => {
+        const client = clientPath(memoryDir, p);
+        return !have.has(client) && !(0, ledgerPresentation_1.isLedgerPreamble)(opening, isLedger) && (!inScope || inScope.has(opening, client));
+    }).map(toResult), closedEvents, includeHistory);
+    // An event is dated by when it was saved, as search_memory dates it: its time was in the record's heading alone, which
+    // a client is not shown (ledgerPresentation.ts). Read from the stored record, before it is shown as the user's words,
+    // which may hold a line that looks like a record's heading.
+    const eventTime = (0, ledgerEventTime_1.createLedgerEventClock)(db);
+    const boundTime = (0, orchestrator_1.createNoteTimeBound)(db);
+    const results = (0, ledgerPresentation_1.presentLedgerEvents)([...prose, ...named].map(r => {
+        const derived = isLedger(r.notePath) ? eventTime(r) : undefined;
+        if (derived === undefined)
+            return r;
+        const session_date = boundTime(path_1.default.resolve(memoryDir, r.notePath), derived);
+        return { ...r, session_date };
+    }), isLedger, ledgerRecords);
+    return {
+        ok: true,
+        results,
+        ...(scopeNorm ? { scope: scopeNorm } : {}),
+    };
+}
+/**
+ * What lookup_entity actually searches — all four mechanisms, in order, because
+ * a narrower description would be a false contract: a term can resolve on a
+ * substring of a canonical name or on a substring of a source note path, and
+ * none of them see words inside note text.
+ */
+exports.ENTITY_LOOKUP_SURFACE = 'registered entities, matched in order by exact canonical name, frontmatter alias, substring of a canonical name, or substring of a source note path — never words inside note text';
+function engineLookupEntity(db, vault, name, scope) {
+    const scopeNorm = scope === undefined ? undefined : (0, scope_1.normalizeScope)(scope);
+    // The scope constrains the SEARCH, not just the answer: the same restriction
+    // narrows the candidate queries and the count, so a scoped lookup cannot
+    // distinguish "no such entity" from "exists in another scope" — the two must
+    // be byte-identical responses or the miss reason is a cross-scope oracle.
+    // A note's entity is judged by its note's path, a remembered fact's subject by
+    // the scopes its facts were saved in (entityScopeFilter), and with no scope while it has a fact (liveEntityFilter): a
+    // subject whose every fact is retired is named nowhere.
+    const filter = scopeNorm === undefined ? (0, entityLookup_1.liveEntityFilter)() : (0, entityLookup_1.entityScopeFilter)(scopeNorm);
+    const row = (0, entityLookup_1.lookupEntity)(db, name, filter);
+    const miss = () => {
+        return {
+            ok: true,
+            entity: null,
+            reason: 'no-entity-match',
+            searched: exports.ENTITY_LOOKUP_SURFACE,
+            // Scoped to the same restriction as the lookup: never a workspace-wide
+            // count handed to a caller that may only read one scope. The same count
+            // memory_status reports, so the two never name different numbers.
+            entitiesIndexed: (0, inventory_1.countEntities)(db, vault, filter),
+            note: `Nothing in ${scopeNorm ?? 'this workspace'} is registered as an entity matching "${name}". lookup_entity searches ${exports.ENTITY_LOOKUP_SURFACE}, so a word that appears only inside saved note text (a daily note, a saved fact body) is not an entity. Use search_text for that word, or search_memory for meaning where the workspace has semantic search.`,
+            ...(scopeNorm === undefined ? {} : { scope: scopeNorm }),
+        };
+    };
+    if (!row)
+        return miss();
+    const sourceNote = clientPath(vault.memoryDir, row.sourceNote);
+    if (scopeNorm !== undefined) {
+        return {
+            ok: true,
+            entity: { canonicalName: row.canonicalName, type: row.type, aliases: row.aliases, sourceNote },
+            ...(scopeNorm === 'global' ? {} : { scope: scopeNorm }),
+        };
+    }
+    return {
+        ok: true,
+        entity: {
+            canonicalName: row.canonicalName,
+            type: row.type,
+            aliases: row.aliases,
+            sourceNote,
+        },
+    };
+}
+function engineStatus(db, engine, vault) {
+    const counts = (0, inventory_1.countInventory)(db, vault);
+    // What cannot answer right now, said rather than left for the agent to infer from empty results. Absent when healthy.
+    const failed = semanticFailures.get(engine);
+    const degraded = {
+        ...(keywordIndexReady(db) ? {} : { keyword: 'keyword search cannot run: its index is missing' }),
+        ...(failed ? { semantic: `the last search by meaning failed at ${failed.at}: ${failed.reason}` } : {}),
+    };
+    return {
+        ...counts,
+        embeddingDims: engine.dims,
+        // Whether semantic search can answer now: false with no provider, and false until the embeddings are whole.
+        semantic: semanticReady(engine),
+        ...(engine.provider && engine.vectorIndex && !semanticReady(engine)
+            ? { embeddingRebuild: { chunksLeft: engine.vectorIndex.space() === (0, space_1.embeddingSpace)(engine.provider) ? (0, rebuild_1.chunksWithoutVectors)(db) : counts.chunks, chunks: counts.chunks } }
+            : {}),
+        // With an embedding provider, the chunks search by meaning cannot find, because they have no vector: a save whose
+        // embedding failed (the next save, or a search, embeds them), notes an import wrote (the background job embeds
+        // them), or a rebuild that is not done.
+        ...(engine.vectorIndex ? { chunksWithoutVectors: (0, rebuild_1.chunksWithoutVectors)(db) } : {}),
+        ...(Object.keys(degraded).length > 0 ? { degraded } : {}),
+    };
+}
+/** `interactive`: an agent is waiting on this pass, so embedding is held to the write budget (see EmbedBudgets). */
+function indexOptionsFor(engine, interactive = false) {
+    return engine.provider && engine.vectorIndex
+        ? {
+            embeddingProvider: engine.provider,
+            vectorIndex: engine.vectorIndex,
+            ...(interactive && engine.embedBudgets && Number.isFinite(engine.embedBudgets.writeMs) ? { embedBudgetMs: engine.embedBudgets.writeMs } : {}),
+        }
+        : {};
+}
+/**
+ * Files an index pass has in flight at once. A file holds its slot while its embedding request is out, so the slot count is
+ * how many requests an index pass can batch: with the coalescing provider (coalescing.ts) the requests stay at four in
+ * flight whatever this is. Value set by the measurements in engine/docs/perf (see the note there).
+ */
+exports.INDEX_FILE_CONCURRENCY = 48;
+/**
+ * `interactive`: an agent waits on this pass (the `index_memory` tool), so a file whose embedding the service will not give
+ * within the write budget is left unfinished and the pass returns; a background job then embeds it
+ * (`engineFinishIndexInBackground`), however long the service takes. Unset: the provider waits as long as its own retries take.
+ */
+async function engineIndex(db, config, engine, pass = {}) {
+    const stats = await (0, pipeline_1.indexVault)(db, config.memoryDir, indexOptionsFor(engine, pass.interactive === true), exports.INDEX_FILE_CONCURRENCY);
+    if (stats.errors > 0 && pass.interactive)
+        engineFinishIndexInBackground(db, config, engine);
+    return stats;
+}
+/**
+ * What this process last reconciled each file with the index at: the file's stamp when `indexFile` last
+ * succeeded on it. Kept per database, since another index has reconciled nothing.
+ */
+const reconciledStamps = new WeakMap();
+/** Per database: the stamp each note over the size cap had when it was last named, so it is said once per change and not on every save. */
+const tooLargeStamps = new WeakMap();
+/**
+ * Per database: the files a pass left unsettled, whose stamp this process does not record as reconciled (a file changed
+ * within a clock tick of its scan, or whose pass failed or was not reached). A save checks them again, as a full scan
+ * would, because it cannot tell from the files it wrote that they are done.
+ */
+const unsettledFiles = new WeakMap();
+/**
+ * Per database: when this process last checked every file of the vault (a full pass). Until one has run, a save walks the
+ * vault (the first pass in a process); after that a save walks it again once FULL_VAULT_CHECK_MS has passed.
+ */
+const walkedAt = new WeakMap();
+/**
+ * How long note saves check only the files they wrote before every file of the vault is checked again. An edit made
+ * outside the engine (in Obsidian, say) is therefore seen by a save once this long has passed since the last full check;
+ * index_memory sees it at once.
+ */
+exports.FULL_VAULT_CHECK_MS = 30_000;
+/** Every stat field a change shows in. The change time is what catches a rewrite that puts the modification time back. */
+const stampOf = (f) => `${f.mtimeMs}:${f.ctimeMs}:${f.size}:${f.ino}`;
+/**
+ * A timestamp is only as fine as the file system's clock tick (two seconds on the coarsest), so a file written,
+ * scanned and written again within one tick, to the same size, carries the same stamp both times; the same goes
+ * for a rewrite that puts the modification time back, within a tick of the first change. A file whose
+ * modification or change time is this close to a scan is therefore not trusted by its stamp: the next scan
+ * reads it again.
+ */
+exports.TIMESTAMP_TICK_MS = 2_000;
+const stampIsTrusted = (f, scanStart) => Math.min(Math.abs(scanStart - f.mtimeMs), Math.abs(scanStart - f.ctimeMs)) >= exports.TIMESTAMP_TICK_MS; // false for a failed stat (NaN)
+/** Run `work` over `items`, `limit` at a time. */
+async function inPool(items, limit, work) {
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (next < items.length)
+            await work(items[next++]);
+    }));
+}
+/**
+ * Bring the index up to date with the vault as a full `indexVault` would, reading only what changed. A file goes
+ * through the same `indexFile` a full pass runs when the index has no finished row for it, or its stamp
+ * (modification time, change time, size, inode) differs from the one recorded when this process last reconciled
+ * it; an unchanged file costs a stat, not a read and a hash, and `indexFile` skips an unchanged file by its hash.
+ *
+ * Which files a pass checks depends on who runs it. A save passes `touched`, the files it wrote (the day's note, the
+ * ledger): then only those, the files still owed (unsettled, see `unsettledFiles`) and the ones the index has no
+ * finished row for are stat'ed, so the vault is not walked and no other note's path is loaded, and a save costs the
+ * same however many notes and events the workspace holds. Without `touched` the vault is walked and every file
+ * stat'ed: that is what a search that finishes a save's unfinished files and a background job run, and what starts
+ * each process. A save walks it too when the last full check is FULL_VAULT_CHECK_MS or more old, so an edit, a new
+ * file, a rename or a delete made outside the engine is seen by a save within that long (and at once by
+ * `index_memory`, a full pass that reads every file). A file changed within a clock tick of its scan is not recorded
+ * as reconciled (see TIMESTAMP_TICK_MS), so the next check reads it again. Like a full pass it leaves a deleted or
+ * renamed-away file's rows in the index. The one change a stamp cannot show is a rewrite that leaves all four fields
+ * as they were, which takes setting the system clock back.
+ */
+async function engineIndexChanged(db, config, engine, 
+/** When the scan starts, on the clock the files' times are read against; a test passes a later one to age its files. */
+scanStart = Date.now(), 
+/**
+ * `interactive`: an agent waits on this pass (a save's, or a read that finishes what a save left), so a file whose
+ * embedding the service will not give within the write budget is left unfinished and the pass returns; a background
+ * job then embeds it (`engineFinishIndexInBackground`), however long the service takes.
+ *
+ * `touched`: the absolute paths of the files the calling save wrote. Given, the pass checks those and the files it
+ * still owes, and walks no vault, unless the last full check is FULL_VAULT_CHECK_MS old or there has been none in this
+ * process.
+ */
+pass = {}) {
+    const options = indexOptionsFor(engine, pass.interactive === true);
+    let reconciled = reconciledStamps.get(db);
+    if (!reconciled) {
+        reconciled = new Map();
+        reconciledStamps.set(db, reconciled);
+    }
+    const known = reconciled;
+    let named = tooLargeStamps.get(db);
+    if (!named) {
+        named = new Map();
+        tooLargeStamps.set(db, named);
+    }
+    const tooLarge = named;
+    let unsettled = unsettledFiles.get(db);
+    if (!unsettled) {
+        unsettled = new Set();
+        unsettledFiles.set(db, unsettled);
+    }
+    let candidates;
+    let isFinished;
+    const now = Date.now();
+    const lastCheck = walkedAt.get(db);
+    const checkedRecently = lastCheck !== undefined && now >= lastCheck && now - lastCheck < exports.FULL_VAULT_CHECK_MS;
+    if (pass.touched !== undefined && checkedRecently) {
+        // What the vault's rules leave out is never checked: the walk never reaches it, so neither does a save.
+        const ignored = await (0, walker_1.vaultIgnores)(config.memoryDir);
+        const owed = [...pass.touched, ...unsettled];
+        if ((0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck().get() > 0) {
+            owed.push(...(0, statements_1.prepareOnce)(db, "SELECT path FROM notes WHERE note_hash = ''").pluck().all());
+        }
+        candidates = (0, walker_1.statVaultFiles)([...new Set(owed)].filter(file => !ignored(file)));
+        // A file that is gone or ignored has nothing left to settle: a later pass finds nothing there either.
+        const present = new Set(candidates.map(f => f.file));
+        for (const file of unsettled)
+            if (!present.has(file))
+                unsettled.delete(file);
+        const hash = (0, statements_1.prepareOnce)(db, "SELECT note_hash FROM notes WHERE path = ?").pluck();
+        isFinished = (file) => {
+            const h = hash.get(file);
+            return typeof h === 'string' && h !== '';
+        };
+    }
+    else {
+        candidates = await (0, walker_1.walkVaultStats)(config.memoryDir);
+        const finished = new Set(db.prepare("SELECT path FROM notes WHERE note_hash != ''").pluck().all());
+        isFinished = (file) => finished.has(file);
+        walkedAt.set(db, now);
+        unsettled.clear(); // rebuilt below from every file's stamp
+    }
+    const changed = candidates.filter(f => tooLarge.get(f.file) !== stampOf(f) && (!isFinished(f.file) || known.get(f.file) !== stampOf(f)));
+    let indexed = 0;
+    let errors = 0;
+    let failure;
+    // Together: each file that embeds waits on the provider for a round trip, and a save that changed the
+    // daily note and the ledger should wait for one round trip, not two.
+    await inPool(changed, exports.INDEX_FILE_CONCURRENCY, async (f) => {
+        try {
+            await (0, pipeline_1.indexFile)(db, f.file, options);
+            if (stampIsTrusted(f, scanStart))
+                known.set(f.file, stampOf(f));
+            else
+                known.delete(f.file);
+            tooLarge.delete(f.file);
+            indexed++;
+        }
+        catch (err) {
+            if (err instanceof pipeline_1.NoteTooLargeError) {
+                // Not a failure to retry: it stays out of the index until the file changes, and is said once per change.
+                known.delete(f.file);
+                tooLarge.set(f.file, stampOf(f));
+                console.warn(`[index] ${f.file}: ${err.message}`);
+                return;
+            }
+            known.delete(f.file);
+            if (err.code === 'ENOENT')
+                return;
+            errors++;
+            failure ??= failureReason(err, 300);
+            console.error(`[index] error indexing ${f.file}:`, err);
+        }
+    });
+    // Settle what was checked: a file whose stamp is recorded (or that is named too large for its stamp) is done; any
+    // other is owed, and the next save checks it again.
+    for (const f of candidates) {
+        if (known.get(f.file) === stampOf(f) || tooLarge.get(f.file) === stampOf(f))
+            unsettled.delete(f.file);
+        else
+            unsettled.add(f.file);
+    }
+    // What the pass could not finish is not dropped: it is finished in the background, once the service answers.
+    if (errors > 0 && pass.interactive)
+        engineFinishIndexInBackground(db, config, engine);
+    return { total: changed.length, indexed, errors, ...(failure === undefined ? {} : { failure }) };
+}
+/**
+ * The files whose last index pass did not finish: their embedding failed, so keyword search finds their chunks and
+ * search by meaning does not, until a later pass embeds them (see indexFile). A partial index answers it (migration
+ * 15), so asking costs those few rows, usually none, however many notes there are.
+ */
+exports.UNFINISHED_FILES = "SELECT COUNT(*) FROM notes WHERE note_hash = ''";
+/** After a search could not finish them, how long searches leave the unfinished files before trying again. */
+exports.UNFINISHED_RETRY_MS = 30_000;
+/** Per index: the epoch ms before which a search does not try the unfinished files again. */
+const unfinishedRetryAfter = new WeakMap();
+/**
+ * Finish, for a search, the files a save left unfinished. A save whose embedding failed (the provider was down)
+ * keeps its chunks, which keyword search finds, and leaves its file unfinished for a later pass to embed: the next
+ * save's, or this one, which a search runs once the provider has embedded its query (`asked`: the search did, or, on
+ * an index with no vector yet, which a search does not ask, it is asked here). So what was saved during an outage is
+ * found by meaning as soon as the provider is back, not only after the next save; and a search waits on a provider
+ * that is down no longer than for its own query, which then fails, so the search says semantic search failed rather
+ * than that nothing matched. It is engineIndexChanged's full pass (no `touched`: every file is checked, as a save's first
+ * pass in a process does). When it leaves a file unfinished although
+ * the provider answers (a chunk it refuses), searches leave the files for UNFINISHED_RETRY_MS, so that costs a search a
+ * provider call at most that often; saves go on retrying them as before. Throws only what embedding the query throws.
+ * True when a file was finished, so the search runs again and finds its chunks.
+ */
+async function finishUnfinishedFiles(db, memoryDir, engine, 
+/** The provider as a read sees it: its calls end within the read budget. */
+provider, query, asked) {
+    const unfinished = (0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck();
+    const before = unfinished.get();
+    if (before === 0 || Date.now() < (unfinishedRetryAfter.get(db) ?? 0))
+        return false;
+    // Asked here only to learn that it answers: the search that follows embeds the query and reports it as before.
+    if (!asked.queryVector)
+        await (provider.embedQuery ? provider.embedQuery(query) : provider.embed([query]));
+    const finish = async () => {
+        try {
+            await engineIndexChanged(db, { memoryDir }, engine, undefined, { interactive: true });
+        }
+        catch (err) {
+            console.error('[index] finishing the unfinished files failed:', err);
+        }
+        const after = unfinished.get();
+        if (after > 0)
+            unfinishedRetryAfter.set(db, Date.now() + exports.UNFINISHED_RETRY_MS);
+        else
+            unfinishedRetryAfter.delete(db);
+        return after < before;
+    };
+    // The pass takes each file's lock, which a save that is embedding holds; a read does not wait for it beyond its own
+    // budget. A pass that is still running then goes on without it, and the read is answered with what is indexed now.
+    const budgetMs = engine.embedBudgets?.readMs;
+    if (budgetMs === undefined || !Number.isFinite(budgetMs))
+        return finish();
+    const running = finish();
+    let timer;
+    const late = new Promise((resolve) => { timer = setTimeout(() => resolve(false), budgetMs); });
+    try {
+        const finished = await Promise.race([running, late]);
+        if (!finished)
+            unfinishedRetryAfter.set(db, Date.now() + exports.UNFINISHED_RETRY_MS);
+        return finished;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+const backgroundIndexes = new WeakMap();
+/** The first wait is short: a save that just failed is usually a service that is already back. */
+exports.BACKGROUND_INDEX_FIRST_DELAY_MS = 1_000;
+/** After an attempt that failed: 30 s, doubling, up to 5 min (the wait a rebuild has). */
+const backgroundIndexRetryDelayMs = (outage) => Math.min(300_000, 30_000 * 2 ** outage);
+/**
+ * Finish, in the background, what a save could not: the chunks it kept for keyword search that have no vector, and the
+ * files left unfinished for them. The embedding is the provider's patient one (its own retries and backoff, however long
+ * they take) and holds no lock, so a service that is down costs only this task's time; the pass that follows takes each
+ * file's lock for as long as it takes to record that its chunks have their vectors. When an attempt fails it waits
+ * (30 s, doubling, up to 5 min) and tries again, so a failed index is queued and never dropped, and it ends only when
+ * nothing is left or `abort` is called. A job already running for this index is returned rather than started twice.
+ * An index whose vectors are not whole (a rebuild is embedding all of them) is left to that rebuild.
+ */
+function engineFinishIndexInBackground(db, config, engine, options = {}) {
+    const { provider, vectorIndex } = engine;
+    if (!provider || !vectorIndex)
+        return null;
+    const running = backgroundIndexes.get(db);
+    if (running)
+        return running;
+    const { firstDelayMs = exports.BACKGROUND_INDEX_FIRST_DELAY_MS, retryDelayMs = backgroundIndexRetryDelayMs } = options;
+    const controller = new AbortController();
+    const unfinished = (0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck();
+    let waiting = false;
+    const wait = async (ms) => {
+        waiting = true;
+        try {
+            await (0, rebuild_1.sleep)(ms, controller.signal);
+        }
+        finally {
+            waiting = false;
+        }
+    };
+    const run = async () => {
+        await wait(firstDelayMs);
+        for (let outage = 0; !controller.signal.aborted; outage++) {
+            try {
+                if (semanticReady(engine)) {
+                    await (0, rebuild_1.embedMissingVectors)(db, vectorIndex, provider, { signal: controller.signal });
+                    if (controller.signal.aborted)
+                        return;
+                    // Their vectors are in: this records it, and embeds only what a save has changed since (within the write budget).
+                    await engineIndexChanged(db, config, engine, undefined, { interactive: true });
+                    if (unfinished.get() === 0 && (0, rebuild_1.chunksWithoutVectors)(db) === 0) {
+                        if (outage > 0)
+                            console.log('[index] the embedding service answers again; what was saved while it did not is findable by meaning now');
+                        return;
+                    }
+                }
+                else if (!vectorIndex.rebuilding()) {
+                    return; // another model's space: a rebuild is the only thing that can make it whole, and it is not this job's
+                }
+                throw new Error('the index is not finished yet');
+            }
+            catch (err) {
+                if (controller.signal.aborted)
+                    return;
+                const delay = retryDelayMs(outage);
+                console.error(`[index] finishing the index in the background failed (${failureReason(err, 200)}); ${(0, rebuild_1.chunksWithoutVectors)(db)} chunk(s) have no vector yet; trying again in ${Math.round(delay / 1000)} s`);
+                await wait(delay);
+            }
+        }
+    };
+    const job = {
+        abort: () => controller.abort(),
+        waiting: () => waiting,
+        done: run().catch((err) => console.error('[index] background indexing failed:', err)).finally(() => {
+            if (backgroundIndexes.get(db) === job)
+                backgroundIndexes.delete(db);
+        }),
+    };
+    backgroundIndexes.set(db, job);
+    return job;
+}
+/** The job `engineFinishIndexInBackground` started for this index that has not ended yet, or null; starts none. */
+function runningBackgroundIndex(db) {
+    return backgroundIndexes.get(db) ?? null;
+}
+/**
+ * Whether anything is waiting for the embedding service: a file left unfinished, or a chunk with no vector in an index
+ * that is otherwise whole. Asked at start, so what an earlier process (or an idle exit) left is finished.
+ */
+function hasIndexWorkToFinish(db, engine) {
+    if (!engine.provider || !engine.vectorIndex || !semanticReady(engine))
+        return false;
+    return (0, statements_1.prepareOnce)(db, exports.UNFINISHED_FILES).pluck().get() > 0 || (0, rebuild_1.chunksWithoutVectors)(db) > 0;
+}
+async function engineRememberNote(config, text, date, source, scope, 
+/** The user's time zone, when the client names it: the note is then filed under their day, at their time. */
+timeZone) {
+    // One reading of the clock gives the entry both its day and its time.
+    const clock = (0, writer_1.entryClock)(new Date(), timeZone);
+    const day = date ?? clock.date;
+    const dir = (0, scope_1.dailyDirForScope)((0, scope_1.normalizeScope)(scope));
+    const written = await (0, writer_1.appendDailyEntry)(config.memoryDir, day, { text, source: source ?? 'mcp' }, dir, clock);
+    const file = clientPath(config.memoryDir, written.filePath);
+    // The id that reads, retires or restores this entry (memory_map's), so forgetting it needs no lookup (live dogfood
+    // 2026-10-03, D12).
+    const id = (0, memoryMap_1.appendedEntryId)(file.split(path_1.default.sep).join('/'), written.entry);
+    return { ok: true, file, scope: (0, scope_1.normalizeScope)(scope), ...(id ? { id } : {}) };
+}