studiograph 1.3.48-next.21 → 1.3.48-next.210

package/dist/services/vector-service.js

@@ -18,22 +18,185 @@
 import { join } from 'path';
 import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
 import * as lancedb from '@lancedb/lancedb';
+import { embeddedAssetIds } from '../core/embeds.js';
+import { isCaptionableImage } from './assets/base.js';
+import { tryResolve } from '../core/secrets/read-flip.js';
 const VOYAGE_API_URL = 'https://api.voyageai.com/v1/embeddings';
 const VOYAGE_MODEL = 'voyage-3';
 const VECTOR_DIM = 1024;
 const TABLE_NAME = 'entities';
+/**
+ * Dedicated LanceDB table for an asset's MUTABLE index state (decision A1).
+ *
+ * Why a separate table, not columns on `entities` and not asset frontmatter:
+ *  - Frontmatter is immutable + git-committed; writing `index_status` there
+ *    would fire one async git commit per asset and defeat the import batching.
+ *    (The immutable `indexable` flag DOES live in frontmatter — see P3a.)
+ *  - New columns on the shared `entities` table force a schema migration on
+ *    every existing row. A side table is additive and touches only assets.
+ */
+const INDEX_STATE_TABLE = 'asset_index_state';
+/**
+ * Derivation-logic version. BUMP this when caption/extraction output changes so
+ * the boot sweep RE-derives every asset (Codex migration rule: a row at a stale
+ * version is re-derived, never treated as "done"). Legacy assets have NO row at
+ * all — also re-derived, never skipped.
+ */
+export const ASSET_INDEX_VERSION = 1;
 export class VectorService {
     dbPath;
-    voyageApiKey;
+    /**
+     * Legacy fallback key passed via constructor for tests or for early-
+     * boot code paths that run before the secrets store is initialized.
+     * Live calls prefer `tryResolve('voyage.api_key')` so values written
+     * to the store via the UI / CLI take effect without a restart.
+     */
+    fallbackVoyageApiKey;
     db;
     table;
+    /** Lazily-opened side table for mutable asset index state (A1). */
+    stateTable;
+    stateInitialized = false;
     lastIndexedPath;
     initialized = false;
+    // ── Single-flight guards (one in-flight init per resource) ──
+    // `init()` and `initStateTable()` both open the LanceDB connection and can be
+    // invoked concurrently (boot incremental reindex via upsert() races the
+    // fire-and-forget asset sweep via getAllIndexStates()). Without
+    // single-flighting, both would pass their boolean guard before either await
+    // resolved, double-connect, and double-`createTable` → "table already exists"
+    // / CommitConflict. Each promise is created once and awaited by all callers.
+    dbPromise;
+    initPromise;
+    stateInitPromise;
+    // ── Write serialization ──
+    // LanceDB uses optimistic concurrency: two `mergeInsert(...).execute()` (or
+    // delete) calls racing on the SAME table throw a CommitConflict and one loses.
+    // This singleton VectorService is shared by foreground entity writes
+    // (graph.ts fire-and-forget upsert/delete) AND the live asset-index drain
+    // (re-upsert after writing a caption sidecar). Without serialization those two
+    // sources can collide on the `entities` table — and the drain would record the
+    // loser as a terminal `failed` index state. `withWrite()` chains every mutating
+    // op onto a single per-instance promise so they run one at a time, in order.
+    writeChain = Promise.resolve();
+    /**
+     * Fired exactly once per asset, when `ensureInitialAssetIndexState` seeds a
+     * fresh `pending` row — i.e. an indexable IMAGE assets.db row that needs
+     * captioning. The asset-creation paths (graph.uploadAsset for import +
+     * manager/editor uploads, asset-finalize-service for presign) call
+     * ensureInitialAssetIndexState after writing the assets.db row, so the live
+     * caption drain registers here instead of being wired at each call site. It
+     * fires only for images and only on the first seed (a re-seed early-returns),
+     * so it can't loop the drain.
+     */
+    onAssetPendingCb;
+    /**
+     * Workspace-global asset ownership index (assets.db). Injected by the
+     * WorkspaceManager so `upsert` can resolve the captions of images an entry
+     * embeds (`resolveEmbeddedCaptions`) and fold them into the entry's search
+     * text. Optional — tests / early-boot paths run without it (no caption fold).
+     */
+    assetIndex;
+    // ── Operator-visible health (surfaced via /api/health/dirty) ──
+    // Tracked here, not at the call sites, because every fire-and-forget
+    // upsert/delete in graph.ts catches its own promise to avoid blocking
+    // writes — the operator needs a single rolled-up signal instead of
+    // grepping logs for `[vector] index failed`.
+    failureCount = 0;
+    inFlight = 0;
+    lastFailureAt = null;
+    lastFailure = null;
     constructor(dbPath, voyageApiKey) {
         this.dbPath = dbPath;
-        this.voyageApiKey = voyageApiKey;
+        this.fallbackVoyageApiKey = voyageApiKey;
         this.lastIndexedPath = join(dbPath, 'last-indexed.json');
     }
+    /**
+     * Resolve the Voyage API key fresh on each call: store > constructor
+     * fallback. Lets the operator flip the key via UI / CLI without a
+     * server restart.
+     */
+    get voyageApiKey() {
+        const fromStore = tryResolve('voyage.api_key');
+        return fromStore ?? this.fallbackVoyageApiKey;
+    }
+    /** Begin tracking an in-flight op. Pair with `recordSuccess`/`recordFailure`. */
+    beginOp() { this.inFlight++; }
+    recordSuccess() {
+        if (this.inFlight > 0)
+            this.inFlight--;
+    }
+    recordFailure(op, entity, err) {
+        if (this.inFlight > 0)
+            this.inFlight--;
+        this.failureCount++;
+        this.lastFailureAt = new Date().toISOString();
+        this.lastFailure = {
+            op,
+            entity,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+    /**
+     * Snapshot for /api/health. Healthy = the last op succeeded (or none have
+     * run yet) AND there is no abnormal in-flight backlog. The failure count
+     * is informational; a single transient Voyage 503 shouldn't permanently
+     * mark the service unhealthy, but operators can see the trend.
+     */
+    getHealth() {
+        return {
+            healthy: this.lastFailure === null,
+            failureCount: this.failureCount,
+            inFlight: this.inFlight,
+            lastFailureAt: this.lastFailureAt,
+            lastFailure: this.lastFailure,
+            embeddingsEnabled: !!this.voyageApiKey,
+        };
+    }
+    /** Inject the workspace-global asset ownership index (assets.db). Lets
+     *  `upsert` fold embedded-image captions into an entry's search text. */
+    setAssetIndex(index) {
+        this.assetIndex = index;
+    }
+    /**
+     * The captions of every immutable-keyed image an entry embeds, joined for
+     * folding into the entry's FTS/embedding text (asset-as-embed, decision D2).
+     *
+     * The caption + OCR text is derived once per image and stored in
+     * `assets.db.caption`, keyed by the immutable assetId (NOT in a sidecar or
+     * the body). An entry becomes findable by an embedded image's visual content
+     * by reading those captions back here — for the `material` body embed, the
+     * `reference` thumbnail, a note body image, etc. Each captioned image's text
+     * is read from the single assets.db row, so the same image embedded by N
+     * entries enriches all N without duplicating the stored caption.
+     *
+     * Forward-only (entry → the assets IT embeds) — no reverse index, no
+     * cross-repo ownership claim. Best-effort: returns undefined without an index
+     * or when nothing embedded is captioned.
+     */
+    resolveEmbeddedCaptions(entity) {
+        const index = this.assetIndex;
+        if (!index)
+            return undefined;
+        // `embeddedAssetIds` (core/embeds.ts) is the SHARED definition of "which
+        // assets does this entry embed" — body + string frontmatter, immutable-key
+        // urls only — also used by the sweep's embedder re-index, so the two never
+        // disagree on coverage. It applies a cheap substring pre-check, so a no-asset
+        // entry costs no regex.
+        const ids = embeddedAssetIds(entity.document?.content ?? '', entity.data);
+        if (ids.size === 0)
+            return undefined;
+        const captions = [];
+        for (const id of ids) {
+            try {
+                const caption = index.get(id)?.caption?.trim();
+                if (caption)
+                    captions.push(caption);
+            }
+            catch { /* best-effort: a lookup failure must not fail the index */ }
+        }
+        return captions.length ? captions.join('\n') : undefined;
+    }
     /**
      * Open/create the LanceDB table and ensure the FTS index exists.
      * Call once before using the service.
@@ -41,11 +204,44 @@ export class VectorService {
     async init() {
         if (this.initialized)
             return;
-        mkdirSync(this.dbPath, { recursive: true });
-        this.db = await lancedb.connect(this.dbPath);
-        const tableNames = await this.db.tableNames();
+        // Single-flight: concurrent callers share one in-flight init.
+        if (!this.initPromise)
+            this.initPromise = this.doInit();
+        await this.initPromise;
+    }
+    /**
+     * Run a mutating LanceDB op under the per-instance write lock so concurrent
+     * writers (foreground entity writes vs. the asset-index drain) can't collide
+     * on a shared table's optimistic-concurrency commit. Ops run strictly in the
+     * order `withWrite` is called; a throwing op rejects to ITS caller without
+     * poisoning the chain for the next writer.
+     */
+    withWrite(fn) {
+        const run = this.writeChain.then(fn, fn);
+        // Keep the chain alive regardless of this op's outcome (swallow on the
+        // chain copy only; the real result/throw still goes to the caller).
+        this.writeChain = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    /**
+     * Open (once) the shared LanceDB connection. Both `init()` and
+     * `initStateTable()` go through this so they never race to assign `this.db`.
+     */
+    async ensureDb() {
+        if (this.db)
+            return this.db;
+        if (!this.dbPromise) {
+            mkdirSync(this.dbPath, { recursive: true });
+            this.dbPromise = lancedb.connect(this.dbPath);
+        }
+        this.db = await this.dbPromise;
+        return this.db;
+    }
+    async doInit() {
+        const db = await this.ensureDb();
+        const tableNames = await db.tableNames();
         if (tableNames.includes(TABLE_NAME)) {
-            this.table = await this.db.openTable(TABLE_NAME);
+            this.table = await db.openTable(TABLE_NAME);
         }
         else {
             // Create table with a seed record (LanceDB infers schema from first batch)
@@ -58,7 +254,7 @@ export class VectorService {
                 vector: new Array(VECTOR_DIM).fill(0),
                 updated_at: new Date().toISOString(),
             };
-            this.table = await this.db.createTable(TABLE_NAME, [seed]);
+            this.table = await db.createTable(TABLE_NAME, [seed]);
             // Create FTS index on text column
             await this.table.createIndex('text', {
                 config: lancedb.Index.fts(),
@@ -68,6 +264,156 @@ export class VectorService {
         }
         this.initialized = true;
     }
+    // ─── Asset index state (mutable, LanceDB-only; keyed by assetId) ─────────────
+    /**
+     * Open/create the side table for asset index state. Independent of `init()`
+     * so state-only callers (and tests) don't pay for the entities table. Lazy
+     * because most workspaces never need it until the first asset is indexed.
+     */
+    async initStateTable() {
+        if (this.stateInitialized)
+            return;
+        // Single-flight: concurrent state readers/writers share one in-flight init,
+        // so two callers can't both take the createTable branch (which would throw
+        // "table already exists" on the loser).
+        if (!this.stateInitPromise)
+            this.stateInitPromise = this.doInitStateTable();
+        await this.stateInitPromise;
+    }
+    async doInitStateTable() {
+        const db = await this.ensureDb();
+        const names = await db.tableNames();
+        if (names.includes(INDEX_STATE_TABLE)) {
+            this.stateTable = await db.openTable(INDEX_STATE_TABLE);
+        }
+        else {
+            const seed = {
+                id: '__seed__', repo: '', entity_id: '', index_status: 'pending',
+                indexed_at: '', index_version: 0, reason: '',
+            };
+            this.stateTable = await db.createTable(INDEX_STATE_TABLE, [seed]);
+            await this.stateTable.delete(`id = '__seed__'`);
+        }
+        this.stateInitialized = true;
+    }
+    toState(r) {
+        return {
+            assetId: r.entity_id, // legacy column name; holds the immutable UUID
+            repo: r.repo,
+            status: r.index_status,
+            indexedAt: r.indexed_at,
+            version: r.index_version,
+            reason: r.reason,
+        };
+    }
+    /**
+     * Read the `text` actually indexed for one entity (the value `buildEntityText`
+     * produced — markdown body plus, for assets, the extracted sidecar). Null when
+     * no row exists. This is the searchable content of record, independent of FTS
+     * index freshness, so it's the deterministic way to assert "the asset's body
+     * reached the index" (e.g. the docx stub-upsert-race regression test).
+     */
+    async getIndexedText(repo, entityType, entityId) {
+        await this.init();
+        const id = `${repo}/${entityType}/${entityId}`.replace(/'/g, "''");
+        const rows = await this.table.query().where(`id = '${id}'`).limit(1).toArray();
+        const row = rows.find((r) => r.id !== '__seed__');
+        return row ? row.text : null;
+    }
+    /** Read one asset's index state, or null when no row exists (never-derived). */
+    async getIndexState(assetId) {
+        await this.initStateTable();
+        const id = assetId.replace(/'/g, "''");
+        const rows = await this.stateTable.query().where(`id = '${id}'`).limit(1).toArray();
+        const row = rows.find((r) => r.id !== '__seed__');
+        return row ? this.toState(row) : null;
+    }
+    /**
+     * Load every index-state row into a Map keyed by assetId. The boot sweep uses
+     * this to decide, per image asset, whether a row is missing (never-derived),
+     * stale (old version), or unfinished (pending) — without re-deriving the
+     * whole set (PERF1).
+     */
+    async getAllIndexStates() {
+        await this.initStateTable();
+        const rows = await this.stateTable.query().toArray();
+        const map = new Map();
+        for (const r of rows) {
+            if (r.id === '__seed__')
+                continue;
+            // Key by the authoritative PK `id` (== assetId for current rows), matching
+            // getIndexState/setIndexState. Legacy entity-keyed rows from the old
+            // pipeline have `id = '<repo>/<entityId>'`, which can never equal a UUID
+            // assetId, so they stay inert here (the sweep looks up by assetId) rather
+            // than being mistaken for a current row via their bare `entity_id`.
+            map.set(r.id, this.toState(r));
+        }
+        return map;
+    }
+    /** Upsert an asset's index state. Stamps `indexed_at` + current version unless overridden. */
+    async setIndexState(assetId, status, opts = {}) {
+        await this.initStateTable();
+        const record = {
+            id: assetId,
+            repo: opts.repo ?? '',
+            entity_id: assetId,
+            index_status: status,
+            indexed_at: opts.indexedAt ?? new Date().toISOString(),
+            index_version: opts.version ?? ASSET_INDEX_VERSION,
+            reason: opts.reason ?? '',
+        };
+        await this.withWrite(() => this.stateTable
+            .mergeInsert('id')
+            .whenMatchedUpdateAll()
+            .whenNotMatchedInsertAll()
+            .execute([record]));
+    }
+    /** Drop an asset's index-state row (e.g. when the asset is deleted). */
+    async deleteIndexState(assetId) {
+        await this.initStateTable();
+        const id = assetId.replace(/'/g, "''");
+        await this.withWrite(() => this.stateTable.delete(`id = '${id}'`));
+    }
+    /**
+     * Register the single sink for "a new indexable image needs captioning."
+     * The live caption drain (AssetIndexQueue) registers here; the chokepoint
+     * fires from `ensureInitialAssetIndexState`, covering every image-upload path
+     * at once. Replaces per-call-site enqueue wiring.
+     */
+    setOnAssetPending(cb) {
+        this.onAssetPendingCb = cb;
+    }
+    /**
+     * Seed the initial index state for a freshly-stored IMAGE asset, IF it has no
+     * row yet, and notify the live caption drain. Called by the asset-creation
+     * paths (graph.uploadAsset, asset-finalize-service) after the assets.db row
+     * is written — keyed by the immutable assetId, never an entity.
+     *
+     * State *transitions* (full/failed) are owned by the caption sweep; this only
+     * establishes the `pending` starting point + fires the drain. Only raster
+     * images are captioned: SVG stays transitional (sanitize-html not wired);
+     * documents/opaque binaries carry no caption. A re-seed (row already exists)
+     * early-returns so the drain can't loop. Best-effort: a state-write failure
+     * must not fail the upload.
+     */
+    async ensureInitialAssetIndexState(assetId, opts) {
+        if (!isCaptionableImage(opts.mediaType, opts.filename))
+            return;
+        try {
+            const existing = await this.getIndexState(assetId);
+            if (existing)
+                return;
+            await this.setIndexState(assetId, 'pending', { repo: opts.repo });
+            try {
+                this.onAssetPendingCb?.(assetId);
+            }
+            catch { /* non-fatal */ }
+        }
+        catch (err) {
+            // eslint-disable-next-line no-console
+            console.warn(`[vector] asset index-state seed failed for ${assetId}: ${err instanceof Error ? err.message : err}`);
+        }
+    }
     /**
      * Index or re-index a single entity.
      * Safe to call on create, update, or as part of bulk indexing.
@@ -75,7 +421,7 @@ export class VectorService {
     async upsert(repoName, entity) {
         await this.init();
         const id = `${repoName}/${entity.entityType}/${entity.id}`;
-        const text = buildEntityText(entity);
+        const text = buildEntityText(entity, this.resolveEmbeddedCaptions(entity));
         const vector = this.voyageApiKey
             ? await this.generateEmbedding(text)
             : null;
@@ -89,24 +435,29 @@ export class VectorService {
             vector: vector ?? new Array(VECTOR_DIM).fill(0),
             updated_at: new Date().toISOString(),
         };
-        await this.table
+        await this.withWrite(() => this.table
             .mergeInsert('id')
             .whenMatchedUpdateAll()
             .whenNotMatchedInsertAll()
-            .execute([record]);
+            .execute([record]));
     }
     /**
      * Bulk upsert — used by `studiograph index`. Batches Voyage API calls.
      */
-    async upsertMany(repoName, entities) {
+    async upsertMany(repoName, entities, opts = {}) {
         await this.init();
         if (entities.length === 0)
             return;
-        const texts = entities.map(e => buildEntityText(e));
+        const texts = entities.map(e => buildEntityText(e, this.resolveEmbeddedCaptions(e)));
         // Batch embeddings (Voyage supports up to 128 per request)
         let vectors = new Array(entities.length).fill(null);
         if (this.voyageApiKey) {
-            vectors = await this.generateEmbeddingsBatch(texts);
+            vectors = await this.generateEmbeddingsBatch(texts, opts.onProgress);
+        }
+        else {
+            // No embedding step — report the whole batch as "done" once so
+            // progress UIs that wait on at least one callback don't sit at 0%.
+            opts.onProgress?.(entities.length, entities.length);
         }
         const records = entities.map((entity, i) => ({
             id: `${repoName}/${entity.entityType}/${entity.id}`,
@@ -117,11 +468,11 @@ export class VectorService {
             vector: vectors[i] ?? new Array(VECTOR_DIM).fill(0),
             updated_at: new Date().toISOString(),
         }));
-        await this.table
+        await this.withWrite(() => this.table
             .mergeInsert('id')
             .whenMatchedUpdateAll()
             .whenNotMatchedInsertAll()
-            .execute(records);
+            .execute(records));
     }
     /**
      * Remove an entity from the index.
@@ -129,7 +480,48 @@ export class VectorService {
     async delete(repoName, entityType, entityId) {
         await this.init();
         const id = `${repoName}/${entityType}/${entityId}`;
-        await this.table.delete(`id = '${id.replace(/'/g, "''")}'`);
+        await this.withWrite(() => this.table.delete(`id = '${id.replace(/'/g, "''")}'`));
+        // No asset index-state reap here: under asset-as-embed the caption state is
+        // keyed by the immutable assetId and survives deletion of any entry that
+        // merely embeds the asset. The state row is reaped only when the asset
+        // itself is deleted (graph.deleteAssetById → vectorService.deleteIndexState).
+    }
+    /**
+     * Remove an entity by `(repo, id)` alone, without needing its type.
+     *
+     * Used by the serve-boot incremental indexer: when git diff reports a
+     * deletion, the file is gone from disk so its type can't be resolved
+     * from frontmatter. The vector store has a discrete `entity_id` column
+     * so we can match on `repo + entity_id` without falling back to a LIKE
+     * pattern on the synthetic id.
+     *
+     * Safe across type renames: a single id that legitimately had two type
+     * rows (e.g. type changed mid-edit and both old + new still exist) gets
+     * both wiped — which is the right behavior for incremental rebuild
+     * since the next upsert recreates the current type's row.
+     */
+    async deleteByRepoAndId(repoName, entityId) {
+        await this.init();
+        const r = repoName.replace(/'/g, "''");
+        const e = entityId.replace(/'/g, "''");
+        await this.withWrite(() => this.table.delete(`repo = '${r}' AND entity_id = '${e}'`));
+        // No asset index-state reap here either — see delete(): asset state is
+        // assetId-keyed and decoupled from the entry's lifecycle.
+    }
+    /**
+     * Best-effort `deleteIndexState` for the asset-delete path: the blob/index
+     * delete is the authoritative op, so a failure to reap the state row must not
+     * turn a successful asset delete into a thrown error. Called by
+     * graph.deleteAssetById.
+     */
+    async deleteIndexStateSafe(assetId) {
+        try {
+            await this.deleteIndexState(assetId);
+        }
+        catch (err) {
+            // eslint-disable-next-line no-console
+            console.warn(`[vector] index-state cleanup failed for ${assetId}: ${err instanceof Error ? err.message : err}`);
+        }
     }
     /**
      * Search the index.
@@ -140,22 +532,35 @@ export class VectorService {
         const limit = options.limit ?? 10;
         try {
             let rows;
+            // Repo scoping is a PRE-filter pushed into the query, not a post-filter on
+            // the result page. Filtering after `.limit()` silently drops a scoped
+            // repo's matches whenever the global top-K is dominated by other repos —
+            // e.g. a `repos:['studiograph']` search returning nothing while the same
+            // query unscoped ranks a studiograph doc #1. `where` applies BEFORE the
+            // limit so the K results are already within scope.
+            const repoWhere = options.repos && options.repos.length > 0
+                ? options.repos.map(r => `repo = '${r.replace(/'/g, "''")}'`).join(' OR ')
+                : null;
             if (this.voyageApiKey) {
                 // Tier 2: hybrid search
                 const queryVector = await this.generateEmbedding(query);
-                rows = await this.table
-                    .search(queryVector)
+                let q = this.table.search(queryVector);
+                if (repoWhere)
+                    q = q.where(repoWhere);
+                rows = await q
                     .limit(limit * 2) // fetch extra for re-ranking
                     .toArray();
             }
             else {
                 // Tier 1: FTS only
-                rows = await this.table
-                    .search(query, 'text')
+                let q = this.table.search(query, 'text');
+                if (repoWhere)
+                    q = q.where(repoWhere);
+                rows = await q
                     .limit(limit)
                     .toArray();
             }
-            let results = rows
+            const results = rows
                 .filter((r) => r.id !== '__seed__' && r.entity_id !== '')
                 .map((r, i) => ({
                 repoName: r.repo,
@@ -163,9 +568,6 @@ export class VectorService {
                 entityId: r.entity_id,
                 score: r._distance != null ? 1 / (1 + r._distance) : 1 - i * 0.01,
             }));
-            if (options.repos && options.repos.length > 0) {
-                results = results.filter(r => options.repos.includes(r.repoName));
-            }
             return results.slice(0, limit);
         }
         catch {
@@ -188,9 +590,13 @@ export class VectorService {
         }
     }
     /**
-     * Record the current time as the last index time for a repo.
+     * Record the last index time for a repo.
+     *
+     * Production callers omit `isoTimestamp` and the current time is used.
+     * Tests pass an explicit ISO timestamp so they can rewind the clock and
+     * assert the git-log `--since` window picks up the right commits.
      */
-    setLastIndexedTime(repoName) {
+    setLastIndexedTime(repoName, isoTimestamp) {
         mkdirSync(this.dbPath, { recursive: true });
         let data = {};
         if (existsSync(this.lastIndexedPath)) {
@@ -199,7 +605,7 @@ export class VectorService {
             }
             catch { /* ignore */ }
         }
-        data[repoName] = new Date().toISOString();
+        data[repoName] = isoTimestamp ?? new Date().toISOString();
         writeFileSync(this.lastIndexedPath, JSON.stringify(data, null, 2) + '\n');
     }
     /** Whether Voyage AI embeddings are enabled (Tier 2). */
@@ -211,10 +617,11 @@ export class VectorService {
         const results = await this.generateEmbeddingsBatch([text]);
         return results[0] ?? new Array(VECTOR_DIM).fill(0);
     }
-    async generateEmbeddingsBatch(texts) {
+    async generateEmbeddingsBatch(texts, onProgress) {
         // Voyage supports up to 128 inputs per request
         const BATCH_SIZE = 128;
         const all = [];
+        const total = texts.length;
         for (let i = 0; i < texts.length; i += BATCH_SIZE) {
             const batch = texts.slice(i, i + BATCH_SIZE);
             const response = await fetch(VOYAGE_API_URL, {
@@ -231,6 +638,7 @@ export class VectorService {
             }
             const json = await response.json();
             all.push(...json.data.map(d => d.embedding));
+            onProgress?.(all.length, total);
         }
         return all;
     }
@@ -246,10 +654,15 @@ export class VectorService {
 export async function reindexWorkspace(workspacePath, repos, onProgress) {
     const { loadUserConfig } = await import('../core/user-config.js');
     const { BaseGraphManager } = await import('../core/graph.js');
+    const { AssetIndexService } = await import('./assets/asset-index-service.js');
     const { homedir } = await import('os');
     const userConfig = loadUserConfig();
     const dbPath = userConfig.vector_db_path || join(homedir(), '.studiograph', 'vector');
     const vectorService = new VectorService(dbPath, userConfig.voyage_api_key);
+    // Wire the assets.db handle so a cold reindex still folds embedded-image
+    // captions into entry search text (otherwise it would overwrite the server's
+    // caption-folded rows with caption-less ones).
+    vectorService.setAssetIndex(new AssetIndexService(workspacePath));
     await vectorService.init();
     const log = onProgress ?? (() => { });
     let total = 0;
@@ -277,13 +690,151 @@ export async function reindexWorkspace(workspacePath, repos, onProgress) {
     }
     return total;
 }
+export async function incrementalReindexWorkspace(workspaceManager, repos, workspacePath, opts = {}) {
+    const vectorService = workspaceManager.vectorService;
+    if (!vectorService) {
+        return { total: 0, fresh: false };
+    }
+    const log = opts.log ?? (() => { });
+    const { execFileSync } = await import('child_process');
+    let total = 0;
+    let anyFreshRepo = false;
+    const staleEntities = [];
+    // Pre-pass: figure out the total first-time entity count so the
+    // progress callback has a stable denominator. Cheap — `graph.list()`
+    // doesn't re-read files, it returns the in-memory graph state the
+    // workspace manager already maintains.
+    let firstTimeTotal = 0;
+    if (opts.onFirstTimeProgress) {
+        for (const repo of repos) {
+            if (vectorService.getLastIndexedTime(repo.name))
+                continue;
+            try {
+                firstTimeTotal += workspaceManager.getGraph(repo.name).list().length;
+            }
+            catch {
+                // tolerate per-repo failure; the real loop below will log it
+            }
+        }
+    }
+    let firstTimeDone = 0;
+    for (const repo of repos) {
+        const lastIndexed = vectorService.getLastIndexedTime(repo.name);
+        // Per-repo first-time index. Catches three cases:
+        //   1. Very first `serve` boot (every repo missing a timestamp).
+        //   2. New folder added between serve boots (existing workspace +
+        //      new repo with no timestamp). Pre-fix this was skipped here
+        //      and never indexed unless the operator ran a full reindex.
+        //   3. Vector DB rebuilt from scratch but workspace has existing
+        //      folders.
+        if (!lastIndexed) {
+            try {
+                const graph = workspaceManager.getGraph(repo.name);
+                const entities = graph.list();
+                if (entities.length > 0) {
+                    await vectorService.upsertMany(repo.name, entities, {
+                        onProgress: (batchDone) => {
+                            opts.onFirstTimeProgress?.(firstTimeDone + batchDone, firstTimeTotal);
+                        },
+                    });
+                    firstTimeDone += entities.length;
+                    total += entities.length;
+                }
+                vectorService.setLastIndexedTime(repo.name);
+                anyFreshRepo = true;
+            }
+            catch (err) {
+                log(`failed to index ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
+            }
+            continue;
+        }
+        // Established repo — diff git log since the last index timestamp and
+        // collect a list of stale (repo, id) tuples to upsert / delete below.
+        const repoPath = join(workspacePath, repo.path);
+        if (!existsSync(repoPath))
+            continue;
+        try {
+            const output = execFileSync('git', ['log', `--since=${lastIndexed}`, '--name-status', '--format=', '--', '*.md'], { cwd: repoPath, encoding: 'utf-8', stdio: 'pipe' }).trim();
+            if (!output)
+                continue;
+            for (const line of output.split('\n')) {
+                // Accept any markdown path. Layouts in the wild:
+                //   - flat:    `entity-id.md`
+                //   - typed:   `<type>/entity-id.md`
+                //   - nested:  `folder/sub/entity-id.md`
+                // STU-131 normalized on flat for new entities; older typed layouts
+                // and arbitrary nested folders still appear in mixed repos. Entity
+                // type lives in frontmatter (resolved via graph.getById for
+                // upserts; not needed for deletes since we now match by `(repo,
+                // entity_id)` columns). Pre-fix this regex required exactly one
+                // slash with a `\w+` type segment and missed all three layouts,
+                // plus hyphenated types like `case-study`.
+                const m = line.match(/^([AMD])\s+(.+\.md)$/);
+                if (!m)
+                    continue;
+                const [, status, path] = m;
+                // basename minus `.md`. Entity IDs are unique within a repo.
+                const id = path.split('/').pop().replace(/\.md$/, '');
+                if (!id)
+                    continue;
+                staleEntities.push({ repoName: repo.name, entityId: id, deleted: status === 'D' });
+            }
+        }
+        catch (err) {
+            log(`failed to diff ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
+        }
+    }
+    if (staleEntities.length === 0) {
+        // Nothing changed via git diff. If we did first-time work above, that
+        // counts as a fresh run; otherwise this was a no-op incremental boot.
+        return { total, fresh: anyFreshRepo };
+    }
+    // Dedupe: if an entity appears multiple times (e.g. M then D), the last
+    // entry wins. Order matters — git log is newest-first by default, so we
+    // reverse before reducing to get the newest state per (repo, id).
+    const seen = new Map();
+    for (const item of staleEntities.slice().reverse()) {
+        seen.set(`${item.repoName}/${item.entityId}`, item);
+    }
+    for (const item of seen.values()) {
+        try {
+            if (item.deleted) {
+                // Delete by (repo, id) — type isn't recoverable from a deleted file.
+                // The new column-based matcher in vectorService handles it without
+                // a frontmatter lookup.
+                await vectorService.deleteByRepoAndId(item.repoName, item.entityId);
+            }
+            else {
+                const graph = workspaceManager.getGraph(item.repoName);
+                const entity = graph.getById(item.entityId);
+                await vectorService.upsert(item.repoName, entity);
+                total++;
+            }
+        }
+        catch {
+            // Entity may have been deleted then re-created; safe to skip.
+        }
+    }
+    // Stamp every repo we touched. Empty-diff repos already had a timestamp
+    // (we skipped the rebuild path) — re-stamping is cheap and keeps the
+    // "since" window tight for the next boot.
+    for (const repo of repos) {
+        vectorService.setLastIndexedTime(repo.name);
+    }
+    return { total, fresh: anyFreshRepo };
+}
 // ─── Helpers ─────────────────────────────────────────────────────────────────
 /**
  * Build a text representation of an entity for FTS/embedding.
  * Combines entity type, key frontmatter fields, and markdown body.
+ *
+ * `extraText` is appended after the body — used by VectorService to
+ * fold PDF sidecar text into asset-entity embeddings. Kept as a plain
+ * string parameter (not file IO) so this stays pure and unit-testable.
+ *
  * Exported for testing.
  */
-export function buildEntityText(entity) {
+export function buildEntityText(entity, extraText) {
     const parts = [];
     // Entity type and ID as context
     parts.push(`${entity.entityType}: ${entity.data.name || entity.id}`);
@@ -317,6 +868,11 @@ export function buildEntityText(entity) {
         if (plainText)
             parts.push(plainText);
     }
+    if (extraText) {
+        const trimmed = extraText.trim();
+        if (trimmed)
+            parts.push(trimmed);
+    }
     return parts.join('\n');
 }
 //# sourceMappingURL=vector-service.js.map