studiograph 1.3.48-next.2 → 1.3.48-next.201

package/dist/services/vector-service.js

@@ -18,22 +18,164 @@
 import { join } from 'path';
 import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
 import * as lancedb from '@lancedb/lancedb';
+import { tryResolve } from '../core/secrets/read-flip.js';
 const VOYAGE_API_URL = 'https://api.voyageai.com/v1/embeddings';
 const VOYAGE_MODEL = 'voyage-3';
 const VECTOR_DIM = 1024;
 const TABLE_NAME = 'entities';
+/**
+ * Dedicated LanceDB table for an asset's MUTABLE index state (decision A1).
+ *
+ * Why a separate table, not columns on `entities` and not asset frontmatter:
+ *  - Frontmatter is immutable + git-committed; writing `index_status` there
+ *    would fire one async git commit per asset and defeat the import batching.
+ *    (The immutable `indexable` flag DOES live in frontmatter — see P3a.)
+ *  - New columns on the shared `entities` table force a schema migration on
+ *    every existing row. A side table is additive and touches only assets.
+ */
+const INDEX_STATE_TABLE = 'asset_index_state';
+/**
+ * Derivation-logic version. BUMP this when caption/extraction output changes so
+ * the boot sweep RE-derives every asset (Codex migration rule: a row at a stale
+ * version is re-derived, never treated as "done"). Legacy assets have NO row at
+ * all — also re-derived, never skipped.
+ */
+export const ASSET_INDEX_VERSION = 1;
 export class VectorService {
     dbPath;
-    voyageApiKey;
+    /**
+     * Legacy fallback key passed via constructor for tests or for early-
+     * boot code paths that run before the secrets store is initialized.
+     * Live calls prefer `tryResolve('voyage.api_key')` so values written
+     * to the store via the UI / CLI take effect without a restart.
+     */
+    fallbackVoyageApiKey;
+    workspaceRoot;
     db;
     table;
+    /** Lazily-opened side table for mutable asset index state (A1). */
+    stateTable;
+    stateInitialized = false;
     lastIndexedPath;
     initialized = false;
-    constructor(dbPath, voyageApiKey) {
+    // ── Single-flight guards (one in-flight init per resource) ──
+    // `init()` and `initStateTable()` both open the LanceDB connection and can be
+    // invoked concurrently (boot incremental reindex via upsert() races the
+    // fire-and-forget asset sweep via getAllIndexStates()). Without
+    // single-flighting, both would pass their boolean guard before either await
+    // resolved, double-connect, and double-`createTable` → "table already exists"
+    // / CommitConflict. Each promise is created once and awaited by all callers.
+    dbPromise;
+    initPromise;
+    stateInitPromise;
+    // ── Write serialization ──
+    // LanceDB uses optimistic concurrency: two `mergeInsert(...).execute()` (or
+    // delete) calls racing on the SAME table throw a CommitConflict and one loses.
+    // This singleton VectorService is shared by foreground entity writes
+    // (graph.ts fire-and-forget upsert/delete) AND the live asset-index drain
+    // (re-upsert after writing a caption sidecar). Without serialization those two
+    // sources can collide on the `entities` table — and the drain would record the
+    // loser as a terminal `failed` index state. `withWrite()` chains every mutating
+    // op onto a single per-instance promise so they run one at a time, in order.
+    writeChain = Promise.resolve();
+    /**
+     * Fired exactly once per asset, when `ensureInitialAssetIndexState` seeds a
+     * fresh `pending` row — i.e. an indexable image that needs captioning. This
+     * is the SINGLE chokepoint every asset-creation path funnels through (all
+     * upserts call ensureInitialAssetIndexState), so the live caption drain
+     * registers here instead of being wired at each of the ~5 upload call sites.
+     * It does NOT fire for docs (seeded `full`) or for re-derives (row already
+     * exists → early return), so it can't loop the drain.
+     */
+    onAssetPendingCb;
+    // ── Operator-visible health (surfaced via /api/health/dirty) ──
+    // Tracked here, not at the call sites, because every fire-and-forget
+    // upsert/delete in graph.ts catches its own promise to avoid blocking
+    // writes — the operator needs a single rolled-up signal instead of
+    // grepping logs for `[vector] index failed`.
+    failureCount = 0;
+    inFlight = 0;
+    lastFailureAt = null;
+    lastFailure = null;
+    constructor(dbPath, voyageApiKey, workspaceRoot) {
         this.dbPath = dbPath;
-        this.voyageApiKey = voyageApiKey;
+        this.fallbackVoyageApiKey = voyageApiKey;
+        this.workspaceRoot = workspaceRoot;
         this.lastIndexedPath = join(dbPath, 'last-indexed.json');
     }
+    /**
+     * Resolve the Voyage API key fresh on each call: store > constructor
+     * fallback. Lets the operator flip the key via UI / CLI without a
+     * server restart.
+     */
+    get voyageApiKey() {
+        const fromStore = tryResolve('voyage.api_key');
+        return fromStore ?? this.fallbackVoyageApiKey;
+    }
+    /** Begin tracking an in-flight op. Pair with `recordSuccess`/`recordFailure`. */
+    beginOp() { this.inFlight++; }
+    recordSuccess() {
+        if (this.inFlight > 0)
+            this.inFlight--;
+    }
+    recordFailure(op, entity, err) {
+        if (this.inFlight > 0)
+            this.inFlight--;
+        this.failureCount++;
+        this.lastFailureAt = new Date().toISOString();
+        this.lastFailure = {
+            op,
+            entity,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+    /**
+     * Snapshot for /api/health. Healthy = the last op succeeded (or none have
+     * run yet) AND there is no abnormal in-flight backlog. The failure count
+     * is informational; a single transient Voyage 503 shouldn't permanently
+     * mark the service unhealthy, but operators can see the trend.
+     */
+    getHealth() {
+        return {
+            healthy: this.lastFailure === null,
+            failureCount: this.failureCount,
+            inFlight: this.inFlight,
+            lastFailureAt: this.lastFailureAt,
+            lastFailure: this.lastFailure,
+            embeddingsEnabled: !!this.voyageApiKey,
+        };
+    }
+    /**
+     * For asset entities, any extracted text the upload/caption pipeline wrote
+     * lives in a sidecar at `<workspace>/.studiograph/assets-text/asset/<id>.txt`,
+     * keyed by the asset's ENTITY ID (not the storage UUID). Returns the contents
+     * so `buildEntityText` can fold it into the embedding/FTS input.
+     *
+     * Generalized from PDF-only (P3): docx/xlsx now write the same sidecar at
+     * upload, and image captions/OCR land here too — so the reader is no longer
+     * gated on `asset_kind === 'pdf'`. The markdown body of an asset entity is
+     * intentionally light; without this, document/image assets would be
+     * effectively invisible to RAG.
+     *
+     * Best-effort: missing sidecar / read errors return null. The asset stays
+     * indexed by frontmatter alone.
+     */
+    readAssetSidecar(entity) {
+        if (!this.workspaceRoot)
+            return null;
+        if (entity.entityType !== 'asset')
+            return null;
+        const path = join(this.workspaceRoot, '.studiograph', 'assets-text', 'asset', `${entity.id}.txt`);
+        try {
+            if (!existsSync(path))
+                return null;
+            const text = readFileSync(path, 'utf-8').trim();
+            return text || null;
+        }
+        catch {
+            return null;
+        }
+    }
     /**
      * Open/create the LanceDB table and ensure the FTS index exists.
      * Call once before using the service.
@@ -41,11 +183,44 @@ export class VectorService {
     async init() {
         if (this.initialized)
             return;
-        mkdirSync(this.dbPath, { recursive: true });
-        this.db = await lancedb.connect(this.dbPath);
-        const tableNames = await this.db.tableNames();
+        // Single-flight: concurrent callers share one in-flight init.
+        if (!this.initPromise)
+            this.initPromise = this.doInit();
+        await this.initPromise;
+    }
+    /**
+     * Run a mutating LanceDB op under the per-instance write lock so concurrent
+     * writers (foreground entity writes vs. the asset-index drain) can't collide
+     * on a shared table's optimistic-concurrency commit. Ops run strictly in the
+     * order `withWrite` is called; a throwing op rejects to ITS caller without
+     * poisoning the chain for the next writer.
+     */
+    withWrite(fn) {
+        const run = this.writeChain.then(fn, fn);
+        // Keep the chain alive regardless of this op's outcome (swallow on the
+        // chain copy only; the real result/throw still goes to the caller).
+        this.writeChain = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    /**
+     * Open (once) the shared LanceDB connection. Both `init()` and
+     * `initStateTable()` go through this so they never race to assign `this.db`.
+     */
+    async ensureDb() {
+        if (this.db)
+            return this.db;
+        if (!this.dbPromise) {
+            mkdirSync(this.dbPath, { recursive: true });
+            this.dbPromise = lancedb.connect(this.dbPath);
+        }
+        this.db = await this.dbPromise;
+        return this.db;
+    }
+    async doInit() {
+        const db = await this.ensureDb();
+        const tableNames = await db.tableNames();
         if (tableNames.includes(TABLE_NAME)) {
-            this.table = await this.db.openTable(TABLE_NAME);
+            this.table = await db.openTable(TABLE_NAME);
         }
         else {
             // Create table with a seed record (LanceDB infers schema from first batch)
@@ -58,7 +233,7 @@ export class VectorService {
                 vector: new Array(VECTOR_DIM).fill(0),
                 updated_at: new Date().toISOString(),
             };
-            this.table = await this.db.createTable(TABLE_NAME, [seed]);
+            this.table = await db.createTable(TABLE_NAME, [seed]);
             // Create FTS index on text column
             await this.table.createIndex('text', {
                 config: lancedb.Index.fts(),
@@ -68,6 +243,163 @@ export class VectorService {
         }
         this.initialized = true;
     }
+    // ─── Asset index state (A1 — mutable, LanceDB-only) ──────────────────────────
+    stateRowId(repo, entityId) {
+        return `${repo}/${entityId}`;
+    }
+    /**
+     * Open/create the side table for asset index state. Independent of `init()`
+     * so state-only callers (and tests) don't pay for the entities table. Lazy
+     * because most workspaces never need it until the first asset is indexed.
+     */
+    async initStateTable() {
+        if (this.stateInitialized)
+            return;
+        // Single-flight: concurrent state readers/writers share one in-flight init,
+        // so two callers can't both take the createTable branch (which would throw
+        // "table already exists" on the loser).
+        if (!this.stateInitPromise)
+            this.stateInitPromise = this.doInitStateTable();
+        await this.stateInitPromise;
+    }
+    async doInitStateTable() {
+        const db = await this.ensureDb();
+        const names = await db.tableNames();
+        if (names.includes(INDEX_STATE_TABLE)) {
+            this.stateTable = await db.openTable(INDEX_STATE_TABLE);
+        }
+        else {
+            const seed = {
+                id: '__seed__', repo: '', entity_id: '', index_status: 'pending',
+                indexed_at: '', index_version: 0, reason: '',
+            };
+            this.stateTable = await db.createTable(INDEX_STATE_TABLE, [seed]);
+            await this.stateTable.delete(`id = '__seed__'`);
+        }
+        this.stateInitialized = true;
+    }
+    toState(r) {
+        return {
+            repo: r.repo,
+            entityId: r.entity_id,
+            status: r.index_status,
+            indexedAt: r.indexed_at,
+            version: r.index_version,
+            reason: r.reason,
+        };
+    }
+    /**
+     * Read the `text` actually indexed for one entity (the value `buildEntityText`
+     * produced — markdown body plus, for assets, the extracted sidecar). Null when
+     * no row exists. This is the searchable content of record, independent of FTS
+     * index freshness, so it's the deterministic way to assert "the asset's body
+     * reached the index" (e.g. the docx stub-upsert-race regression test).
+     */
+    async getIndexedText(repo, entityType, entityId) {
+        await this.init();
+        const id = `${repo}/${entityType}/${entityId}`.replace(/'/g, "''");
+        const rows = await this.table.query().where(`id = '${id}'`).limit(1).toArray();
+        const row = rows.find((r) => r.id !== '__seed__');
+        return row ? row.text : null;
+    }
+    /** Read one asset's index state, or null when no row exists (legacy/never-derived). */
+    async getIndexState(repo, entityId) {
+        await this.initStateTable();
+        const id = this.stateRowId(repo, entityId).replace(/'/g, "''");
+        const rows = await this.stateTable.query().where(`id = '${id}'`).limit(1).toArray();
+        const row = rows.find((r) => r.id !== '__seed__');
+        return row ? this.toState(row) : null;
+    }
+    /**
+     * Load every index-state row into a Map keyed by `${repo}/${entityId}`. The
+     * boot sweep uses this to decide, per asset, whether a row is missing
+     * (legacy), stale (old version), or unfinished (pending) — without an
+     * all-asset disk scan (PERF1).
+     */
+    async getAllIndexStates() {
+        await this.initStateTable();
+        const rows = await this.stateTable.query().toArray();
+        const map = new Map();
+        for (const r of rows) {
+            if (r.id === '__seed__')
+                continue;
+            map.set(this.stateRowId(r.repo, r.entity_id), this.toState(r));
+        }
+        return map;
+    }
+    /** Upsert an asset's index state. Stamps `indexed_at` + current version unless overridden. */
+    async setIndexState(repo, entityId, status, opts = {}) {
+        await this.initStateTable();
+        const record = {
+            id: this.stateRowId(repo, entityId),
+            repo,
+            entity_id: entityId,
+            index_status: status,
+            indexed_at: opts.indexedAt ?? new Date().toISOString(),
+            index_version: opts.version ?? ASSET_INDEX_VERSION,
+            reason: opts.reason ?? '',
+        };
+        await this.withWrite(() => this.stateTable
+            .mergeInsert('id')
+            .whenMatchedUpdateAll()
+            .whenNotMatchedInsertAll()
+            .execute([record]));
+    }
+    /** Drop an asset's index-state row (e.g. when the asset is deleted). */
+    async deleteIndexState(repo, entityId) {
+        await this.initStateTable();
+        const id = this.stateRowId(repo, entityId).replace(/'/g, "''");
+        await this.withWrite(() => this.stateTable.delete(`id = '${id}'`));
+    }
+    /**
+     * Seed the initial index state for a freshly (re)indexed asset, IF it has no
+     * row yet. State *transitions* are owned by the caption/sweep pass — this only
+     * establishes the starting point at import time:
+     *  - `indexable !== true` (opaque, or legacy `undefined`) → no row. Opaque
+     *    assets read "filename only" from frontmatter; legacy assets are
+     *    re-derived by the boot sweep, not seeded here.
+     *  - sidecar already on disk (doc extracted synchronously at upload, P3a) → `full`.
+     *  - no sidecar (image awaiting caption) → `pending`.
+     * Best-effort: a state-write failure must not fail the text index.
+     */
+    /**
+     * Register the single sink for "a new indexable image needs captioning."
+     * The live caption drain (AssetIndexQueue) registers here; the chokepoint
+     * fires from `ensureInitialAssetIndexState`, covering every upload path at
+     * once. Replaces the per-call-site enqueue wiring.
+     */
+    setOnAssetPending(cb) {
+        this.onAssetPendingCb = cb;
+    }
+    async ensureInitialAssetIndexState(repo, entity) {
+        if (entity.entityType !== 'asset')
+            return;
+        if (entity.data?.indexable !== true)
+            return;
+        try {
+            const existing = await this.getIndexState(repo, entity.id);
+            if (existing)
+                return;
+            const hasSidecar = this.readAssetSidecar(entity) !== null;
+            await this.setIndexState(repo, entity.id, hasSidecar ? 'full' : 'pending');
+            // Chokepoint: a freshly-seeded `pending` image is the universal signal
+            // that an asset needs caption/extract work. Notify the live drain so it
+            // captions while the server runs — for EVERY ingress (web ×2, MCP ×2,
+            // agent import_file), with no per-call-site wiring. Docs land `full`
+            // (sidecar already on disk) and don't need the drain. Best-effort: a
+            // throwing subscriber must not fail the index write.
+            if (!hasSidecar) {
+                try {
+                    this.onAssetPendingCb?.(repo, entity.id);
+                }
+                catch { /* non-fatal */ }
+            }
+        }
+        catch (err) {
+            // eslint-disable-next-line no-console
+            console.warn(`[vector] asset index-state seed failed for ${repo}/${entity.id}: ${err instanceof Error ? err.message : err}`);
+        }
+    }
     /**
      * Index or re-index a single entity.
      * Safe to call on create, update, or as part of bulk indexing.
@@ -75,7 +407,7 @@ export class VectorService {
     async upsert(repoName, entity) {
         await this.init();
         const id = `${repoName}/${entity.entityType}/${entity.id}`;
-        const text = buildEntityText(entity);
+        const text = buildEntityText(entity, this.readAssetSidecar(entity) ?? undefined);
         const vector = this.voyageApiKey
             ? await this.generateEmbedding(text)
             : null;
@@ -89,24 +421,33 @@ export class VectorService {
             vector: vector ?? new Array(VECTOR_DIM).fill(0),
             updated_at: new Date().toISOString(),
         };
-        await this.table
+        await this.withWrite(() => this.table
             .mergeInsert('id')
             .whenMatchedUpdateAll()
             .whenNotMatchedInsertAll()
-            .execute([record]);
+            .execute([record]));
+        // Seed asset index state at import time (decision A1). New image uploads
+        // land as `pending` so the caption pass (boot sweep) picks them up; docs
+        // whose sidecar was already extracted at upload land as `full`.
+        await this.ensureInitialAssetIndexState(repoName, entity);
     }
     /**
      * Bulk upsert — used by `studiograph index`. Batches Voyage API calls.
      */
-    async upsertMany(repoName, entities) {
+    async upsertMany(repoName, entities, opts = {}) {
         await this.init();
         if (entities.length === 0)
             return;
-        const texts = entities.map(e => buildEntityText(e));
+        const texts = entities.map(e => buildEntityText(e, this.readAssetSidecar(e) ?? undefined));
         // Batch embeddings (Voyage supports up to 128 per request)
         let vectors = new Array(entities.length).fill(null);
         if (this.voyageApiKey) {
-            vectors = await this.generateEmbeddingsBatch(texts);
+            vectors = await this.generateEmbeddingsBatch(texts, opts.onProgress);
+        }
+        else {
+            // No embedding step — report the whole batch as "done" once so
+            // progress UIs that wait on at least one callback don't sit at 0%.
+            opts.onProgress?.(entities.length, entities.length);
         }
         const records = entities.map((entity, i) => ({
             id: `${repoName}/${entity.entityType}/${entity.id}`,
@@ -117,11 +458,11 @@ export class VectorService {
             vector: vectors[i] ?? new Array(VECTOR_DIM).fill(0),
             updated_at: new Date().toISOString(),
         }));
-        await this.table
+        await this.withWrite(() => this.table
             .mergeInsert('id')
             .whenMatchedUpdateAll()
             .whenNotMatchedInsertAll()
-            .execute(records);
+            .execute(records));
     }
     /**
      * Remove an entity from the index.
@@ -129,7 +470,50 @@ export class VectorService {
     async delete(repoName, entityType, entityId) {
         await this.init();
         const id = `${repoName}/${entityType}/${entityId}`;
-        await this.table.delete(`id = '${id.replace(/'/g, "''")}'`);
+        await this.withWrite(() => this.table.delete(`id = '${id.replace(/'/g, "''")}'`));
+        // Reap any asset index-state row keyed by (repo, id). Harmless for
+        // non-asset types (no row exists). Without this the row would survive the
+        // asset's deletion and, on id reuse, make the recreated asset look
+        // already-indexed to the boot sweep (stale-state → silently unsearchable).
+        await this.deleteIndexStateSafe(repoName, entityId);
+    }
+    /**
+     * Remove an entity by `(repo, id)` alone, without needing its type.
+     *
+     * Used by the serve-boot incremental indexer: when git diff reports a
+     * deletion, the file is gone from disk so its type can't be resolved
+     * from frontmatter. The vector store has a discrete `entity_id` column
+     * so we can match on `repo + entity_id` without falling back to a LIKE
+     * pattern on the synthetic id.
+     *
+     * Safe across type renames: a single id that legitimately had two type
+     * rows (e.g. type changed mid-edit and both old + new still exist) gets
+     * both wiped — which is the right behavior for incremental rebuild
+     * since the next upsert recreates the current type's row.
+     */
+    async deleteByRepoAndId(repoName, entityId) {
+        await this.init();
+        const r = repoName.replace(/'/g, "''");
+        const e = entityId.replace(/'/g, "''");
+        await this.withWrite(() => this.table.delete(`repo = '${r}' AND entity_id = '${e}'`));
+        // Reap the asset index-state row too (the git-pull/incremental-delete path
+        // reaches here when a .md is removed outside the app). Same stale-state
+        // hazard as the in-app delete if left behind.
+        await this.deleteIndexStateSafe(repoName, entityId);
+    }
+    /**
+     * Best-effort `deleteIndexState` for the delete paths: the entities-table
+     * delete is the authoritative op, so a failure to reap the (usually absent)
+     * state row must not turn a successful delete into a thrown error.
+     */
+    async deleteIndexStateSafe(repoName, entityId) {
+        try {
+            await this.deleteIndexState(repoName, entityId);
+        }
+        catch (err) {
+            // eslint-disable-next-line no-console
+            console.warn(`[vector] index-state cleanup failed for ${repoName}/${entityId}: ${err instanceof Error ? err.message : err}`);
+        }
     }
     /**
      * Search the index.
@@ -140,22 +524,35 @@ export class VectorService {
         const limit = options.limit ?? 10;
         try {
             let rows;
+            // Repo scoping is a PRE-filter pushed into the query, not a post-filter on
+            // the result page. Filtering after `.limit()` silently drops a scoped
+            // repo's matches whenever the global top-K is dominated by other repos —
+            // e.g. a `repos:['studiograph']` search returning nothing while the same
+            // query unscoped ranks a studiograph doc #1. `where` applies BEFORE the
+            // limit so the K results are already within scope.
+            const repoWhere = options.repos && options.repos.length > 0
+                ? options.repos.map(r => `repo = '${r.replace(/'/g, "''")}'`).join(' OR ')
+                : null;
             if (this.voyageApiKey) {
                 // Tier 2: hybrid search
                 const queryVector = await this.generateEmbedding(query);
-                rows = await this.table
-                    .search(queryVector)
+                let q = this.table.search(queryVector);
+                if (repoWhere)
+                    q = q.where(repoWhere);
+                rows = await q
                     .limit(limit * 2) // fetch extra for re-ranking
                     .toArray();
             }
             else {
                 // Tier 1: FTS only
-                rows = await this.table
-                    .search(query, 'text')
+                let q = this.table.search(query, 'text');
+                if (repoWhere)
+                    q = q.where(repoWhere);
+                rows = await q
                     .limit(limit)
                     .toArray();
             }
-            let results = rows
+            const results = rows
                 .filter((r) => r.id !== '__seed__' && r.entity_id !== '')
                 .map((r, i) => ({
                 repoName: r.repo,
@@ -163,9 +560,6 @@ export class VectorService {
                 entityId: r.entity_id,
                 score: r._distance != null ? 1 / (1 + r._distance) : 1 - i * 0.01,
             }));
-            if (options.repos && options.repos.length > 0) {
-                results = results.filter(r => options.repos.includes(r.repoName));
-            }
             return results.slice(0, limit);
         }
         catch {
@@ -188,9 +582,13 @@ export class VectorService {
         }
     }
     /**
-     * Record the current time as the last index time for a repo.
+     * Record the last index time for a repo.
+     *
+     * Production callers omit `isoTimestamp` and the current time is used.
+     * Tests pass an explicit ISO timestamp so they can rewind the clock and
+     * assert the git-log `--since` window picks up the right commits.
      */
-    setLastIndexedTime(repoName) {
+    setLastIndexedTime(repoName, isoTimestamp) {
         mkdirSync(this.dbPath, { recursive: true });
         let data = {};
         if (existsSync(this.lastIndexedPath)) {
@@ -199,7 +597,7 @@ export class VectorService {
             }
             catch { /* ignore */ }
         }
-        data[repoName] = new Date().toISOString();
+        data[repoName] = isoTimestamp ?? new Date().toISOString();
         writeFileSync(this.lastIndexedPath, JSON.stringify(data, null, 2) + '\n');
     }
     /** Whether Voyage AI embeddings are enabled (Tier 2). */
@@ -211,10 +609,11 @@ export class VectorService {
         const results = await this.generateEmbeddingsBatch([text]);
         return results[0] ?? new Array(VECTOR_DIM).fill(0);
     }
-    async generateEmbeddingsBatch(texts) {
+    async generateEmbeddingsBatch(texts, onProgress) {
         // Voyage supports up to 128 inputs per request
         const BATCH_SIZE = 128;
         const all = [];
+        const total = texts.length;
         for (let i = 0; i < texts.length; i += BATCH_SIZE) {
             const batch = texts.slice(i, i + BATCH_SIZE);
             const response = await fetch(VOYAGE_API_URL, {
@@ -231,6 +630,7 @@ export class VectorService {
             }
             const json = await response.json();
             all.push(...json.data.map(d => d.embedding));
+            onProgress?.(all.length, total);
         }
         return all;
     }
@@ -249,7 +649,7 @@ export async function reindexWorkspace(workspacePath, repos, onProgress) {
     const { homedir } = await import('os');
     const userConfig = loadUserConfig();
     const dbPath = userConfig.vector_db_path || join(homedir(), '.studiograph', 'vector');
-    const vectorService = new VectorService(dbPath, userConfig.voyage_api_key);
+    const vectorService = new VectorService(dbPath, userConfig.voyage_api_key, workspacePath);
     await vectorService.init();
     const log = onProgress ?? (() => { });
     let total = 0;
@@ -277,13 +677,151 @@ export async function reindexWorkspace(workspacePath, repos, onProgress) {
     }
     return total;
 }
+export async function incrementalReindexWorkspace(workspaceManager, repos, workspacePath, opts = {}) {
+    const vectorService = workspaceManager.vectorService;
+    if (!vectorService) {
+        return { total: 0, fresh: false };
+    }
+    const log = opts.log ?? (() => { });
+    const { execFileSync } = await import('child_process');
+    let total = 0;
+    let anyFreshRepo = false;
+    const staleEntities = [];
+    // Pre-pass: figure out the total first-time entity count so the
+    // progress callback has a stable denominator. Cheap — `graph.list()`
+    // doesn't re-read files, it returns the in-memory graph state the
+    // workspace manager already maintains.
+    let firstTimeTotal = 0;
+    if (opts.onFirstTimeProgress) {
+        for (const repo of repos) {
+            if (vectorService.getLastIndexedTime(repo.name))
+                continue;
+            try {
+                firstTimeTotal += workspaceManager.getGraph(repo.name).list().length;
+            }
+            catch {
+                // tolerate per-repo failure; the real loop below will log it
+            }
+        }
+    }
+    let firstTimeDone = 0;
+    for (const repo of repos) {
+        const lastIndexed = vectorService.getLastIndexedTime(repo.name);
+        // Per-repo first-time index. Catches three cases:
+        //   1. Very first `serve` boot (every repo missing a timestamp).
+        //   2. New folder added between serve boots (existing workspace +
+        //      new repo with no timestamp). Pre-fix this was skipped here
+        //      and never indexed unless the operator ran a full reindex.
+        //   3. Vector DB rebuilt from scratch but workspace has existing
+        //      folders.
+        if (!lastIndexed) {
+            try {
+                const graph = workspaceManager.getGraph(repo.name);
+                const entities = graph.list();
+                if (entities.length > 0) {
+                    await vectorService.upsertMany(repo.name, entities, {
+                        onProgress: (batchDone) => {
+                            opts.onFirstTimeProgress?.(firstTimeDone + batchDone, firstTimeTotal);
+                        },
+                    });
+                    firstTimeDone += entities.length;
+                    total += entities.length;
+                }
+                vectorService.setLastIndexedTime(repo.name);
+                anyFreshRepo = true;
+            }
+            catch (err) {
+                log(`failed to index ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
+            }
+            continue;
+        }
+        // Established repo — diff git log since the last index timestamp and
+        // collect a list of stale (repo, id) tuples to upsert / delete below.
+        const repoPath = join(workspacePath, repo.path);
+        if (!existsSync(repoPath))
+            continue;
+        try {
+            const output = execFileSync('git', ['log', `--since=${lastIndexed}`, '--name-status', '--format=', '--', '*.md'], { cwd: repoPath, encoding: 'utf-8', stdio: 'pipe' }).trim();
+            if (!output)
+                continue;
+            for (const line of output.split('\n')) {
+                // Accept any markdown path. Layouts in the wild:
+                //   - flat:    `entity-id.md`
+                //   - typed:   `<type>/entity-id.md`
+                //   - nested:  `folder/sub/entity-id.md`
+                // STU-131 normalized on flat for new entities; older typed layouts
+                // and arbitrary nested folders still appear in mixed repos. Entity
+                // type lives in frontmatter (resolved via graph.getById for
+                // upserts; not needed for deletes since we now match by `(repo,
+                // entity_id)` columns). Pre-fix this regex required exactly one
+                // slash with a `\w+` type segment and missed all three layouts,
+                // plus hyphenated types like `case-study`.
+                const m = line.match(/^([AMD])\s+(.+\.md)$/);
+                if (!m)
+                    continue;
+                const [, status, path] = m;
+                // basename minus `.md`. Entity IDs are unique within a repo.
+                const id = path.split('/').pop().replace(/\.md$/, '');
+                if (!id)
+                    continue;
+                staleEntities.push({ repoName: repo.name, entityId: id, deleted: status === 'D' });
+            }
+        }
+        catch (err) {
+            log(`failed to diff ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
+        }
+    }
+    if (staleEntities.length === 0) {
+        // Nothing changed via git diff. If we did first-time work above, that
+        // counts as a fresh run; otherwise this was a no-op incremental boot.
+        return { total, fresh: anyFreshRepo };
+    }
+    // Dedupe: if an entity appears multiple times (e.g. M then D), the last
+    // entry wins. Order matters — git log is newest-first by default, so we
+    // reverse before reducing to get the newest state per (repo, id).
+    const seen = new Map();
+    for (const item of staleEntities.slice().reverse()) {
+        seen.set(`${item.repoName}/${item.entityId}`, item);
+    }
+    for (const item of seen.values()) {
+        try {
+            if (item.deleted) {
+                // Delete by (repo, id) — type isn't recoverable from a deleted file.
+                // The new column-based matcher in vectorService handles it without
+                // a frontmatter lookup.
+                await vectorService.deleteByRepoAndId(item.repoName, item.entityId);
+            }
+            else {
+                const graph = workspaceManager.getGraph(item.repoName);
+                const entity = graph.getById(item.entityId);
+                await vectorService.upsert(item.repoName, entity);
+                total++;
+            }
+        }
+        catch {
+            // Entity may have been deleted then re-created; safe to skip.
+        }
+    }
+    // Stamp every repo we touched. Empty-diff repos already had a timestamp
+    // (we skipped the rebuild path) — re-stamping is cheap and keeps the
+    // "since" window tight for the next boot.
+    for (const repo of repos) {
+        vectorService.setLastIndexedTime(repo.name);
+    }
+    return { total, fresh: anyFreshRepo };
+}
 // ─── Helpers ─────────────────────────────────────────────────────────────────
 /**
  * Build a text representation of an entity for FTS/embedding.
  * Combines entity type, key frontmatter fields, and markdown body.
+ *
+ * `extraText` is appended after the body — used by VectorService to
+ * fold PDF sidecar text into asset-entity embeddings. Kept as a plain
+ * string parameter (not file IO) so this stays pure and unit-testable.
+ *
  * Exported for testing.
  */
-export function buildEntityText(entity) {
+export function buildEntityText(entity, extraText) {
     const parts = [];
     // Entity type and ID as context
     parts.push(`${entity.entityType}: ${entity.data.name || entity.id}`);
@@ -317,6 +855,11 @@ export function buildEntityText(entity) {
         if (plainText)
             parts.push(plainText);
     }
+    if (extraText) {
+        const trimmed = extraText.trim();
+        if (trimmed)
+            parts.push(trimmed);
+    }
     return parts.join('\n');
 }
 //# sourceMappingURL=vector-service.js.map