studiograph 1.3.48-next.2 → 1.3.48-next.201

package/dist/services/vector-service.d.ts

@@ -16,25 +16,198 @@
  * Last-index timestamps: ~/.studiograph/vector/last-indexed.json
  */
 import type { EntityFile } from '../core/graph.js';
+/**
+ * Derivation-logic version. BUMP this when caption/extraction output changes so
+ * the boot sweep RE-derives every asset (Codex migration rule: a row at a stale
+ * version is re-derived, never treated as "done"). Legacy assets have NO row at
+ * all — also re-derived, never skipped.
+ */
+export declare const ASSET_INDEX_VERSION = 1;
+export type AssetIndexStatus = 'pending' | 'full' | 'failed';
+/** Public, normalized view of an asset's index state. */
+export interface AssetIndexState {
+    repo: string;
+    entityId: string;
+    status: AssetIndexStatus;
+    /** ISO timestamp of the last derivation attempt; '' when never attempted. */
+    indexedAt: string;
+    /** Derivation-logic version this row was written under. */
+    version: number;
+    /** Machine-readable failure reason ('' when not failed). */
+    reason: string;
+}
 export interface VectorSearchResult {
     repoName: string;
     entityType: string;
     entityId: string;
     score: number;
 }
+/**
+ * Snapshot of vector-index health for the /api/health endpoint. Failures are
+ * fire-and-forget at the call sites (we can't block writes on indexing),
+ * so the operator-visible signal lives here instead. Audit (2026-05) flagged
+ * the previous `catch(console.warn)` shape: a steady leak of indexing errors
+ * was undetectable until search results went stale.
+ */
+export interface VectorIndexHealth {
+    /** True when the most recent op succeeded OR no ops have run yet. */
+    healthy: boolean;
+    /** Number of failed ops since process start (or since reset). */
+    failureCount: number;
+    /** Number of in-flight ops not yet acked. Steady growth = backpressure. */
+    inFlight: number;
+    /** ISO timestamp of the most recent failure, if any. */
+    lastFailureAt: string | null;
+    /** Most recent failure message + operation, for log correlation. */
+    lastFailure: {
+        op: 'upsert' | 'delete';
+        entity: string;
+        error: string;
+    } | null;
+    /** Whether Voyage embeddings are configured (Tier 2 — semantic search). */
+    embeddingsEnabled: boolean;
+}
 export declare class VectorService {
     private dbPath;
-    private voyageApiKey?;
+    /**
+     * Legacy fallback key passed via constructor for tests or for early-
+     * boot code paths that run before the secrets store is initialized.
+     * Live calls prefer `tryResolve('voyage.api_key')` so values written
+     * to the store via the UI / CLI take effect without a restart.
+     */
+    private fallbackVoyageApiKey?;
+    private workspaceRoot?;
     private db?;
     private table?;
+    /** Lazily-opened side table for mutable asset index state (A1). */
+    private stateTable?;
+    private stateInitialized;
     private lastIndexedPath;
     private initialized;
-    constructor(dbPath: string, voyageApiKey?: string);
+    private dbPromise?;
+    private initPromise?;
+    private stateInitPromise?;
+    private writeChain;
+    /**
+     * Fired exactly once per asset, when `ensureInitialAssetIndexState` seeds a
+     * fresh `pending` row — i.e. an indexable image that needs captioning. This
+     * is the SINGLE chokepoint every asset-creation path funnels through (all
+     * upserts call ensureInitialAssetIndexState), so the live caption drain
+     * registers here instead of being wired at each of the ~5 upload call sites.
+     * It does NOT fire for docs (seeded `full`) or for re-derives (row already
+     * exists → early return), so it can't loop the drain.
+     */
+    private onAssetPendingCb?;
+    private failureCount;
+    private inFlight;
+    private lastFailureAt;
+    private lastFailure;
+    constructor(dbPath: string, voyageApiKey?: string, workspaceRoot?: string);
+    /**
+     * Resolve the Voyage API key fresh on each call: store > constructor
+     * fallback. Lets the operator flip the key via UI / CLI without a
+     * server restart.
+     */
+    private get voyageApiKey();
+    /** Begin tracking an in-flight op. Pair with `recordSuccess`/`recordFailure`. */
+    beginOp(): void;
+    recordSuccess(): void;
+    recordFailure(op: 'upsert' | 'delete', entity: string, err: unknown): void;
+    /**
+     * Snapshot for /api/health. Healthy = the last op succeeded (or none have
+     * run yet) AND there is no abnormal in-flight backlog. The failure count
+     * is informational; a single transient Voyage 503 shouldn't permanently
+     * mark the service unhealthy, but operators can see the trend.
+     */
+    getHealth(): VectorIndexHealth;
+    /**
+     * For asset entities, any extracted text the upload/caption pipeline wrote
+     * lives in a sidecar at `<workspace>/.studiograph/assets-text/asset/<id>.txt`,
+     * keyed by the asset's ENTITY ID (not the storage UUID). Returns the contents
+     * so `buildEntityText` can fold it into the embedding/FTS input.
+     *
+     * Generalized from PDF-only (P3): docx/xlsx now write the same sidecar at
+     * upload, and image captions/OCR land here too — so the reader is no longer
+     * gated on `asset_kind === 'pdf'`. The markdown body of an asset entity is
+     * intentionally light; without this, document/image assets would be
+     * effectively invisible to RAG.
+     *
+     * Best-effort: missing sidecar / read errors return null. The asset stays
+     * indexed by frontmatter alone.
+     */
+    private readAssetSidecar;
     /**
      * Open/create the LanceDB table and ensure the FTS index exists.
      * Call once before using the service.
      */
     init(): Promise<void>;
+    /**
+     * Run a mutating LanceDB op under the per-instance write lock so concurrent
+     * writers (foreground entity writes vs. the asset-index drain) can't collide
+     * on a shared table's optimistic-concurrency commit. Ops run strictly in the
+     * order `withWrite` is called; a throwing op rejects to ITS caller without
+     * poisoning the chain for the next writer.
+     */
+    private withWrite;
+    /**
+     * Open (once) the shared LanceDB connection. Both `init()` and
+     * `initStateTable()` go through this so they never race to assign `this.db`.
+     */
+    private ensureDb;
+    private doInit;
+    private stateRowId;
+    /**
+     * Open/create the side table for asset index state. Independent of `init()`
+     * so state-only callers (and tests) don't pay for the entities table. Lazy
+     * because most workspaces never need it until the first asset is indexed.
+     */
+    private initStateTable;
+    private doInitStateTable;
+    private toState;
+    /**
+     * Read the `text` actually indexed for one entity (the value `buildEntityText`
+     * produced — markdown body plus, for assets, the extracted sidecar). Null when
+     * no row exists. This is the searchable content of record, independent of FTS
+     * index freshness, so it's the deterministic way to assert "the asset's body
+     * reached the index" (e.g. the docx stub-upsert-race regression test).
+     */
+    getIndexedText(repo: string, entityType: string, entityId: string): Promise<string | null>;
+    /** Read one asset's index state, or null when no row exists (legacy/never-derived). */
+    getIndexState(repo: string, entityId: string): Promise<AssetIndexState | null>;
+    /**
+     * Load every index-state row into a Map keyed by `${repo}/${entityId}`. The
+     * boot sweep uses this to decide, per asset, whether a row is missing
+     * (legacy), stale (old version), or unfinished (pending) — without an
+     * all-asset disk scan (PERF1).
+     */
+    getAllIndexStates(): Promise<Map<string, AssetIndexState>>;
+    /** Upsert an asset's index state. Stamps `indexed_at` + current version unless overridden. */
+    setIndexState(repo: string, entityId: string, status: AssetIndexStatus, opts?: {
+        reason?: string;
+        indexedAt?: string;
+        version?: number;
+    }): Promise<void>;
+    /** Drop an asset's index-state row (e.g. when the asset is deleted). */
+    deleteIndexState(repo: string, entityId: string): Promise<void>;
+    /**
+     * Seed the initial index state for a freshly (re)indexed asset, IF it has no
+     * row yet. State *transitions* are owned by the caption/sweep pass — this only
+     * establishes the starting point at import time:
+     *  - `indexable !== true` (opaque, or legacy `undefined`) → no row. Opaque
+     *    assets read "filename only" from frontmatter; legacy assets are
+     *    re-derived by the boot sweep, not seeded here.
+     *  - sidecar already on disk (doc extracted synchronously at upload, P3a) → `full`.
+     *  - no sidecar (image awaiting caption) → `pending`.
+     * Best-effort: a state-write failure must not fail the text index.
+     */
+    /**
+     * Register the single sink for "a new indexable image needs captioning."
+     * The live caption drain (AssetIndexQueue) registers here; the chokepoint
+     * fires from `ensureInitialAssetIndexState`, covering every upload path at
+     * once. Replaces the per-call-site enqueue wiring.
+     */
+    setOnAssetPending(cb: ((repo: string, entityId: string) => void) | undefined): void;
+    ensureInitialAssetIndexState(repo: string, entity: EntityFile): Promise<void>;
     /**
      * Index or re-index a single entity.
      * Safe to call on create, update, or as part of bulk indexing.
@@ -43,11 +216,37 @@ export declare class VectorService {
     /**
      * Bulk upsert — used by `studiograph index`. Batches Voyage API calls.
      */
-    upsertMany(repoName: string, entities: EntityFile[]): Promise<void>;
+    upsertMany(repoName: string, entities: EntityFile[], opts?: {
+        /** Called after each embedding batch with the cumulative number of
+         *  entities processed so far inside THIS upsertMany call. Lets the
+         *  caller render a progress bar during long first-time indexes. */
+        onProgress?: (done: number, total: number) => void;
+    }): Promise<void>;
     /**
      * Remove an entity from the index.
      */
     delete(repoName: string, entityType: string, entityId: string): Promise<void>;
+    /**
+     * Remove an entity by `(repo, id)` alone, without needing its type.
+     *
+     * Used by the serve-boot incremental indexer: when git diff reports a
+     * deletion, the file is gone from disk so its type can't be resolved
+     * from frontmatter. The vector store has a discrete `entity_id` column
+     * so we can match on `repo + entity_id` without falling back to a LIKE
+     * pattern on the synthetic id.
+     *
+     * Safe across type renames: a single id that legitimately had two type
+     * rows (e.g. type changed mid-edit and both old + new still exist) gets
+     * both wiped — which is the right behavior for incremental rebuild
+     * since the next upsert recreates the current type's row.
+     */
+    deleteByRepoAndId(repoName: string, entityId: string): Promise<void>;
+    /**
+     * Best-effort `deleteIndexState` for the delete paths: the entities-table
+     * delete is the authoritative op, so a failure to reap the (usually absent)
+     * state row must not turn a successful delete into a thrown error.
+     */
+    private deleteIndexStateSafe;
     /**
      * Search the index.
      * Uses hybrid search (vector + FTS) when Voyage key is configured, FTS only otherwise.
@@ -61,9 +260,13 @@ export declare class VectorService {
      */
     getLastIndexedTime(repoName: string): string | undefined;
     /**
-     * Record the current time as the last index time for a repo.
+     * Record the last index time for a repo.
+     *
+     * Production callers omit `isoTimestamp` and the current time is used.
+     * Tests pass an explicit ISO timestamp so they can rewind the clock and
+     * assert the git-log `--since` window picks up the right commits.
      */
-    setLastIndexedTime(repoName: string): void;
+    setLastIndexedTime(repoName: string, isoTimestamp?: string): void;
     /** Whether Voyage AI embeddings are enabled (Tier 2). */
     get hasEmbeddings(): boolean;
     private generateEmbedding;
@@ -80,9 +283,60 @@ export declare function reindexWorkspace(workspacePath: string, repos: Array<{
     name: string;
     path: string;
 }>, onProgress?: (msg: string) => void): Promise<number>;
+/**
+ * Incremental reindex for the `serve` boot path.
+ *
+ * First run (no last-indexed timestamps): full index of every repo, same
+ * as `reindexWorkspace` but reuses the WorkspaceManager's existing graphs
+ * instead of constructing new ones. Subsequent runs: diff each repo via
+ * `git log --name-status` since the last indexed timestamp, then upsert /
+ * delete only the changed entities. Lifted from `studiograph start`'s
+ * private `initSearchIndex` so the logic survives `start.ts` deletion
+ * (Phase C). No spinner — boot path uses a logger callback.
+ */
+/**
+ * Structural slice of `WorkspaceManager` that `incrementalReindexWorkspace`
+ * touches. Declared locally to avoid a vector-service → workspace-manager
+ * import cycle. `getGraph`'s entity-type parameter is typed loosely (any)
+ * because WorkspaceManager.getGraph().get accepts a narrow EntityType
+ * union — accepting `any` here keeps the parameter contravariantly
+ * compatible without dragging the validation module into vector-service.
+ */
+interface ReindexWorkspaceManager {
+    vectorService?: VectorService;
+    getGraph: (repoName: string) => {
+        list: () => EntityFile[];
+        get: (type: any, id: string) => EntityFile;
+        /** Resolve an entity by ID alone — needed by the incremental indexer
+         *  when the path's type segment isn't reliably present (flat layout,
+         *  hyphenated types, arbitrary nested folders per STU-131). */
+        getById: (id: string) => EntityFile;
+    };
+}
+export declare function incrementalReindexWorkspace(workspaceManager: ReindexWorkspaceManager, repos: Array<{
+    name: string;
+    path: string;
+}>, workspacePath: string, opts?: {
+    log?: (msg: string) => void;
+    /** Fired during first-time indexes to drive a progress bar in the
+     *  caller. `done` is cumulative across all first-time repos, `total`
+     *  is the workspace-wide first-time count. Not fired for the
+     *  incremental (git-diff) path — that path is typically too small
+     *  to need progress UI. */
+    onFirstTimeProgress?: (done: number, total: number) => void;
+}): Promise<{
+    total: number;
+    fresh: boolean;
+}>;
 /**
  * Build a text representation of an entity for FTS/embedding.
  * Combines entity type, key frontmatter fields, and markdown body.
+ *
+ * `extraText` is appended after the body — used by VectorService to
+ * fold PDF sidecar text into asset-entity embeddings. Kept as a plain
+ * string parameter (not file IO) so this stays pure and unit-testable.
+ *
  * Exported for testing.
  */
-export declare function buildEntityText(entity: EntityFile): string;
+export declare function buildEntityText(entity: EntityFile, extraText?: string): string;
+export {};