studiograph 1.3.48-next.21 → 1.3.48-next.210

package/dist/services/vector-service.d.ts

@@ -16,25 +16,220 @@
  * Last-index timestamps: ~/.studiograph/vector/last-indexed.json
  */
 import type { EntityFile } from '../core/graph.js';
+import type { AssetIndexService } from './assets/asset-index-service.js';
+/**
+ * Derivation-logic version. BUMP this when caption/extraction output changes so
+ * the boot sweep RE-derives every asset (Codex migration rule: a row at a stale
+ * version is re-derived, never treated as "done"). Legacy assets have NO row at
+ * all — also re-derived, never skipped.
+ */
+export declare const ASSET_INDEX_VERSION = 1;
+export type AssetIndexStatus = 'pending' | 'full' | 'failed';
+/** Public, normalized view of an asset's index state.
+ *  Keyed by the immutable assetId (asset-as-embed): the caption/OCR derivation
+ *  belongs to the stored bytes, not to any entry that embeds them. */
+export interface AssetIndexState {
+    /** Immutable storage UUID (assets.db `asset_id`). */
+    assetId: string;
+    /** Owner repo at seed time — informational (debug/observability). */
+    repo: string;
+    status: AssetIndexStatus;
+    /** ISO timestamp of the last derivation attempt; '' when never attempted. */
+    indexedAt: string;
+    /** Derivation-logic version this row was written under. */
+    version: number;
+    /** Machine-readable failure reason ('' when not failed). */
+    reason: string;
+}
 export interface VectorSearchResult {
     repoName: string;
     entityType: string;
     entityId: string;
     score: number;
 }
+/**
+ * Snapshot of vector-index health for the /api/health endpoint. Failures are
+ * fire-and-forget at the call sites (we can't block writes on indexing),
+ * so the operator-visible signal lives here instead. Audit (2026-05) flagged
+ * the previous `catch(console.warn)` shape: a steady leak of indexing errors
+ * was undetectable until search results went stale.
+ */
+export interface VectorIndexHealth {
+    /** True when the most recent op succeeded OR no ops have run yet. */
+    healthy: boolean;
+    /** Number of failed ops since process start (or since reset). */
+    failureCount: number;
+    /** Number of in-flight ops not yet acked. Steady growth = backpressure. */
+    inFlight: number;
+    /** ISO timestamp of the most recent failure, if any. */
+    lastFailureAt: string | null;
+    /** Most recent failure message + operation, for log correlation. */
+    lastFailure: {
+        op: 'upsert' | 'delete';
+        entity: string;
+        error: string;
+    } | null;
+    /** Whether Voyage embeddings are configured (Tier 2 — semantic search). */
+    embeddingsEnabled: boolean;
+}
 export declare class VectorService {
     private dbPath;
-    private voyageApiKey?;
+    /**
+     * Legacy fallback key passed via constructor for tests or for early-
+     * boot code paths that run before the secrets store is initialized.
+     * Live calls prefer `tryResolve('voyage.api_key')` so values written
+     * to the store via the UI / CLI take effect without a restart.
+     */
+    private fallbackVoyageApiKey?;
     private db?;
     private table?;
+    /** Lazily-opened side table for mutable asset index state (A1). */
+    private stateTable?;
+    private stateInitialized;
     private lastIndexedPath;
     private initialized;
+    private dbPromise?;
+    private initPromise?;
+    private stateInitPromise?;
+    private writeChain;
+    /**
+     * Fired exactly once per asset, when `ensureInitialAssetIndexState` seeds a
+     * fresh `pending` row — i.e. an indexable IMAGE assets.db row that needs
+     * captioning. The asset-creation paths (graph.uploadAsset for import +
+     * manager/editor uploads, asset-finalize-service for presign) call
+     * ensureInitialAssetIndexState after writing the assets.db row, so the live
+     * caption drain registers here instead of being wired at each call site. It
+     * fires only for images and only on the first seed (a re-seed early-returns),
+     * so it can't loop the drain.
+     */
+    private onAssetPendingCb?;
+    /**
+     * Workspace-global asset ownership index (assets.db). Injected by the
+     * WorkspaceManager so `upsert` can resolve the captions of images an entry
+     * embeds (`resolveEmbeddedCaptions`) and fold them into the entry's search
+     * text. Optional — tests / early-boot paths run without it (no caption fold).
+     */
+    private assetIndex?;
+    private failureCount;
+    private inFlight;
+    private lastFailureAt;
+    private lastFailure;
     constructor(dbPath: string, voyageApiKey?: string);
+    /**
+     * Resolve the Voyage API key fresh on each call: store > constructor
+     * fallback. Lets the operator flip the key via UI / CLI without a
+     * server restart.
+     */
+    private get voyageApiKey();
+    /** Begin tracking an in-flight op. Pair with `recordSuccess`/`recordFailure`. */
+    beginOp(): void;
+    recordSuccess(): void;
+    recordFailure(op: 'upsert' | 'delete', entity: string, err: unknown): void;
+    /**
+     * Snapshot for /api/health. Healthy = the last op succeeded (or none have
+     * run yet) AND there is no abnormal in-flight backlog. The failure count
+     * is informational; a single transient Voyage 503 shouldn't permanently
+     * mark the service unhealthy, but operators can see the trend.
+     */
+    getHealth(): VectorIndexHealth;
+    /** Inject the workspace-global asset ownership index (assets.db). Lets
+     *  `upsert` fold embedded-image captions into an entry's search text. */
+    setAssetIndex(index: AssetIndexService | undefined): void;
+    /**
+     * The captions of every immutable-keyed image an entry embeds, joined for
+     * folding into the entry's FTS/embedding text (asset-as-embed, decision D2).
+     *
+     * The caption + OCR text is derived once per image and stored in
+     * `assets.db.caption`, keyed by the immutable assetId (NOT in a sidecar or
+     * the body). An entry becomes findable by an embedded image's visual content
+     * by reading those captions back here — for the `material` body embed, the
+     * `reference` thumbnail, a note body image, etc. Each captioned image's text
+     * is read from the single assets.db row, so the same image embedded by N
+     * entries enriches all N without duplicating the stored caption.
+     *
+     * Forward-only (entry → the assets IT embeds) — no reverse index, no
+     * cross-repo ownership claim. Best-effort: returns undefined without an index
+     * or when nothing embedded is captioned.
+     */
+    private resolveEmbeddedCaptions;
     /**
      * Open/create the LanceDB table and ensure the FTS index exists.
      * Call once before using the service.
      */
     init(): Promise<void>;
+    /**
+     * Run a mutating LanceDB op under the per-instance write lock so concurrent
+     * writers (foreground entity writes vs. the asset-index drain) can't collide
+     * on a shared table's optimistic-concurrency commit. Ops run strictly in the
+     * order `withWrite` is called; a throwing op rejects to ITS caller without
+     * poisoning the chain for the next writer.
+     */
+    private withWrite;
+    /**
+     * Open (once) the shared LanceDB connection. Both `init()` and
+     * `initStateTable()` go through this so they never race to assign `this.db`.
+     */
+    private ensureDb;
+    private doInit;
+    /**
+     * Open/create the side table for asset index state. Independent of `init()`
+     * so state-only callers (and tests) don't pay for the entities table. Lazy
+     * because most workspaces never need it until the first asset is indexed.
+     */
+    private initStateTable;
+    private doInitStateTable;
+    private toState;
+    /**
+     * Read the `text` actually indexed for one entity (the value `buildEntityText`
+     * produced — markdown body plus, for assets, the extracted sidecar). Null when
+     * no row exists. This is the searchable content of record, independent of FTS
+     * index freshness, so it's the deterministic way to assert "the asset's body
+     * reached the index" (e.g. the docx stub-upsert-race regression test).
+     */
+    getIndexedText(repo: string, entityType: string, entityId: string): Promise<string | null>;
+    /** Read one asset's index state, or null when no row exists (never-derived). */
+    getIndexState(assetId: string): Promise<AssetIndexState | null>;
+    /**
+     * Load every index-state row into a Map keyed by assetId. The boot sweep uses
+     * this to decide, per image asset, whether a row is missing (never-derived),
+     * stale (old version), or unfinished (pending) — without re-deriving the
+     * whole set (PERF1).
+     */
+    getAllIndexStates(): Promise<Map<string, AssetIndexState>>;
+    /** Upsert an asset's index state. Stamps `indexed_at` + current version unless overridden. */
+    setIndexState(assetId: string, status: AssetIndexStatus, opts?: {
+        reason?: string;
+        indexedAt?: string;
+        version?: number;
+        repo?: string;
+    }): Promise<void>;
+    /** Drop an asset's index-state row (e.g. when the asset is deleted). */
+    deleteIndexState(assetId: string): Promise<void>;
+    /**
+     * Register the single sink for "a new indexable image needs captioning."
+     * The live caption drain (AssetIndexQueue) registers here; the chokepoint
+     * fires from `ensureInitialAssetIndexState`, covering every image-upload path
+     * at once. Replaces per-call-site enqueue wiring.
+     */
+    setOnAssetPending(cb: ((assetId: string) => void) | undefined): void;
+    /**
+     * Seed the initial index state for a freshly-stored IMAGE asset, IF it has no
+     * row yet, and notify the live caption drain. Called by the asset-creation
+     * paths (graph.uploadAsset, asset-finalize-service) after the assets.db row
+     * is written — keyed by the immutable assetId, never an entity.
+     *
+     * State *transitions* (full/failed) are owned by the caption sweep; this only
+     * establishes the `pending` starting point + fires the drain. Only raster
+     * images are captioned: SVG stays transitional (sanitize-html not wired);
+     * documents/opaque binaries carry no caption. A re-seed (row already exists)
+     * early-returns so the drain can't loop. Best-effort: a state-write failure
+     * must not fail the upload.
+     */
+    ensureInitialAssetIndexState(assetId: string, opts: {
+        mediaType: string;
+        filename: string;
+        repo?: string;
+    }): Promise<void>;
     /**
      * Index or re-index a single entity.
      * Safe to call on create, update, or as part of bulk indexing.
@@ -43,11 +238,38 @@ export declare class VectorService {
     /**
      * Bulk upsert — used by `studiograph index`. Batches Voyage API calls.
      */
-    upsertMany(repoName: string, entities: EntityFile[]): Promise<void>;
+    upsertMany(repoName: string, entities: EntityFile[], opts?: {
+        /** Called after each embedding batch with the cumulative number of
+         *  entities processed so far inside THIS upsertMany call. Lets the
+         *  caller render a progress bar during long first-time indexes. */
+        onProgress?: (done: number, total: number) => void;
+    }): Promise<void>;
     /**
      * Remove an entity from the index.
      */
     delete(repoName: string, entityType: string, entityId: string): Promise<void>;
+    /**
+     * Remove an entity by `(repo, id)` alone, without needing its type.
+     *
+     * Used by the serve-boot incremental indexer: when git diff reports a
+     * deletion, the file is gone from disk so its type can't be resolved
+     * from frontmatter. The vector store has a discrete `entity_id` column
+     * so we can match on `repo + entity_id` without falling back to a LIKE
+     * pattern on the synthetic id.
+     *
+     * Safe across type renames: a single id that legitimately had two type
+     * rows (e.g. type changed mid-edit and both old + new still exist) gets
+     * both wiped — which is the right behavior for incremental rebuild
+     * since the next upsert recreates the current type's row.
+     */
+    deleteByRepoAndId(repoName: string, entityId: string): Promise<void>;
+    /**
+     * Best-effort `deleteIndexState` for the asset-delete path: the blob/index
+     * delete is the authoritative op, so a failure to reap the state row must not
+     * turn a successful asset delete into a thrown error. Called by
+     * graph.deleteAssetById.
+     */
+    deleteIndexStateSafe(assetId: string): Promise<void>;
     /**
      * Search the index.
      * Uses hybrid search (vector + FTS) when Voyage key is configured, FTS only otherwise.
@@ -61,9 +283,13 @@ export declare class VectorService {
      */
     getLastIndexedTime(repoName: string): string | undefined;
     /**
-     * Record the current time as the last index time for a repo.
+     * Record the last index time for a repo.
+     *
+     * Production callers omit `isoTimestamp` and the current time is used.
+     * Tests pass an explicit ISO timestamp so they can rewind the clock and
+     * assert the git-log `--since` window picks up the right commits.
      */
-    setLastIndexedTime(repoName: string): void;
+    setLastIndexedTime(repoName: string, isoTimestamp?: string): void;
     /** Whether Voyage AI embeddings are enabled (Tier 2). */
     get hasEmbeddings(): boolean;
     private generateEmbedding;
@@ -80,9 +306,60 @@ export declare function reindexWorkspace(workspacePath: string, repos: Array<{
     name: string;
     path: string;
 }>, onProgress?: (msg: string) => void): Promise<number>;
+/**
+ * Incremental reindex for the `serve` boot path.
+ *
+ * First run (no last-indexed timestamps): full index of every repo, same
+ * as `reindexWorkspace` but reuses the WorkspaceManager's existing graphs
+ * instead of constructing new ones. Subsequent runs: diff each repo via
+ * `git log --name-status` since the last indexed timestamp, then upsert /
+ * delete only the changed entities. Lifted from `studiograph start`'s
+ * private `initSearchIndex` so the logic survives `start.ts` deletion
+ * (Phase C). No spinner — boot path uses a logger callback.
+ */
+/**
+ * Structural slice of `WorkspaceManager` that `incrementalReindexWorkspace`
+ * touches. Declared locally to avoid a vector-service → workspace-manager
+ * import cycle. `getGraph`'s entity-type parameter is typed loosely (any)
+ * because WorkspaceManager.getGraph().get accepts a narrow EntityType
+ * union — accepting `any` here keeps the parameter contravariantly
+ * compatible without dragging the validation module into vector-service.
+ */
+interface ReindexWorkspaceManager {
+    vectorService?: VectorService;
+    getGraph: (repoName: string) => {
+        list: () => EntityFile[];
+        get: (type: any, id: string) => EntityFile;
+        /** Resolve an entity by ID alone — needed by the incremental indexer
+         *  when the path's type segment isn't reliably present (flat layout,
+         *  hyphenated types, arbitrary nested folders per STU-131). */
+        getById: (id: string) => EntityFile;
+    };
+}
+export declare function incrementalReindexWorkspace(workspaceManager: ReindexWorkspaceManager, repos: Array<{
+    name: string;
+    path: string;
+}>, workspacePath: string, opts?: {
+    log?: (msg: string) => void;
+    /** Fired during first-time indexes to drive a progress bar in the
+     *  caller. `done` is cumulative across all first-time repos, `total`
+     *  is the workspace-wide first-time count. Not fired for the
+     *  incremental (git-diff) path — that path is typically too small
+     *  to need progress UI. */
+    onFirstTimeProgress?: (done: number, total: number) => void;
+}): Promise<{
+    total: number;
+    fresh: boolean;
+}>;
 /**
  * Build a text representation of an entity for FTS/embedding.
  * Combines entity type, key frontmatter fields, and markdown body.
+ *
+ * `extraText` is appended after the body — used by VectorService to
+ * fold PDF sidecar text into asset-entity embeddings. Kept as a plain
+ * string parameter (not file IO) so this stays pure and unit-testable.
+ *
  * Exported for testing.
  */
-export declare function buildEntityText(entity: EntityFile): string;
+export declare function buildEntityText(entity: EntityFile, extraText?: string): string;
+export {};