studiograph 1.3.48-next.2 → 1.3.48-next.201

package/dist/services/asset-index-sweep.js

@@ -0,0 +1,258 @@
+/**
+ * Asset index sweep (P3b) — the boot trigger of the caption / extract pass.
+ *
+ * OV1: this is SEPARATE from the text/FTS reindex. The vector reindex
+ * (boot / git-pull / reconcile) only refreshes the FTS+embedding text for
+ * changed entities; it deliberately does NOT fire vision calls. This sweep is
+ * the one place that drives image captioning (and re-derives legacy document
+ * sidecars), so vision spend is intentional and bounded, never a side effect of
+ * a routine reindex.
+ *
+ * What it does, per indexable asset:
+ *  - PERF1: it reads the unfinished set from the LanceDB index-state table
+ *    (in-memory entity list × state map — no all-asset disk scan).
+ *  - Migration (Codex CRITICAL): an asset with NO state row, or a row at a
+ *    STALE `index_version`, is RE-derived — undefined is never treated as
+ *    "done". An up-to-date `full`/`failed` row at the current version is
+ *    skipped (and `failed` is terminal at boot so we don't re-bill a permanent
+ *    failure every restart).
+ *  - Images → caption + OCR (bounded fan-out, A3). When captioning is
+ *    unreachable (gateway / disabled / non-vision model) the image is left
+ *    honestly `pending` rather than burned as `failed`, so flipping the gate or
+ *    model later picks it up without a version bump.
+ *  - Documents (pdf/docx/xlsx) missing a sidecar → re-extract from bytes
+ *    (bounded), so legacy docs uploaded before P3a become searchable.
+ *  - SVG is explicitly skipped — it stays on the transitional agent path until
+ *    sanitize-html is wired (out of scope for P3).
+ *
+ * Side effects are limited to: sidecar `.txt` writes, a text re-upsert (so the
+ * new sidecar folds into FTS/embeddings), and LanceDB index-state writes. It
+ * never writes frontmatter or git (decision A1) — so it can't fire a commit per
+ * asset.
+ */
+import { extname } from 'path';
+import { lookupByExtension, FALLBACK_ASSET_ENTRY, isIndexable } from '../core/format-registry.js';
+import { extractSidecarText, writeAssetSidecar } from './asset-sidecar-service.js';
+import { captionImage, canCaption } from './asset-caption-service.js';
+import { ASSET_INDEX_VERSION } from './vector-service.js';
+/** Resolve a format from a bare/dotted extension via the registry. */
+function formatForExt(ext) {
+    return (ext ? lookupByExtension(ext) : undefined) ?? FALLBACK_ASSET_ENTRY;
+}
+/** Run `fn` over `items` with a bounded number of concurrent workers. */
+async function mapBounded(items, concurrency, fn) {
+    let next = 0;
+    const worker = async () => {
+        while (true) {
+            const i = next++;
+            if (i >= items.length)
+                return;
+            await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), items.length || 1) }, () => worker()));
+}
+/**
+ * Drive the caption / extraction pass for every asset that needs it. Safe to
+ * call fire-and-forget at boot; resilient to per-asset failure.
+ */
+export async function sweepAssetIndex(deps, opts = {}) {
+    const log = opts.log ?? (() => { });
+    const concurrency = opts.concurrency ?? 2;
+    const result = { scanned: 0, derived: 0, failed: 0, skippedUpToDate: 0, leftPending: 0 };
+    const vec = deps.workspaceManager.vectorService;
+    if (!vec)
+        return result;
+    // A targeted drain (AssetIndexService tick) with an empty set has no work —
+    // skip even the LanceDB read. `undefined` means "consider everything" (boot
+    // recovery), which is NOT the same as an empty set.
+    const onlyAssets = opts.onlyAssets;
+    if (onlyAssets && onlyAssets.size === 0)
+        return result;
+    // PERF1: one query for the whole unfinished set, no per-asset disk scan.
+    const stateMap = await vec.getAllIndexStates();
+    // Collect the work-list from the in-memory graphs.
+    const docItems = [];
+    const imageItems = [];
+    for (const repo of deps.repos) {
+        let graph;
+        try {
+            graph = deps.workspaceManager.getGraph(repo.name);
+        }
+        catch {
+            continue;
+        }
+        let entities;
+        try {
+            entities = graph.list();
+        }
+        catch {
+            continue;
+        }
+        for (const entity of entities) {
+            if (entity.entityType !== 'asset')
+                continue;
+            // Targeted drain: skip assets the caller didn't enqueue.
+            if (onlyAssets && !onlyAssets.has(`${repo.name}/${entity.id}`))
+                continue;
+            const data = entity.data;
+            const assetUrl = data.asset_url ?? '';
+            const ext = String(data.asset_extension || extname(assetUrl) || '').toLowerCase();
+            // SVG is NOT in P3 — leave it on the transitional path (no sanitize yet).
+            if (ext === '.svg')
+                continue;
+            const format = formatForExt(ext);
+            // Frontmatter `indexable` wins when present (P3a); legacy assets fall back
+            // to the registry's declared indexability.
+            const indexable = data.indexable === true || (data.indexable === undefined && isIndexable(format));
+            if (!indexable)
+                continue;
+            const extractor = format.extractor;
+            const kind = extractor === 'image-caption' ? 'image'
+                : (extractor === 'pdf' || extractor === 'docx' || extractor === 'xlsx') ? 'doc'
+                    : null;
+            if (!kind)
+                continue;
+            result.scanned++;
+            const state = stateMap.get(`${repo.name}/${entity.id}`);
+            // Up-to-date = a current-version row that's finished (full) or terminal
+            // (failed). Missing row / stale version / pending ⇒ (re-)derive.
+            const upToDate = !!state
+                && state.version >= ASSET_INDEX_VERSION
+                && (state.status === 'full' || state.status === 'failed');
+            if (upToDate) {
+                result.skippedUpToDate++;
+                continue;
+            }
+            (kind === 'image' ? imageItems : docItems).push({ repo: repo.name, entity, kind, assetUrl, format });
+        }
+    }
+    // Respect the optional safety cap (docs first — they're cheaper than vision).
+    let budget = opts.maxAssets ?? Infinity;
+    const take = (arr) => {
+        if (budget === Infinity)
+            return arr;
+        const slice = arr.slice(0, Math.max(0, budget));
+        budget -= slice.length;
+        return slice;
+    };
+    const docs = take(docItems);
+    const images = take(imageItems);
+    // ── Documents: re-extract a missing sidecar from bytes (legacy migration) ──
+    const docDerivations = new Array(docs.length);
+    await mapBounded(docs.map((item, i) => ({ item, i })), concurrency, async ({ item, i }) => {
+        try {
+            const graph = deps.workspaceManager.getGraph(item.repo);
+            const buffer = await graph.downloadOwnedAsset(item.entity.id, item.assetUrl);
+            // Reuse the format resolved during collection — re-resolving here risked
+            // a different extractor than the one that classified this item as a doc.
+            const { text } = await extractSidecarText(buffer, item.format.extractor, item.entity.id);
+            docDerivations[i] = { item, text: text || null, reason: text ? '' : 'no-text' };
+        }
+        catch (err) {
+            docDerivations[i] = { item, text: null, reason: err instanceof Error ? err.message.slice(0, 120) : 'error' };
+        }
+    });
+    // ── Images: caption + OCR, gated + bounded ──
+    const captionGate = deps.captioningEnabled
+        ? (deps.captionConfig ? canCaption(deps.captionConfig) : { ok: false, reason: 'no-model' })
+        : { ok: false, reason: 'disabled' };
+    const imageDerivations = [];
+    if (captionGate.ok) {
+        const captionConfig = deps.captionConfig;
+        const out = new Array(images.length);
+        await mapBounded(images.map((item, i) => ({ item, i })), concurrency, async ({ item, i }) => {
+            try {
+                const graph = deps.workspaceManager.getGraph(item.repo);
+                const buffer = await graph.downloadOwnedAsset(item.entity.id, item.assetUrl);
+                const mime = String(item.entity.data.asset_mime || 'image/png');
+                const caption = await captionImage(buffer, mime, captionConfig, { completeFn: opts.completeFn });
+                out[i] = caption.status === 'full' && caption.text
+                    ? { item, text: caption.text, reason: '' }
+                    : { item, text: null, reason: caption.reason ?? 'error' };
+            }
+            catch (err) {
+                out[i] = { item, text: null, reason: err instanceof Error ? err.message.slice(0, 120) : 'error' };
+            }
+        });
+        imageDerivations.push(...out);
+    }
+    else if (images.length > 0) {
+        // Degrade gracefully (option a): can't caption now. Seed a `pending` row so
+        // the badge is honestly filename-only; it transitions when the gate flips
+        // (config enabled / model switched), no version bump needed.
+        log(`captioning skipped (${captionGate.reason}); ${images.length} image(s) left pending (filename-only)`);
+        for (const item of images) {
+            try {
+                // Every image here already failed the upToDate gate, so its row (if any)
+                // is missing, stale-version, or pending. Write `pending` unless it's
+                // ALREADY pending — this seeds missing rows AND downgrades a stale
+                // `full`/`failed` row so the persisted status matches what we report,
+                // while skipping current-pending rows to avoid re-stamping every boot.
+                const existing = stateMap.get(`${item.repo}/${item.entity.id}`);
+                if (existing?.status !== 'pending') {
+                    await vec.setIndexState(item.repo, item.entity.id, 'pending', { reason: captionGate.reason });
+                }
+                result.leftPending++;
+            }
+            catch { /* best-effort */ }
+        }
+    }
+    // ── Persist serially (one LanceDB writer at a time) ──
+    // LanceDB uses optimistic concurrency, so two mergeInserts racing on the same
+    // table conflict and one loses — derive in parallel, persist one at a time.
+    for (const d of [...docDerivations, ...imageDerivations]) {
+        if (!d)
+            continue;
+        const ref = `${d.item.repo}/${d.item.entity.id}`;
+        if (!d.text) {
+            // The derive itself produced no text → honest `failed` (filename-only).
+            try {
+                await vec.setIndexState(d.item.repo, d.item.entity.id, 'failed', { reason: d.reason || 'error' });
+            }
+            catch (err) {
+                log(`persist(failed-state) failed for ${ref}: ${err instanceof Error ? err.message : err}`);
+            }
+            result.failed++;
+            continue;
+        }
+        // We have caption/extract text. Make the content live first (sidecar +
+        // text re-index), THEN stamp `full`. Splitting the two means a status-stamp
+        // hiccup can't mislabel an already-searchable asset as `failed` (it isn't),
+        // and can't re-bill vision on every boot for content that's already indexed.
+        try {
+            writeAssetSidecar(deps.workspacePath, d.item.entity.id, d.text);
+            await vec.upsert(d.item.repo, d.item.entity); // re-index text incl. new sidecar
+        }
+        catch (err) {
+            // The sidecar/index write itself failed. This is a TRANSIENT infra error
+            // (a LanceDB commit conflict, a disk hiccup) — NOT "this asset can't be
+            // captioned." The caption text is in hand; the only thing that failed is
+            // persisting it. Do NOT stamp terminal `failed` (Codex P1): that would
+            // permanently skip a perfectly captionable asset until a version bump.
+            // Leave it `pending` so the next sweep retries the cheap persist (the
+            // caption itself is re-derived, but that's the honest cost of not having
+            // landed the text). result.failed counts the attempt for observability.
+            log(`persist(index) failed for ${ref} (transient; will retry next sweep): ${err instanceof Error ? err.message : err}`);
+            try {
+                await vec.setIndexState(d.item.repo, d.item.entity.id, 'pending', { reason: 'index-write-retry' });
+            }
+            catch { /* best-effort */ }
+            result.failed++;
+            continue;
+        }
+        // Content is live. Stamp `full`; if the stamp alone fails, the content is
+        // still searchable — count it as derived and accept a rare re-derive next
+        // boot rather than mislabeling a searchable asset `failed`.
+        try {
+            await vec.setIndexState(d.item.repo, d.item.entity.id, 'full');
+        }
+        catch (err) {
+            log(`persist(full-state) failed for ${ref} (content is indexed; will re-stamp next sweep): ${err instanceof Error ? err.message : err}`);
+        }
+        result.derived++;
+    }
+    log(`[index-sweep] scanned=${result.scanned} derived=${result.derived} failed=${result.failed} skipped=${result.skippedUpToDate} pending=${result.leftPending}`);
+    return result;
+}
+//# sourceMappingURL=asset-index-sweep.js.map

package/dist/services/asset-index-sweep.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asset-index-sweep.js","sourceRoot":"","sources":["../../src/services/asset-index-sweep.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,WAAW,EAAoB,MAAM,4BAA4B,CAAC;AACpH,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AACnF,OAAO,EAAE,YAAY,EAAE,UAAU,EAAmD,MAAM,4BAA4B,CAAC;AACvH,OAAO,EAAE,mBAAmB,EAAsB,MAAM,qBAAqB,CAAC;AAmE9E,sEAAsE;AACtE,SAAS,YAAY,CAAC,GAAW;IAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,oBAAoB,CAAC;AAC5E,CAAC;AAED,yEAAyE;AACzE,KAAK,UAAU,UAAU,CAAI,KAAU,EAAE,WAAmB,EAAE,EAA8B;IAC1F,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACnH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAyB,EACzB,OAA+B,EAAE;IAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAA0B,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IAEhH,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;IAChD,IAAI,CAAC,GAAG;QAAE,OAAO,MAAM,CAAC;IAExB,4EAA4E;IAC5E,4EAA4E;IAC5E,oDAAoD;IACpD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACnC,IAAI,UAAU,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAEvD,yEAAyE;IACzE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;IAE/C,mDAAmD;IACnD,MAAM,QAAQ,GAAe,EAAE,CAAC;IAChC,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,KAAiB,CAAC;QACtB,IAAI,CAAC;YACH,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,QAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,UAAU,KAAK,OAAO;gBAAE,SAAS;YAC5C,yDAAyD;YACzD,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBAAE,SAAS;YACzE,MAAM,IAAI,GAAG,MAAM,CAAC,IAA2B,CAAC;YAChD,MAAM,QAAQ,GAAW,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAElF,0EAA0E;YAC1E,IAAI,GAAG,KAAK,MAAM;gBAAE,SAAS;YAE7B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACjC,2EAA2E;YAC3E,2CAA2C;YAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;YACnG,IAAI,CAAC,SAAS;gBAAE,SAAS;YAEzB,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YACnC,MAAM,IAAI,GACR,SAAS,KAAK,eAAe,CAAC,CAAC,CAAC,OAAO;gBACvC,CAAC,CAAC,CAAC,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,MAAM,IAAI,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;oBAC/E,CAAC,CAAC,IAAI,CAAC;YACT,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,CAAC,OAAO,EAAE,CAAC;YAEjB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;YACxD,wEAAwE;YACxE,iEAAiE;YACjE,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK;mBACnB,KAAK,CAAC,OAAO,IAAI,mBAAmB;mBACpC,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;YAC5D,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,eAAe,EAAE,CAAC;gBACzB,SAAS;YACX,CAAC;YAED,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,IAAI,MAAM,GAAG,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC;IACxC,MAAM,IAAI,GAAG,CAAI,GAAQ,EAAO,EAAE;QAChC,IAAI,MAAM,KAAK,QAAQ;YAAE,OAAO,GAAG,CAAC;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAQhC,8EAA8E;IAC9E,MAAM,cAAc,GAAiB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;QACxF,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC7E,yEAAyE;YACzE,yEAAyE;YACzE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACzF,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;QAClF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/G,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,+CAA+C;IAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB;QACxC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAmB,EAAE,CAAC;QACpG,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAmB,EAAE,CAAC;IAE/C,MAAM,gBAAgB,GAAiB,EAAE,CAAC;IAC1C,IAAI,WAAW,CAAC,EAAE,EAAE,CAAC;QACnB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAc,CAAC;QAC1C,MAAM,GAAG,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnD,MAAM,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;YAC1F,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC7E,MAAM,IAAI,GAAG,MAAM,CAAE,IAAI,CAAC,MAAM,CAAC,IAAY,CAAC,UAAU,IAAI,WAAW,CAAC,CAAC;gBACzE,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;gBACjG,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI;oBAChD,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE;oBAC1C,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;YACpG,CAAC;QACH,CAAC,CAAC,CAAC;QACH,gBAAgB,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IAChC,CAAC;SAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,4EAA4E;QAC5E,0EAA0E;QAC1E,6DAA6D;QAC7D,GAAG,CAAC,uBAAuB,WAAW,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,wCAAwC,CAAC,CAAC;QAC1G,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,yEAAyE;gBACzE,qEAAqE;gBACrE,mEAAmE;gBACnE,sEAAsE;gBACtE,uEAAuE;gBACvE,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;gBAChE,IAAI,QAAQ,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;oBACnC,MAAM,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBAChG,CAAC;gBACD,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,8EAA8E;IAC9E,4EAA4E;IAC5E,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,gBAAgB,CAAC,EAAE,CAAC;QACzD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACZ,wEAAwE;YACxE,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAC;YACpG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,oCAAoC,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,uEAAuE;QACvE,4EAA4E;QAC5E,4EAA4E;QAC5E,6EAA6E;QAC7E,IAAI,CAAC;YACH,iBAAiB,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,kCAAkC;QAClF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,yEAAyE;YACzE,wEAAwE;YACxE,yEAAyE;YACzE,uEAAuE;YACvE,uEAAuE;YACvE,sEAAsE;YACtE,yEAAyE;YACzE,wEAAwE;YACxE,GAAG,CAAC,6BAA6B,GAAG,wCAAwC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACxH,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;YACrG,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,0EAA0E;QAC1E,0EAA0E;QAC1E,4DAA4D;QAC5D,IAAI,CAAC;YACH,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,kCAAkC,GAAG,oDAAoD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3I,CAAC;QACD,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,yBAAyB,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,OAAO,WAAW,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,eAAe,YAAY,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACjK,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/services/asset-lifecycle-check.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Boot-time backstop for the MCP pre-signed asset upload flow.
+ *
+ * Pre-signed PUT URLs land bytes under `pending/{assetId}/{filename}`.
+ * Finalize promotes them to canonical keys + deletes pending. Two cases
+ * leave bytes orphaned in `pending/`:
+ *   - agent presigns but never PUTs (URL expires, blob never existed)
+ *   - agent PUTs but never finalizes (CopyObject + delete-pending skipped)
+ *
+ * The bucket's lifecycle rule (`pending/` TTL 24h) is the only mechanism
+ * bounding storage cost for these. This check runs once per workspace per
+ * boot, confirms the rule exists, and emits a structured WARN if missing.
+ * It does NOT crash boot — operators get a loud reminder instead. See
+ * docs/r2-lifecycle.md for the dashboard steps.
+ *
+ * Auto-fix is intentionally NOT attempted: the R2 IAM token used by the
+ * server typically lacks `s3:PutBucketLifecycleConfiguration` (which would
+ * let any compromised deploy disable orphan cleanup). One-shot operator
+ * action per bucket is the right boundary.
+ */
+import type { WorkspaceManager } from '../core/workspace-manager.js';
+export declare function checkAssetLifecycleRule(wm: WorkspaceManager): Promise<void>;

package/dist/services/asset-lifecycle-check.js

@@ -0,0 +1,80 @@
+/**
+ * Boot-time backstop for the MCP pre-signed asset upload flow.
+ *
+ * Pre-signed PUT URLs land bytes under `pending/{assetId}/{filename}`.
+ * Finalize promotes them to canonical keys + deletes pending. Two cases
+ * leave bytes orphaned in `pending/`:
+ *   - agent presigns but never PUTs (URL expires, blob never existed)
+ *   - agent PUTs but never finalizes (CopyObject + delete-pending skipped)
+ *
+ * The bucket's lifecycle rule (`pending/` TTL 24h) is the only mechanism
+ * bounding storage cost for these. This check runs once per workspace per
+ * boot, confirms the rule exists, and emits a structured WARN if missing.
+ * It does NOT crash boot — operators get a loud reminder instead. See
+ * docs/r2-lifecycle.md for the dashboard steps.
+ *
+ * Auto-fix is intentionally NOT attempted: the R2 IAM token used by the
+ * server typically lacks `s3:PutBucketLifecycleConfiguration` (which would
+ * let any compromised deploy disable orphan cleanup). One-shot operator
+ * action per bucket is the right boundary.
+ */
+import { log } from '../utils/log.js';
+const PENDING_PREFIX = 'pending/';
+const MAX_TTL_DAYS = 1;
+export async function checkAssetLifecycleRule(wm) {
+    // The check uses the first available graph as a lens onto the workspace's
+    // asset config (every graph shares the same AssetService instance under
+    // the hood, so any one will do). When the workspace has no folders yet
+    // (fresh init), there's nothing to check — skip silently.
+    let assetService;
+    for (const repoName of wm.getRepoNames()) {
+        try {
+            const graph = wm.getGraph(repoName);
+            assetService = graph.getAssetService();
+            if (assetService)
+                break;
+        }
+        catch { /* keep looking */ }
+    }
+    if (!assetService)
+        return;
+    // Only R2 has lifecycle; local FS has nothing to check.
+    if (assetService.getStorage() !== 'r2')
+        return;
+    let rules;
+    try {
+        rules = await assetService.getBucketLifecycle();
+    }
+    catch (err) {
+        // Permission-denied, R2 outage, network error at boot — degrade. The
+        // alternative (crash boot) would be a worse failure mode: a transient
+        // R2 hiccup shouldn't prevent the workspace from serving.
+        log.warn('[asset-lifecycle] could not verify pending/ lifecycle rule — continuing', {
+            event: 'asset-lifecycle-check-failed',
+            err: err instanceof Error ? err.message : String(err),
+        });
+        return;
+    }
+    const ok = (rules ?? []).some(rule => {
+        // Disabled rules don't fire; an operator who turned the rule off
+        // expects orphans to NOT be reaped. Codex P2 (2026-05-28).
+        if (!rule.enabled)
+            return false;
+        if (rule.expirationDays === undefined)
+            return false;
+        if (rule.expirationDays > MAX_TTL_DAYS)
+            return false;
+        // Empty/missing prefix matches everything → also covers pending/.
+        if (!rule.prefix)
+            return true;
+        return PENDING_PREFIX.startsWith(rule.prefix);
+    });
+    if (!ok) {
+        log.warn('[asset-lifecycle] WARNING: no R2 lifecycle rule covers prefix=pending/ with TTL≤24h', {
+            event: 'asset-lifecycle-missing',
+            hint: 'Set the rule via the Cloudflare R2 dashboard. See docs/r2-lifecycle.md.',
+            rulesFound: rules?.length ?? 0,
+        });
+    }
+}
+//# sourceMappingURL=asset-lifecycle-check.js.map

package/dist/services/asset-lifecycle-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asset-lifecycle-check.js","sourceRoot":"","sources":["../../src/services/asset-lifecycle-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAEtC,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAAoB;IAChE,0EAA0E;IAC1E,wEAAwE;IACxE,uEAAuE;IACvE,0DAA0D;IAC1D,IAAI,YAAY,CAAC;IACjB,KAAK,MAAM,QAAQ,IAAI,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACpC,YAAY,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,YAAY;gBAAE,MAAM;QAC1B,CAAC;QAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,YAAY;QAAE,OAAO;IAE1B,wDAAwD;IACxD,IAAI,YAAY,CAAC,UAAU,EAAE,KAAK,IAAI;QAAE,OAAO;IAE/C,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC,kBAAkB,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,qEAAqE;QACrE,sEAAsE;QACtE,0DAA0D;QAC1D,GAAG,CAAC,IAAI,CACN,yEAAyE,EACzE;YACE,KAAK,EAAE,8BAA8B;YACrC,GAAG,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACtD,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACnC,iEAAiE;QACjE,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACpD,IAAI,IAAI,CAAC,cAAc,GAAG,YAAY;YAAE,OAAO,KAAK,CAAC;QACrD,kEAAkE;QAClE,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC9B,OAAO,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,GAAG,CAAC,IAAI,CACN,qFAAqF,EACrF;YACE,KAAK,EAAE,yBAAyB;YAChC,IAAI,EAAE,yEAAyE;YAC/E,UAAU,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;SAC/B,CACF,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/services/asset-listing.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Shared repo-asset listing.
+ *
+ * Builds the list of assets owned by entries in a single repo, merging the two
+ * storage schemes (legacy `{type}/{id}/{filename}` blobs surfaced by the
+ * backend's listAll, and immutable-keyed assets tracked only in the ownership
+ * index). Extracted from the asset-api route so the HTTP endpoints and the MCP
+ * `list_assets` tool return an identical shape — including a render-safe
+ * `embed` string callers can paste straight into an entry body.
+ */
+import type { WorkspaceManager } from '../core/workspace-manager.js';
+/** Owner kinds that mean "free-standing" — the asset is owned by a folder or
+ *  the workspace, not by an entity, so it has no owning markdown file and is
+ *  deleted/managed by its assetId rather than an entity coordinate. */
+export declare const FREE_STANDING_OWNER_TYPES: Set<string>;
+export declare function isFreeStandingOwner(ownerType: string): boolean;
+export interface RepoAsset {
+    filename: string;
+    size: number;
+    contentType: string;
+    url: string;
+    /** Render-safe markdown embed: `![filename](url)`. Paste verbatim. */
+    embed: string;
+    /** Immutable asset id when the asset is index-tracked; null for legacy blobs. */
+    assetId: string | null;
+    /** Raw owner kind from the index: an entity type when the asset is embedded
+     *  in an entity, or 'folder'/'workspace' when the asset is free-standing
+     *  (owned by a folder/workspace, not an entity). Drives free-standing
+     *  detection on the client (render + delete-by-assetId routing). */
+    ownerType: string;
+    entityType: string;
+    entityId: string;
+    entityName: string;
+    mediaType: string;
+    /** Owning entity's `created_at` (ISO), or null if absent. For sorting. */
+    createdAt: string | null;
+}
+/**
+ * Build the asset list for a single repo. Returns [] if the repo's graph can't
+ * be loaded (right for the bulk endpoint; single-repo callers should check
+ * repo existence themselves first if they want a 404).
+ *
+ * Algorithm (works for both local and R2 backends):
+ *   1. One listAllAssets() call — single filesystem walk or one paginated S3
+ *      ListObjectsV2 request. No HEAD round-trips, no markdown parsing.
+ *   2. Resolve each asset's entity markdown path via the graph's file index
+ *      (not a `{plural}/{id}.md` guess, which misses root/nested entries).
+ *   3. Merge in immutable-keyed assets from the ownership index — these live
+ *      under an `{assetId}/` prefix the backend's listAll can't see.
+ */
+export declare function listRepoAssets(repo: string, workspaceManager: WorkspaceManager): Promise<RepoAsset[]>;
+/** A `RepoAsset` tagged with the folder (repo) that owns it. */
+export type WorkspaceAsset = RepoAsset & {
+    repo: string;
+};
+/** Drop the cached legacy-bucket walk(s) (tests; post-delete/cleanup refresh). */
+export declare function clearLegacyBucketCache(): void;
+/**
+ * List assets across MANY repos for the workspace-wide manager — index-first.
+ *
+ * The fast path is the local ownership index (`assets.db`): one cheap scan per
+ * repo yields every immutable-keyed asset with no network I/O. On top of that we
+ * fold in any surviving *legacy* blobs via the cached bucket walk above (see
+ * `getLegacyBucketSnapshot`) so un-migrated assets still appear. Dedup prefers
+ * the immutable entry, so a legacy blob that has already been migrated collapses
+ * into its index row rather than double-listing.
+ *
+ * A legacy blob whose owning entity exists in K accessible repos surfaces under
+ * each of those K (entity ids can — rarely — collide across folders), matching
+ * the old per-repo behaviour.
+ */
+export declare function listWorkspaceAssets(repos: string[], workspaceManager: WorkspaceManager): Promise<WorkspaceAsset[]>;