studiograph 1.3.48-next.21 → 1.3.48-next.210

package/dist/services/asset-index-sweep.js

@@ -0,0 +1,249 @@
+/**
+ * Asset index sweep (asset-as-embed) — the boot/recovery trigger of the image
+ * caption / OCR pass.
+ *
+ * OV1: this is SEPARATE from the text/FTS reindex. The vector reindex (boot /
+ * git-pull / reconcile) only refreshes the FTS+embedding text for changed
+ * entries; it deliberately does NOT fire vision calls. This sweep is the one
+ * place that drives image captioning, so vision spend is intentional and
+ * bounded, never a side effect of a routine reindex.
+ *
+ * Asset-as-embed: derivation is keyed by the immutable assetId (the assets.db
+ * row), NOT an `asset` entity — that type was retired by Phase F. Per indexable
+ * IMAGE row (assets.db `media_type === 'image'`, non-SVG):
+ *  - PERF1: reads the unfinished set from the LanceDB index-state table (one
+ *    query) and skips rows already `full`/`failed` at the current version.
+ *  - Migration (Codex CRITICAL): a row with NO state, or a row at a STALE
+ *    `index_version`, is RE-derived — undefined is never treated as "done".
+ *    `failed` at the current version is terminal (don't re-bill every boot).
+ *  - Caption + OCR (bounded fan-out, A3). When captioning is unreachable
+ *    (gateway / disabled / non-vision model) the image is left honestly
+ *    `pending`, so flipping the gate or model later picks it up.
+ *  - SVG is skipped — transitional until sanitize-html is wired.
+ *
+ * On a successful caption the text is stored in `assets.db.caption` (decision
+ * D1 — DB column, NO sidecar; survives deletion of any embedding entry) and the
+ * entries that EMBED the asset are re-upserted so the caption folds into their
+ * search text (decision D2). The expensive work (vision, byte downloads) runs
+ * CONCURRENTLY (bounded); the LanceDB index-state + embedder writes run SERIALLY
+ * through `vectorService` (optimistic concurrency — racing mergeInserts conflict).
+ */
+import { captionImage, canCaption } from './asset-caption-service.js';
+import { ASSET_INDEX_VERSION } from './vector-service.js';
+import { isCaptionableImage } from './assets/base.js';
+import { embeddedAssetIds } from '../core/embeds.js';
+/** Run `fn` over `items` with a bounded number of concurrent workers. */
+async function mapBounded(items, concurrency, fn) {
+    let next = 0;
+    const worker = async () => {
+        while (true) {
+            const i = next++;
+            if (i >= items.length)
+                return;
+            await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), items.length || 1) }, () => worker()));
+}
+/**
+ * Re-upsert every entry that embeds ANY freshly-captioned asset so the stored
+ * captions (already in assets.db.caption) fold into the entries' search text
+ * (decision D2). ONE pass per repo for the whole `captioned` batch: scan each
+ * entry's embedded assetIds (the SAME `embeddedAssetIds` definition the fold
+ * uses — body + frontmatter, no field exclusions) and re-upsert it if it embeds
+ * any captioned asset. O(repos × entities) for the batch, not per-asset, and an
+ * entry embedding several captioned images is re-upserted once. Best-effort +
+ * isolated; the upserts serialize through `vectorService`.
+ */
+async function reindexEmbedders(deps, captioned, log) {
+    const vec = deps.workspaceManager.vectorService;
+    if (!vec || captioned.size === 0)
+        return;
+    for (const repo of deps.repos) {
+        let graph;
+        try {
+            graph = deps.workspaceManager.getGraph(repo.name);
+        }
+        catch {
+            continue;
+        }
+        let entities;
+        try {
+            entities = graph.list();
+        }
+        catch {
+            continue;
+        }
+        for (const e of entities) {
+            const ids = embeddedAssetIds(e.document?.content ?? '', e.data);
+            if (ids.size === 0)
+                continue;
+            let embedsCaptioned = false;
+            for (const id of ids) {
+                if (captioned.has(id)) {
+                    embedsCaptioned = true;
+                    break;
+                }
+            }
+            if (!embedsCaptioned)
+                continue;
+            try {
+                await vec.upsert(repo.name, e);
+            }
+            catch (err) {
+                log(`reindex embedder ${repo.name}/${e.id} failed: ${err instanceof Error ? err.message : err}`);
+            }
+        }
+    }
+}
+/**
+ * Drive the caption / OCR pass for every image asset that needs it. Safe to
+ * call fire-and-forget at boot; resilient to per-asset failure.
+ */
+export async function sweepAssetIndex(deps, opts = {}) {
+    const log = opts.log ?? (() => { });
+    const concurrency = opts.concurrency ?? 2;
+    const result = { scanned: 0, derived: 0, failed: 0, skippedUpToDate: 0, leftPending: 0 };
+    const vec = deps.workspaceManager.vectorService;
+    if (!vec)
+        return result;
+    // A targeted drain with an empty set has no work — skip even the LanceDB read.
+    // `undefined` means "consider everything" (boot recovery), NOT an empty set.
+    const onlyAssets = opts.onlyAssets;
+    if (onlyAssets && onlyAssets.size === 0)
+        return result;
+    const assetIndex = deps.workspaceManager.getAssetIndex();
+    // PERF1: one query for the whole unfinished set, no per-asset re-derive.
+    const stateMap = await vec.getAllIndexStates();
+    // Collect the work-list from assets.db image rows across the swept repos.
+    const items = [];
+    for (const repo of deps.repos) {
+        let rows;
+        try {
+            rows = assetIndex.listByRepo(repo.name);
+        }
+        catch {
+            continue;
+        }
+        for (const row of rows) {
+            // Only raster images are captioned (SVG stays transitional). Shared with
+            // the live trigger so the drain and boot sweep can't disagree.
+            if (!isCaptionableImage(row.mediaType, row.filename))
+                continue;
+            // Targeted drain: skip assets the caller didn't enqueue.
+            if (onlyAssets && !onlyAssets.has(row.assetId))
+                continue;
+            result.scanned++;
+            const state = stateMap.get(row.assetId);
+            // Up-to-date = a current-version row that's finished (full) or terminal
+            // (failed). Missing row / stale version / pending ⇒ (re-)derive. D1: there
+            // is no sidecar to stat for an image — the assetId-keyed state row IS the
+            // truth (the caption lives in assets.db.caption).
+            const upToDate = !!state
+                && state.version >= ASSET_INDEX_VERSION
+                && (state.status === 'full' || state.status === 'failed');
+            if (upToDate) {
+                result.skippedUpToDate++;
+                continue;
+            }
+            items.push({ assetId: row.assetId, repo: repo.name, filename: row.filename, contentType: row.contentType });
+        }
+    }
+    // Respect the optional safety cap.
+    const images = opts.maxAssets != null ? items.slice(0, Math.max(0, opts.maxAssets)) : items;
+    // Caption reachability gate (per-workspace flag + model capability).
+    const captionGate = deps.captioningEnabled
+        ? (deps.captionConfig ? canCaption(deps.captionConfig) : { ok: false, reason: 'no-model' })
+        : { ok: false, reason: 'disabled' };
+    if (!captionGate.ok) {
+        // Degrade gracefully (option a): can't caption now. Seed/keep a `pending`
+        // row so the image is honestly filename-only; it transitions when the gate
+        // flips (config enabled / model switched), no version bump needed.
+        if (images.length > 0) {
+            log(`captioning skipped (${captionGate.reason}); ${images.length} image(s) left pending (filename-only)`);
+            for (const item of images) {
+                try {
+                    const existing = stateMap.get(item.assetId);
+                    if (existing?.status !== 'pending') {
+                        await vec.setIndexState(item.assetId, 'pending', { reason: captionGate.reason, repo: item.repo });
+                    }
+                    result.leftPending++;
+                }
+                catch { /* best-effort */ }
+            }
+        }
+        log(`[index-sweep] scanned=${result.scanned} derived=0 failed=0 skipped=${result.skippedUpToDate} pending=${result.leftPending}`);
+        return result;
+    }
+    // ── Caption + OCR each image, gated + bounded ──
+    const captionConfig = deps.captionConfig;
+    const derivations = new Array(images.length);
+    await mapBounded(images.map((item, i) => ({ item, i })), concurrency, async ({ item, i }) => {
+        try {
+            const graph = deps.workspaceManager.getGraph(item.repo);
+            const buffer = await graph.downloadAssetById(item.assetId, item.filename);
+            const caption = await captionImage(buffer, item.contentType || 'image/png', captionConfig, { completeFn: opts.completeFn });
+            derivations[i] = caption.status === 'full' && caption.text
+                ? { item, text: caption.text, reason: '' }
+                : { item, text: null, reason: caption.reason ?? 'error' };
+        }
+        catch (err) {
+            derivations[i] = { item, text: null, reason: err instanceof Error ? err.message.slice(0, 120) : 'error' };
+        }
+    });
+    // ── Persist serially (one LanceDB writer at a time) ──
+    const captioned = new Set(); // assetIds whose caption landed this sweep
+    for (const d of derivations) {
+        if (!d)
+            continue;
+        const ref = `${d.item.repo}/${d.item.assetId}`;
+        if (!d.text) {
+            // The caption produced no text → honest `failed` (filename-only).
+            try {
+                await vec.setIndexState(d.item.assetId, 'failed', { reason: d.reason || 'error', repo: d.item.repo });
+            }
+            catch (err) {
+                log(`persist(failed-state) failed for ${ref}: ${err instanceof Error ? err.message : err}`);
+            }
+            result.failed++;
+            continue;
+        }
+        // We have caption text. Store it in assets.db FIRST, then stamp `full`.
+        // Splitting the two means a status-stamp hiccup can't mislabel an
+        // already-stored caption as `failed`. The embedder re-index runs once for
+        // the whole batch AFTER this loop (so every caption is in place first).
+        try {
+            assetIndex.setCaption(d.item.assetId, d.text);
+        }
+        catch (err) {
+            // A TRANSIENT infra error (sqlite hiccup) — NOT "this image can't be
+            // captioned." The caption text is in hand; only persisting it failed.
+            // Leave it `pending` so the next sweep retries. Counted for observability.
+            log(`persist(caption) failed for ${ref} (transient; will retry next sweep): ${err instanceof Error ? err.message : err}`);
+            try {
+                await vec.setIndexState(d.item.assetId, 'pending', { reason: 'caption-write-retry', repo: d.item.repo });
+            }
+            catch { /* best-effort */ }
+            result.failed++;
+            continue;
+        }
+        // Caption is stored. Stamp `full`; if the stamp alone fails the caption is
+        // still in assets.db — count it as derived and accept a rare re-derive next
+        // boot rather than mislabeling a stored caption `failed`.
+        try {
+            await vec.setIndexState(d.item.assetId, 'full', { repo: d.item.repo });
+        }
+        catch (err) {
+            log(`persist(full-state) failed for ${ref} (caption is stored; will re-stamp next sweep): ${err instanceof Error ? err.message : err}`);
+        }
+        captioned.add(d.item.assetId);
+        result.derived++;
+    }
+    // Fold the freshly-stored captions into the entries that embed them — one pass
+    // per repo for the whole batch. Done after the persist loop so every caption
+    // is in assets.db before any embedder is re-indexed.
+    await reindexEmbedders(deps, captioned, log);
+    log(`[index-sweep] scanned=${result.scanned} derived=${result.derived} failed=${result.failed} skipped=${result.skippedUpToDate} pending=${result.leftPending}`);
+    return result;
+}
+//# sourceMappingURL=asset-index-sweep.js.map

package/dist/services/asset-index-sweep.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asset-index-sweep.js","sourceRoot":"","sources":["../../src/services/asset-index-sweep.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAmD,MAAM,4BAA4B,CAAC;AACvH,OAAO,EAAE,mBAAmB,EAAsB,MAAM,qBAAqB,CAAC;AAE9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAoErD,yEAAyE;AACzE,KAAK,UAAU,UAAU,CAAI,KAAU,EAAE,WAAmB,EAAE,EAA8B;IAC1F,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAC9B,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACnH,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,gBAAgB,CAC7B,IAAyB,EACzB,SAA8B,EAC9B,GAA0B;IAE1B,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;IAChD,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO;IACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,KAAiB,CAAC;QACtB,IAAI,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9E,IAAI,QAAsB,CAAC;QAC3B,IAAI,CAAC;YAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS;QAAC,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAChE,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YAC7B,IAAI,eAAe,GAAG,KAAK,CAAC;YAC5B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBAAC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAAC,eAAe,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;YAAC,CAAC;YACnF,IAAI,CAAC,eAAe;gBAAE,SAAS;YAC/B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,oBAAoB,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,YAAY,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACnG,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAyB,EACzB,OAA+B,EAAE;IAEjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACnC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAA0B,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IAEhH,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC;IAChD,IAAI,CAAC,GAAG;QAAE,OAAO,MAAM,CAAC;IAExB,+EAA+E;IAC/E,6EAA6E;IAC7E,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;IACnC,IAAI,UAAU,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAEvD,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC;IAEzD,yEAAyE;IACzE,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,iBAAiB,EAAE,CAAC;IAE/C,0EAA0E;IAC1E,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC;QACT,IAAI,CAAC;YACH,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,yEAAyE;YACzE,+DAA+D;YAC/D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAC/D,yDAAyD;YACzD,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEzD,MAAM,CAAC,OAAO,EAAE,CAAC;YAEjB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACxC,wEAAwE;YACxE,2EAA2E;YAC3E,0EAA0E;YAC1E,kDAAkD;YAClD,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK;mBACnB,KAAK,CAAC,OAAO,IAAI,mBAAmB;mBACpC,CAAC,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;YAC5D,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,eAAe,EAAE,CAAC;gBACzB,SAAS;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAE5F,qEAAqE;IACrE,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB;QACxC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAmB,EAAE,CAAC;QACpG,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,UAAmB,EAAE,CAAC;IAE/C,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;QACpB,0EAA0E;QAC1E,2EAA2E;QAC3E,mEAAmE;QACnE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,uBAAuB,WAAW,CAAC,MAAM,MAAM,MAAM,CAAC,MAAM,wCAAwC,CAAC,CAAC;YAC1G,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE,MAAM,KAAK,SAAS,EAAE,CAAC;wBACnC,MAAM,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;oBACpG,CAAC;oBACD,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvB,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,GAAG,CAAC,yBAAyB,MAAM,CAAC,OAAO,+BAA+B,MAAM,CAAC,eAAe,YAAY,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAClI,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAc,CAAC;IAE1C,MAAM,WAAW,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;QAC1F,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1E,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,WAAW,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5H,WAAW,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI;gBACxD,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE;gBAC1C,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,WAAW,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5G,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,wDAAwD;IACxD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC,CAAC,2CAA2C;IAChF,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/C,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACZ,kEAAkE;YAClE,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACxG,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,oCAAoC,GAAG,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,wEAAwE;QACxE,kEAAkE;QAClE,0EAA0E;QAC1E,wEAAwE;QACxE,IAAI,CAAC;YACH,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,qEAAqE;YACrE,sEAAsE;YACtE,2EAA2E;YAC3E,GAAG,CAAC,+BAA+B,GAAG,wCAAwC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAC1H,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3G,CAAC;YAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;YAC7B,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,2EAA2E;QAC3E,4EAA4E;QAC5E,0DAA0D;QAC1D,IAAI,CAAC;YACH,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,kCAAkC,GAAG,mDAAmD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1I,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,+EAA+E;IAC/E,6EAA6E;IAC7E,qDAAqD;IACrD,MAAM,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;IAE7C,GAAG,CAAC,yBAAyB,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,OAAO,WAAW,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,eAAe,YAAY,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IACjK,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/services/asset-lifecycle-check.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Boot-time backstop for the MCP pre-signed asset upload flow.
+ *
+ * Pre-signed PUT URLs land bytes under `pending/{assetId}/{filename}`.
+ * Finalize promotes them to canonical keys + deletes pending. Two cases
+ * leave bytes orphaned in `pending/`:
+ *   - agent presigns but never PUTs (URL expires, blob never existed)
+ *   - agent PUTs but never finalizes (CopyObject + delete-pending skipped)
+ *
+ * The bucket's lifecycle rule (`pending/` TTL 24h) is the only mechanism
+ * bounding storage cost for these. This check runs once per workspace per
+ * boot, confirms the rule exists, and emits a structured WARN if missing.
+ * It does NOT crash boot — operators get a loud reminder instead. See
+ * docs/r2-lifecycle.md for the dashboard steps.
+ *
+ * Auto-fix is intentionally NOT attempted: the R2 IAM token used by the
+ * server typically lacks `s3:PutBucketLifecycleConfiguration` (which would
+ * let any compromised deploy disable orphan cleanup). One-shot operator
+ * action per bucket is the right boundary.
+ */
+import type { WorkspaceManager } from '../core/workspace-manager.js';
+export declare function checkAssetLifecycleRule(wm: WorkspaceManager): Promise<void>;

package/dist/services/asset-lifecycle-check.js

@@ -0,0 +1,80 @@
+/**
+ * Boot-time backstop for the MCP pre-signed asset upload flow.
+ *
+ * Pre-signed PUT URLs land bytes under `pending/{assetId}/{filename}`.
+ * Finalize promotes them to canonical keys + deletes pending. Two cases
+ * leave bytes orphaned in `pending/`:
+ *   - agent presigns but never PUTs (URL expires, blob never existed)
+ *   - agent PUTs but never finalizes (CopyObject + delete-pending skipped)
+ *
+ * The bucket's lifecycle rule (`pending/` TTL 24h) is the only mechanism
+ * bounding storage cost for these. This check runs once per workspace per
+ * boot, confirms the rule exists, and emits a structured WARN if missing.
+ * It does NOT crash boot — operators get a loud reminder instead. See
+ * docs/r2-lifecycle.md for the dashboard steps.
+ *
+ * Auto-fix is intentionally NOT attempted: the R2 IAM token used by the
+ * server typically lacks `s3:PutBucketLifecycleConfiguration` (which would
+ * let any compromised deploy disable orphan cleanup). One-shot operator
+ * action per bucket is the right boundary.
+ */
+import { log } from '../utils/log.js';
+const PENDING_PREFIX = 'pending/';
+const MAX_TTL_DAYS = 1;
+export async function checkAssetLifecycleRule(wm) {
+    // The check uses the first available graph as a lens onto the workspace's
+    // asset config (every graph shares the same AssetService instance under
+    // the hood, so any one will do). When the workspace has no folders yet
+    // (fresh init), there's nothing to check — skip silently.
+    let assetService;
+    for (const repoName of wm.getRepoNames()) {
+        try {
+            const graph = wm.getGraph(repoName);
+            assetService = graph.getAssetService();
+            if (assetService)
+                break;
+        }
+        catch { /* keep looking */ }
+    }
+    if (!assetService)
+        return;
+    // Only R2 has lifecycle; local FS has nothing to check.
+    if (assetService.getStorage() !== 'r2')
+        return;
+    let rules;
+    try {
+        rules = await assetService.getBucketLifecycle();
+    }
+    catch (err) {
+        // Permission-denied, R2 outage, network error at boot — degrade. The
+        // alternative (crash boot) would be a worse failure mode: a transient
+        // R2 hiccup shouldn't prevent the workspace from serving.
+        log.warn('[asset-lifecycle] could not verify pending/ lifecycle rule — continuing', {
+            event: 'asset-lifecycle-check-failed',
+            err: err instanceof Error ? err.message : String(err),
+        });
+        return;
+    }
+    const ok = (rules ?? []).some(rule => {
+        // Disabled rules don't fire; an operator who turned the rule off
+        // expects orphans to NOT be reaped. Codex P2 (2026-05-28).
+        if (!rule.enabled)
+            return false;
+        if (rule.expirationDays === undefined)
+            return false;
+        if (rule.expirationDays > MAX_TTL_DAYS)
+            return false;
+        // Empty/missing prefix matches everything → also covers pending/.
+        if (!rule.prefix)
+            return true;
+        return PENDING_PREFIX.startsWith(rule.prefix);
+    });
+    if (!ok) {
+        log.warn('[asset-lifecycle] WARNING: no R2 lifecycle rule covers prefix=pending/ with TTL≤24h', {
+            event: 'asset-lifecycle-missing',
+            hint: 'Set the rule via the Cloudflare R2 dashboard. See docs/r2-lifecycle.md.',
+            rulesFound: rules?.length ?? 0,
+        });
+    }
+}
+//# sourceMappingURL=asset-lifecycle-check.js.map

package/dist/services/asset-lifecycle-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asset-lifecycle-check.js","sourceRoot":"","sources":["../../src/services/asset-lifecycle-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAEtC,MAAM,cAAc,GAAG,UAAU,CAAC;AAClC,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAAoB;IAChE,0EAA0E;IAC1E,wEAAwE;IACxE,uEAAuE;IACvE,0DAA0D;IAC1D,IAAI,YAAY,CAAC;IACjB,KAAK,MAAM,QAAQ,IAAI,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACpC,YAAY,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,YAAY;gBAAE,MAAM;QAC1B,CAAC;QAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,YAAY;QAAE,OAAO;IAE1B,wDAAwD;IACxD,IAAI,YAAY,CAAC,UAAU,EAAE,KAAK,IAAI;QAAE,OAAO;IAE/C,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC,kBAAkB,EAAE,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,qEAAqE;QACrE,sEAAsE;QACtE,0DAA0D;QAC1D,GAAG,CAAC,IAAI,CACN,yEAAyE,EACzE;YACE,KAAK,EAAE,8BAA8B;YACrC,GAAG,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACtD,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACnC,iEAAiE;QACjE,2DAA2D;QAC3D,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,IAAI,CAAC,cAAc,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACpD,IAAI,IAAI,CAAC,cAAc,GAAG,YAAY;YAAE,OAAO,KAAK,CAAC;QACrD,kEAAkE;QAClE,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC9B,OAAO,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,GAAG,CAAC,IAAI,CACN,qFAAqF,EACrF;YACE,KAAK,EAAE,yBAAyB;YAChC,IAAI,EAAE,yEAAyE;YAC/E,UAAU,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;SAC/B,CACF,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/services/asset-listing.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Shared repo-asset listing.
+ *
+ * Builds the list of assets owned by entries in a single repo, merging the two
+ * storage schemes (legacy `{type}/{id}/{filename}` blobs surfaced by the
+ * backend's listAll, and immutable-keyed assets tracked only in the ownership
+ * index). Extracted from the asset-api route so the HTTP endpoints and the MCP
+ * `list_assets` tool return an identical shape — including a render-safe
+ * `embed` string callers can paste straight into an entry body.
+ */
+import type { WorkspaceManager } from '../core/workspace-manager.js';
+/** Owner kinds that mean "free-standing" — the asset is owned by a folder or
+ *  the workspace, not by an entity, so it has no owning markdown file and is
+ *  deleted/managed by its assetId rather than an entity coordinate. */
+export declare const FREE_STANDING_OWNER_TYPES: Set<string>;
+export declare function isFreeStandingOwner(ownerType: string): boolean;
+export interface RepoAsset {
+    filename: string;
+    size: number;
+    contentType: string;
+    url: string;
+    /** Render-safe markdown embed: `![filename](url)`. Paste verbatim. */
+    embed: string;
+    /** Immutable asset id when the asset is index-tracked; null for legacy blobs. */
+    assetId: string | null;
+    /** Raw owner kind from the index: an entity type when the asset is embedded
+     *  in an entity, or 'folder'/'workspace' when the asset is free-standing
+     *  (owned by a folder/workspace, not an entity). Drives free-standing
+     *  detection on the client (render + delete-by-assetId routing). */
+    ownerType: string;
+    entityType: string;
+    entityId: string;
+    entityName: string;
+    mediaType: string;
+    /** Owning entity's `created_at` (ISO), or null if absent. For sorting. */
+    createdAt: string | null;
+}
+/**
+ * Build the asset list for a single repo. Returns [] if the repo's graph can't
+ * be loaded (right for the bulk endpoint; single-repo callers should check
+ * repo existence themselves first if they want a 404).
+ *
+ * Algorithm (works for both local and R2 backends):
+ *   1. One listAllAssets() call — single filesystem walk or one paginated S3
+ *      ListObjectsV2 request. No HEAD round-trips, no markdown parsing.
+ *   2. Resolve each asset's entity markdown path via the graph's file index
+ *      (not a `{plural}/{id}.md` guess, which misses root/nested entries).
+ *   3. Merge in immutable-keyed assets from the ownership index — these live
+ *      under an `{assetId}/` prefix the backend's listAll can't see.
+ */
+export declare function listRepoAssets(repo: string, workspaceManager: WorkspaceManager): Promise<RepoAsset[]>;
+/** A `RepoAsset` tagged with the folder (repo) that owns it. */
+export type WorkspaceAsset = RepoAsset & {
+    repo: string;
+};
+/** Drop the cached legacy-bucket walk(s) (tests; post-delete/cleanup refresh). */
+export declare function clearLegacyBucketCache(): void;
+/**
+ * List assets across MANY repos for the workspace-wide manager — index-first.
+ *
+ * The fast path is the local ownership index (`assets.db`): one cheap scan per
+ * repo yields every immutable-keyed asset with no network I/O. On top of that we
+ * fold in any surviving *legacy* blobs via the cached bucket walk above (see
+ * `getLegacyBucketSnapshot`) so un-migrated assets still appear. Dedup prefers
+ * the immutable entry, so a legacy blob that has already been migrated collapses
+ * into its index row rather than double-listing.
+ *
+ * A legacy blob whose owning entity exists in K accessible repos surfaces under
+ * each of those K (entity ids can — rarely — collide across folders), matching
+ * the old per-repo behaviour.
+ */
+export declare function listWorkspaceAssets(repos: string[], workspaceManager: WorkspaceManager): Promise<WorkspaceAsset[]>;