sliftutils 1.4.47 → 1.4.49

package/storage/BulkDatabase2/LoadedIndex.ts

@@ -0,0 +1,402 @@
+import type { FileStorage } from "../FileFolderAPI";
+import { ABSENT, BaseBulkDatabaseReader, EMPTY_BUFFER, KEY_COLUMN, loadBulkDatabase } from "./BulkDatabaseFormat";
+import { blockCache, GetRange } from "./blockCache";
+import { STREAM_EXTENSION, StreamEntry, parseStream, streamReaderFromEntries } from "./streamLog";
+import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { blue, red } from "socket-function/src/formatting/logColors";
+
+export type BulkFileInfo = { fileName: string; level: number; timestamp: number };
+export type StreamFileInfo = { fileName: string; timestamp: number };
+export type StreamReaderCacheEntry = { readSize: number; parsedPos: number; entries: StreamEntry[] };
+
+export class MissingFileError extends Error { }
+class FilesChangedError extends Error { }
+
+export type ResolvedReader = {
+    rowCount: number;
+    totalBytes: number;
+    keys: string[];
+    rawKeyCount: number;
+    readerCount: number;
+    columns: { column: string; byteSize: number }[];
+    keyTimes: Map<string, number>;
+    deleteTimes: Map<string, number>;
+    getColumn: (column: string) => Promise<{ key: string; value: unknown; time: number }[]>;
+    getSingleField: (key: string, column: string) => Promise<{ value: unknown; time: number } | undefined>;
+};
+
+export type SubReaderCaches = {
+    bulk: Map<string, BaseBulkDatabaseReader>;
+    stream: Map<string, StreamReaderCacheEntry>;
+};
+
+const MAX_READ_ATTEMPTS = 8;
+
+function nullJoin(a: string, b: string): string {
+    return a + String.fromCharCode(0) + b;
+}
+
+// A loaded snapshot of one collection's on-disk state: the resolved reader + the per-file decoded sub-
+// readers it stitched, the stream file sizes/times it observed, and the sync (reactive) caches that
+// surface column/field reads. Built once via LoadedIndex.build — eagerly, so the swap into a fresh
+// LoadedIndex never blocks a read on a synchronous rebuild.
+export class LoadedIndex<T extends { key: string }> {
+    private constructor(
+        public readonly name: string,
+        public readonly storage: FileStorage,
+        public readonly bulkFiles: BulkFileInfo[],
+        public readonly streamFiles: StreamFileInfo[],
+        public readonly reader: ResolvedReader,
+        public readonly streamTimes: Map<string, number>,
+        public readonly streamSizes: Map<string, number>,
+        public readonly streamRowsOnDisk: number,
+        public readonly streamBytesOnDisk: number,
+        public readonly subCaches: SubReaderCaches,
+    ) {
+        this.keys = new Set(reader.keys);
+        this.fileSet = new Set([...bulkFiles.map(f => f.fileName), ...streamFiles.map(f => f.fileName)]);
+    }
+
+    public readonly keys: Set<string>;
+    public readonly fileSet: Set<string>;
+
+    private baseColumns = new Map<string, { key: string; value: unknown; time: number }[]>();
+    private baseColumnsLoading = new Set<string>();
+    private baseFields = new Map<string, { value: unknown; time: number } | undefined>();
+    private baseFieldsLoading = new Set<string>();
+    // Last-known values from the previous index, served until this index loads its own fresh value, so
+    // a swap doesn't flash empty in sync reads. Cleared per-entry when the fresh value lands.
+    private staleBaseColumns = new Map<string, { key: string; value: unknown; time: number }[]>();
+    private staleBaseFields = new Map<string, { value: unknown; time: number } | undefined>();
+
+    get totalBytes(): number { return this.reader.totalBytes; }
+    get rowCount(): number { return this.reader.rowCount; }
+
+    isLive(key: string): boolean { return this.keys.has(key); }
+
+    static async build<T extends { key: string }>(config: {
+        name: string;
+        storage: FileStorage;
+        bulkFiles: BulkFileInfo[];
+        streamFiles: StreamFileInfo[];
+        subCaches: SubReaderCaches;
+        onUnreadableFile?: (file: BulkFileInfo, message: string) => Promise<void>;
+    }): Promise<LoadedIndex<T>> {
+        const start = Date.now();
+        let { bulkFiles, streamFiles } = config;
+        let attempt = 0;
+        while (true) {
+            const tolerateMissing = attempt >= MAX_READ_ATTEMPTS;
+            try {
+                let filesChanged = false;
+                const [bulkReadersRaw, streamData] = await Promise.all([
+                    Promise.all(bulkFiles.map(async f => {
+                        try {
+                            return await loadFileReader(config.name, config.storage, f, config.subCaches.bulk);
+                        } catch (e) {
+                            if (e instanceof MissingFileError) { filesChanged = true; return undefined; }
+                            if (config.onUnreadableFile) await config.onUnreadableFile(f, (e as Error).message);
+                            return undefined;
+                        }
+                    })),
+                    loadStreamEntries(config.name, config.storage, streamFiles, config.subCaches.stream),
+                ]);
+                if (streamData.missing) filesChanged = true;
+                if (filesChanged && !tolerateMissing) throw new FilesChangedError();
+
+                const bulkReaders = bulkReadersRaw.filter((r): r is BaseBulkDatabaseReader => !!r);
+                const readers: BaseBulkDatabaseReader[] = [];
+                let streamTimes = new Map<string, number>();
+                const ordered = orderStreamEntries(streamData.entries);
+                if (ordered.length) {
+                    const stream = streamReaderFromEntries(ordered, streamData.totalBytes);
+                    readers.push(stream.reader);
+                    streamTimes = stream.times;
+                }
+                readers.push(...bulkReaders);
+                const reader = joinBulkDatabases(readers);
+
+                pruneSubCaches(bulkFiles, streamFiles, config.subCaches);
+
+                const elapsed = Date.now() - start;
+                if (elapsed > 50) {
+                    let bytesRead = streamData.totalBytes;
+                    for (const r of bulkReaders) bytesRead += r.columns.find(c => c.column === KEY_COLUMN)?.byteSize ?? 0;
+                    console.log(`${blue(config.name)} loaded in ${red(formatTime(elapsed))} (${blue(formatNumber(reader.rowCount))} rows, ${bulkFiles.length} bulk + ${streamFiles.length} stream files, read ${blue(formatNumber(bytesRead))}B)`);
+                }
+
+                return new LoadedIndex<T>(
+                    config.name,
+                    config.storage,
+                    bulkFiles,
+                    streamFiles,
+                    reader,
+                    streamTimes,
+                    streamData.sizes,
+                    streamData.entries.length,
+                    streamData.totalBytes,
+                    config.subCaches,
+                );
+            } catch (e) {
+                if (e instanceof FilesChangedError) {
+                    attempt++;
+                    continue;
+                }
+                throw e;
+            }
+        }
+    }
+
+    // Adopts the previous index's loaded base values as STALE on this one, so a sync read mid-swap can
+    // serve the old value (patched with the current overlay) instead of flashing empty while the fresh
+    // load runs. Per-entry: dropped when the fresh value lands here.
+    inheritStaleFrom(prev: LoadedIndex<T>): void {
+        for (const [k, v] of prev.baseColumns) this.staleBaseColumns.set(k, v);
+        for (const [k, v] of prev.staleBaseColumns) if (!this.staleBaseColumns.has(k)) this.staleBaseColumns.set(k, v);
+        for (const [k, v] of prev.baseFields) this.staleBaseFields.set(k, v);
+        for (const [k, v] of prev.staleBaseFields) if (!this.staleBaseFields.has(k)) this.staleBaseFields.set(k, v);
+    }
+
+    async getColumn(column: string): Promise<{ key: string; value: unknown; time: number }[]> {
+        return this.reader.getColumn(column);
+    }
+
+    async getSingleField(key: string, column: string): Promise<{ value: unknown; time: number } | undefined> {
+        return this.reader.getSingleField(key, column);
+    }
+
+    ensureBaseColumn(column: string, onLoaded: () => void): void {
+        if (this.baseColumns.has(column) || this.baseColumnsLoading.has(column)) return;
+        this.baseColumnsLoading.add(column);
+        void (async () => {
+            try {
+                const value = await this.reader.getColumn(column);
+                this.baseColumns.set(column, value);
+                this.staleBaseColumns.delete(column);
+                this.baseColumnsLoading.delete(column);
+                onLoaded();
+            } catch (e) {
+                this.baseColumnsLoading.delete(column);
+                console.warn(`${this.name}.getColumnSync(${JSON.stringify(column)}) load failed, will retry: ${(e as Error).message}`);
+            }
+        })();
+    }
+
+    ensureBaseField(key: string, column: string, onLoaded: () => void): void {
+        const ck = nullJoin(column, key);
+        if (this.baseFields.has(ck) || this.baseFieldsLoading.has(ck)) return;
+        this.baseFieldsLoading.add(ck);
+        void (async () => {
+            try {
+                const value = await this.reader.getSingleField(key, column);
+                this.baseFields.set(ck, value);
+                this.staleBaseFields.delete(ck);
+                this.baseFieldsLoading.delete(ck);
+                onLoaded();
+            } catch (e) {
+                this.baseFieldsLoading.delete(ck);
+                console.warn(`${this.name}.getSingleFieldSync(${JSON.stringify(key)}, ${JSON.stringify(column)}) load failed, will retry: ${(e as Error).message}`);
+            }
+        })();
+    }
+
+    // Returns the loaded value (fresh if available, else last-known stale) for the column, or undefined
+    // if neither is present yet. The boolean indicates whether the returned value is fresh; callers may
+    // skip caching a stale value.
+    getBaseColumn(column: string): { entries: { key: string; value: unknown; time: number }[]; fresh: boolean } | undefined {
+        const fresh = this.baseColumns.get(column);
+        if (fresh) return { entries: fresh, fresh: true };
+        const stale = this.staleBaseColumns.get(column);
+        if (stale) return { entries: stale, fresh: false };
+        return undefined;
+    }
+
+    getBaseField(key: string, column: string): { value: { value: unknown; time: number } | undefined; fresh: boolean; loaded: boolean } {
+        const ck = nullJoin(column, key);
+        if (this.baseFields.has(ck)) return { value: this.baseFields.get(ck), fresh: true, loaded: true };
+        if (this.staleBaseFields.has(ck)) return { value: this.staleBaseFields.get(ck), fresh: false, loaded: true };
+        return { value: undefined, fresh: false, loaded: false };
+    }
+
+    isBaseColumnLoaded(column: string): boolean {
+        return this.baseColumns.has(column) || this.staleBaseColumns.has(column);
+    }
+
+    isBaseFieldLoaded(key: string, column: string): boolean {
+        const ck = nullJoin(column, key);
+        return this.baseFields.has(ck) || this.staleBaseFields.has(ck);
+    }
+
+    // Drops in-memory loaded values for this index, so a subsequent read reloads fresh from disk. Used
+    // by reloadFromDisk() — a hard reset, no stale fallback survives.
+    dropLoadedValues(): void {
+        this.baseColumns.clear();
+        this.baseColumnsLoading.clear();
+        this.baseFields.clear();
+        this.baseFieldsLoading.clear();
+        this.staleBaseColumns.clear();
+        this.staleBaseFields.clear();
+    }
+}
+
+export async function makeRawGetRange(storage: FileStorage, fileName: string): Promise<{ rawGetRange: GetRange; size: number }> {
+    const info = await storage.getInfo(fileName);
+    if (!info) throw new MissingFileError(`bulk file ${fileName} is missing`);
+    const rawGetRange: GetRange = async (start, end) => {
+        if (end <= start) return EMPTY_BUFFER;
+        const buf = await storage.getRange(fileName, { start, end });
+        if (!buf) throw new MissingFileError(`range [${start}, ${end}) of ${fileName} is missing`);
+        return buf;
+    };
+    return { rawGetRange, size: info.size };
+}
+
+export async function loadFileReader(name: string, storage: FileStorage, f: BulkFileInfo, cache: Map<string, BaseBulkDatabaseReader>): Promise<BaseBulkDatabaseReader> {
+    const cached = cache.get(f.fileName);
+    if (cached) return cached;
+    const raw = await makeRawGetRange(storage, f.fileName);
+    const fileId = nullJoin(name, f.fileName);
+    const opened = await blockCache.open(fileId, raw.size, raw.rawGetRange);
+    const reader = await loadBulkDatabase({ totalBytes: opened.uncompressedSize, getRange: opened.getRange });
+    cache.set(f.fileName, reader);
+    return reader;
+}
+
+export async function loadStreamEntries(
+    name: string,
+    storage: FileStorage,
+    streamFiles: StreamFileInfo[],
+    cache: Map<string, StreamReaderCacheEntry>,
+): Promise<{ entries: { time: number; fileName: string; entry: StreamEntry }[]; totalBytes: number; missing: boolean; sizes: Map<string, number> }> {
+    const sizes = new Map<string, number>();
+    if (!streamFiles.length) return { entries: [], totalBytes: 0, missing: false, sizes };
+    let missing = false;
+    const perFile = await Promise.all(streamFiles.map(async (f): Promise<{ fileName: string; size: number; entries: StreamEntry[] } | undefined> => {
+        try {
+            const info = await storage.getInfo(f.fileName);
+            if (!info) { missing = true; return undefined; }
+            const size = info.size;
+            sizes.set(f.fileName, size);
+            const cached = cache.get(f.fileName);
+            if (cached && cached.readSize === size) return { fileName: f.fileName, size, entries: cached.entries };
+            if (size === 0) { cache.set(f.fileName, { readSize: 0, parsedPos: 0, entries: [] }); return { fileName: f.fileName, size: 0, entries: [] }; }
+            // Append-only: a larger size means new bytes at the tail, parse only those.
+            if (cached && size > cached.readSize) {
+                const suffix = await storage.getRange(f.fileName, { start: cached.parsedPos, end: size });
+                if (!suffix) { missing = true; return undefined; }
+                const parsed = parseStream(suffix);
+                if (parsed.badBytes > 0) console.warn(`${name} stream file ${f.fileName} had ${parsed.badBytes} trailing bad/incomplete bytes (stopped reading there)`);
+                const entries = parsed.entries.length ? cached.entries.concat(parsed.entries) : cached.entries;
+                const parsedPos = cached.parsedPos + (suffix.length - parsed.badBytes);
+                cache.set(f.fileName, { readSize: size, parsedPos, entries });
+                return { fileName: f.fileName, size, entries };
+            }
+            const buffer = await storage.getRange(f.fileName, { start: 0, end: size });
+            if (!buffer) { missing = true; return undefined; }
+            const parsed = parseStream(buffer);
+            if (parsed.badBytes > 0) console.warn(`${name} stream file ${f.fileName} had ${parsed.badBytes} trailing bad/incomplete bytes (stopped reading there)`);
+            cache.set(f.fileName, { readSize: size, parsedPos: size - parsed.badBytes, entries: parsed.entries });
+            return { fileName: f.fileName, size, entries: parsed.entries };
+        } catch {
+            missing = true;
+            return undefined;
+        }
+    }));
+    const entries: { time: number; fileName: string; entry: StreamEntry }[] = [];
+    let totalBytes = 0;
+    for (const pf of perFile) {
+        if (!pf) continue;
+        totalBytes += pf.size;
+        for (const entry of pf.entries) entries.push({ time: entry.time, fileName: pf.fileName, entry });
+    }
+    return { entries, totalBytes, missing, sizes };
+}
+
+export function orderStreamEntries(entries: { time: number; fileName: string; entry: StreamEntry }[]): StreamEntry[] {
+    entries.sort((a, b) => {
+        if (a.time !== b.time) return a.time - b.time;
+        return a.fileName < b.fileName && -1 || a.fileName > b.fileName && 1 || 0;
+    });
+    return entries.map(e => e.entry);
+}
+
+function pruneSubCaches(bulkFiles: BulkFileInfo[], streamFiles: StreamFileInfo[], subCaches: SubReaderCaches) {
+    const liveBulk = new Set(bulkFiles.map(f => f.fileName));
+    const liveStream = new Set(streamFiles.map(f => f.fileName));
+    for (const n of subCaches.bulk.keys()) if (!liveBulk.has(n)) subCaches.bulk.delete(n);
+    for (const n of subCaches.stream.keys()) if (!liveStream.has(n)) subCaches.stream.delete(n);
+}
+
+function joinBulkDatabases(databases: BaseBulkDatabaseReader[]): ResolvedReader {
+    const deleteTime = new Map<string, number>();
+    for (const db of databases) {
+        if (!db.deleteTimes) continue;
+        for (const [key, t] of db.deleteTimes) deleteTime.set(key, Math.max(deleteTime.get(key) ?? -Infinity, t));
+    }
+    const keyTime = new Map<string, number>();
+    for (const db of databases) {
+        for (const [key, t] of db.keyTimes) keyTime.set(key, Math.max(keyTime.get(key) ?? -Infinity, t));
+    }
+    const delOf = (key: string) => deleteTime.get(key) ?? -Infinity;
+    const keys: string[] = [];
+    for (const [key, t] of keyTime) if (t > delOf(key)) keys.push(key);
+
+    const columns: { column: string; byteSize: number }[] = [];
+    const columnByName = new Map<string, { column: string; byteSize: number }>();
+    for (const db of databases) {
+        for (const col of db.columns) {
+            let existing = columnByName.get(col.column);
+            if (!existing) {
+                existing = { column: col.column, byteSize: 0 };
+                columnByName.set(col.column, existing);
+                columns.push(existing);
+            }
+            existing.byteSize += col.byteSize;
+        }
+    }
+
+    return {
+        totalBytes: databases.reduce((acc, db) => acc + db.totalBytes, 0),
+        rowCount: keys.length,
+        keys,
+        rawKeyCount: databases.reduce((acc, db) => acc + db.keyTimes.size + (db.deleteTimes?.size ?? 0), 0),
+        readerCount: databases.length,
+        columns,
+        keyTimes: keyTime,
+        deleteTimes: deleteTime,
+        async getColumn(column) {
+            const perReader = await Promise.all(databases.map(async db => {
+                if (!db.columns.some(c => c.column === column)) return undefined;
+                const entries = await db.getColumn(column);
+                return new Map(entries.map(e => [e.key, { value: e.value, time: e.time }]));
+            }));
+            return keys.map(key => {
+                let bestTime = -Infinity;
+                let bestVal: unknown;
+                let found = false;
+                for (const m of perReader) {
+                    const cell = m && m.get(key);
+                    if (!cell || cell.value === ABSENT) continue;
+                    if (cell.time > bestTime) { bestTime = cell.time; bestVal = cell.value; found = true; }
+                }
+                const live = found && bestTime > delOf(key);
+                return { key, value: live ? bestVal : undefined, time: live ? bestTime : (keyTime.get(key) ?? 0) };
+            });
+        },
+        async getSingleField(key, column) {
+            const kt = keyTime.get(key);
+            if (kt === undefined || kt <= delOf(key)) return undefined;
+            let bestTime = -Infinity;
+            let bestVal: unknown;
+            let found = false;
+            for (const db of databases) {
+                if (!db.columns.some(c => c.column === column)) continue;
+                const r = await db.getSingleField(key, column);
+                if (r === ABSENT) continue;
+                if (r.time > bestTime) { bestTime = r.time; bestVal = r.value; found = true; }
+            }
+            const live = found && bestTime > delOf(key);
+            return { value: live ? bestVal : undefined, time: live ? bestTime : kt };
+        },
+    };
+}

package/storage/BulkDatabase2/WriteOverlay.d.ts

@@ -0,0 +1,30 @@
+export declare const DELETED: unique symbol;
+export type OverlayEntry = {
+    time: number;
+    value: Record<string, unknown> | typeof DELETED;
+};
+export declare class WriteOverlay {
+    private entries;
+    get size(): number;
+    get(key: string): OverlayEntry | undefined;
+    has(key: string): boolean;
+    keys(): IterableIterator<string>;
+    [Symbol.iterator](): IterableIterator<[string, OverlayEntry]>;
+    writeRow(key: string, row: Record<string, unknown>, time: number, wasLive: boolean): {
+        invalidatedColumns: Iterable<string> | "all";
+    };
+    deleteKey(key: string, time: number, wasLive: boolean): {
+        invalidatedColumns: Iterable<string> | "all";
+    };
+    clear(): void;
+    sweepCovered(authority: (key: string) => number): void;
+    patchColumn(base: {
+        key: string;
+        value: unknown;
+        time: number;
+    }[], column: string): {
+        key: string;
+        value: unknown;
+        time: number;
+    }[];
+}

package/storage/BulkDatabase2/WriteOverlay.ts

@@ -0,0 +1,52 @@
+export const DELETED = Symbol("deleted");
+
+export type OverlayEntry = { time: number; value: Record<string, unknown> | typeof DELETED };
+
+// In-memory pending mutations layered on top of a LoadedIndex. Each entry carries the write's unique
+// timestamp so cross-tab writes can be ordered + the swap can drop entries the new index already
+// covers. Methods that mutate report which columns invalidate ("all" if the key's liveness flipped),
+// so the host can drop the right caches without scanning every column.
+export class WriteOverlay {
+    private entries = new Map<string, OverlayEntry>();
+
+    get size(): number { return this.entries.size; }
+    get(key: string): OverlayEntry | undefined { return this.entries.get(key); }
+    has(key: string): boolean { return this.entries.has(key); }
+    keys(): IterableIterator<string> { return this.entries.keys(); }
+    [Symbol.iterator]() { return this.entries[Symbol.iterator](); }
+
+    writeRow(key: string, row: Record<string, unknown>, time: number, wasLive: boolean): { invalidatedColumns: Iterable<string> | "all" } {
+        const existing = this.entries.get(key);
+        const value = existing && existing.value !== DELETED ? { ...existing.value, ...row } : { ...row };
+        this.entries.set(key, { time, value });
+        return { invalidatedColumns: wasLive ? Object.keys(row) : "all" };
+    }
+
+    deleteKey(key: string, time: number, wasLive: boolean): { invalidatedColumns: Iterable<string> | "all" } {
+        this.entries.set(key, { time, value: DELETED });
+        return { invalidatedColumns: wasLive ? "all" : [] };
+    }
+
+    clear(): void { this.entries.clear(); }
+
+    // Drop entries the new index already reflects (its keyTime or deleteTime ≥ the entry's time). The
+    // remaining entries are writes the index doesn't yet see — they have to keep serving reads.
+    sweepCovered(authority: (key: string) => number): void {
+        for (const [key, entry] of this.entries) {
+            if (entry.time <= authority(key)) this.entries.delete(key);
+        }
+    }
+
+    // Apply this overlay onto the base column entries. A column-unset overlay entry leaves the base
+    // value+time in place (a partial write only overrides the columns it set).
+    patchColumn(base: { key: string; value: unknown; time: number }[], column: string): { key: string; value: unknown; time: number }[] {
+        if (this.entries.size === 0) return base;
+        const map = new Map(base.map(e => [e.key, { value: e.value, time: e.time }]));
+        for (const [key, entry] of this.entries) {
+            if (entry.value === DELETED) { map.delete(key); continue; }
+            if (column in entry.value) map.set(key, { value: entry.value[column], time: entry.time });
+            else if (!map.has(key)) map.set(key, { value: undefined, time: entry.time });
+        }
+        return [...map].map(([key, v]) => ({ key, value: v.value, time: v.time }));
+    }
+}

package/storage/BulkDatabase2/blockCache.d.ts

@@ -6,6 +6,7 @@ export declare class BlockCache {
     private blocks;
     private indexes;
     clear(): void;
+    evict(fileId: string): void;
     private touch;
     private readIndex;
     open(fileId: string, fileSize: number, rawGetRange: GetRange): Promise<{

package/storage/BulkDatabase2/blockCache.ts

@@ -68,6 +68,12 @@ export class BlockCache {
         this.indexes.clear();
     }
 
+    public evict(fileId: string) {
+        this.indexes.delete(fileId);
+        const prefix = fileId + ":";
+        for (const key of this.blocks.keys()) if (key.startsWith(prefix)) this.blocks.delete(key);
+    }
+
     private touch(key: string, value: Promise<Buffer>) {
         this.blocks.delete(key);
         this.blocks.set(key, value);

package/storage/BulkDatabase2/streamLog.ts

@@ -1,5 +1,5 @@
 import cborx from "cbor-x";
-import { ABSENT, BaseBulkDatabaseReader } from "./BulkDatabaseFormat";
+import { ABSENT, BaseBulkDatabaseReader, ColumnIndex, EMPTY_BUFFER, encodeValue, RawCell, TYPE_ABSENT_TAG } from "./BulkDatabaseFormat";
 
 // Tier-0 streaming format: an append log of whole-row writes and deletes (row-format, not columnar),
 // so small mutations are a single cheap append instead of rewriting a columnar file. Each block is:
@@ -104,6 +104,7 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
         if (t < minTime) minTime = t;
         if (t > maxTime) maxTime = t;
     }
+    const keyIndexMap = buildKeyIndex(keys);
     let reader: BaseBulkDatabaseReader = {
         totalBytes,
         rowCount: keys.length,
@@ -126,6 +127,82 @@ export function streamReaderFromEntries(entries: StreamEntry[], totalBytes: numb
             if (!row || !(column in row)) return ABSENT;
             return { value: row[column], time: times.get(key) || 0 };
         },
+        // Stream values are decoded objects (CBOR), so unlike a bulk file we have to encode them to raw
+        // cells here. Stream data is small (tier-0, size-capped), so this is cheap. A column the row never
+        // set is omitted (ABSENT → fall through); a stored undefined is kept (a real clear).
+        async getRawColumn(column) {
+            let map = new Map<string, RawCell>();
+            for (let key of keys) {
+                let row = byKey.get(key);
+                if (!row || !(column in row)) continue;
+                map.set(key, encodeValue(row[column]));
+            }
+            return map;
+        },
+        // Per-column index in the same shape a bulk file exposes, materialized in memory once (stream data
+        // is tier-0, size-capped, so the encode is cheap). Lets the planned merge treat stream + bulk
+        // sources uniformly: it loads each (source, column) index, uses offsets/types to plan the output's
+        // byte layout, and reads contiguous runs from this index's in-memory data buffer during execute.
+        async getColumnIndex(column) {
+            return getStreamColumnIndex(keys, byKey, column);
+        },
+        rowOfKey(key) {
+            return keyIndexMap.get(key);
+        },
     };
     return { reader, times };
 }
+
+// keys[] is the column-side row order for this stream reader; rowOfKey is used by the planner to look up
+// a winning cell's source row without going through a column read. Stays consistent with getColumnIndex,
+// which builds its offsets/types over the same `keys` array in the same order.
+function buildKeyIndex(keys: string[]): Map<string, number> {
+    const m = new Map<string, number>();
+    for (let i = 0; i < keys.length; i++) m.set(keys[i], i);
+    return m;
+}
+
+// Build a stream-side ColumnIndex (offsets + types + a contiguous-row byte slicer) over the in-memory
+// merged rows. Encodes each row's value for this column once into a single backing buffer; readValueBytes
+// then slices it. Cached per (reader, column) so a merge planning + execution pass doesn't re-encode.
+function getStreamColumnIndex(keys: string[], byKey: Map<string, Record<string, unknown>>, column: string): ColumnIndex {
+    let cached = columnIndexCache.get(byKey)?.get(column);
+    if (cached) return cached;
+    const offsets = new Uint32Array(keys.length + 1);
+    const types = new Uint8Array(keys.length);
+    const parts: Buffer[] = [];
+    let pos = 0;
+    for (let i = 0; i < keys.length; i++) {
+        const row = byKey.get(keys[i]);
+        offsets[i] = pos;
+        if (!row || !(column in row)) {
+            types[i] = TYPE_ABSENT_TAG;
+            continue;
+        }
+        const { type, bytes } = encodeValue(row[column]);
+        types[i] = type;
+        parts.push(bytes);
+        pos += bytes.length;
+    }
+    offsets[keys.length] = pos;
+    const data = parts.length ? Buffer.concat(parts) : EMPTY_BUFFER;
+    const idx: ColumnIndex = {
+        offsets,
+        types,
+        async readValueBytes(startRow, endRow) {
+            if (endRow <= startRow) return EMPTY_BUFFER;
+            const start = offsets[startRow];
+            const end = offsets[endRow];
+            if (end <= start) return EMPTY_BUFFER;
+            return data.subarray(start, end);
+        },
+    };
+    let perReader = columnIndexCache.get(byKey);
+    if (!perReader) { perReader = new Map(); columnIndexCache.set(byKey, perReader); }
+    perReader.set(column, idx);
+    return idx;
+}
+
+// Per stream-reader cache of its column indexes. Keyed by the reader's byKey map (a fresh
+// streamReaderFromEntries call produces a fresh map, so this is effectively per-reader).
+const columnIndexCache = new WeakMap<Map<string, Record<string, unknown>>, Map<string, ColumnIndex>>();