vastgrid-js 0.0.0-stage → 0.2.0

package/dist/engine.js

@@ -0,0 +1,1022 @@
+/**
+ * An engine over tables in memory (docs/spec.md §8): it answers the requests of §8.2 about tables that
+ * its host gives it as arrays of typed JSON values (§2.2), as Arrow, as text files or as Parquet files, with views in
+ * the order of §3.3.
+ */
+import { answerId, checkPlan, checkRequest, encodeBlock, errorResponse, FUNCTIONS, keyText, MEASURES, MESSAGE_LIMITS, PlanError, ProtocolError, planHash, planOptions, SpecError, sameCellValue, textOf, } from "vastgrid-spec";
+import { rankColumn, rankColumnPausing, sortRows } from "./order.js";
+import { Summary } from "./summary.js";
+import { checkTable, isArrowTable, isFileTable, isObject, isParquetTable, isWorkbook, shown, TableProblem, valueChecker, vectorOf, vectorValue, } from "./table.js";
+/** The formats that `export` writes (§8.2), answered as the file's bytes. */
+const EXPORT_FORMATS = Object.freeze(["csv", "tsv", "json", "xlsx"]);
+/**
+ * What the engine evaluates (§4.10): every plan of §4.1, with every function of §4.4 and every measure of §4.5;
+ * `summarize`, `find`, `locate` and `export` of CSV, TSV, JSON and xlsx as data.
+ */
+export const MEMORY_CAPABILITIES = Object.freeze({
+    plan: Object.freeze({
+        derive: true,
+        where: true,
+        aggregate: true,
+        pivot: true,
+        having: true,
+        sort: true,
+        search: true,
+    }),
+    functions: Object.freeze([...FUNCTIONS]),
+    measures: Object.freeze([...MEASURES]),
+    summarize: true,
+    find: true,
+    locate: true,
+    write: false,
+    save: false,
+    export: Object.freeze({ formats: EXPORT_FORMATS, delivery: "data" }),
+    preview: false,
+    partialViews: false,
+    changes: "poll",
+});
+/** A copy of the capabilities, for a message; tables that the host makes editable are written (§7). */
+function capabilities(write = false) {
+    return {
+        ...MEMORY_CAPABILITIES,
+        plan: { ...MEMORY_CAPABILITIES.plan },
+        export: { formats: [...EXPORT_FORMATS], delivery: "data" },
+        functions: [...FUNCTIONS],
+        measures: [...MEASURES],
+        write,
+    };
+}
+const PLAN_OPTIONS = planOptions(capabilities());
+/** Views that a table keeps, the least recently used forgotten first (§8.3). */
+const KEPT_VIEWS = 8;
+/** An error of §8.7 that a request is answered with. */
+class Failure extends Error {
+    code;
+    extra;
+    constructor(code, message, extra = {}) {
+        super(message);
+        this.code = code;
+        this.extra = extra;
+        this.name = "Failure";
+    }
+}
+/** A typed column with the values written to it (§7.2) over it; it reads as JSON, as the host's rows do. */
+function writtenVector(base, written) {
+    return {
+        length: base.length,
+        at: (row) => {
+            const value = written.get(row);
+            return value === undefined ? base.at(row) : value;
+        },
+    };
+}
+/**
+ * Adds the names of columns that a plan names (§4.1) to `names`: in expressions, keys, measures, pivots and sorts.
+ * True when it searches (§3.5), which reads every column.
+ */
+function planNames(value, names) {
+    if (Array.isArray(value))
+        return value.reduce((search, item) => planNames(item, names) || search, false);
+    if (!isObject(value))
+        return false;
+    let search = typeof value.search === "string";
+    for (const [key, item] of Object.entries(value)) {
+        if ((key === "col" || key === "column") && typeof item === "string")
+            names.add(item);
+        else if ((key === "keys" || key === "columns") && Array.isArray(item)) {
+            for (const name of item)
+                if (typeof name === "string")
+                    names.add(name);
+        }
+        else if (planNames(item, names))
+            search = true;
+    }
+    return search;
+}
+/** The position of a row that a view leaves out, in `View.positions`. */
+const NO_POSITION = 0xffff_ffff;
+/** The work that a request does before it lets other messages in, such as a `cancel` of it (§8.6), in ms. */
+const SLICE = 40;
+/**
+ * Lets other messages in when the request has worked for a slice since it last did, and stops it when it was
+ * cancelled (§8.6): a cancel that came in while the request waited for a file is seen at once. Requests that go
+ * through many rows call it between batches of them.
+ */
+async function pause(running) {
+    if (running.cancelled)
+        throw new Failure("cancelled", "The request was cancelled (§8.6).");
+    if (performance.now() - running.yielded < SLICE)
+        return;
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    running.yielded = performance.now();
+    if (running.cancelled)
+        throw new Failure("cancelled", "The request was cancelled (§8.6).");
+}
+/** The names that a plan sorts or groups by, or counts the distinct values of; derived columns are left out. */
+function rankedNames(plan) {
+    const { aggregate } = plan;
+    const names = aggregate
+        ? [
+            ...aggregate.keys,
+            ...(aggregate.pivot?.columns ?? []),
+            ...aggregate.measures.flatMap((measure) => measure.fn === "count_distinct" && measure.column ? [measure.column] : []),
+        ]
+        : (plan.sort ?? []).map((key) => key.column);
+    const derived = new Set((plan.derive ?? []).map((item) => item.name));
+    return names.filter((name) => !derived.has(name));
+}
+// The revision of each table that hosts give: its data never changes, so neither does its revision.
+const revisions = new WeakMap();
+let revisionCount = 0;
+function revisionOf(table) {
+    let revision = revisions.get(table);
+    if (revision === undefined) {
+        revision = `memory-${++revisionCount}`;
+        revisions.set(table, revision);
+    }
+    return revision;
+}
+/**
+ * The string cut to `limit` code points, or null when it has no more than that (§8.4). A string of no more
+ * UTF-16 units than the limit has no more code points either.
+ */
+function cut(text, limit) {
+    if (text.length <= limit)
+        return null;
+    let units = 0;
+    for (let points = 0; points < limit && units < text.length; points++) {
+        units += text.codePointAt(units) > 0xffff ? 2 : 1;
+    }
+    return units < text.length ? text.slice(0, units) : null;
+}
+/** `[start, end)` spans within `[0, count)`, sorted, with those that overlap or touch merged. */
+function mergeRanges(ranges, count) {
+    const spans = ranges
+        .map(([start, end]) => [Math.min(start, count), Math.min(end, count)])
+        .filter(([start, end]) => start < end)
+        .sort((a, b) => a[0] - b[0]);
+    const merged = [];
+    for (const span of spans) {
+        const last = merged[merged.length - 1];
+        if (last && span[0] <= last[1])
+            last[1] = Math.max(last[1], span[1]);
+        else
+            merged.push([span[0], span[1]]);
+    }
+    return merged;
+}
+class Engine {
+    sources;
+    /** Tables by ref, while a table ID is open on them. */
+    prepared = new Map();
+    /** Files being read by ref, which every open of the file waits for. */
+    reading = new Map();
+    /** Tables by table ID. */
+    tables = new Map();
+    running = new Map();
+    listeners = new Set();
+    nextTable = 1;
+    constructor(options) {
+        if (!isObject(options) || !isObject(options.tables)) {
+            throw new TypeError("memoryEngine takes { tables }, the tables that open opens by ref.");
+        }
+        this.sources = new Map(Object.entries(options.tables));
+    }
+    subscribe(listener) {
+        this.listeners.add(listener);
+        return () => {
+            this.listeners.delete(listener);
+        };
+    }
+    async handle(message) {
+        let request;
+        try {
+            request = checkRequest(message);
+        }
+        catch (error) {
+            if (error instanceof ProtocolError) {
+                return errorResponse(answerId(message), "invalid_request", error.message, {
+                    path: error.path,
+                    reason: error.reason,
+                });
+            }
+            return errorResponse(answerId(message), "internal", String(error));
+        }
+        if (request.op === "cancel") {
+            const running = this.running.get(request.request);
+            if (running)
+                running.cancelled = true;
+            return undefined;
+        }
+        const running = { cancelled: false, yielded: performance.now(), releases: [] };
+        this.running.set(request.id, running);
+        try {
+            return { v: 2, id: request.id, result: await this.answer(request, running) };
+        }
+        catch (error) {
+            if (error instanceof Failure)
+                return errorResponse(request.id, error.code, error.message, error.extra);
+            if (error instanceof PlanError) {
+                return errorResponse(request.id, "invalid_plan", error.message, { path: error.path, reason: error.reason });
+            }
+            if (error instanceof ProtocolError) {
+                return errorResponse(request.id, "invalid_request", error.message, { path: error.path, reason: error.reason });
+            }
+            if (error instanceof SpecError)
+                return errorResponse(request.id, error.code, error.message);
+            return errorResponse(request.id, "internal", error instanceof Error ? error.message : String(error));
+        }
+        finally {
+            if (this.running.get(request.id) === running)
+                this.running.delete(request.id);
+            for (const release of running.releases)
+                release();
+        }
+    }
+    async answer(request, running) {
+        switch (request.op) {
+            case "open":
+                return this.open(request.ref, request.read);
+            case "info": {
+                const id = request.table;
+                return this.info(id, this.table(request));
+            }
+            case "changes":
+                return { revision: this.table(request).revision, edits: { ...this.table(request).edits } };
+            case "reload": {
+                // Tables in memory have no file that changes (§7.6): the table is as it is.
+                const id = request.table;
+                return this.info(id, this.table(request));
+            }
+            case "close": {
+                const table = this.table(request);
+                this.tables.delete(request.table);
+                // A table that no ID names any more forgets its ranks and views, unless it holds written values, which
+                // every later request sees (§7.2); opening it again checks it again.
+                if (--table.open === 0 && table.written.size === 0) {
+                    this.prepared.delete(table.key);
+                    table.reader?.close();
+                }
+                return null;
+            }
+            case "rows":
+                return this.rows(request, running);
+            case "summarize":
+                return this.summarize(request, running);
+            case "find":
+                return this.find(request, running);
+            case "locate":
+                return this.locate(request, running);
+            case "export":
+                return this.export(request, running);
+            case "write":
+                return this.write(request, running);
+            case "save":
+                // A table in memory has no file to write back (§7.5).
+                this.table(request);
+                throw new Failure("save_unsupported", "This engine keeps tables in memory and does not save them (capabilities.save is false).");
+            default:
+                throw new Failure("invalid_request", `This engine does not offer ${request.op} (§4.10).`, {
+                    path: "/op",
+                    reason: "unknown_op",
+                });
+        }
+    }
+    async open(ref, read) {
+        if (!this.sources.has(ref))
+            throw new Failure("not_allowed", `The host does not open ${shown(ref)}.`);
+        const source = this.sources.get(ref);
+        // Arrow tables and files are read by modules of their own, loaded before the table is looked up so that two
+        // opens of one table do not both make it.
+        const arrow = isArrowTable(source) ? await import("./arrow.js") : null;
+        const file = isFileTable(source);
+        const parquet = isParquetTable(source);
+        const typed = arrow || file || parquet ? await import("./numbers.js") : null;
+        // A workbook is read once for each set of options of `read`; other tables ignore them.
+        const workbook = file && isWorkbook(source) ? await import("./workbook.js") : null;
+        const options = workbook ? workbook.readOptionsOf(read) : null;
+        const key = options ? `${ref}\u0000${JSON.stringify(options)}` : ref;
+        let table = this.prepared.get(key);
+        if (!table) {
+            let checked;
+            try {
+                checked = arrow
+                    ? arrow.checkArrow(source.arrow)
+                    : workbook && options
+                        ? await this.reads(key, () => workbook.readSheet(source, options))
+                        : file
+                            ? await this.reads(key, async () => (await import("./text.js")).readFile(source))
+                            : parquet
+                                ? await this.reads(key, async () => (await import("./parquet.js")).ParquetFile.open(source).then((file) => file.checked()))
+                                : checkTable(source);
+            }
+            catch (error) {
+                if (!(error instanceof TableProblem))
+                    throw error;
+                const at = error.path === "" ? "" : `at ${error.path}, `;
+                throw new Failure(error.code, `The table ${shown(ref)} cannot be read: ${at}${error.message}`);
+            }
+            // Another open of the file may have made the table while the file was read.
+            table = this.prepared.get(key) ?? this.prepare(ref, key, source, checked, typed);
+        }
+        const id = `t${this.nextTable++}`;
+        this.tables.set(id, table);
+        table.open++;
+        return this.info(id, table);
+    }
+    /**
+     * Reads a table given as a file, once for every open that waits for it: a text file (text.ts), a sheet of a
+     * workbook (workbook.ts), or the footer of a Parquet file (parquet.ts).
+     */
+    reads(key, read) {
+        let reading = this.reading.get(key);
+        if (!reading) {
+            reading = read().finally(() => this.reading.delete(key));
+            this.reading.set(key, reading);
+        }
+        return reading;
+    }
+    /** A table that is opened from its host's table, checked or read. */
+    prepare(ref, key, source, checked, typed) {
+        const { columns } = checked;
+        const table = {
+            ref,
+            key,
+            columns,
+            index: new Map(columns.map((column, index) => [column.name, index])),
+            rows: checked.rows,
+            vectors: checked.vectors,
+            reader: checked.reader ?? null,
+            typed,
+            rowCount: checked.rowCount,
+            revision: revisionOf(source),
+            editable: source.editable === true,
+            edits: { version: 0, cells: 0 },
+            written: new Set(),
+            keys: new Map(),
+            ranks: columns.map(() => undefined),
+            checkers: columns.map((column, index) => valueChecker(column, index)),
+            values: columns.map(() => undefined),
+            overlays: columns.map(() => undefined),
+            views: new Map(),
+            open: 0,
+            workbook: checked.workbook ?? null,
+        };
+        // What the engine made of a column of a file that is forgotten, it makes again when a request reads it whole.
+        if (table.reader) {
+            table.reader.forgotten = (index) => {
+                table.values[index] = undefined;
+            };
+        }
+        this.prepared.set(key, table);
+        return table;
+    }
+    /** TableInfo (§8.2). */
+    info(id, table) {
+        const info = {
+            table: id,
+            revision: table.revision,
+            edits: { ...table.edits },
+            status: "ready",
+            progress: 1,
+            rowCount: table.rowCount,
+            readableRows: table.rowCount,
+            columns: table.columns.map((column) => ({ ...column })),
+            capabilities: capabilities(table.editable),
+        };
+        const { workbook } = table;
+        if (workbook) {
+            if (workbook.notes.length > 0)
+                info.notes = [...workbook.notes];
+            info.sheets = workbook.sheets.map((sheet) => ({ ...sheet }));
+            info.sheet = workbook.sheet;
+        }
+        return info;
+    }
+    /** The table a request names, at the revision it names when it names one (§8.3). */
+    table(request) {
+        const table = this.tables.get(request.table);
+        if (!table)
+            throw new Failure("table_not_found", `There is no open table ${shown(request.table)}.`);
+        if ("revision" in request && request.revision !== table.revision) {
+            throw new Failure("revision_changed", `The table's revision is ${table.revision}; read info again.`);
+        }
+        return table;
+    }
+    /** Answers `load_failed` for a value that does not follow §2.2, at its JSON Pointer in the table. */
+    loadFailed(table, error) {
+        throw new Failure(error.code, `The table ${shown(table.ref)} cannot be read: at ${error.path}, ${error.message}`);
+    }
+    /**
+     * Checks every value of a column, once, before something reads all of it, and gives its values by row ID: of a
+     * table given as rows, an array of the engine's own; of one given as Arrow or as a file, its column.
+     */
+    valuesOf(table, index) {
+        const made = table.values[index];
+        if (made)
+            return made;
+        const check = table.checkers[index];
+        let values;
+        try {
+            if (table.vectors) {
+                values = table.vectors[index];
+                const column = table.columns[index];
+                // Numbers are checked as numbers; a value found wrong is then read, which fails as its JSON would.
+                if (values.numbers && table.typed) {
+                    const row = table.typed.firstInvalid(column, values.numbers, values.validity ?? null);
+                    if (row >= 0)
+                        check(vectorValue(values, column, index, row), row);
+                }
+                else {
+                    for (let row = 0; row < table.rowCount; row++)
+                        check(vectorValue(values, column, index, row), row);
+                }
+            }
+            else {
+                const rows = table.rows;
+                const array = new Array(rows.length);
+                for (let row = 0; row < rows.length; row++) {
+                    const value = rows[row][index];
+                    check(value, row);
+                    array[row] = value;
+                }
+                values = vectorOf(array);
+            }
+        }
+        catch (error) {
+            if (error instanceof TableProblem)
+                this.loadFailed(table, error);
+            throw error;
+        }
+        const written = table.overlays[index];
+        if (written)
+            values = writtenVector(values, written);
+        table.values[index] = values;
+        return values;
+    }
+    /**
+     * Reads whole the columns that a request reads whole, before it reads them, of a table whose file is read as
+     * requests need it (lazy.ts); the other tables hold all their columns.
+     */
+    async readColumns(table, indexes, running) {
+        if (!table.reader)
+            return;
+        try {
+            running.releases.push(await table.reader.read(indexes, () => pause(running)));
+        }
+        catch (error) {
+            if (error instanceof TableProblem)
+                this.loadFailed(table, error);
+            throw error;
+        }
+    }
+    /**
+     * The values of a table's column at rows, in their order, checked when its column is not (§2.2): of a table whose
+     * file is read as requests need it, read by the parts of the file that hold them unless the column was read whole.
+     */
+    async cellsOf(table, index, rows) {
+        if (table.values[index] || !table.reader)
+            return Array.from(rows, (row) => this.cell(table, row, index));
+        let read;
+        try {
+            read = await table.reader.cells(index, rows);
+            const written = table.overlays[index];
+            const check = table.checkers[index];
+            return Array.from(rows, (row, at) => {
+                const value = written?.get(row);
+                if (value !== undefined)
+                    return value;
+                check(read[at], row);
+                return read[at];
+            });
+        }
+        catch (error) {
+            if (error instanceof TableProblem)
+                this.loadFailed(table, error);
+            throw error;
+        }
+    }
+    /** The value of a table's column at a row as the table holds it, checked when its column is not (§2.2). */
+    cell(table, row, index) {
+        const values = table.values[index];
+        if (values)
+            return values.at(row);
+        const written = table.overlays[index]?.get(row);
+        if (written !== undefined)
+            return written;
+        try {
+            const value = table.vectors
+                ? vectorValue(table.vectors[index], table.columns[index], index, row)
+                : table.rows[row][index];
+            table.checkers[index](value, row);
+            return value;
+        }
+        catch (error) {
+            if (error instanceof TableProblem)
+                this.loadFailed(table, error);
+            throw error;
+        }
+    }
+    /**
+     * Writes a cell: into the engine's own array of a column of a table given as rows, or over a typed column, which
+     * stays as it was and is not copied.
+     */
+    setCell(table, row, index, value) {
+        if (!table.vectors) {
+            this.valuesOf(table, index)[row] = value;
+            return;
+        }
+        let written = table.overlays[index];
+        if (!written) {
+            written = new Map();
+            table.overlays[index] = written;
+            const made = table.values[index];
+            if (made)
+                table.values[index] = writtenVector(made, written);
+        }
+        written.set(row, value);
+    }
+    ranksOf(table, index) {
+        let ranks = table.ranks[index];
+        if (!ranks) {
+            const column = table.columns[index];
+            const values = this.valuesOf(table, index);
+            // A typed column is ranked from its numbers, until a write makes it an array.
+            const write = values.numbers && table.typed ? table.typed.numberWriter(column, values.numbers) : null;
+            ranks = rankColumn(column, values, write);
+            table.ranks[index] = ranks;
+        }
+        return ranks;
+    }
+    /**
+     * Ranks a column as `ranksOf` does, with pauses while text is ranked: the ranks of a write's column made while the
+     * write came are not kept.
+     */
+    async rankPausing(table, index, running) {
+        if (table.ranks[index])
+            return;
+        const { version } = table.edits;
+        const column = table.columns[index];
+        const values = this.valuesOf(table, index);
+        const write = values.numbers && table.typed ? table.typed.numberWriter(column, values.numbers) : null;
+        const ranks = await rankColumnPausing(column, values, write, () => pause(running));
+        if (table.edits.version === version)
+            table.ranks[index] ??= ranks;
+    }
+    /** The view of a request's plan (§4.7, §8.3): checked against the table, kept by its hash. */
+    async view(table, plan, running) {
+        const checked = checkPlan(plan, table.columns, PLAN_OPTIONS);
+        const hash = await planHash(checked.plan);
+        if (running.cancelled)
+            throw new Failure("cancelled", "The request was cancelled (§8.6).");
+        const kept = table.views.get(hash);
+        if (kept) {
+            table.views.delete(hash);
+            table.views.set(hash, kept);
+            return kept;
+        }
+        if (table.reader) {
+            const names = new Set();
+            const every = planNames(checked.plan, names);
+            const named = [...names].filter((name) => table.index.has(name));
+            await this.readColumns(table, every ? table.columns.keys() : named.map((name) => table.index.get(name)), running);
+        }
+        // The columns that the view sorts or groups by are ranked first, which a cancel stops.
+        for (const name of rankedNames(checked.plan)) {
+            const index = table.index.get(name);
+            if (index !== undefined)
+                await this.rankPausing(table, index, running);
+        }
+        const count = table.rowCount;
+        const ranks = (name) => this.ranksOf(table, table.index.get(name));
+        let shape;
+        if (checked.derive.length > 0 || checked.where || checked.aggregate) {
+            const { planView } = await import("./plan.js");
+            shape = planView(checked, {
+                count,
+                values: (name) => this.valuesOf(table, table.index.get(name)),
+                ranks,
+                rankColumn,
+                sortRows,
+                sortKeys: (ranksOf) => this.sortKeys(checked, ranksOf),
+            });
+        }
+        else {
+            shape = {
+                columns: checked.columns,
+                derived: new Map(),
+                grouped: false,
+                tableRows: count,
+                rowCount: count,
+                order: sortRows(count, this.sortKeys(checked, ranks), null),
+            };
+        }
+        const view = { hash, ...shape, firsts: new Map() };
+        table.views.set(hash, view);
+        if (table.views.size > KEPT_VIEWS)
+            table.views.delete(table.views.keys().next().value);
+        // A view is made to its end, and kept; a cancel that came while it was made stops the request after it.
+        await pause(running);
+        return view;
+    }
+    /** The sort keys of a plan (§3.3); a key on a column that an earlier key sorted by cannot break a tie. */
+    sortKeys(checked, ranksOf) {
+        const used = new Set();
+        const keys = [];
+        for (const key of checked.sort) {
+            const name = key.column.name;
+            if (used.has(name))
+                continue;
+            used.add(name);
+            keys.push({ ranks: ranksOf(name), descending: key.direction === "desc" });
+        }
+        return keys;
+    }
+    /**
+     * The columns that a request names, which must be columns of the view: the index of a table's column, or the
+     * values of a derived one or of a column of groups.
+     */
+    columnsOf(table, view, names) {
+        return names.map((name, at) => {
+            const derived = view.derived.get(name);
+            if (derived)
+                return { name, column: derived.column, index: -1, values: vectorOf(derived.values) };
+            const index = table.index.get(name);
+            if (index === undefined || !view.columns.some((column) => column.name === name)) {
+                throw new Failure("invalid_request", `The view has no column ${shown(name)}.`, {
+                    path: `/columns/${at}`,
+                    reason: "syntax",
+                });
+            }
+            return { name, column: table.columns[index], index, values: null };
+        });
+    }
+    async rows(request, running) {
+        const table = this.table(request);
+        const view = await this.view(table, request.plan, running);
+        const names = request.columns;
+        const sources = this.columnsOf(table, view, names);
+        const start = request.start;
+        const end = Math.max(start, Math.min(request.end, view.rowCount));
+        const limit = request.textLimit ?? MESSAGE_LIMITS.textLimitDefault;
+        const columns = sources.map((source) => source.column);
+        const rowIds = [];
+        for (let position = start; position < end; position++)
+            rowIds.push(view.order ? view.order[position] : position);
+        if (table.reader) {
+            try {
+                await table.reader.prepare(sources.flatMap((source) => (source.values ? [] : [source.index])), rowIds, () => pause(running));
+            }
+            catch (error) {
+                if (error instanceof TableProblem)
+                    this.loadFailed(table, error);
+                throw error;
+            }
+        }
+        // The values read are checked, of the table's columns not checked whole (§2.2).
+        const cells = await Promise.all(sources.map((source) => (source.values ? null : this.cellsOf(table, source.index, rowIds))));
+        const values = [];
+        // By name in a Map: a column may be named "__proto__".
+        const truncated = new Map();
+        for (let offset = 0; offset < rowIds.length; offset++) {
+            const row = rowIds[offset];
+            values.push(sources.map(({ values }, at) => {
+                const value = values ? values.at(row) : cells[at][offset];
+                if (typeof value !== "string" || columns[at].type !== "string")
+                    return value;
+                const shorter = cut(value, limit);
+                if (shorter === null)
+                    return value;
+                const offsets = truncated.get(names[at]);
+                if (!offsets)
+                    truncated.set(names[at], [offset]);
+                else if (offsets[offsets.length - 1] !== offset)
+                    offsets.push(offset);
+                return shorter;
+            }));
+        }
+        return {
+            view: {
+                hash: view.hash,
+                status: "ready",
+                rowCount: view.rowCount,
+                rowCountExact: true,
+                readableRows: view.rowCount,
+                columns: view.columns.map((column) => ({ ...column })),
+                edits: { ...table.edits },
+            },
+            start,
+            end,
+            rowIds: view.grouped ? null : rowIds,
+            block: { bytes: encodeBlock(columns, values) },
+            ...(truncated.size > 0 ? { truncated: Object.fromEntries(truncated) } : {}),
+        };
+    }
+    async summarize(request, running) {
+        const table = this.table(request);
+        const view = await this.view(table, request.plan, running);
+        // The cells of ranges × columns: each cell once, however the ranges overlap or a column repeats.
+        const sources = new Map(this.columnsOf(table, view, request.columns).map((item) => [item.name, item]));
+        const spans = mergeRanges(request.ranges, view.rowCount);
+        const rows = new Uint32Array(spans.reduce((total, [start, end]) => total + end - start, 0));
+        let at = 0;
+        for (const [start, end] of spans) {
+            for (let position = start; position < end; position++)
+                rows[at++] = view.order ? view.order[position] : position;
+        }
+        await this.readColumns(table, [...sources.values()].filter((source) => !source.values).map((source) => source.index), running);
+        const summary = new Summary();
+        for (const { column, index, values } of sources.values()) {
+            summary.add(column, values ?? this.valuesOf(table, index), rows);
+        }
+        return summary.result();
+    }
+    /**
+     * The next cell of the view whose text form contains `text` (§8.2): row by row, in a row in the order of
+     * `columns`, from the cell after `from` (before it, with −1), wrapping around and ending with `from` itself.
+     */
+    async find(request, running) {
+        const table = this.table(request);
+        const view = await this.view(table, request.plan, running);
+        const sources = this.columnsOf(table, view, request.columns);
+        const text = request.text;
+        if (text === "" || view.rowCount === 0 || sources.length === 0)
+            return null;
+        const sensitive = request.caseSensitive === true;
+        const { mapCase } = await import("./evaluate.js");
+        const needle = sensitive ? text : mapCase(text, false);
+        await this.readColumns(table, sources.filter((source) => !source.values).map((source) => source.index), running);
+        const values = sources.map((source) => source.values ?? this.valuesOf(table, source.index));
+        const matches = (position, at) => {
+            const value = values[at].at(view.order ? view.order[position] : position);
+            if (value === null)
+                return false;
+            const column = sources[at].column;
+            const form = column.type === "string" ? value : textOf(value, column);
+            return (sensitive ? form : mapCase(form, false)).includes(needle);
+        };
+        const count = view.rowCount;
+        const width = sources.length;
+        const forward = request.direction === 1;
+        const from = request.from;
+        const row = Math.min(from.row, count - 1);
+        const column = Math.min(from.column, width);
+        // The cells of the row of `from` after it (before it, with −1).
+        if (forward) {
+            for (let at = column + 1; at < width; at++)
+                if (matches(row, at))
+                    return { row, column: at };
+        }
+        else {
+            for (let at = column - 1; at >= 0; at--)
+                if (matches(row, at))
+                    return { row, column: at };
+        }
+        // Then the other rows, in order, wrapping around at the end.
+        for (let step = 1; step < count; step++) {
+            if (step % 1_024 === 0)
+                await pause(running);
+            const position = forward ? (row + step) % count : (row - step + count) % count;
+            if (forward) {
+                for (let at = 0; at < width; at++)
+                    if (matches(position, at))
+                        return { row: position, column: at };
+            }
+            else {
+                for (let at = width - 1; at >= 0; at--)
+                    if (matches(position, at))
+                        return { row: position, column: at };
+            }
+        }
+        // Last, the cells of the row of `from` up to `from` itself.
+        if (forward) {
+            for (let at = 0; at <= Math.min(column, width - 1); at++)
+                if (matches(row, at))
+                    return { row, column: at };
+        }
+        else {
+            for (let at = width - 1; at >= column; at--)
+                if (matches(row, at))
+                    return { row, column: at };
+        }
+        return null;
+    }
+    /**
+     * The view's rows as a file (§8.2), answered as its bytes (`delivery` `data`): CSV, TSV and JSON as the DuckDB
+     * engine writes them, and xlsx. The engine has no files, so a `target` is not allowed.
+     */
+    async export(request, running) {
+        const table = this.table(request);
+        const format = request.format;
+        if (!EXPORT_FORMATS.includes(format)) {
+            throw new Failure("invalid_request", `This engine does not export ${format} (§4.10).`, {
+                path: "/format",
+                reason: "syntax",
+            });
+        }
+        if (request.target !== undefined) {
+            throw new Failure("not_allowed", "This engine answers exports as data and writes no files: leave out target.");
+        }
+        const view = await this.view(table, request.plan, running);
+        const names = request.columns ?? view.columns.map((column) => column.name);
+        const sources = this.columnsOf(table, view, names);
+        await this.readColumns(table, sources.filter((source) => !source.values).map((source) => source.index), running);
+        const values = sources.map((source) => source.values ?? this.valuesOf(table, source.index));
+        const { writeExport } = await import("./export.js");
+        const bytes = await writeExport({
+            format: format,
+            columns: sources.map((source) => source.column),
+            headers: request.headers ?? names,
+            numberFormats: request.numberFormats ?? null,
+            rowCount: view.rowCount,
+            row: (position) => {
+                const row = view.order ? view.order[position] : position;
+                return values.map((vector) => vector.at(row));
+            },
+            pause: () => pause(running),
+        });
+        return { target: null, rows: view.rowCount, data: { bytes } };
+    }
+    async locate(request, running) {
+        const table = this.table(request);
+        const view = await this.view(table, request.plan, running);
+        const keys = request.keys;
+        let column;
+        if (request.rowKey !== undefined) {
+            column = table.index.get(request.rowKey);
+            if (column === undefined) {
+                throw new Failure("invalid_request", `The table has no column ${shown(request.rowKey)}.`, {
+                    path: "/rowKey",
+                    reason: "syntax",
+                });
+            }
+        }
+        else if (keys.some((key) => "value" in key)) {
+            throw new Failure("invalid_request", "Keys by value need rowKey, the column of their values (§5.3).", {
+                path: "/rowKey",
+                reason: "syntax",
+            });
+        }
+        const name = request.rowKey;
+        // In a view of groups, values are looked up among the groups' keys; groups have no row IDs.
+        const group = view.grouped ? view.derived.get(name) : undefined;
+        if (column !== undefined && !group && keys.some((key) => "value" in key)) {
+            await this.readColumns(table, [column], running);
+        }
+        let ranks;
+        return {
+            rows: keys.map((key) => {
+                if ("rowId" in key)
+                    return view.grouped ? null : this.positionOf(view, key.rowId);
+                if (view.grouped && !group)
+                    return null;
+                // The first position in the view of a row whose value equals the key's (§3.1).
+                ranks ??= group ? rankColumn(group.column, vectorOf(group.values)) : this.ranksOf(table, column);
+                const rank = ranks.rankOf(key.value);
+                if (rank < 0)
+                    return null;
+                const first = this.firstsOf(view, name, ranks)[rank];
+                return first < 0 ? null : first;
+            }),
+        };
+    }
+    /** The row ID of a row key (§5.3): `{ rowId }`, or `{ value }` of `rowKey`'s column; null when no row has it. */
+    rowOf(table, key, column) {
+        if ("rowId" in key)
+            return Number.isSafeInteger(key.rowId) && key.rowId < table.rowCount ? key.rowId : null;
+        if (column === undefined)
+            return null;
+        let byText = table.keys.get(column);
+        if (!byText) {
+            const values = this.valuesOf(table, column);
+            byText = new Map();
+            for (let row = values.length - 1; row >= 0; row--) {
+                const value = values.at(row);
+                if (value !== null)
+                    byText.set(keyText({ value }), row);
+            }
+            table.keys.set(column, byText);
+        }
+        return byText.get(keyText(key)) ?? null;
+    }
+    /**
+     * Writes cells (§7.2): every update or none. Values are checked as their column's type, `previous` against the
+     * cell's value now, and the written rows replace the host's rows in the engine's own list of rows.
+     */
+    async write(request, running) {
+        const table = this.table(request);
+        const updates = request.updates;
+        let keyColumn;
+        if (request.rowKey !== undefined) {
+            keyColumn = table.index.get(request.rowKey);
+            if (keyColumn === undefined) {
+                throw new Failure("invalid_request", `The table has no column ${shown(request.rowKey)}.`, {
+                    path: "/rowKey",
+                    reason: "syntax",
+                });
+            }
+        }
+        if (keyColumn !== undefined && updates.some((update) => "value" in update.key)) {
+            await this.readColumns(table, [keyColumn], running);
+        }
+        const width = table.columns.length;
+        const changes = [];
+        for (const [at, update] of updates.entries()) {
+            const index = table.index.get(update.column);
+            const column = index === undefined ? undefined : table.columns[index];
+            if (index === undefined ||
+                !column ||
+                !table.editable ||
+                column.readOnly ||
+                column.type === "interval" ||
+                column.type === "binary") {
+                throw new Failure("read_only", `The cells of ${shown(update.column)} are not written (§7.1).`, {
+                    details: { column: update.column },
+                });
+            }
+            const row = this.rowOf(table, update.key, keyColumn);
+            if (row === null) {
+                throw new Failure("edit_conflict", "No row has this key now.", { details: { update: at } });
+            }
+            try {
+                table.checkers[index](update.value, row);
+            }
+            catch (error) {
+                if (!(error instanceof TableProblem))
+                    throw error;
+                throw new Failure("invalid_value", `${shown(update.value)} is not a value of ${shown(update.column)}.`, {
+                    path: `/updates/${at}/value`,
+                    reason: "type",
+                    details: { update: at },
+                });
+            }
+            const [now] = await this.cellsOf(table, index, [row]);
+            if (update.previous !== undefined && !sameCellValue(now, update.previous, column)) {
+                throw new Failure("edit_conflict", "The cell holds another value now.", { details: { update: at } });
+            }
+            changes.push({ row, index, value: update.value });
+        }
+        if (changes.length === 0)
+            return { updated: 0, edits: { ...table.edits } };
+        // The host's rows, Arrow columns and files stay as they were: writes change the engine's own values.
+        for (const { row, index, value } of changes) {
+            this.setCell(table, row, index, value);
+            table.written.add(row * width + index);
+            table.ranks[index] = undefined;
+            table.keys.delete(index);
+        }
+        table.views.clear();
+        table.edits = { version: table.edits.version + 1, cells: table.written.size };
+        for (const [id, open] of this.tables) {
+            if (open !== table)
+                continue;
+            this.emit({ v: 2, event: "changed", table: id, revision: table.revision, edits: { ...table.edits } });
+        }
+        return { updated: changes.length, edits: { ...table.edits } };
+    }
+    emit(event) {
+        for (const listener of this.listeners)
+            listener(event);
+    }
+    /** The position of a row in a view, or null when the view does not have it. */
+    positionOf(view, rowId) {
+        if (rowId >= view.tableRows)
+            return null;
+        if (!view.order)
+            return rowId;
+        if (!view.positions) {
+            // Rows that the view leaves out have no position.
+            const positions = new Uint32Array(view.tableRows).fill(NO_POSITION);
+            view.order.forEach((row, position) => {
+                positions[row] = position;
+            });
+            view.positions = positions;
+        }
+        const position = view.positions[rowId];
+        return position === NO_POSITION ? null : position;
+    }
+    /** The first position in a view of each rank of a column, −1 for ranks that no row of the view has. */
+    firstsOf(view, column, ranks) {
+        let firsts = view.firsts.get(column);
+        if (!firsts) {
+            firsts = new Int32Array(ranks.distinct + 2).fill(-1);
+            for (let position = 0; position < view.rowCount; position++) {
+                const rank = ranks.ranks[view.order ? view.order[position] : position];
+                if (firsts[rank] < 0)
+                    firsts[rank] = position;
+            }
+            view.firsts.set(column, firsts);
+        }
+        return firsts;
+    }
+}
+/**
+ * An engine over tables in memory. The host gives the tables by the `ref` that `open` takes; the engine checks
+ * the form of a table against §2 when it is first opened, and its values as it reads them (a column whole before
+ * it sorts or sums it), answering `load_failed` with the JSON Pointer of a value that does not follow §2.2. It
+ * reads the rows as they are, without copying them, so they must not change afterwards.
+ */
+export function memoryEngine(options) {
+    return new Engine(options);
+}
+//# sourceMappingURL=engine.js.map