vastgrid-js 0.0.0-stage → 0.2.0

package/dist/text.js

@@ -0,0 +1,1204 @@
+/**
+ * Tables given as text files (CSV, TSV and JSON Lines in UTF-8), read by a module of their own that the engine
+ * loads with the first such table, so that pages whose tables are rows do not load it. A file is read in slices,
+ * first to find its columns, the types that all their values fit, the memory they take and where every 1,024th row
+ * starts. When its columns fit `maxBytes`, it is read again to keep its values in typed columns, numbers in typed
+ * arrays and text as UTF-8, without making rows of JSON; otherwise its rows are read by blocks of 1,024 rows as
+ * requests need them, and columns whole only when a request goes through every row of them (lazy.ts).
+ */
+import { dateText, daysFromCivil, floatJson, timestampText, timeText, } from "vastgrid-spec";
+import { FileColumns } from "./lazy.js";
+import { HIGH, LOW, TWO_32 } from "./order.js";
+import { shown, TableProblem, } from "./table.js";
+/** Bytes read at a time. */
+const CHUNK_BYTES = 4 * 1024 * 1024;
+/** Rows of a block, the part of a file that rows are read by when its columns are not kept. */
+const BLOCK_ROWS = 1024;
+/** The bytes of the blocks read for rows that are kept, the most recently used. */
+const CACHE_BYTES = 32 * 1024 * 1024;
+/** The memory that a table's columns may take by default (`FileTable.maxBytes`). */
+const MAX_BYTES = 2 ** 30;
+const TAB = 0x09;
+const LF = 0x0a;
+const CR = 0x0d;
+const QUOTE = 0x22;
+const PLUS = 0x2b;
+const MINUS = 0x2d;
+const DOT = 0x2e;
+const ZERO = 0x30;
+const NINE = 0x39;
+const COLON = 0x3a;
+/** `e`, which a byte ORed with 0x20 is when it is `e` or `E`. */
+const LOWER_E = 0x65;
+/** The types that a value fits, as bits. */
+const BOOLEAN = 1;
+const INTEGER = 2;
+const BIGINT = 4;
+const FLOAT = 8;
+const TIME = 16;
+const DATE = 32;
+const TIMESTAMP = 64;
+const TIMESTAMPTZ = 128;
+const TEMPORAL = TIME | DATE | TIMESTAMP | TIMESTAMPTZ;
+const EVERY = 255;
+/** The types of a column in the order they are tried: the first that every value of the column fits. */
+const TYPES = [
+    [BOOLEAN, "boolean"],
+    [INTEGER, "integer"],
+    [BIGINT, "bigint"],
+    [FLOAT, "float"],
+    [TIME, "time"],
+    [DATE, "date"],
+    [TIMESTAMP, "timestamp"],
+    [TIMESTAMPTZ, "timestamptz"],
+];
+const FORMATS = new Map([
+    ["csv", "csv"],
+    ["tsv", "tsv"],
+    ["tab", "tsv"],
+    ["jsonl", "jsonl"],
+    ["ndjson", "jsonl"],
+]);
+/** The words of floats that are not numbers, as DuckDB reads them. */
+const FLOAT_WORDS = new Map(["", "+", "-"].flatMap((sign) => [
+    [`${sign}nan`, Number.NaN],
+    [`${sign}inf`, sign === "-" ? -Infinity : Infinity],
+    [`${sign}infinity`, sign === "-" ? -Infinity : Infinity],
+]));
+const INT64_MIN = -(2n ** 63n);
+const INT64_MAX = 2n ** 63n - 1n;
+const DAY_MICROS = 86_400_000_000;
+const FIRST_DAY = daysFromCivil(1, 1, 1);
+const LAST_DAY = daysFromCivil(9999, 12, 31);
+/** Text of fields and lines, which the first reading has checked as UTF-8; it keeps a byte order mark. */
+const decoder = new TextDecoder("utf-8", { ignoreBOM: true, fatal: true });
+const encoder = new TextEncoder();
+function found(name) {
+    return { name, fits: EVERY, values: 0, empty: 0, bytes: 0, nested: 0 };
+}
+/**
+ * Reads a table given as a text file into typed columns. Throws a `TableProblem` at the member of the table at
+ * fault, or at `/rows/{row}/{column}` (`/rows/{row}` for a line of JSON Lines) for text that is not UTF-8 or a
+ * line that is not JSON. `fields` gives the column of each field of a CSV file, whose values fit its type, or null
+ * for a field that is left out (workbook.ts); by default each field is a column of the type its values fit.
+ */
+export async function readFile(table, chunkBytes = CHUNK_BYTES, fields) {
+    const { file } = table;
+    if (file === null ||
+        typeof file !== "object" ||
+        typeof file.size !== "number" ||
+        typeof file.slice !== "function") {
+        throw new TableProblem("/file", "file is a Blob or a File.");
+    }
+    const format = table.format ?? FORMATS.get(/\.([^.]+)$/.exec(String(file.name ?? ""))?.[1].toLowerCase() ?? "");
+    if (format !== "csv" && format !== "tsv" && format !== "jsonl") {
+        throw new TableProblem("/format", table.format === undefined
+            ? `The file's name (${shown(file.name ?? null)}) does not tell its format: give format, csv, tsv, jsonl or xlsx.`
+            : "format is csv, tsv, jsonl or xlsx.");
+    }
+    const { delimiter, header = true, maxBytes = MAX_BYTES } = table;
+    if (delimiter !== undefined) {
+        const code = typeof delimiter === "string" && delimiter.length === 1 ? delimiter.charCodeAt(0) : -1;
+        if (format !== "csv" || code < 0 || code >= 0x80 || code === QUOTE || code === CR || code === LF) {
+            throw new TableProblem("/delimiter", "delimiter is one ASCII character other than a quote or a line break, for CSV.");
+        }
+    }
+    if (typeof header !== "boolean")
+        throw new TableProblem("/header", "header is a boolean.");
+    if (typeof maxBytes !== "number" || !(maxBytes > 0)) {
+        throw new TableProblem("/maxBytes", "maxBytes is a number of bytes.");
+    }
+    const csv = format === "jsonl" ? null : { separator: format === "tsv" ? TAB : (delimiter?.charCodeAt(0) ?? null), header };
+    const survey = csv ? await surveyCsv(file, csv, chunkBytes) : await surveyJson(file, chunkBytes);
+    const { rows } = survey;
+    const fieldOf = fields
+        ? fields.flatMap((column, field) => (column ? [field] : []))
+        : survey.found.map((_, field) => field);
+    const kept = fieldOf.map((field) => survey.found[field] ?? found(""));
+    const columns = fields ? fieldOf.map((field) => ({ ...fields[field] })) : kept.map(columnOf);
+    const total = columns.reduce((sum, column, index) => sum + sizeOf(column, kept[index], rows), 0);
+    if (total > maxBytes) {
+        const text = csv ? { separator: survey.separator, header: csv.header } : null;
+        return new TextColumns(file, text, columns, { ...survey, found: kept }, fieldOf, maxBytes, chunkBytes).checked();
+    }
+    const built = columns.map((column, index) => build(column, kept[index], rows));
+    const read = csv
+        ? await keepCsv(file, survey.separator, csv.header, byField(built, fieldOf), rows, chunkBytes)
+        : await keepJson(file, built, rows, chunkBytes);
+    if (read !== rows)
+        throw changed();
+    return { columns, rows: null, vectors: built.map((column) => vectorOf(column, rows)), rowCount: rows };
+}
+/** The columns of a CSV file's fields, by field: null for the fields that are not columns. */
+function byField(columns, fieldOf) {
+    const fields = new Array(fieldOf.length === 0 ? 0 : Math.max(...fieldOf) + 1).fill(null);
+    fieldOf.forEach((field, index) => {
+        fields[field] = columns[index];
+    });
+    return fields;
+}
+function changed() {
+    return new TableProblem("/file", "The file changed while it was read.");
+}
+/**
+ * A text file whose columns would take more than `maxBytes`: its rows are read by blocks of `BLOCK_ROWS` rows, from
+ * where the first reading found that each starts, and its columns whole when a request goes through every row of
+ * them, within `maxBytes`.
+ */
+class TextColumns extends FileColumns {
+    file;
+    text;
+    fieldOf;
+    chunkBytes;
+    found;
+    starts;
+    /** Blocks of rows read, the most recently used last, and the bytes of the file that they hold. */
+    cache = new Map();
+    cached = 0;
+    constructor(file, text, columns, survey, 
+    /** The field of a CSV file that each column is. */
+    fieldOf, maxBytes, chunkBytes) {
+        super(columns, survey.rows, maxBytes);
+        this.file = file;
+        this.text = text;
+        this.fieldOf = fieldOf;
+        this.chunkBytes = chunkBytes;
+        this.found = survey.found;
+        this.starts = survey.starts;
+    }
+    partOf(row) {
+        return Math.floor(row / BLOCK_ROWS);
+    }
+    /** The rows of a view are read by blocks wherever they lie: a block is small, and reading a column whole reads the file. */
+    get spread() {
+        return Number.POSITIVE_INFINITY;
+    }
+    sizeOf(index) {
+        return sizeOf(this.columns[index], this.found[index], this.rowCount);
+    }
+    /** The bytes of the file that a block's rows lie in. */
+    spanOf(block) {
+        return { start: this.starts[block], end: this.starts[block + 1] ?? this.file.size };
+    }
+    /** A block's columns, read once and kept while they fit the cache. */
+    block(block) {
+        const span = this.spanOf(block);
+        const size = span.end - span.start;
+        let entry = this.cache.get(block);
+        if (entry) {
+            this.cache.delete(block);
+        }
+        else {
+            const rows = Math.min(BLOCK_ROWS, this.rowCount - block * BLOCK_ROWS);
+            // A column's text takes the block's bytes at most, and is cut to its length once read.
+            const built = this.columns.map((column, index) => build(column, { ...this.found[index], values: 0, empty: 0, bytes: size }, rows));
+            entry = this.keep(built, rows, span).then(() => built.map((column) => {
+                if (column.text)
+                    column.text = column.text.slice(0, column.cursor);
+                return vectorOf(column, rows);
+            }));
+            this.cached += size;
+            // A failed read is not kept.
+            entry.catch(() => {
+                if (this.cache.get(block) === entry) {
+                    this.cache.delete(block);
+                    this.cached -= size;
+                }
+            });
+        }
+        this.cache.set(block, entry);
+        for (const [old] of this.cache) {
+            if (this.cached <= CACHE_BYTES || old === block)
+                break;
+            const { start, end } = this.spanOf(old);
+            this.cache.delete(old);
+            this.cached -= end - start;
+        }
+        return entry;
+    }
+    /** Reads rows of the file into the columns that are not null, and fails when they are not the rows it had. */
+    async keep(built, rows, span) {
+        const { file, text, chunkBytes } = this;
+        const read = text
+            ? await keepCsv(file, text.separator, text.header && span.start === undefined, byField(built, this.fieldOf), rows, chunkBytes, span)
+            : await keepJson(file, built, rows, chunkBytes, span);
+        if (read !== rows)
+            throw changed();
+    }
+    async readCells(index, rows) {
+        const blocks = new Map();
+        for (let at = 0; at < rows.length; at++) {
+            const block = this.partOf(rows[at]);
+            if (!blocks.has(block))
+                blocks.set(block, this.block(block));
+        }
+        const read = new Map();
+        for (const [block, columns] of blocks)
+            read.set(block, (await columns)[index]);
+        return Array.from(rows, (row) => read.get(this.partOf(row)).at(row % BLOCK_ROWS));
+    }
+    async readWhole(indexes, pause) {
+        const built = this.columns.map(() => null);
+        for (const index of indexes)
+            built[index] = build(this.columns[index], this.found[index], this.rowCount);
+        await this.keep(built, this.rowCount, { pause });
+        return indexes.map((index) => vectorOf(built[index], this.rowCount));
+    }
+    close() {
+        this.cache.clear();
+        this.cached = 0;
+    }
+}
+/** A column of the values that the first reading found. */
+function columnOf(column) {
+    const { name } = column;
+    if (column.values > 0) {
+        for (const [bit, type] of TYPES)
+            if (column.fits & bit)
+                return { name, type };
+        // Objects and lists of JSON, and only they, make a column of their JSON text (§2.1).
+        if (column.nested === column.values)
+            return { name, type: "string", sourceType: "JSON", readOnly: true };
+    }
+    return { name, type: "string" };
+}
+/** The values of a column that are not null. */
+function present(column, found) {
+    return column.type === "string" ? found.values + found.empty : found.values;
+}
+/** The bytes that a column's values take in memory, `Infinity` for text of more than 4 GiB, which is not kept. */
+function sizeOf(column, found, rows) {
+    return column.type === "string" && found.bytes > 0xffff_ffff ? Infinity : memoryOf(column, found, rows);
+}
+/** The bytes that a column's values take in memory. */
+function memoryOf(column, found, rows) {
+    const validity = present(column, found) < rows ? Math.ceil(rows / 8) : 0;
+    switch (column.type) {
+        case "boolean":
+            return rows + validity;
+        case "integer":
+        case "date":
+            return 4 * rows + validity;
+        case "string":
+            return 4 * (rows + 1) + found.bytes + validity;
+        default:
+            return 8 * rows + validity;
+    }
+}
+function build(column, found, rows) {
+    const validity = present(column, found) < rows ? new Uint8Array(Math.ceil(rows / 8)) : null;
+    const built = {
+        column,
+        validity,
+        numbers: null,
+        halves: null,
+        flags: null,
+        offsets: null,
+        text: null,
+        cursor: 0,
+    };
+    switch (column.type) {
+        case "boolean":
+            built.flags = new Uint8Array(rows);
+            break;
+        case "integer":
+        case "date":
+            built.numbers = new Int32Array(rows);
+            break;
+        case "float":
+            built.numbers = new Float64Array(rows);
+            break;
+        case "string":
+            built.offsets = new Uint32Array(rows + 1);
+            built.text = new Uint8Array(found.bytes);
+            break;
+        default:
+            built.numbers = new BigInt64Array(rows);
+            built.halves = new Uint32Array(built.numbers.buffer);
+    }
+    return built;
+}
+/** The JSON (§2.2) of a value of a logical type from the number that holds it. */
+function numberJson(type) {
+    switch (type) {
+        case "integer":
+            return (value) => value;
+        case "float":
+            return (value) => floatJson(value);
+        case "date":
+            return (value) => dateText(value);
+        case "bigint":
+            return (value) => (Number.isSafeInteger(Number(value)) ? Number(value) : value.toString());
+        case "time":
+            return (value) => timeText(value);
+        case "timestamp":
+            return (value) => timestampText(value);
+        default:
+            return (value) => `${timestampText(value)}+00`;
+    }
+}
+/** The text of UTF-8 bytes; short text of ASCII without a decoder. */
+function textOf(bytes, start, end) {
+    if (end - start <= 32) {
+        let text = "";
+        for (let index = start; index < end; index++) {
+            const code = bytes[index];
+            if (code >= 0x80)
+                return decoder.decode(bytes.subarray(start, end));
+            text += String.fromCharCode(code);
+        }
+        return text;
+    }
+    return decoder.decode(bytes.subarray(start, end));
+}
+function vectorOf(built, rows) {
+    const { validity } = built;
+    const valid = (row) => validity === null || ((validity[row >> 3] >> (row & 7)) & 1) === 1;
+    const { numbers, flags, offsets, text } = built;
+    let at;
+    if (numbers) {
+        const json = numberJson(built.column.type);
+        at = (row) => (valid(row) ? json(numbers[row]) : null);
+    }
+    else if (flags) {
+        at = (row) => (valid(row) ? flags[row] === 1 : null);
+    }
+    else {
+        const ends = offsets;
+        const bytes = text;
+        at = (row) => (valid(row) ? textOf(bytes, ends[row], ends[row + 1]) : null);
+    }
+    return { length: rows, numbers, validity, at };
+}
+/** Sets a row's bit of validity. */
+function mark(built, row) {
+    if (built.validity)
+        built.validity[row >> 3] |= 1 << (row & 7);
+}
+/**
+ * Reads a file in slices from after its byte order mark, giving each to `read` after the bytes that the last one
+ * left, with where they start in the file: `read` gives how many of them it used. A row longer than a slice makes
+ * the next slice as long, so that its bytes are not read again and again.
+ */
+async function eachSlice(file, chunkBytes, read, span = {}) {
+    const size = span.end ?? file.size;
+    const bytesOf = async (start, end) => {
+        try {
+            return new Uint8Array(await file.slice(start, end).arrayBuffer());
+        }
+        catch (error) {
+            throw new TableProblem("/file", `The file cannot be read: ${error.message}`);
+        }
+    };
+    let offset = span.start ?? 0;
+    if (span.start === undefined) {
+        const head = await bytesOf(0, Math.min(size, 3));
+        if (head[0] === 0xef && head[1] === 0xbb && head[2] === 0xbf)
+            offset = 3;
+        else if ((head[0] === 0xff && head[1] === 0xfe) || (head[0] === 0xfe && head[1] === 0xff)) {
+            throw new TableProblem("/file", "The file is UTF-16 text; save it as UTF-8.");
+        }
+    }
+    let rest = null;
+    for (let first = true;; first = false) {
+        if (!first)
+            await span.pause?.();
+        const end = Math.min(size, offset + Math.max(chunkBytes, rest?.length ?? 0));
+        let bytes = await bytesOf(offset, end);
+        if (rest) {
+            const joined = new Uint8Array(rest.length + bytes.length);
+            joined.set(rest);
+            joined.set(bytes, rest.length);
+            bytes = joined;
+        }
+        const last = end >= size;
+        const used = read(bytes, last, end - bytes.length);
+        if (last)
+            return;
+        rest = used < bytes.length ? bytes.subarray(used) : null;
+        offset = end;
+    }
+}
+/**
+ * The length of the UTF-8 sequence that starts with a byte of 0x80 or more at `index`: 2 to 4 bytes, 0 when the
+ * bytes are not UTF-8 (overlong forms, surrogates and code points beyond U+10FFFF are not), −1 when `end` cuts it.
+ */
+function sequenceLength(bytes, index, end) {
+    const lead = bytes[index];
+    let length;
+    let low = 0x80;
+    let high = 0xbf;
+    if (lead < 0xc2)
+        return 0;
+    if (lead < 0xe0)
+        length = 2;
+    else if (lead < 0xf0) {
+        length = 3;
+        if (lead === 0xe0)
+            low = 0xa0;
+        else if (lead === 0xed)
+            high = 0x9f;
+    }
+    else if (lead < 0xf5) {
+        length = 4;
+        if (lead === 0xf0)
+            low = 0x90;
+        else if (lead === 0xf4)
+            high = 0x8f;
+    }
+    else
+        return 0;
+    for (let next = 1; next < length; next++) {
+        if (index + next >= end)
+            return -1;
+        const byte = bytes[index + next];
+        if (byte < low || byte > high)
+            return 0;
+        low = 0x80;
+        high = 0xbf;
+    }
+    return length;
+}
+/** The fields of a row of CSV: where each starts and ends in the bytes, and whether it holds a quote. */
+class Fields {
+    starts = new Int32Array(16);
+    ends = new Int32Array(16);
+    quoted = new Uint8Array(16);
+    count = 0;
+    add(start, end, quoted) {
+        if (this.count === this.starts.length) {
+            const grow = (array, make) => {
+                const grown = make(array.length * 2);
+                grown.set(array);
+                return grown;
+            };
+            this.starts = grow(this.starts, (length) => new Int32Array(length));
+            this.ends = grow(this.ends, (length) => new Int32Array(length));
+            this.quoted = grow(this.quoted, (length) => new Uint8Array(length));
+        }
+        this.starts[this.count] = start;
+        this.ends[this.count] = end;
+        this.quoted[this.count] = quoted;
+        this.count++;
+    }
+}
+/** Bytes that are not UTF-8, in the row being read, in its field `field`. */
+class NotUtf8 {
+    field;
+    constructor(field) {
+        this.field = field;
+    }
+}
+/**
+ * Reads the rows of CSV bytes, calling `row` with each row's fields, and gives where the first row that is not
+ * whole starts (all of the bytes when they are the last). A quote switches quoting on and off wherever it is, so
+ * that a doubled quote within quotes is a quote; a line break outside quotes ends a row, without a carriage return
+ * before it, and an empty line is no row. Throws `NotUtf8` for bytes that are not UTF-8.
+ */
+function scanCsv(bytes, separator, last, fields, row) {
+    const end = bytes.length;
+    let rowStart = 0;
+    let fieldStart = 0;
+    let inQuotes = false;
+    let quoted = 0;
+    const endRow = (fieldEnd) => {
+        fields.add(fieldStart, fieldEnd > fieldStart && bytes[fieldEnd - 1] === CR ? fieldEnd - 1 : fieldEnd, quoted);
+        if (fields.count > 1 || fields.ends[0] > fields.starts[0] || quoted)
+            row();
+        fields.count = 0;
+        quoted = 0;
+    };
+    fields.count = 0;
+    for (let index = 0; index < end; index++) {
+        const code = bytes[index];
+        if (code === QUOTE) {
+            inQuotes = !inQuotes;
+            quoted = 1;
+        }
+        else if (code >= 0x80) {
+            const length = sequenceLength(bytes, index, end);
+            if (length > 0)
+                index += length - 1;
+            else if (length < 0 && !last)
+                return rowStart;
+            else
+                throw new NotUtf8(fields.count);
+        }
+        else if (!inQuotes) {
+            if (code === separator) {
+                fields.add(fieldStart, index, quoted);
+                fieldStart = index + 1;
+                quoted = 0;
+            }
+            else if (code === LF) {
+                endRow(index);
+                fieldStart = rowStart = index + 1;
+            }
+        }
+    }
+    if (!last)
+        return rowStart;
+    if (fieldStart < end || fields.count > 0 || quoted)
+        endRow(end);
+    return end;
+}
+/** Where `unquote` writes the text of a field. */
+let unquoted = new Uint8Array(256);
+/**
+ * Writes the text of a field that holds a quote into `unquoted`, and gives its length: quotes switch quoting, and
+ * a doubled quote within quotes is a quote.
+ */
+function unquote(bytes, start, end) {
+    if (unquoted.length < end - start)
+        unquoted = new Uint8Array(Math.max(end - start, unquoted.length * 2));
+    let length = 0;
+    let inQuotes = false;
+    for (let index = start; index < end; index++) {
+        const code = bytes[index];
+        if (code !== QUOTE)
+            unquoted[length++] = code;
+        else if (inQuotes && index + 1 < end && bytes[index + 1] === QUOTE) {
+            unquoted[length++] = QUOTE;
+            index++;
+        }
+        else
+            inQuotes = !inQuotes;
+    }
+    return length;
+}
+/** The separator of `,`, tab, `;` and `|` that splits the first line into the most fields; `,` when none does. */
+function detectSeparator(bytes) {
+    const candidates = [0x2c, TAB, 0x3b, 0x7c];
+    const counts = [0, 0, 0, 0];
+    let inQuotes = false;
+    for (let index = 0; index < bytes.length; index++) {
+        const code = bytes[index];
+        if (code === QUOTE)
+            inQuotes = !inQuotes;
+        else if (!inQuotes) {
+            if (code === LF)
+                break;
+            const at = candidates.indexOf(code);
+            if (at >= 0)
+                counts[at]++;
+        }
+    }
+    let best = 0;
+    for (let at = 1; at < counts.length; at++)
+        if (counts[at] > counts[best])
+            best = at;
+    return candidates[best];
+}
+/** ASCII bytes as text. */
+function ascii(bytes, start, end) {
+    return end - start > 64 ? decoder.decode(bytes.subarray(start, end)) : textOf(bytes, start, end);
+}
+function isDigit(code) {
+    return code >= ZERO && code <= NINE;
+}
+/** The value of the text of an integer of no more than 15 digits. */
+function integerValue(bytes, start, end) {
+    let index = start;
+    const negative = bytes[index] === MINUS;
+    if (negative || bytes[index] === PLUS)
+        index++;
+    let value = 0;
+    for (; index < end; index++)
+        value = value * 10 + (bytes[index] - ZERO);
+    return negative ? -value : value;
+}
+/** Powers of ten that are floats exactly. */
+const POWERS = Array.from({ length: 23 }, (_, power) => Number(`1e${power}`));
+/**
+ * The float of the text of a number: a decimal of up to 15 digits without an exponent is its digits divided by a
+ * power of ten, both exact, which rounds once as `Number` does (Clinger's fast path); others are read by `Number`.
+ */
+function floatValue(bytes, start, end) {
+    let index = start;
+    const negative = bytes[index] === MINUS;
+    if (negative || bytes[index] === PLUS)
+        index++;
+    let mantissa = 0;
+    let count = 0;
+    let scale = 0;
+    for (; index < end && isDigit(bytes[index]); index++, count++)
+        mantissa = mantissa * 10 + bytes[index] - ZERO;
+    if (index < end && bytes[index] === DOT) {
+        for (index++; index < end && isDigit(bytes[index]); index++, count++, scale++) {
+            mantissa = mantissa * 10 + bytes[index] - ZERO;
+        }
+    }
+    if (index < end || count > 15) {
+        const text = ascii(bytes, start, end);
+        const value = Number(text);
+        return Number.isNaN(value) ? (FLOAT_WORDS.get(text.toLowerCase()) ?? value) : value;
+    }
+    const value = mantissa / POWERS[scale];
+    return negative ? -value : value;
+}
+/** The types that the text of an integer fits, with its digits from `digits`. */
+function integerFits(bytes, start, digits, end) {
+    let first = digits;
+    while (first < end - 1 && bytes[first] === ZERO)
+        first++;
+    const count = end - first;
+    if (count <= 9)
+        return INTEGER | BIGINT | FLOAT;
+    if (count <= 15) {
+        const value = integerValue(bytes, start, end);
+        return value >= -0x8000_0000 && value <= 0x7fff_ffff ? INTEGER | BIGINT | FLOAT : BIGINT | FLOAT;
+    }
+    if (count <= 19) {
+        const value = BigInt(ascii(bytes, start, end));
+        return value >= INT64_MIN && value <= INT64_MAX ? BIGINT | FLOAT : FLOAT;
+    }
+    return FLOAT;
+}
+/** Whether the text from `index`, after the whole digits of a number, ends it as a float: a fraction, an exponent. */
+function floatRest(bytes, index, end, whole) {
+    let at = index;
+    let fraction = 0;
+    if (bytes[at] === DOT) {
+        const from = ++at;
+        while (at < end && isDigit(bytes[at]))
+            at++;
+        fraction = at - from;
+    }
+    if (!whole && fraction === 0)
+        return false;
+    if (at < end && (bytes[at] | 0x20) === LOWER_E) {
+        at++;
+        if (at < end && (bytes[at] === PLUS || bytes[at] === MINUS))
+            at++;
+        const from = at;
+        while (at < end && isDigit(bytes[at]))
+            at++;
+        if (at === from)
+            return false;
+    }
+    return at === end;
+}
+/**
+ * The types that the text of a field fits, of those in `wanted`: booleans (`true`, `false` in any case), integers
+ * by their range, floats (with `nan`, `inf` and `infinity`), and times, dates and timestamps (`temporalFits`).
+ */
+function textFits(bytes, start, end, wanted) {
+    const first = bytes[start];
+    const signed = first === PLUS || first === MINUS;
+    let index = signed ? start + 1 : start;
+    const digits = index;
+    while (index < end && isDigit(bytes[index]))
+        index++;
+    const whole = index - digits;
+    if (index === end)
+        return whole > 0 ? wanted & integerFits(bytes, start, digits, end) : 0;
+    const next = bytes[index];
+    if (wanted & FLOAT && (next === DOT || (next | 0x20) === LOWER_E) && floatRest(bytes, index, end, whole > 0)) {
+        return FLOAT;
+    }
+    if (wanted & TEMPORAL &&
+        !signed &&
+        ((whole === 2 && next === COLON) || (whole === 4 && next === MINUS)) &&
+        end - start <= 40) {
+        return wanted & temporalFits(bytes, start, end);
+    }
+    if (wanted & (BOOLEAN | FLOAT) && whole === 0 && end - start <= 9) {
+        const word = ascii(bytes, start, end).toLowerCase();
+        if (word === "true" || word === "false")
+            return wanted & BOOLEAN;
+        if (FLOAT_WORDS.has(word))
+            return wanted & FLOAT;
+    }
+    return 0;
+}
+const MONTH_DAYS = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+const FRACTION_SCALE = [100_000, 10_000, 1000, 100, 10, 1];
+const SPACE = 0x20;
+const UPPER_T = 0x54;
+const UPPER_Z = 0x5a;
+/** The days and the microseconds of the day of the text that `temporalFits` last read, and where its clock ended. */
+let readDays = 0;
+let readMicros = 0;
+let clockEnd = 0;
+/** The number of two digits at `at`, or −1. */
+function twoDigits(bytes, at) {
+    const tens = bytes[at] - ZERO;
+    const ones = bytes[at + 1] - ZERO;
+    return tens >= 0 && tens <= 9 && ones >= 0 && ones <= 9 ? tens * 10 + ones : -1;
+}
+/**
+ * Microseconds since midnight of `HH:MM:SS` with up to 6 fraction digits at `at`, before `end`, or −1; sets
+ * `clockEnd` to where it ends.
+ */
+function clockAt(bytes, at, end) {
+    if (end - at < 8 || bytes[at + 2] !== COLON || bytes[at + 5] !== COLON)
+        return -1;
+    const hours = twoDigits(bytes, at);
+    const minutes = twoDigits(bytes, at + 3);
+    const seconds = twoDigits(bytes, at + 6);
+    if (hours < 0 || hours > 23 || minutes < 0 || minutes > 59 || seconds < 0 || seconds > 59)
+        return -1;
+    let micros = (hours * 3600 + minutes * 60 + seconds) * 1_000_000;
+    let index = at + 8;
+    if (index < end && bytes[index] === DOT) {
+        const from = ++index;
+        while (index < end && isDigit(bytes[index])) {
+            if (index - from === 6)
+                return -1;
+            micros += (bytes[index] - ZERO) * FRACTION_SCALE[index - from];
+            index++;
+        }
+        if (index === from)
+            return -1;
+    }
+    clockEnd = index;
+    return micros;
+}
+/**
+ * The type of the text of a time (`HH:MM:SS` with up to 6 fraction digits), a date (`YYYY-MM-DD` of the years 0001
+ * to 9999, which also fits timestamps), a timestamp (a date, a space or `T`, and a time) or a timestamp with `Z` or
+ * an offset (`+HH`, `+HH:MM` or `+HHMM`), which is an instant; 0 for other text. Keeps the value in `readDays` and
+ * `readMicros`, in UTC for an instant.
+ */
+function temporalFits(bytes, start, end) {
+    if (bytes[start + 2] === COLON) {
+        const micros = clockAt(bytes, start, end);
+        if (micros < 0 || clockEnd !== end)
+            return 0;
+        readMicros = micros;
+        return TIME;
+    }
+    if (end - start < 10 || bytes[start + 4] !== MINUS || bytes[start + 7] !== MINUS)
+        return 0;
+    const century = twoDigits(bytes, start);
+    const decade = twoDigits(bytes, start + 2);
+    const month = twoDigits(bytes, start + 5);
+    const day = twoDigits(bytes, start + 8);
+    if (century < 0 || decade < 0 || month < 1 || month > 12 || day < 1)
+        return 0;
+    const year = century * 100 + decade;
+    const leap = month === 2 && year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
+    if (year < 1 || day > MONTH_DAYS[month - 1] + (leap ? 1 : 0))
+        return 0;
+    const days = daysFromCivil(year, month, day);
+    if (end - start === 10) {
+        readDays = days;
+        readMicros = 0;
+        return DATE | TIMESTAMP;
+    }
+    const separator = bytes[start + 10];
+    if (separator !== SPACE && separator !== UPPER_T)
+        return 0;
+    let micros = clockAt(bytes, start + 11, end);
+    if (micros < 0)
+        return 0;
+    let index = clockEnd;
+    if (index === end) {
+        readDays = days;
+        readMicros = micros;
+        return TIMESTAMP;
+    }
+    const sign = bytes[index];
+    if (sign === UPPER_Z) {
+        if (index + 1 !== end)
+            return 0;
+    }
+    else {
+        if ((sign !== PLUS && sign !== MINUS) || end - index < 3)
+            return 0;
+        const hours = twoDigits(bytes, index + 1);
+        let minutes = 0;
+        index += 3;
+        if (index < end) {
+            if (bytes[index] === COLON)
+                index++;
+            minutes = end - index === 2 ? twoDigits(bytes, index) : -1;
+        }
+        if (hours < 0 || hours > 23 || minutes < 0 || minutes > 59)
+            return 0;
+        micros -= (sign === MINUS ? -1 : 1) * (hours * 3600 + minutes * 60) * 1_000_000;
+    }
+    const shift = Math.floor(micros / DAY_MICROS);
+    readDays = days + shift;
+    readMicros = micros - shift * DAY_MICROS;
+    return readDays >= FIRST_DAY && readDays <= LAST_DAY ? TIMESTAMPTZ : 0;
+}
+/** Where `temporalText` writes a string. */
+const temporal = new Uint8Array(40);
+/** `temporalFits` of a string of JSON. */
+function temporalText(text) {
+    if (text.length > temporal.length)
+        return 0;
+    for (let index = 0; index < text.length; index++) {
+        const code = text.charCodeAt(index);
+        if (code >= 0x80)
+            return 0;
+        temporal[index] = code;
+    }
+    return temporalFits(temporal, 0, text.length);
+}
+/** Reads a CSV or TSV file the first time: its separator, its columns with what their values fit, and its rows. */
+async function surveyCsv(file, options, chunkBytes) {
+    const fields = new Fields();
+    const columns = [];
+    const starts = [];
+    let header = options.header ? null : [];
+    let separator = options.separator;
+    let rows = 0;
+    let bytes = new Uint8Array(0);
+    let base = 0;
+    const row = () => {
+        if (header === null) {
+            header = [];
+            for (let index = 0; index < fields.count; index++) {
+                const start = fields.starts[index];
+                const end = fields.ends[index];
+                header.push(fields.quoted[index]
+                    ? decoder.decode(unquoted.subarray(0, unquote(bytes, start, end)))
+                    : decoder.decode(bytes.subarray(start, end)));
+            }
+            return;
+        }
+        // A row's first field starts where the row does.
+        if (rows % BLOCK_ROWS === 0)
+            starts.push(base + fields.starts[0]);
+        for (let index = 0; index < fields.count; index++) {
+            columns[index] ??= found("");
+            const column = columns[index];
+            let source = bytes;
+            let start = fields.starts[index];
+            let end = fields.ends[index];
+            if (fields.quoted[index]) {
+                end = unquote(bytes, start, end);
+                start = 0;
+                source = unquoted;
+                if (end === 0) {
+                    column.empty++;
+                    continue;
+                }
+            }
+            else if (start === end)
+                continue;
+            column.values++;
+            column.bytes += end - start;
+            if (column.fits !== 0)
+                column.fits &= textFits(source, start, end, column.fits);
+        }
+        rows++;
+    };
+    await eachSlice(file, chunkBytes, (slice, last, at) => {
+        bytes = slice;
+        base = at;
+        separator ??= detectSeparator(slice);
+        try {
+            return scanCsv(slice, separator, last, fields, row);
+        }
+        catch (error) {
+            if (!(error instanceof NotUtf8))
+                throw error;
+            throw header === null
+                ? new TableProblem("/file", "The first line, which names the columns, is not UTF-8 text.")
+                : new TableProblem(`/rows/${rows}/${error.field}`, "The text is not UTF-8 (§2.2).");
+        }
+    });
+    const names = (header ?? []);
+    const width = Math.max(names.length, columns.length);
+    const used = new Set();
+    for (let index = 0; index < width; index++) {
+        const base = names[index] || `column${index}`;
+        let name = base;
+        for (let suffix = 1; used.has(name); suffix++)
+            name = `${base}_${suffix}`;
+        used.add(name);
+        columns[index] = columns[index] ?? found("");
+        columns[index].name = name;
+    }
+    return { separator: separator ?? 0x2c, found: columns, rows, starts };
+}
+/** Writes the text of a field into a column as its type, from the first reading's finding that it fits. */
+function putText(built, bytes, start, end, row) {
+    switch (built.column.type) {
+        case "boolean":
+            built.flags[row] = (bytes[start] | 0x20) === 0x74 ? 1 : 0;
+            break;
+        case "integer":
+            built.numbers[row] = integerValue(bytes, start, end);
+            break;
+        case "bigint":
+            if (end - start <= 15)
+                putInt64(built, row, integerValue(bytes, start, end));
+            else
+                built.numbers[row] = BigInt(ascii(bytes, start, end));
+            break;
+        case "float":
+            built.numbers[row] = floatValue(bytes, start, end);
+            break;
+        case "string":
+            built.text.set(bytes.subarray(start, end), built.cursor);
+            built.cursor += end - start;
+            break;
+        default:
+            if (!temporalFits(bytes, start, end))
+                throw changed();
+            putTemporal(built, row);
+    }
+    mark(built, row);
+}
+/** Writes what `temporalFits` last read into a column of a time, a date or a timestamp. */
+function putTemporal(built, row) {
+    if (built.column.type === "date")
+        built.numbers[row] = readDays;
+    else if (built.column.type === "time")
+        putInt64(built, row, readMicros);
+    else {
+        // Microseconds since 1970-01-01 are safe integers for about 285 years either side of it.
+        const micros = readDays * DAY_MICROS + readMicros;
+        if (Number.isSafeInteger(micros))
+            putInt64(built, row, micros);
+        else
+            built.numbers[row] = BigInt(readDays) * BigInt(DAY_MICROS) + BigInt(readMicros);
+    }
+}
+/** Writes a safe integer into a column of 64-bit integers, as its halves. */
+function putInt64(built, row, value) {
+    const halves = built.halves;
+    const high = Math.floor(value / TWO_32);
+    halves[row * 2 + HIGH] = high;
+    halves[row * 2 + LOW] = value - high * TWO_32;
+}
+/**
+ * Reads a CSV or TSV file the second time, or a span of it, keeping its values in the columns that are not null;
+ * gives the rows read.
+ */
+async function keepCsv(file, separator, header, built, rowCount, chunkBytes, span) {
+    const fields = new Fields();
+    const strings = built.filter((column) => column?.offsets);
+    let skip = header;
+    let rows = 0;
+    let bytes = new Uint8Array(0);
+    const row = () => {
+        if (skip) {
+            skip = false;
+            return;
+        }
+        if (rows === rowCount)
+            throw changed();
+        const count = Math.min(fields.count, built.length);
+        for (let index = 0; index < count; index++) {
+            const column = built[index];
+            if (!column)
+                continue;
+            let source = bytes;
+            let start = fields.starts[index];
+            let end = fields.ends[index];
+            if (fields.quoted[index]) {
+                end = unquote(bytes, start, end);
+                start = 0;
+                source = unquoted;
+                // A quoted empty field is an empty string, and null in a column of another type.
+                if (end === 0 && !column.offsets)
+                    continue;
+            }
+            else if (start === end)
+                continue;
+            putText(column, source, start, end, rows);
+        }
+        for (const column of strings)
+            column.offsets[rows + 1] = column.cursor;
+        rows++;
+    };
+    await eachSlice(file, chunkBytes, (slice, last) => {
+        bytes = slice;
+        return scanCsv(slice, separator, last, fields, row);
+    }, span);
+    return rows;
+}
+/** The bytes of a string as UTF-8, or −1 when it holds a lone surrogate. */
+function utf8Length(text) {
+    let length = text.length;
+    for (let index = 0; index < text.length; index++) {
+        const code = text.charCodeAt(index);
+        if (code < 0x80)
+            continue;
+        if (code < 0x800)
+            length += 1;
+        else if (code < 0xd800 || code > 0xdfff)
+            length += 2;
+        else if (code < 0xdc00 && (text.charCodeAt(index + 1) & 0xfc00) === 0xdc00) {
+            length += 2;
+            index++;
+        }
+        else
+            return -1;
+    }
+    return length;
+}
+/**
+ * Calls `line` with each whole line of the bytes, without its line break, and gives where the first line that is
+ * not whole starts (all of the bytes when they are the last).
+ */
+function eachLine(bytes, last, line) {
+    let start = 0;
+    for (;;) {
+        const newline = bytes.indexOf(LF, start);
+        const end = newline < 0 ? bytes.length : newline;
+        if (newline < 0 && !last)
+            return start;
+        if (newline >= 0 || start < end)
+            line(start, end > start && bytes[end - 1] === CR ? end - 1 : end);
+        if (newline < 0)
+            return bytes.length;
+        start = newline + 1;
+    }
+}
+/** The object of a line of JSON Lines, or null for a line of white space. */
+function parseLine(bytes, start, end, row) {
+    let text;
+    try {
+        text = decoder.decode(bytes.subarray(start, end));
+    }
+    catch {
+        throw new TableProblem(`/rows/${row}`, "The line is not UTF-8 text (§2.2).");
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch (error) {
+        if (text.trim() === "")
+            return null;
+        throw new TableProblem(`/rows/${row}`, `The line is not JSON: ${error.message}`);
+    }
+    if (value === null || typeof value !== "object" || Array.isArray(value)) {
+        throw new TableProblem(`/rows/${row}`, "A line of JSON Lines is an object of the row's values by column name.");
+    }
+    return value;
+}
+/** The types that a number of JSON fits. */
+function numberFits(value) {
+    if (!Number.isInteger(value))
+        return FLOAT;
+    if (value >= -0x8000_0000 && value <= 0x7fff_ffff)
+        return INTEGER | BIGINT | FLOAT;
+    return Number.isSafeInteger(value) ? BIGINT | FLOAT : FLOAT;
+}
+/** The text of a value of JSON in a string column: strings as they are, other values as JSON. */
+function jsonText(value) {
+    return typeof value === "string" ? value : JSON.stringify(value);
+}
+/** Reads a file of JSON Lines the first time: its columns by first appearance, with what their values fit, and its rows. */
+async function surveyJson(file, chunkBytes) {
+    const columns = [];
+    const index = new Map();
+    const starts = [];
+    let rows = 0;
+    let bytes = new Uint8Array(0);
+    let base = 0;
+    const line = (start, end) => {
+        if (start === end)
+            return;
+        const object = parseLine(bytes, start, end, rows);
+        if (!object)
+            return;
+        if (rows % BLOCK_ROWS === 0)
+            starts.push(base + start);
+        for (const key in object) {
+            let at = index.get(key);
+            if (at === undefined) {
+                if (utf8Length(key) < 0)
+                    throw new TableProblem(`/rows/${rows}`, "A name holds a lone surrogate (§2.2).");
+                at = columns.length;
+                index.set(key, at);
+                columns.push(found(key));
+            }
+            const value = object[key];
+            if (value === null)
+                continue;
+            const column = columns[at];
+            column.values++;
+            if (typeof value === "object") {
+                column.fits = 0;
+                column.nested++;
+                column.bytes += utf8Length(JSON.stringify(value));
+                continue;
+            }
+            if (typeof value === "string") {
+                const length = utf8Length(value);
+                if (length < 0) {
+                    throw new TableProblem(`/rows/${rows}/${at}`, "The string holds a lone surrogate, which is not Unicode text (§2.2).");
+                }
+                column.bytes += length;
+                if (column.fits !== 0)
+                    column.fits &= temporalText(value);
+            }
+            else {
+                column.bytes += jsonText(value).length;
+                column.fits &= typeof value === "boolean" ? BOOLEAN : numberFits(value);
+            }
+        }
+        rows++;
+    };
+    await eachSlice(file, chunkBytes, (slice, last, at) => {
+        bytes = slice;
+        base = at;
+        return eachLine(slice, last, line);
+    });
+    return { separator: 0, found: columns, rows, starts };
+}
+/**
+ * Reads a file of JSON Lines the second time, or a span of it, keeping its values in the columns that are not null;
+ * gives the rows read.
+ */
+async function keepJson(file, built, rowCount, chunkBytes, span) {
+    const index = new Map(built.flatMap((column, at) => (column ? [[column.column.name, at]] : [])));
+    const strings = built.filter((column) => column?.offsets);
+    let rows = 0;
+    let bytes = new Uint8Array(0);
+    const line = (start, end) => {
+        if (start === end)
+            return;
+        const object = parseLine(bytes, start, end, rows);
+        if (!object)
+            return;
+        if (rows === rowCount)
+            throw changed();
+        for (const key in object) {
+            const value = object[key];
+            const at = index.get(key);
+            if (value === null || at === undefined)
+                continue;
+            const column = built[at];
+            switch (column.column.type) {
+                case "boolean":
+                    column.flags[rows] = value ? 1 : 0;
+                    break;
+                case "integer":
+                case "float":
+                    column.numbers[rows] = value;
+                    break;
+                case "bigint":
+                    putInt64(column, rows, value);
+                    break;
+                case "string": {
+                    const { written } = encoder.encodeInto(jsonText(value), column.text.subarray(column.cursor));
+                    column.cursor += written;
+                    break;
+                }
+                default:
+                    if (!temporalText(value))
+                        throw changed();
+                    putTemporal(column, rows);
+            }
+            mark(column, rows);
+        }
+        for (const column of strings)
+            column.offsets[rows + 1] = column.cursor;
+        rows++;
+    };
+    await eachSlice(file, chunkBytes, (slice, last) => {
+        bytes = slice;
+        return eachLine(slice, last, line);
+    }, span);
+    return rows;
+}
+//# sourceMappingURL=text.js.map