vastgrid-js 0.0.0-stage → 0.2.0

package/dist/table.js

@@ -0,0 +1,154 @@
+/**
+ * Tables in memory: what a host gives the engine, and the checks that make them tables of docs/spec.md
+ * §2: columns of §2.1 with unique names, and rows of typed JSON values (§2.2).
+ */
+import { isValueOf, LOGICAL_TYPES, typeName } from "vastgrid-spec";
+/** An array of typed JSON as a vector, which arrays are: they have `length` and `at`. */
+export function vectorOf(values) {
+    return values;
+}
+/**
+ * A table that does not follow §2, or a file that cannot be read, which the engine answers with `load_failed`
+ * (§8.7); a file that changed while it was open with `file_changed`.
+ */
+export class TableProblem extends Error {
+    path;
+    code;
+    constructor(path, message, code = "load_failed") {
+        super(message);
+        this.path = path;
+        this.code = code;
+        this.name = "TableProblem";
+    }
+}
+const COLUMN_MEMBERS = ["name", "type", "precision", "scale", "bits", "unit", "sourceType", "readOnly"];
+const UNITS = ["s", "ms", "us", "ns"];
+export function isObject(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function isInteger(value, min, max) {
+    return typeof value === "number" && Number.isInteger(value) && value >= min && value <= max;
+}
+/** A value as a problem shows it: its JSON, cut to a length that reads. */
+export function shown(value) {
+    const text = value === undefined ? "undefined" : (JSON.stringify(value) ?? String(value));
+    return text.length > 60 ? `${text.slice(0, 57)}...` : text;
+}
+/** Checks a column of a table (§2.1), and gives a copy of it. */
+function checkColumn(value, path) {
+    if (!isObject(value))
+        throw new TableProblem(path, "A column is an object: { name, type, … } (§2.1).");
+    for (const key of Object.keys(value)) {
+        if (!COLUMN_MEMBERS.includes(key))
+            throw new TableProblem(`${path}/${key}`, `"${key}" is not a member of a column (§2.1).`);
+    }
+    const { name, type } = value;
+    if (typeof name !== "string")
+        throw new TableProblem(`${path}/name`, "A column's name is a string.");
+    if (typeof type !== "string" || !LOGICAL_TYPES.includes(type)) {
+        throw new TableProblem(`${path}/type`, `${shown(type)} is not a type of §2.1.`);
+    }
+    if (type === "decimal") {
+        if (!isInteger(value.precision, 1, 38))
+            throw new TableProblem(`${path}/precision`, "A decimal's precision is from 1 to 38.");
+        if (!isInteger(value.scale, 0, value.precision))
+            throw new TableProblem(`${path}/scale`, "A decimal's scale is from 0 to its precision.");
+    }
+    else {
+        for (const member of ["precision", "scale"]) {
+            if (value[member] !== undefined)
+                throw new TableProblem(`${path}/${member}`, "Only decimals have a precision and a scale.");
+        }
+    }
+    if (value.bits !== undefined && (type !== "float" || (value.bits !== 32 && value.bits !== 64))) {
+        throw new TableProblem(`${path}/bits`, "bits is 32 or 64, for floats.");
+    }
+    if (value.unit !== undefined && (type !== "timestamp" || !UNITS.includes(value.unit))) {
+        throw new TableProblem(`${path}/unit`, "unit is s, ms, us or ns, for timestamps.");
+    }
+    if (value.sourceType !== undefined && typeof value.sourceType !== "string") {
+        throw new TableProblem(`${path}/sourceType`, "sourceType is a string.");
+    }
+    if (value.readOnly !== undefined && typeof value.readOnly !== "boolean") {
+        throw new TableProblem(`${path}/readOnly`, "readOnly is a boolean.");
+    }
+    return { ...value };
+}
+/** Throws a `TableProblem` at `path(index)` for the first column whose name an earlier column has. */
+export function checkNames(columns, path) {
+    const names = new Set();
+    columns.forEach((column, index) => {
+        if (names.has(column.name))
+            throw new TableProblem(path(index), `"${column.name}" names two columns.`);
+        names.add(column.name);
+    });
+}
+/** Whether a host's table is given as Arrow (`ArrowTable`), which `checkArrow` of arrow.ts checks. */
+export function isArrowTable(table) {
+    return isObject(table) && "arrow" in table;
+}
+/** Whether a host's table is given as a Parquet file (`ParquetTable`), which parquet.ts reads. */
+export function isParquetTable(table) {
+    return isObject(table) && "parquet" in table;
+}
+/** Whether a host's table is given as a text file (`FileTable`), which `readFile` of text.ts reads. */
+export function isFileTable(table) {
+    return isObject(table) && "file" in table;
+}
+/** Whether a table given as a file is an Excel workbook, which workbook.ts reads. */
+export function isWorkbook(table) {
+    if (table.format !== undefined)
+        return table.format === "xlsx";
+    const name = isObject(table.file) ? table.file.name : undefined;
+    return typeof name === "string" && /\.xls[xm]$/i.test(name);
+}
+/**
+ * Checks the form of a table given as rows against §2: its columns, with unique names, and rows of one value per
+ * column. Gives copies of the columns. Throws a `TableProblem` with the JSON Pointer of the first member at fault.
+ * The values are checked as they are read (`valueChecker`), so that opening a table of millions of rows costs
+ * nothing per value.
+ */
+export function checkTable(table) {
+    if (!isObject(table))
+        throw new TableProblem("", "A table is an object: { columns, rows }, { arrow }, { file } or { parquet }.");
+    if (!Array.isArray(table.columns))
+        throw new TableProblem("/columns", "columns is a list of columns (§2.1).");
+    const columns = table.columns.map((column, index) => checkColumn(column, `/columns/${index}`));
+    checkNames(columns, (index) => `/columns/${index}/name`);
+    const rows = table.rows;
+    if (!Array.isArray(rows))
+        throw new TableProblem("/rows", "rows is a list of rows.");
+    for (let row = 0; row < rows.length; row++) {
+        const values = rows[row];
+        if (!Array.isArray(values) || values.length !== columns.length) {
+            throw new TableProblem(`/rows/${row}`, `A row is a list of ${columns.length} values, one per column.`);
+        }
+    }
+    return { columns, rows, vectors: null, rowCount: rows.length };
+}
+/**
+ * Checks the values of a column of a table (§2.2): a function of a value and its row that throws a
+ * `TableProblem` at `/rows/{row}/{index}` for a value that is not of the column's type.
+ */
+export function valueChecker(column, index) {
+    return (value, row) => {
+        if (value !== null &&
+            !((typeof value === "string" || typeof value === "number" || typeof value === "boolean") &&
+                isValueOf(value, column))) {
+            throw new TableProblem(`/rows/${row}/${index}`, `${shown(value)} is not a ${typeName(column)} value of column "${column.name}" (§2.2).`);
+        }
+    };
+}
+/**
+ * The value of a column of an Arrow table at a row. A value that has no typed JSON, such as text that is not
+ * UTF-8 or a time beyond a day, throws a `TableProblem` at `/rows/{row}/{index}`.
+ */
+export function vectorValue(vector, column, index, row) {
+    try {
+        return vector.at(row);
+    }
+    catch (error) {
+        throw new TableProblem(`/rows/${row}/${index}`, `the value is not a ${typeName(column)} value of column "${column.name}" (§2.2): ${error.message}`);
+    }
+}
+//# sourceMappingURL=table.js.map

package/dist/table.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"table.js","sourceRoot":"","sources":["../src/table.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAe,SAAS,EAAkB,aAAa,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAehG,yFAAyF;AACzF,MAAM,UAAU,QAAQ,CAAC,MAA4B;IACnD,OAAO,MAA2B,CAAC;AACrC,CAAC;AAwGD;;;GAGG;AACH,MAAM,OAAO,YAAa,SAAQ,KAAK;IAE1B,IAAI;IAEJ,IAAI;IAHf,YACW,IAAY,EACrB,OAAe,EACN,IAAI,GAAmC,aAAa;QAE7D,KAAK,CAAC,OAAO,CAAC,CAAC;oBAJN,IAAI;oBAEJ,IAAI;QAGb,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;AACxG,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEtC,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,EAAE,GAAW,EAAE,GAAW;IACzD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC9F,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,KAAK,CAAC,KAAc;IAClC,MAAM,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1F,OAAO,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,KAAc,EAAE,IAAY;IAC/C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,YAAY,CAAC,IAAI,EAAE,kDAAkD,CAAC,CAAC;IACvG,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,IAAI,GAAG,uCAAuC,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAC7B,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,OAAO,EAAE,8BAA8B,CAAC,CAAC;IACrG,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAE,aAAmC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACrF,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,OAAO,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,YAAY,EAAE,wCAAwC,CAAC,CAAC;QACxF,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;YAC7C,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC7F,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,MAAM,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,SAAS;gBAC7B,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,IAAI,MAAM,EAAE,EAAE,6CAA6C,CAAC,CAAC;QAC/F,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC;QAC/F,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,OAAO,EAAE,+BAA+B,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAc,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,OAAO,EAAE,0CAA0C,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC3E,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,aAAa,EAAE,yBAAyB,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACxE,MAAM,IAAI,YAAY,CAAC,GAAG,IAAI,WAAW,EAAE,wBAAwB,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,EAAE,GAAG,KAAK,EAAuB,CAAC;AAC3C,CAAC;AAcD,sGAAsG;AACtG,MAAM,UAAU,UAAU,CAAC,OAA0B,EAAE,IAA+B;IACpF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,MAAM,CAAC,IAAI,sBAAsB,CAAC,CAAC;QACvG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,sGAAsG;AACtG,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,IAAI,KAAK,CAAC;AAC7C,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,CAAC;AAC/C,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,WAAW,CAAC,KAAc;IACxC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,KAAK,CAAC;AAC5C,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,UAAU,CAAC,KAAgB;IACzC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC;IAC/D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAChE,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,KAAc;IACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,EAAE,EAAE,8EAA8E,CAAC,CAAC;IAC7G,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,YAAY,CAAC,UAAU,EAAE,sCAAsC,CAAC,CAAC;IAC9G,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/F,UAAU,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,KAAK,OAAO,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,YAAY,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAC;IACrF,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YAC/D,MAAM,IAAI,YAAY,CAAC,SAAS,GAAG,EAAE,EAAE,sBAAsB,OAAO,CAAC,MAAM,0BAA0B,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AACjE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,KAAa;IACxD,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACpB,IACE,KAAK,KAAK,IAAI;YACd,CAAC,CACC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,CAAC;gBACtF,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CACzB,EACD,CAAC;YACD,MAAM,IAAI,YAAY,CACpB,SAAS,GAAG,IAAI,KAAK,EAAE,EACvB,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,IAAI,WAAW,CACxF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,MAAc,EAAE,KAAa,EAAE,GAAW;IACpF,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,YAAY,CACpB,SAAS,GAAG,IAAI,KAAK,EAAE,EACvB,sBAAsB,QAAQ,CAAC,MAAM,CAAC,qBAAqB,MAAM,CAAC,IAAI,aAAc,KAAe,CAAC,OAAO,EAAE,CAC9G,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/text.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Tables given as text files (CSV, TSV and JSON Lines in UTF-8), read by a module of their own that the engine
+ * loads with the first such table, so that pages whose tables are rows do not load it. A file is read in slices,
+ * first to find its columns, the types that all their values fit, the memory they take and where every 1,024th row
+ * starts. When its columns fit `maxBytes`, it is read again to keep its values in typed columns, numbers in typed
+ * arrays and text as UTF-8, without making rows of JSON; otherwise its rows are read by blocks of 1,024 rows as
+ * requests need them, and columns whole only when a request goes through every row of them (lazy.ts).
+ */
+import { type Column } from "vastgrid-spec";
+import { type CheckedTable, type FileTable } from "./table.js";
+/**
+ * Reads a table given as a text file into typed columns. Throws a `TableProblem` at the member of the table at
+ * fault, or at `/rows/{row}/{column}` (`/rows/{row}` for a line of JSON Lines) for text that is not UTF-8 or a
+ * line that is not JSON. `fields` gives the column of each field of a CSV file, whose values fit its type, or null
+ * for a field that is left out (workbook.ts); by default each field is a column of the type its values fit.
+ */
+export declare function readFile(table: FileTable, chunkBytes?: number, fields?: readonly (Column | null)[]): Promise<CheckedTable>;
+//# sourceMappingURL=text.d.ts.map

package/dist/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EACL,KAAK,MAAM,EAOZ,MAAM,eAAe,CAAC;AAIvB,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,SAAS,EAMf,MAAM,YAAY,CAAC;AAsGpB;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,EAChB,UAAU,SAAc,EACxB,MAAM,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,GAClC,OAAO,CAAC,YAAY,CAAC,CAsDvB"}