qvdjs 2.2.1 → 2.2.3

package/dist/index.js

@@ -1189,6 +1189,212 @@ var init_bitUtils = __esm({
   }
 });
 
+// src/util/seenNumbers.js
+function firstJudgementRow(rows) {
+  return Math.min(WINDOW_ROWS, Math.max(MIN_JUDGED_LOOKUPS, Math.floor(rows / 2)));
+}
+function slotOf(value, mask) {
+  const product = value * MULTIPLIER;
+  return product < EXACT && product > -EXACT ? product & mask : ((product | 0) ^ (product * UPPER_HALF | 0)) & mask;
+}
+function slotsFor(count, most) {
+  let slots = MIN_SLOTS;
+  while (slots < count && slots < most) {
+    slots *= 2;
+  }
+  return slots;
+}
+function shareOf(columns) {
+  let slots = MAX_SLOTS;
+  while (slots > MIN_SLOTS && slots * columns > TOTAL_SLOTS) {
+    slots /= 2;
+  }
+  return slots;
+}
+function symbolPassTable(rows, pool) {
+  return pool.take(MIN_SLOTS) ? new SeenNumbers(MIN_SLOTS, slotsFor(rows, MAX_SLOTS), SYMBOL_PASS_MISS_LIMIT, pool) : null;
+}
+function numericCellsAfter(data, column, stopped, numbers) {
+  const most = stopped.cells + data.length - stopped.row;
+  if (!indexPassWants(numbers, most)) {
+    return most;
+  }
+  let cells = stopped.cells;
+  for (let row = stopped.row; row < data.length; row++) {
+    if (typeof data[row]?.[column] === "number") {
+      cells++;
+    }
+  }
+  return cells;
+}
+function numbersIn(map) {
+  let numbers = 0;
+  for (const key of map.keys()) {
+    if (typeof key === "number") {
+      numbers++;
+    }
+  }
+  return numbers;
+}
+function indexPassWants(numbers, cells) {
+  return numbers > 0 && numbers <= INDEX_PASS_MISS_LIMIT * cells;
+}
+function indexPassTable(numbers, cells, symbols, share, pool) {
+  const slots = Math.min(slotsFor(numbers * SLOTS_PER_NUMBER, share), slotsFor(cells, share));
+  return pool.take(slots) ? new SeenNumbers(slots, slots, INDEX_PASS_MISS_LIMIT - numbers / cells, pool, symbols) : null;
+}
+function endWindow(tables, row = 0, stopped = null) {
+  for (let column = 0; column < tables.length; column++) {
+    const table = tables[column];
+    if (table === null || table === NO_TABLE_YET) {
+      continue;
+    }
+    const lookups = table.cells - table.judgedCells;
+    if (lookups < MIN_JUDGED_LOOKUPS) {
+      continue;
+    }
+    if (table.misses > (table.judged ? table.limit : FIRST_WINDOW_MISS_LIMIT) * lookups) {
+      tables[column] = null;
+      table.pool.give(table.values.length);
+      if (stopped !== null) {
+        stopped[column] = { row, cells: table.cells };
+      }
+    } else {
+      table.judgedCells = table.cells;
+      table.misses = 0;
+      table.judged = true;
+    }
+  }
+}
+var MULTIPLIER, EXACT, UPPER_HALF, WINDOW_ROWS, MIN_JUDGED_LOOKUPS, MIN_SLOTS, MAX_SLOTS, TOTAL_SLOTS, SLOTS_PER_NUMBER, FIRST_WINDOW_MISS_LIMIT, SYMBOL_PASS_MISS_LIMIT, INDEX_PASS_MISS_LIMIT, SlotPool, SeenNumbers, NO_TABLE_YET;
+var init_seenNumbers = __esm({
+  "src/util/seenNumbers.js"() {
+    MULTIPLIER = 2654435761;
+    EXACT = 2 ** 53;
+    UPPER_HALF = 2 ** -32;
+    WINDOW_ROWS = 16384;
+    MIN_JUDGED_LOOKUPS = 1024;
+    __name(firstJudgementRow, "firstJudgementRow");
+    MIN_SLOTS = 64;
+    MAX_SLOTS = 65536;
+    TOTAL_SLOTS = 2 ** 21;
+    SLOTS_PER_NUMBER = 8;
+    FIRST_WINDOW_MISS_LIMIT = 0.9;
+    SYMBOL_PASS_MISS_LIMIT = 0.3;
+    INDEX_PASS_MISS_LIMIT = 0.75;
+    __name(slotOf, "slotOf");
+    __name(slotsFor, "slotsFor");
+    __name(shareOf, "shareOf");
+    SlotPool = class {
+      static {
+        __name(this, "SlotPool");
+      }
+      /**
+       * @param {number} [slots] How many there are to give.
+       */
+      constructor(slots = TOTAL_SLOTS) {
+        this.left = slots;
+      }
+      /**
+       * Takes slots, if there are that many left.
+       *
+       * @param {number} slots How many.
+       * @return {boolean} Whether they were taken.
+       */
+      take(slots) {
+        if (slots > this.left) {
+          return false;
+        }
+        this.left -= slots;
+        return true;
+      }
+      /**
+       * Gives slots back.
+       *
+       * @param {number} slots How many.
+       */
+      give(slots) {
+        this.left += slots;
+      }
+    };
+    SeenNumbers = class _SeenNumbers {
+      static {
+        __name(this, "SeenNumbers");
+      }
+      /**
+       * @param {number} slots How many slots: a power of two from `MIN_SLOTS` to `MAX_SLOTS`, already taken from
+       *   `pool`.
+       * @param {number} maxSlots The most it may grow to, in the symbol table pass.
+       * @param {number} limit The share of its lookups a judged window may miss.
+       * @param {SlotPool} pool Where its slots came from, and where they go back when it is switched off.
+       * @param {number} [symbols] In the index table pass, the column's symbols, one bit each. -1, the default, in
+       *   the symbol table pass.
+       */
+      constructor(slots, maxSlots, limit, pool, symbols = -1) {
+        this.values = new Float64Array(slots).fill(NaN);
+        this.indices = new Int32Array(symbols >= 0 ? slots : 0);
+        this.met = new Uint32Array(symbols >= 0 ? Math.ceil(symbols / 32) : 0);
+        this.mask = slots - 1;
+        this.maxSlots = maxSlots;
+        this.limit = limit;
+        this.pool = pool;
+        this.cells = 0;
+        this.judgedCells = 0;
+        this.misses = 0;
+        this.judged = symbols >= 0;
+        this.inserted = 0;
+      }
+      /**
+       * Remembers a number the symbol table pass has just put in the map, growing the table instead once more
+       * than an eighth of its slots would hold distinct numbers - if its pool has the slots. A table the pool
+       * cannot grow stops trying, and keeps remembering in the slots it has.
+       *
+       * @param {number} slot The number's slot, from `slotOf`.
+       * @param {number} value The number.
+       * @return {SeenNumbers} The table to use from now on: this one, or a larger, empty one.
+       */
+      remember(slot, value) {
+        this.inserted++;
+        if (this.inserted * SLOTS_PER_NUMBER > this.values.length && this.values.length < this.maxSlots) {
+          const slots = Math.min(this.values.length * 4, this.maxSlots);
+          if (this.pool.take(slots - this.values.length)) {
+            const grown = new _SeenNumbers(slots, this.maxSlots, this.limit, this.pool);
+            grown.inserted = this.inserted;
+            grown.cells = this.cells;
+            grown.judgedCells = this.judgedCells;
+            grown.misses = this.misses;
+            grown.judged = this.judged;
+            return grown;
+          }
+          this.maxSlots = this.values.length;
+        }
+        this.values[slot] = value;
+        return this;
+      }
+      /**
+       * Marks a symbol met through this table, in the index table pass.
+       *
+       * @param {number} index The symbol the map answered.
+       * @return {boolean} Whether it had been met before; false for a first sight.
+       */
+      metBefore(index) {
+        const word = index >>> 5;
+        const bit = 1 << (index & 31);
+        const before = (this.met[word] & bit) !== 0;
+        this.met[word] |= bit;
+        return before;
+      }
+    };
+    NO_TABLE_YET = new SeenNumbers(1, 1, 0, new SlotPool(1));
+    __name(symbolPassTable, "symbolPassTable");
+    __name(numericCellsAfter, "numericCellsAfter");
+    __name(numbersIn, "numbersIn");
+    __name(indexPassWants, "indexPassWants");
+    __name(indexPassTable, "indexPassTable");
+    __name(endWindow, "endWindow");
+  }
+});
+
 // src/QvdFileWriter.js
 var QvdFileWriter_exports = {};
 __export(QvdFileWriter_exports, {
@@ -1476,6 +1682,7 @@ var init_QvdFileWriter = __esm({
     init_cellRules();
     init_symbolBytes();
     init_storedSymbols();
+    init_seenNumbers();
     __name(notXmlIndex, "notXmlIndex");
     __name(checkHeaderText, "checkHeaderText");
     __name(checkHeaderTexts, "checkHeaderTexts");
@@ -1938,7 +2145,14 @@ var init_QvdFileWriter = __esm({
         const byCells = state.map((column) => column.byCell);
         const byObjects = state.map((column) => column.byObject);
         const containsNull = columns.map(() => false);
+        const pool = new SlotPool();
+        const seen = columns.map(() => NO_TABLE_YET);
+        const firstJudgement = firstJudgementRow(numRows);
+        const stopped = columns.map(() => null);
         for (let row = 0; row < numRows; row++) {
+          if (row % WINDOW_ROWS === 0 && row !== 0 || row === firstJudgement) {
+            endWindow(seen, row, stopped);
+          }
           const values = data[row];
           if (values !== null && values !== void 0 && !Array.isArray(values)) {
             throw new QvdValidationError("Each row must be an array of values", {
@@ -1969,7 +2183,34 @@ var init_QvdFileWriter = __esm({
               }
               continue;
             }
+            let table = null;
+            let slot = 0;
+            if (typeof value === "number") {
+              table = seen[column];
+              if (table !== null) {
+                table.cells++;
+                slot = slotOf(value, table.mask);
+                if (table.values[slot] === value) {
+                  continue;
+                }
+                table.misses++;
+                if (table === NO_TABLE_YET) {
+                  table = symbolPassTable(numRows, pool);
+                  seen[column] = table;
+                  if (table === null) {
+                    stopped[column] = { row, cells: 0 };
+                  } else {
+                    table.cells = 1;
+                    table.misses = 1;
+                    slot = slotOf(value, table.mask);
+                  }
+                }
+              }
+            }
             if (byCells[column].has(value)) {
+              if (table !== null) {
+                table.values[slot] = value;
+              }
               continue;
             }
             if (typeof value === "number") {
@@ -1989,6 +2230,9 @@ var init_QvdFileWriter = __esm({
               refuseCell(value, columns[column], row, this._path);
             }
             byCells[column].set(value, slotForCell(state[column], value, row, this._path));
+            if (table !== null && typeof value === "number") {
+              seen[column] = table.remember(slot, value);
+            }
           }
         }
         const columnBuffers = [];
@@ -1997,10 +2241,12 @@ var init_QvdFileWriter = __esm({
           const { keys, texts } = state[column];
           const kinds = new Uint8Array(keys.length);
           let byteLength = 0;
+          let numberKeys = 0;
           for (let slot = 0; slot < keys.length; slot++) {
             const key = keys[slot];
             const text = texts[slot];
             if (typeof key === "number") {
+              numberKeys++;
               if (numberProblem(key) !== null) {
                 checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
               }
@@ -2029,10 +2275,20 @@ var init_QvdFileWriter = __esm({
           this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
           this._symbolCounts?.push(keys.length);
           this._symbolFacts?.push(state[column].facts);
+          const numbers = state[column].byCell === state[column].byKey ? numberKeys : numbersIn(state[column].byCell);
+          const table = seen[column];
           this._symbolIndexByValue?.push({
             byCell: state[column].byCell,
             byKey: state[column].byKey,
-            byObject: state[column].byObject
+            byObject: state[column].byObject,
+            numbers,
+            cells: table === NO_TABLE_YET ? 0 : table !== null ? table.cells : numericCellsAfter(
+              data,
+              column,
+              /** @type {{row: number, cells: number}} */
+              stopped[column],
+              numbers
+            )
           });
           symbolsOffset += byteLength;
           this._emitProgress("symbol-table", column + 1, numColumns);
@@ -2082,7 +2338,7 @@ var init_QvdFileWriter = __esm({
           const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
           const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
           const lookup = this._symbolIndexByValue[column];
-          const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
+          const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : [lookup.byCell, lookup.byKey, lookup.byObject];
           for (const map of maps) {
             for (const index of map.values()) {
               if (index + nullShift > maxStoredIndex) {
@@ -2108,14 +2364,44 @@ var init_QvdFileWriter = __esm({
         const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
         const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
         const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
+        const symbolCounts = this._symbolCounts;
+        const wanted = this._symbolIndexByValue.map((lookup) => indexPassWants(lookup.numbers, lookup.cells));
+        const share = shareOf(wanted.filter(Boolean).length);
+        const pool = new SlotPool();
+        const seen = this._symbolIndexByValue.map(
+          (lookup, column) => wanted[column] ? indexPassTable(lookup.numbers, lookup.cells, symbolCounts[column], share, pool) : null
+        );
+        const firstJudgement = firstJudgementRow(numRows);
         for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
+          if (row % WINDOW_ROWS === 0 && row !== 0 || row === firstJudgement) {
+            endWindow(seen);
+          }
           const values = data[row];
           for (let column = 0; column < numColumns; column++) {
             const value = values?.[column];
             if (value === null || value === void 0) {
               continue;
             }
-            const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
+            const table = typeof value === "number" ? seen[column] : null;
+            let index;
+            if (table !== null) {
+              table.cells++;
+              const slot = slotOf(value, table.mask);
+              if (table.values[slot] === value) {
+                index = table.indices[slot];
+              } else {
+                index = byCells[column].get(value);
+                if (index !== void 0) {
+                  table.values[slot] = value;
+                  table.indices[slot] = index;
+                  if (table.metBefore(index)) {
+                    table.misses++;
+                  }
+                }
+              }
+            } else {
+              index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
+            }
             if (index === void 0) {
               throw new QvdValidationError("A value is missing from the symbol table", {
                 field: columns[column],
@@ -5129,12 +5415,14 @@ var init_QvdFileReader = __esm({
        * viewer scrolls to - and the header is already in hand. `parseHeaderOnly` has to have run.
        *
        * @param {QvdRowWindow} window The rows the read would cover, normalised.
-       * @param {{chunkSize?: number|null, fields?: Array<string>|null, materialisesRows?: boolean}} [options]
-       *   `chunkSize` for an `iterate()`; `fields` and `materialisesRows` to ask about a read other than the
-       *   one this reader was built for, which is what a `QvdFile` does per call.
+       * @param {{chunkSize?: number|null, fields?: Array<string>|null, materialisesRows?: boolean,
+       *   paging?: boolean}} [options] `chunkSize` for an `iterate()`; `fields` and `materialisesRows` to ask
+       *   about a read other than the one this reader was built for, which is what a `QvdFile` does per call;
+       *   `paging` to say whether the read being asked about is a paging read, which defaults to whether this
+       *   reader is one.
        * @return {any} The answer - see `checkMemory`.
        */
-      checkParsed(window, { chunkSize = null, fields = void 0, materialisesRows = void 0 } = {}) {
+      checkParsed(window, { chunkSize = null, fields = void 0, materialisesRows = void 0, paging = this._symbolCache !== null } = {}) {
         assert4(this._header && this._allFields, "The QVD file header has not been parsed.");
         const header = this._header["QvdTableHeader"];
         const totalRows = headerInteger(header["NoOfRecords"]);
@@ -5145,15 +5433,13 @@ var init_QvdFileReader = __esm({
         const resolved = resolveWindow(window, totalRows);
         const windowRows = resolved.limit;
         const liveRows = chunkSize === null ? null : { rows: chunkSize * 2, perChunk: 2 };
-        const analysisAhead = this._analysisAhead(window, resolved, totalRows, symbolTableLength);
+        const analysisAhead = paging ? false : this._analysisWouldRun(window, resolved, totalRows, symbolTableLength);
         const measured = getMemoryBudget();
         const ask = /* @__PURE__ */ __name((asked, rows) => {
-          const bytes = symbolBytesOf(this._fieldsHeldAfter(asked, this._allFields), symbolTableLength);
-          const read = symbolBytesOf(this._fieldsReadBy(asked), symbolTableLength);
-          const retained = symbolBytesOf(
-            this._fieldsHeldAfter(asked, this._allFields).slice(asked.length),
-            symbolTableLength
-          );
+          const held = paging ? this._fieldsHeldAfter(asked, this._allFields) : asked;
+          const bytes = symbolBytesOf(held, symbolTableLength);
+          const read = symbolBytesOf(paging ? this._fieldsReadBy(asked) : asked, symbolTableLength);
+          const retained = paging ? symbolBytesOf(held.slice(asked.length), symbolTableLength) : 0;
           return checkMemory({
             measured,
             symbolTableSize: bytes,
@@ -5164,7 +5450,7 @@ var init_QvdFileReader = __esm({
             materialisesRows: builds,
             live: liveRows,
             // A paging read keeps whole columns, so it is charged for whole columns - see `estimateMemoryUsage`.
-            wholeSymbols: this._symbolCache !== null,
+            wholeSymbols: paging,
             retainedBytes: retained,
             bytesHeld: this._bytesHeld(read, rows, recordSize, liveRows, analysisAhead, bytes - retained),
             // What it reads from the file, which is not what it holds: the symbol areas it has still to read,
@@ -5499,12 +5785,16 @@ var init_QvdFile = __esm({
       /**
        * What a read of this file would cost, and whether it fits - with no I/O at all.
        *
-       * The same answer `QvdDataFrame.checkRead()` gives, from the header this file already holds, so a
-       * viewer can size a page before asking for it without touching the disk.
+       * Without `chunkSize` it asks about a page, and answers for the page this file would read next: the
+       * columns it names plus the ones earlier pages decoded and kept, since those are part of what the page
+       * costs. With `chunkSize` it asks about an `iterate()` of the file instead - which this object cannot
+       * make, but a caller holding it may want to weigh - and answers exactly as `QvdDataFrame.checkRead()`
+       * would, from the header this file already holds rather than by reading it again.
        *
        * @param {{offset?: number, limit?: number|null, maxRows?: number|null, fields?: Array<string>|null,
        *   as?: 'rows'|'columns', chunkSize?: number|null}} [options] The read being asked about - the same
-       *   bag `rows()` takes, plus `as` and `chunkSize` to say which shape of read it is.
+       *   bag `rows()` takes, plus `as` to say which shape of page, and `chunkSize` to ask about an
+       *   `iterate()` of the file rather than a page.
        * @return {any} The answer - `fits`, `reason`, `estimate`, `budget`, `exact`, `suggestions`.
        * @throws {QvdValidationError} If the file is closed, or an option's value is not valid.
        */
@@ -5523,7 +5813,7 @@ var init_QvdFile = __esm({
         if (chunkSize !== null) {
           requireChunkSize(chunkSize, this._options.path);
         }
-        const reader = this._readers[as] ?? this._reader;
+        const reader = chunkSize !== null ? this._reader : this._readers[as] ?? this._reader;
         return reader.checkParsed(normaliseWindow(windowFrom(options), this._options.path), {
           chunkSize,
           // Resolved here rather than left to the reader, exactly as `_page` resolves it. A warm reader is
@@ -5532,7 +5822,12 @@ var init_QvdFile = __esm({
           // file after a page naming one of them, it reported 27,490 bytes for a read that costs 108,160:
           // understating, which is the direction that approves a read the read then refuses.
           fields: options.fields === void 0 ? this._options.fields ?? null : options.fields,
-          materialisesRows: as === "rows"
+          materialisesRows: as === "rows",
+          // Said outright rather than read off the reader. The header reader has paging on, because a check
+          // made before the first page has to be priced as that page will be - so it cannot tell an `iterate()`
+          // question from a page one by its own mode, and routing to it alone still priced the iterate as a
+          // page wherever a bounded window would have been filtered.
+          ...chunkSize === null ? {} : { paging: false }
         });
       }
       /**