qvdjs 2.2.2 → 2.2.4

package/README.md

@@ -42,28 +42,31 @@ thing worth reading:
 
 ## Checked against Qlik Sense
 
-qvdjs is compared with Qlik Sense itself, not only with its own reading of the format. 36 fields in
-7 datasets - integers, doubles, text (non-ASCII, empty and number-like included), booleans, dates,
-timestamps and NULLs - are defined twice, as Qlik load-script expressions and as the JavaScript
-values a caller would write. Qlik Sense and qvdjs each write them to a QVD, and three things are
-checked:
+qvdjs is compared with Qlik Sense itself, not only with its own reading of the format. 48 fields in
+8 datasets - integers, doubles, text (non-ASCII, empty and number-like included), booleans, dates,
+timestamps, and NULLs beside numbers, text, dates and timestamps - are defined twice, as Qlik
+load-script expressions and as the JavaScript values a caller would write. Qlik Sense and qvdjs each
+write them to a QVD, and three things are checked:
 
 - **qvdjs writes the content Qlik writes** - the same symbols, of the same types, with the same text,
   on the same rows.
-- **Given Qlik's tags and number formats, qvdjs writes them unchanged** - the bit layout, below, is
-  the only part of a field's header left to differ.
+- **Given Qlik's tags and number formats, qvdjs writes them unchanged** - the bit layout and a tag the
+  values contradict, below, are the only parts of a field's header left to differ.
 - **Qlik reads qvdjs's file as it reads its own** - loaded into Qlik and stored again, with `Text()`,
   `Num()`, `IsNum()`, `IsText()`, `IsNull()` and `Year()` the same on every row.
 
 The live comparison runs on demand against a Qlik Sense server; its last run, against Qlik Sense
-Enterprise on Windows build 50699, passed for all 36 fields. Qlik's files from that run are kept in
+Enterprise on Windows build 50699, passed for all 48 fields. Qlik's files from that run are kept in
 the test suite, so **every CI run compares qvdjs's output with them again** - on Linux, Windows and
 macOS, under Node 22, 24 and 26 - along with reference files Qlik wrote for booleans, dates and
 timestamps.
 
-Two differences are by design: Qlik often gives a field more bits in the index than its values need,
-where qvdjs uses the fewest, and qvdjs adds no tags or number formats of its own - Qlik works tags out
-when it loads a file. QVD is also QlikView's format, but QlikView is not part of the comparison.
+Three differences are by design. Qlik often gives a field more bits in the index than its values need,
+where qvdjs uses the fewest. qvdjs adds no tags or number formats of its own - Qlik works tags out when
+it loads a file, and keeps any the file carries. And qvdjs drops a tag the values contradict, even one
+Qlik wrote: Qlik tags dates outside 1980 to 2080 as text when their field has no date format, although
+its own `IsNum()` reads every one as a number. QVD is also QlikView's format, but QlikView is not part
+of the comparison.
 
 → [Checked against Qlik Sense](https://qvdjs.ptarmiganlabs.com/v2.0/overview/checked-against-qlik-sense/)
 

package/dist/index.cjs

@@ -1201,6 +1201,212 @@ var init_bitUtils = __esm({
   }
 });
 
+// src/util/seenNumbers.js
+function firstJudgementRow(rows) {
+  return Math.min(WINDOW_ROWS, Math.max(MIN_JUDGED_LOOKUPS, Math.floor(rows / 2)));
+}
+function slotOf(value, mask) {
+  const product = value * MULTIPLIER;
+  return product < EXACT && product > -EXACT ? product & mask : ((product | 0) ^ (product * UPPER_HALF | 0)) & mask;
+}
+function slotsFor(count, most) {
+  let slots = MIN_SLOTS;
+  while (slots < count && slots < most) {
+    slots *= 2;
+  }
+  return slots;
+}
+function shareOf(columns) {
+  let slots = MAX_SLOTS;
+  while (slots > MIN_SLOTS && slots * columns > TOTAL_SLOTS) {
+    slots /= 2;
+  }
+  return slots;
+}
+function symbolPassTable(rows, pool) {
+  return pool.take(MIN_SLOTS) ? new SeenNumbers(MIN_SLOTS, slotsFor(rows, MAX_SLOTS), SYMBOL_PASS_MISS_LIMIT, pool) : null;
+}
+function numericCellsAfter(data, column, stopped, numbers) {
+  const most = stopped.cells + data.length - stopped.row;
+  if (!indexPassWants(numbers, most)) {
+    return most;
+  }
+  let cells = stopped.cells;
+  for (let row = stopped.row; row < data.length; row++) {
+    if (typeof data[row]?.[column] === "number") {
+      cells++;
+    }
+  }
+  return cells;
+}
+function numbersIn(map) {
+  let numbers = 0;
+  for (const key of map.keys()) {
+    if (typeof key === "number") {
+      numbers++;
+    }
+  }
+  return numbers;
+}
+function indexPassWants(numbers, cells) {
+  return numbers > 0 && numbers <= INDEX_PASS_MISS_LIMIT * cells;
+}
+function indexPassTable(numbers, cells, symbols, share, pool) {
+  const slots = Math.min(slotsFor(numbers * SLOTS_PER_NUMBER, share), slotsFor(cells, share));
+  return pool.take(slots) ? new SeenNumbers(slots, slots, INDEX_PASS_MISS_LIMIT - numbers / cells, pool, symbols) : null;
+}
+function endWindow(tables, row = 0, stopped = null) {
+  for (let column = 0; column < tables.length; column++) {
+    const table = tables[column];
+    if (table === null || table === NO_TABLE_YET) {
+      continue;
+    }
+    const lookups = table.cells - table.judgedCells;
+    if (lookups < MIN_JUDGED_LOOKUPS) {
+      continue;
+    }
+    if (table.misses > (table.judged ? table.limit : FIRST_WINDOW_MISS_LIMIT) * lookups) {
+      tables[column] = null;
+      table.pool.give(table.values.length);
+      if (stopped !== null) {
+        stopped[column] = { row, cells: table.cells };
+      }
+    } else {
+      table.judgedCells = table.cells;
+      table.misses = 0;
+      table.judged = true;
+    }
+  }
+}
+var MULTIPLIER, EXACT, UPPER_HALF, WINDOW_ROWS, MIN_JUDGED_LOOKUPS, MIN_SLOTS, MAX_SLOTS, TOTAL_SLOTS, SLOTS_PER_NUMBER, FIRST_WINDOW_MISS_LIMIT, SYMBOL_PASS_MISS_LIMIT, INDEX_PASS_MISS_LIMIT, SlotPool, SeenNumbers, NO_TABLE_YET;
+var init_seenNumbers = __esm({
+  "src/util/seenNumbers.js"() {
+    MULTIPLIER = 2654435761;
+    EXACT = 2 ** 53;
+    UPPER_HALF = 2 ** -32;
+    WINDOW_ROWS = 16384;
+    MIN_JUDGED_LOOKUPS = 1024;
+    __name(firstJudgementRow, "firstJudgementRow");
+    MIN_SLOTS = 64;
+    MAX_SLOTS = 65536;
+    TOTAL_SLOTS = 2 ** 21;
+    SLOTS_PER_NUMBER = 8;
+    FIRST_WINDOW_MISS_LIMIT = 0.9;
+    SYMBOL_PASS_MISS_LIMIT = 0.3;
+    INDEX_PASS_MISS_LIMIT = 0.75;
+    __name(slotOf, "slotOf");
+    __name(slotsFor, "slotsFor");
+    __name(shareOf, "shareOf");
+    SlotPool = class {
+      static {
+        __name(this, "SlotPool");
+      }
+      /**
+       * @param {number} [slots] How many there are to give.
+       */
+      constructor(slots = TOTAL_SLOTS) {
+        this.left = slots;
+      }
+      /**
+       * Takes slots, if there are that many left.
+       *
+       * @param {number} slots How many.
+       * @return {boolean} Whether they were taken.
+       */
+      take(slots) {
+        if (slots > this.left) {
+          return false;
+        }
+        this.left -= slots;
+        return true;
+      }
+      /**
+       * Gives slots back.
+       *
+       * @param {number} slots How many.
+       */
+      give(slots) {
+        this.left += slots;
+      }
+    };
+    SeenNumbers = class _SeenNumbers {
+      static {
+        __name(this, "SeenNumbers");
+      }
+      /**
+       * @param {number} slots How many slots: a power of two from `MIN_SLOTS` to `MAX_SLOTS`, already taken from
+       *   `pool`.
+       * @param {number} maxSlots The most it may grow to, in the symbol table pass.
+       * @param {number} limit The share of its lookups a judged window may miss.
+       * @param {SlotPool} pool Where its slots came from, and where they go back when it is switched off.
+       * @param {number} [symbols] In the index table pass, the column's symbols, one bit each. -1, the default, in
+       *   the symbol table pass.
+       */
+      constructor(slots, maxSlots, limit, pool, symbols = -1) {
+        this.values = new Float64Array(slots).fill(NaN);
+        this.indices = new Int32Array(symbols >= 0 ? slots : 0);
+        this.met = new Uint32Array(symbols >= 0 ? Math.ceil(symbols / 32) : 0);
+        this.mask = slots - 1;
+        this.maxSlots = maxSlots;
+        this.limit = limit;
+        this.pool = pool;
+        this.cells = 0;
+        this.judgedCells = 0;
+        this.misses = 0;
+        this.judged = symbols >= 0;
+        this.inserted = 0;
+      }
+      /**
+       * Remembers a number the symbol table pass has just put in the map, growing the table instead once more
+       * than an eighth of its slots would hold distinct numbers - if its pool has the slots. A table the pool
+       * cannot grow stops trying, and keeps remembering in the slots it has.
+       *
+       * @param {number} slot The number's slot, from `slotOf`.
+       * @param {number} value The number.
+       * @return {SeenNumbers} The table to use from now on: this one, or a larger, empty one.
+       */
+      remember(slot, value) {
+        this.inserted++;
+        if (this.inserted * SLOTS_PER_NUMBER > this.values.length && this.values.length < this.maxSlots) {
+          const slots = Math.min(this.values.length * 4, this.maxSlots);
+          if (this.pool.take(slots - this.values.length)) {
+            const grown = new _SeenNumbers(slots, this.maxSlots, this.limit, this.pool);
+            grown.inserted = this.inserted;
+            grown.cells = this.cells;
+            grown.judgedCells = this.judgedCells;
+            grown.misses = this.misses;
+            grown.judged = this.judged;
+            return grown;
+          }
+          this.maxSlots = this.values.length;
+        }
+        this.values[slot] = value;
+        return this;
+      }
+      /**
+       * Marks a symbol met through this table, in the index table pass.
+       *
+       * @param {number} index The symbol the map answered.
+       * @return {boolean} Whether it had been met before; false for a first sight.
+       */
+      metBefore(index) {
+        const word = index >>> 5;
+        const bit = 1 << (index & 31);
+        const before = (this.met[word] & bit) !== 0;
+        this.met[word] |= bit;
+        return before;
+      }
+    };
+    NO_TABLE_YET = new SeenNumbers(1, 1, 0, new SlotPool(1));
+    __name(symbolPassTable, "symbolPassTable");
+    __name(numericCellsAfter, "numericCellsAfter");
+    __name(numbersIn, "numbersIn");
+    __name(indexPassWants, "indexPassWants");
+    __name(indexPassTable, "indexPassTable");
+    __name(endWindow, "endWindow");
+  }
+});
+
 // src/QvdFileWriter.js
 var QvdFileWriter_exports = {};
 __export(QvdFileWriter_exports, {
@@ -1488,6 +1694,7 @@ var init_QvdFileWriter = __esm({
     init_cellRules();
     init_symbolBytes();
     init_storedSymbols();
+    init_seenNumbers();
     __name(notXmlIndex, "notXmlIndex");
     __name(checkHeaderText, "checkHeaderText");
     __name(checkHeaderTexts, "checkHeaderTexts");
@@ -1950,7 +2157,14 @@ var init_QvdFileWriter = __esm({
         const byCells = state.map((column) => column.byCell);
         const byObjects = state.map((column) => column.byObject);
         const containsNull = columns.map(() => false);
+        const pool = new SlotPool();
+        const seen = columns.map(() => NO_TABLE_YET);
+        const firstJudgement = firstJudgementRow(numRows);
+        const stopped = columns.map(() => null);
         for (let row = 0; row < numRows; row++) {
+          if (row % WINDOW_ROWS === 0 && row !== 0 || row === firstJudgement) {
+            endWindow(seen, row, stopped);
+          }
           const values = data[row];
           if (values !== null && values !== void 0 && !Array.isArray(values)) {
             throw new exports.QvdValidationError("Each row must be an array of values", {
@@ -1981,7 +2195,34 @@ var init_QvdFileWriter = __esm({
               }
               continue;
             }
+            let table = null;
+            let slot = 0;
+            if (typeof value === "number") {
+              table = seen[column];
+              if (table !== null) {
+                table.cells++;
+                slot = slotOf(value, table.mask);
+                if (table.values[slot] === value) {
+                  continue;
+                }
+                table.misses++;
+                if (table === NO_TABLE_YET) {
+                  table = symbolPassTable(numRows, pool);
+                  seen[column] = table;
+                  if (table === null) {
+                    stopped[column] = { row, cells: 0 };
+                  } else {
+                    table.cells = 1;
+                    table.misses = 1;
+                    slot = slotOf(value, table.mask);
+                  }
+                }
+              }
+            }
             if (byCells[column].has(value)) {
+              if (table !== null) {
+                table.values[slot] = value;
+              }
               continue;
             }
             if (typeof value === "number") {
@@ -2001,6 +2242,9 @@ var init_QvdFileWriter = __esm({
               refuseCell(value, columns[column], row, this._path);
             }
             byCells[column].set(value, slotForCell(state[column], value, row, this._path));
+            if (table !== null && typeof value === "number") {
+              seen[column] = table.remember(slot, value);
+            }
           }
         }
         const columnBuffers = [];
@@ -2009,10 +2253,12 @@ var init_QvdFileWriter = __esm({
           const { keys, texts } = state[column];
           const kinds = new Uint8Array(keys.length);
           let byteLength = 0;
+          let numberKeys = 0;
           for (let slot = 0; slot < keys.length; slot++) {
             const key = keys[slot];
             const text = texts[slot];
             if (typeof key === "number") {
+              numberKeys++;
               if (numberProblem(key) !== null) {
                 checkNumber(key, null, { column: columns[column], file: this._path, stage: "buildSymbolTable" });
               }
@@ -2041,10 +2287,20 @@ var init_QvdFileWriter = __esm({
           this._symbolTableMetadata?.push([symbolsOffset, byteLength, containsNull[column]]);
           this._symbolCounts?.push(keys.length);
           this._symbolFacts?.push(state[column].facts);
+          const numbers = state[column].byCell === state[column].byKey ? numberKeys : numbersIn(state[column].byCell);
+          const table = seen[column];
           this._symbolIndexByValue?.push({
             byCell: state[column].byCell,
             byKey: state[column].byKey,
-            byObject: state[column].byObject
+            byObject: state[column].byObject,
+            numbers,
+            cells: table === NO_TABLE_YET ? 0 : table !== null ? table.cells : numericCellsAfter(
+              data,
+              column,
+              /** @type {{row: number, cells: number}} */
+              stopped[column],
+              numbers
+            )
           });
           symbolsOffset += byteLength;
           this._emitProgress("symbol-table", column + 1, numColumns);
@@ -2094,7 +2350,7 @@ var init_QvdFileWriter = __esm({
           const maxStoredIndex = symbolCount === 0 ? 0 : symbolCount - 1 + nullShift;
           const bitWidth = maxStoredIndex === 0 ? 0 : 32 - Math.clz32(maxStoredIndex);
           const lookup = this._symbolIndexByValue[column];
-          const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : Object.values(lookup);
+          const maps = lookup.byCell === lookup.byKey ? [lookup.byKey, lookup.byObject] : [lookup.byCell, lookup.byKey, lookup.byObject];
           for (const map of maps) {
             for (const index of map.values()) {
               if (index + nullShift > maxStoredIndex) {
@@ -2120,14 +2376,44 @@ var init_QvdFileWriter = __esm({
         const byCells = this._symbolIndexByValue.map((lookup) => lookup.byCell);
         const byKeys = this._symbolIndexByValue.map((lookup) => lookup.byKey);
         const byObjects = this._symbolIndexByValue.map((lookup) => lookup.byObject);
+        const symbolCounts = this._symbolCounts;
+        const wanted = this._symbolIndexByValue.map((lookup) => indexPassWants(lookup.numbers, lookup.cells));
+        const share = shareOf(wanted.filter(Boolean).length);
+        const pool = new SlotPool();
+        const seen = this._symbolIndexByValue.map(
+          (lookup, column) => wanted[column] ? indexPassTable(lookup.numbers, lookup.cells, symbolCounts[column], share, pool) : null
+        );
+        const firstJudgement = firstJudgementRow(numRows);
         for (let row = 0, recordBase = 0; row < numRows; row++, recordBase += recordByteSize) {
+          if (row % WINDOW_ROWS === 0 && row !== 0 || row === firstJudgement) {
+            endWindow(seen);
+          }
           const values = data[row];
           for (let column = 0; column < numColumns; column++) {
             const value = values?.[column];
             if (value === null || value === void 0) {
               continue;
             }
-            const index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
+            const table = typeof value === "number" ? seen[column] : null;
+            let index;
+            if (table !== null) {
+              table.cells++;
+              const slot = slotOf(value, table.mask);
+              if (table.values[slot] === value) {
+                index = table.indices[slot];
+              } else {
+                index = byCells[column].get(value);
+                if (index !== void 0) {
+                  table.values[slot] = value;
+                  table.indices[slot] = index;
+                  if (table.metBefore(index)) {
+                    table.misses++;
+                  }
+                }
+              }
+            } else {
+              index = typeof value === "object" ? byObjects[column].get(value) ?? byKeys[column].get(value.number) : byCells[column].get(value);
+            }
             if (index === void 0) {
               throw new exports.QvdValidationError("A value is missing from the symbol table", {
                 field: columns[column],