qvdjs 0.10.0 → 0.10.1

package/README.md

@@ -135,6 +135,43 @@ console.log(df.head(5));
 The above example loads the _qvdjs_ library and parses an example QVD file. A QVD file is typically loaded using the static
 `QvdDataFrame.fromQvd` function of the `QvdDataFrame` class itself. After loading the file's content, numerous methods and properties are available to work with the parsed data.
 
+### Three ways to open a file
+
+`fromQvd` is the general one, and often not the one you want. A QVD is an XML header, then a
+symbol table of every distinct value, then a bit-packed index table of one code per cell — so how
+far into the file a read has to go is what separates these, and all three take the same options.
+
+| You want                                | Call                              | How far it reads                                                       |
+| --------------------------------------- | --------------------------------- | ---------------------------------------------------------------------- |
+| Rows, to index, iterate or write back    | `QvdDataFrame.fromQvd(path)`      | Everything, and materialises every row                                 |
+| A few columns of a large file            | `QvdColumnTable.fromQvd(path)`    | Everything, but stops before building rows — 141 MiB against 385 MiB   |
+| Only the schema: names, row count, types | `QvdDataFrame.readMetadata(path)` | The header alone. Constant cost, whatever the file's size              |
+
+```javascript
+import {QvdDataFrame, QvdColumnTable} from 'qvdjs';
+
+// What is in this file? Costs the same whether it is 20 KB or 20 GB.
+const {columns, rowCount} = await QvdDataFrame.readMetadata('sales.qvd');
+console.log(`${rowCount} rows x ${columns.length} columns`);
+
+// Sum one column without ever building a row.
+const table = await QvdColumnTable.fromQvd('sales.qvd');
+let total = 0;
+for (const value of table.column('amount')) {
+  if (typeof value === 'number') total += value;
+}
+
+// Rows, when you want rows.
+const df = await QvdDataFrame.fromQvd('sales.qvd');
+console.log(df.head(5));
+```
+
+Reaching for `fromQvd` when you wanted one of the other two is the common mistake, and
+`fromQvd(path, {maxRows: 0})` is not a substitute for `readMetadata`: it loads no rows but still
+reads and parses the whole symbol table, which grows with the data. See
+[QvdColumnTable](#qvdcolumntable) and the [API Documentation](#api-documentation) for the
+details and the measurements behind the table above.
+
 ### Lazy Loading
 
 For large QVD files, you can load only a specific number of rows to improve performance and reduce memory usage. The library implements **lazy loading** - it reads only the necessary portions of the file from disk, not the entire file.

package/dist/index.cjs

@@ -957,6 +957,24 @@ var init_memoryUtils = __esm({
 });
 
 // src/util/validationUtils.js
+function validateHeaderStructure(headerObj, filePath, stage) {
+  const tableHeader = headerObj?.["QvdTableHeader"];
+  if (tableHeader === null || typeof tableHeader !== "object" || Array.isArray(tableHeader)) {
+    throw new exports.QvdCorruptedError("The XML header contains no usable QvdTableHeader element", {
+      rootElements: headerObj && typeof headerObj === "object" ? Object.keys(headerObj) : [],
+      file: filePath,
+      stage
+    });
+  }
+  const symbolTableLength = parseInt(tableHeader["Offset"], 10);
+  if (isNaN(symbolTableLength) || !Number.isSafeInteger(symbolTableLength) || symbolTableLength < 0) {
+    throw new exports.QvdCorruptedError("Invalid symbol table offset", {
+      offset: tableHeader["Offset"],
+      file: filePath,
+      stage
+    });
+  }
+}
 function validateSymbolTableSizeEarly(symbolTableLength, filePath) {
   const heapLimit = getHeapLimit();
   const MAX_SYMBOL_TABLE_SIZE = heapLimit * 0.125;
@@ -1861,6 +1879,7 @@ var init_QvdFileReader = __esm({
             stage: "readData"
           });
         }
+        validateHeaderStructure(headerObj, this._path, "readData");
         const symbolTableOffset = headerEndIndex;
         const symbolTableLength = parseInt(headerObj["QvdTableHeader"]["Offset"], 10);
         const indexTableOffset = symbolTableOffset + symbolTableLength;
@@ -1984,14 +2003,26 @@ var init_QvdFileReader = __esm({
             stage: "parseHeader"
           });
         }
+        validateHeaderStructure(this._header, this._path, "parseHeader");
         const fields = this._header["QvdTableHeader"]?.["Fields"]?.["QvdFieldHeader"];
-        const fieldCount = fields === void 0 || fields === null ? 0 : Array.isArray(fields) ? fields.length : 1;
-        if (fieldCount === 0) {
+        const fieldList = fields === void 0 || fields === null ? [] : Array.isArray(fields) ? fields : [fields];
+        if (fieldList.length === 0) {
           throw new exports.QvdCorruptedError("The QVD file header declares no fields", {
             file: this._path,
             stage: "parseHeader"
           });
         }
+        const malformedIndex = fieldList.findIndex(
+          (field) => field === null || typeof field !== "object" || Array.isArray(field)
+        );
+        if (malformedIndex !== -1) {
+          throw new exports.QvdCorruptedError("The QVD file header declares a field with no properties", {
+            fieldIndex: malformedIndex,
+            fieldCount: fieldList.length,
+            file: this._path,
+            stage: "parseHeader"
+          });
+        }
         this._headerOffset = headerBeginIndex;
         this._symbolTableOffset = headerEndIndex;
         this._indexTableOffset = this._symbolTableOffset + parseInt(this._header["QvdTableHeader"]["Offset"], 10);