strtok3 10.3.5 → 10.4.1

package/lib/index.d.ts

@@ -1,10 +1,10 @@
 import type { Readable } from 'node:stream';
-import type { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { type ITokenizerOptions } from './core.js';
-import { FileTokenizer } from "./FileTokenizer.js";
-export { FileTokenizer } from './FileTokenizer.js';
+import { FileTokenizer } from './FileTokenizer.js';
+import type { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+export type { IGetToken, IToken } from '@tokenizer/token';
 export * from './core.js';
-export type { IToken, IGetToken } from '@tokenizer/token';
+export { FileTokenizer } from './FileTokenizer.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.

package/lib/index.js

@@ -1,8 +1,8 @@
 import { stat as fsStat } from 'node:fs/promises';
 import { fromStream as coreFromStream } from './core.js';
-import { FileTokenizer } from "./FileTokenizer.js";
-export { FileTokenizer } from './FileTokenizer.js';
+import { FileTokenizer } from './FileTokenizer.js';
 export * from './core.js';
+export { FileTokenizer } from './FileTokenizer.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.

package/lib/stream/AbstractStreamReader.js

@@ -1,4 +1,4 @@
-import { EndOfStreamError, AbortError } from "./Errors.js";
+import { AbortError, EndOfStreamError } from './Errors.js';
 export class AbstractStreamReader {
     constructor() {
         this.endOfStream = false;
@@ -38,8 +38,9 @@ export class AbstractStreamReader {
         // consume peeked data first
         while (this.peekQueue.length > 0 && remaining > 0) {
             const peekData = this.peekQueue.pop(); // Front of queue
-            if (!peekData)
+            if (!peekData) {
                 throw new Error('peekData should be defined');
+            }
             const lenCopy = Math.min(peekData.length, remaining);
             buffer.set(peekData.subarray(0, lenCopy), bytesRead);
             bytesRead += lenCopy;
@@ -59,8 +60,9 @@ export class AbstractStreamReader {
                 throw new AbortError();
             }
             const chunkLen = await this.readFromStream(buffer.subarray(bytesRead), mayBeLess);
-            if (chunkLen === 0)
+            if (chunkLen === 0) {
                 break;
+            }
             bytesRead += chunkLen;
         }
         if (!mayBeLess && bytesRead < buffer.length) {

package/lib/stream/Errors.js

@@ -5,12 +5,12 @@ export const defaultMessages = 'End-Of-Stream';
 export class EndOfStreamError extends Error {
     constructor() {
         super(defaultMessages);
-        this.name = "EndOfStreamError";
+        this.name = 'EndOfStreamError';
     }
 }
 export class AbortError extends Error {
-    constructor(message = "The operation was aborted") {
+    constructor(message = 'The operation was aborted') {
         super(message);
-        this.name = "AbortError";
+        this.name = 'AbortError';
     }
 }

package/lib/stream/StreamReader.d.ts

@@ -1,5 +1,5 @@
 import type { Readable } from 'node:stream';
-import { AbstractStreamReader } from "./AbstractStreamReader.js";
+import { AbstractStreamReader } from './AbstractStreamReader.js';
 /**
  * Node.js Readable Stream Reader
  * Ref: https://nodejs.org/api/stream.html#readable-streams
@@ -7,9 +7,9 @@ import { AbstractStreamReader } from "./AbstractStreamReader.js";
 export declare class StreamReader extends AbstractStreamReader {
     private s;
     /**
-     * Deferred used for postponed read request (as not data is yet available to read)
+     * Pending requests waiting for data, tracked until completion or cancellation
      */
-    private deferred;
+    private pendingRequests;
     constructor(s: Readable);
     /**
      * Read chunk from stream
@@ -23,6 +23,7 @@ export declare class StreamReader extends AbstractStreamReader {
      * @param request Deferred read request
      */
     private readDeferred;
+    private removeRequest;
     private reject;
     abort(): Promise<void>;
     close(): Promise<void>;

package/lib/stream/StreamReader.js

@@ -1,6 +1,6 @@
-import { AbortError, } from './Errors.js';
+import { AbstractStreamReader } from './AbstractStreamReader.js';
 import { Deferred } from './Deferred.js';
-import { AbstractStreamReader } from "./AbstractStreamReader.js";
+import { AbortError } from './Errors.js';
 /**
  * Node.js Readable Stream Reader
  * Ref: https://nodejs.org/api/stream.html#readable-streams
@@ -10,20 +10,26 @@ export class StreamReader extends AbstractStreamReader {
         super();
         this.s = s;
         /**
-         * Deferred used for postponed read request (as not data is yet available to read)
+         * Pending requests waiting for data, tracked until completion or cancellation
          */
-        this.deferred = null;
+        this.pendingRequests = new Set();
         if (!s.read || !s.once) {
             throw new Error('Expected an instance of stream.Readable');
         }
         this.s.once('end', () => {
             this.endOfStream = true;
-            if (this.deferred) {
-                this.deferred.resolve(0);
+            for (const request of this.pendingRequests) {
+                this.removeRequest(request);
+                request.deferred.resolve(0);
             }
         });
         this.s.once('error', err => this.reject(err));
-        this.s.once('close', () => this.abort());
+        this.s.once('close', () => {
+            // Normal EOF may leave peeked bytes that the caller still needs to consume.
+            if (!this.endOfStream) {
+                void this.abort();
+            }
+        });
     }
     /**
      * Read chunk from stream
@@ -32,8 +38,9 @@ export class StreamReader extends AbstractStreamReader {
      * @returns Number of bytes read
      */
     async readFromStream(buffer, mayBeLess) {
-        if (buffer.length === 0)
+        if (buffer.length === 0) {
             return 0;
+        }
         const readBuffer = this.s.read(buffer.length);
         if (readBuffer) {
             buffer.set(readBuffer);
@@ -42,12 +49,11 @@ export class StreamReader extends AbstractStreamReader {
         const request = {
             buffer,
             mayBeLess,
-            deferred: new Deferred()
+            deferred: new Deferred(),
+            onReadable: () => this.readDeferred(request)
         };
-        this.deferred = request.deferred;
-        this.s.once('readable', () => {
-            this.readDeferred(request);
-        });
+        this.pendingRequests.add(request);
+        this.s.once('readable', request.onReadable);
         return request.deferred.promise;
     }
     /**
@@ -55,23 +61,38 @@ export class StreamReader extends AbstractStreamReader {
      * @param request Deferred read request
      */
     readDeferred(request) {
-        const readBuffer = this.s.read(request.buffer.length);
-        if (readBuffer) {
-            request.buffer.set(readBuffer);
-            request.deferred.resolve(readBuffer.length);
-            this.deferred = null;
+        if (this.interrupted || !this.pendingRequests.has(request)) {
+            return;
         }
-        else {
-            this.s.once('readable', () => {
-                this.readDeferred(request);
-            });
+        try {
+            const readBuffer = this.s.read(request.buffer.length);
+            // Reading may synchronously emit an error or close event that cancels the request.
+            if (!this.pendingRequests.has(request)) {
+                return;
+            }
+            if (readBuffer) {
+                request.buffer.set(readBuffer);
+                this.removeRequest(request);
+                request.deferred.resolve(readBuffer.length);
+            }
+            else {
+                this.s.once('readable', request.onReadable);
+            }
         }
+        catch (error) {
+            this.reject(error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    removeRequest(request) {
+        this.s.removeListener('readable', request.onReadable);
+        this.pendingRequests.delete(request);
     }
     reject(err) {
         this.interrupted = true;
-        if (this.deferred) {
-            this.deferred.reject(err);
-            this.deferred = null;
+        this.peekQueue = [];
+        for (const request of this.pendingRequests) {
+            this.removeRequest(request);
+            request.deferred.reject(err);
         }
     }
     async abort() {

package/lib/stream/WebStreamByobReader.js

@@ -11,10 +11,12 @@ export class WebStreamByobReader extends WebStreamReader {
      * @protected Bytes read
      */
     async readFromStream(buffer, mayBeLess) {
-        if (buffer.length === 0)
+        if (buffer.length === 0) {
             return 0;
-        // @ts-expect-error
-        const result = await this.reader.read(new Uint8Array(buffer.length), { min: mayBeLess ? undefined : buffer.length });
+        }
+        const result = await this.reader.read(new Uint8Array(buffer.length), {
+            min: mayBeLess ? undefined : buffer.length
+        });
         if (result.done) {
             this.endOfStream = result.done;
         }

package/lib/stream/WebStreamDefaultReader.d.ts

@@ -1,4 +1,4 @@
-import { AbstractStreamReader } from "./AbstractStreamReader.js";
+import { AbstractStreamReader } from './AbstractStreamReader.js';
 export declare class WebStreamDefaultReader extends AbstractStreamReader {
     private reader;
     private buffer;

package/lib/stream/WebStreamDefaultReader.js

@@ -1,5 +1,5 @@
+import { AbstractStreamReader } from './AbstractStreamReader.js';
 import { EndOfStreamError } from './Errors.js';
-import { AbstractStreamReader } from "./AbstractStreamReader.js";
 export class WebStreamDefaultReader extends AbstractStreamReader {
     constructor(reader) {
         super();
@@ -28,8 +28,9 @@ export class WebStreamDefaultReader extends AbstractStreamReader {
      * @protected Bytes read
      */
     async readFromStream(buffer, mayBeLess) {
-        if (buffer.length === 0)
+        if (buffer.length === 0) {
             return 0;
+        }
         let totalBytesRead = 0;
         // Serve from the internal buffer first
         if (this.buffer) {
@@ -56,7 +57,11 @@ export class WebStreamDefaultReader extends AbstractStreamReader {
         return this.reader.cancel();
     }
     async close() {
-        await this.abort();
-        this.reader.releaseLock();
+        try {
+            await this.abort();
+        }
+        finally {
+            this.reader.releaseLock();
+        }
     }
 }

package/lib/stream/WebStreamReader.d.ts

@@ -1,4 +1,4 @@
-import { AbstractStreamReader } from "./AbstractStreamReader.js";
+import { AbstractStreamReader } from './AbstractStreamReader.js';
 export declare abstract class WebStreamReader extends AbstractStreamReader {
     protected reader: ReadableStreamDefaultReader | ReadableStreamBYOBReader;
     constructor(reader: ReadableStreamDefaultReader | ReadableStreamBYOBReader);

package/lib/stream/WebStreamReader.js

@@ -1,4 +1,4 @@
-import { AbstractStreamReader } from "./AbstractStreamReader.js";
+import { AbstractStreamReader } from './AbstractStreamReader.js';
 export class WebStreamReader extends AbstractStreamReader {
     constructor(reader) {
         super();

package/lib/stream/WebStreamReaderFactory.js

@@ -2,7 +2,7 @@ import { WebStreamByobReader } from './WebStreamByobReader.js';
 import { WebStreamDefaultReader } from './WebStreamDefaultReader.js';
 export function makeWebStreamReader(stream) {
     try {
-        const reader = stream.getReader({ mode: "byob" });
+        const reader = stream.getReader({ mode: 'byob' });
         if (reader instanceof ReadableStreamDefaultReader) {
             // Fallback to default reader in case `mode: byob` is ignored
             return new WebStreamDefaultReader(reader);

package/lib/stream/index.d.ts

@@ -1,6 +1,6 @@
+export type { IStreamReader } from './AbstractStreamReader.js';
 export { AbortError, EndOfStreamError } from './Errors.js';
 export { StreamReader } from './StreamReader.js';
 export { WebStreamByobReader } from './WebStreamByobReader.js';
 export { WebStreamDefaultReader } from './WebStreamDefaultReader.js';
-export type { IStreamReader } from './AbstractStreamReader.js';
 export { type AnyWebByteStream, makeWebStreamReader } from './WebStreamReaderFactory.js';

package/lib/types.d.ts

@@ -123,6 +123,13 @@ export interface ITokenizer {
 }
 export type OnClose = () => Promise<void>;
 export interface ITokenizerOptions {
+    /**
+     * Maximum bytes in a dynamically sized tokenizer buffer or stream lookahead span.
+     * Set once when creating the tokenizer; applies to subsequent reads and peeks.
+     * Disabled by default (Infinity). Must be a positive safe integer or Infinity.
+     * Does not limit total process memory, caller-owned buffers, or decoded token values.
+     */
+    maxBufferSize?: number;
     /**
      * Pass additional file information to the tokenizer
      */

package/package.json

@@ -1,94 +1,95 @@
-{
-  "name": "strtok3",
-  "version": "10.3.5",
-  "description": "A promise based streaming tokenizer",
-  "author": {
-    "name": "Borewit",
-    "url": "https://github.com/Borewit"
-  },
-  "funding": {
-    "type": "github",
-    "url": "https://github.com/sponsors/Borewit"
-  },
-  "scripts": {
-    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
-    "compile-src": "tsc -p lib  --sourceMap false",
-    "compile-test": "tsc -p test",
-    "compile": "yarn run compile-src && yarn run compile-test",
-    "build": "yarn run clean && yarn run compile",
-    "prepublishOnly": "yarn run build",
-    "eslint": "eslint lib test",
-    "lint:md": "remark -u preset-lint-recommended .",
-    "lint:ts": "biome check",
-    "lint": "yarn run lint:md && yarn run lint:ts",
-    "fix": "yarn run biome lint --write",
-    "test": "mocha",
-    "bun:test": "bun run --bun test",
-    "test-coverage": "c8 yarn run test",
-    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
-    "start": "yarn run compile && yarn run lint && yarn run cover-test",
-    "update-biome": "yarn add -D --exact @biomejs/biome && npx @biomejs/biome migrate --write"
-  },
-  "engines": {
-    "node": ">=18"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/Borewit/strtok3.git"
-  },
-  "license": "MIT",
-  "type": "module",
-  "exports": {
-    ".": {
-      "node": "./lib/index.js",
-      "default": "./lib/core.js"
-    },
-    "./core": "./lib/core.js"
-  },
-  "types": "lib/index.d.ts",
-  "files": [
-    "lib/**/*.js",
-    "lib/**/*.d.ts"
-  ],
-  "bugs": {
-    "url": "https://github.com/Borewit/strtok3/issues"
-  },
-  "dependencies": {
-    "@tokenizer/token": "^0.3.0"
-  },
-  "devDependencies": {
-    "@biomejs/biome": "2.4.8",
-    "@types/chai": "^5.2.2",
-    "@types/chai-as-promised": "^8.0.2",
-    "@types/debug": "^4.1.12",
-    "@types/mocha": "^10.0.10",
-    "@types/node": "^24.1.0",
-    "c8": "^11.0.0",
-    "chai": "^5.2.1",
-    "chai-as-promised": "^8.0.2",
-    "del-cli": "^7.0.0",
-    "mocha": "^11.7.5",
-    "node-readable-to-web-readable-stream": "^0.4.2",
-    "remark-cli": "^12.0.1",
-    "remark-preset-lint-recommended": "^7.0.1",
-    "token-types": "^6.1.2",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.8.3",
-    "uint8array-extras": "^1.4.0"
-  },
-  "keywords": [
-    "tokenizer",
-    "reader",
-    "token",
-    "async",
-    "promise",
-    "parser",
-    "decoder",
-    "binary",
-    "endian",
-    "uint",
-    "stream",
-    "streaming"
-  ],
-  "packageManager": "yarn@4.9.2"
-}
+{
+  "name": "strtok3",
+  "version": "10.4.1",
+  "description": "A promise based streaming tokenizer",
+  "author": {
+    "name": "Borewit",
+    "url": "https://github.com/Borewit"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/Borewit"
+  },
+  "scripts": {
+    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
+    "compile-src": "tsc -p lib  --sourceMap false",
+    "compile-test": "tsc -p test",
+    "compile": "yarn run compile-src && yarn run compile-test",
+    "build": "yarn run clean && yarn run compile",
+    "prepublishOnly": "yarn run build",
+    "eslint": "eslint lib test",
+    "lint:md": "remark -u preset-lint-recommended .",
+    "lint:ts": "biome check",
+    "lint": "yarn run lint:md && yarn run lint:ts",
+    "fix": "yarn run lint:ts:fix",
+    "test": "mocha",
+    "bun:test": "bun run --bun test",
+    "test-coverage": "c8 yarn run test",
+    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
+    "start": "yarn run compile && yarn run lint && yarn run cover-test",
+    "update-biome": "yarn add -D --exact @biomejs/biome && npx @biomejs/biome migrate --write",
+    "lint:ts:fix": "biome check --write"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Borewit/strtok3.git"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": {
+      "node": "./lib/index.js",
+      "default": "./lib/core.js"
+    },
+    "./core": "./lib/core.js"
+  },
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib/**/*.js",
+    "lib/**/*.d.ts"
+  ],
+  "bugs": {
+    "url": "https://github.com/Borewit/strtok3/issues"
+  },
+  "dependencies": {
+    "@tokenizer/token": "^0.3.0"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.15",
+    "@types/chai": "^5.2.2",
+    "@types/chai-as-promised": "^8.0.2",
+    "@types/debug": "^4.1.12",
+    "@types/mocha": "^10.0.10",
+    "@types/node": "^24.19.1",
+    "c8": "^12.0.0",
+    "chai": "^6.3.0",
+    "chai-as-promised": "^8.0.2",
+    "del-cli": "^7.0.0",
+    "mocha": "^11",
+    "node-readable-to-web-readable-stream": "^0.4.2",
+    "remark-cli": "^12.0.1",
+    "remark-preset-lint-recommended": "^7.0.1",
+    "token-types": "^6.1.2",
+    "tsx": "^4.23.15",
+    "typescript": "^7.0.2",
+    "uint8array-extras": "^1.6.0"
+  },
+  "keywords": [
+    "tokenizer",
+    "reader",
+    "token",
+    "async",
+    "promise",
+    "parser",
+    "decoder",
+    "binary",
+    "endian",
+    "uint",
+    "stream",
+    "streaming"
+  ],
+  "packageManager": "yarn@4.18.1"
+}