strtok3 10.3.4 → 10.4.0

package/lib/AbortSignal.d.ts

@@ -0,0 +1,2 @@
+/** Reject an already-cancelled operation before acquiring tokenizer resources. */
+export declare function throwIfAborted(signal?: AbortSignal): void;

package/lib/AbortSignal.js

@@ -0,0 +1,7 @@
+import { AbortError } from './stream/Errors.js';
+/** Reject an already-cancelled operation before acquiring tokenizer resources. */
+export function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw new AbortError();
+    }
+}

package/lib/AbstractTokenizer.d.ts

@@ -1,5 +1,5 @@
-import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 import type { IGetToken, IToken } from '@tokenizer/token';
+import type { IFileInfo, IReadChunkOptions, ITokenizer, ITokenizerOptions } from './types.js';
 interface INormalizedReadChunkOptions extends IReadChunkOptions {
     length: number;
     position: number;
@@ -10,7 +10,10 @@ interface INormalizedReadChunkOptions extends IReadChunkOptions {
  */
 export declare abstract class AbstractTokenizer implements ITokenizer {
     private onClose?;
+    private abortSignal?;
+    private abortListener?;
     private numBuffer;
+    protected readonly maxBufferSize: number;
     abstract fileInfo: IFileInfo;
     /**
      * Tokenizer-stream position
@@ -22,6 +25,8 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      * @protected
      */
     protected constructor(options?: ITokenizerOptions);
+    private addAbortListener;
+    protected removeAbortListener(): void;
     abstract supportsRandomAccess(): boolean;
     /**
      * Read buffer from tokenizer
@@ -50,7 +55,7 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
      * @returns Promise with token data
      */
-    peekToken<Value>(token: IGetToken<Value>, position?: number): Promise<Value>;
+    peekToken<Value>(token: IGetToken<Value>, position?: number | null): Promise<Value>;
     /**
      * Read a numeric token from the stream
      * @param token - Numeric token
@@ -65,11 +70,44 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
     peekNumber(token: IToken<number>): Promise<number>;
     /**
      * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to ignore
+     * @param length - Number of bytes to ignore.  Must be ≥ 0.
      * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
      */
     ignore(length: number): Promise<number>;
     close(): Promise<void>;
+    /**
+     * Validate a byte length or position before using it in buffer calculations.
+     * @param value - Value that must be a non-negative safe integer
+     * @param name - Label used in the validation error
+     * @throws RangeError if the value is negative or not a safe integer
+     */
+    protected validateLength(value: number, name: string): void;
+    /**
+     * Validate a requested buffer size against the configured allocation limit.
+     * @param length - Requested buffer size in bytes
+     * @throws RangeError if the length is negative or not a safe integer
+     * @throws BufferSizeError if the length exceeds maxBufferSize
+     */
+    protected validateBufferSize(length: number): void;
+    /**
+     * Validate a token request before allocating a buffer or consuming input.
+     * @param length - Token length in bytes
+     * @param position - Byte offset where the token starts
+     * @param peek - Whether to include skipped stream bytes in the lookahead limit
+     * @throws RangeError if a length or position is negative or not a safe integer
+     * @throws BufferSizeError if the token buffer or stream lookahead exceeds maxBufferSize
+     * @throws Error if a stream request starts before the current position
+     * @throws EndOfStreamError if the token extends beyond the known input size
+     */
+    private validateTokenRead;
+    /**
+     * Apply defaults and validate buffer read or peek options.
+     * @param uint8Array - Target buffer whose length is the default read length
+     * @param options - Requested length, position and partial-read behavior
+     * @returns Validated options with an explicit length, position and mayBeLess flag
+     * @throws RangeError if a length or position is invalid, or the length exceeds the target buffer
+     * @throws Error if a stream request starts before the current position
+     */
     protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): INormalizedReadChunkOptions;
     abort(): Promise<void>;
 }

package/lib/AbstractTokenizer.js

@@ -1,3 +1,5 @@
+import { throwIfAborted } from './AbortSignal.js';
+import { BufferSizeError, getMaxBufferSize } from './BufferSizeError.js';
 import { EndOfStreamError } from './stream/index.js';
 /**
  * Core tokenizer
@@ -14,13 +16,29 @@ export class AbstractTokenizer {
          * Tokenizer-stream position
          */
         this.position = 0;
+        this.maxBufferSize = getMaxBufferSize(options);
+        throwIfAborted(options?.abortSignal);
         this.onClose = options?.onClose;
         if (options?.abortSignal) {
-            options.abortSignal.addEventListener('abort', () => {
-                this.abort();
+            this.addAbortListener(options.abortSignal, () => {
+                void this.abort().catch(() => {
+                    // Signal events cannot propagate cancellation errors; explicit abort/close calls still reject.
+                });
             });
         }
     }
+    addAbortListener(signal, listener) {
+        this.abortSignal = signal;
+        this.abortListener = listener;
+        signal.addEventListener('abort', listener, { once: true });
+    }
+    removeAbortListener() {
+        if (this.abortListener) {
+            this.abortSignal?.removeEventListener('abort', this.abortListener);
+        }
+        this.abortSignal = undefined;
+        this.abortListener = undefined;
+    }
     /**
      * Read a token from the tokenizer-stream
      * @param token - The token to read
@@ -28,10 +46,12 @@ export class AbstractTokenizer {
      * @returns Promise with token data
      */
     async readToken(token, position = this.position) {
+        this.validateTokenRead(token.len, position);
         const uint8Array = new Uint8Array(token.len);
         const len = await this.readBuffer(uint8Array, { position });
-        if (len < token.len)
+        if (len < token.len) {
             throw new EndOfStreamError();
+        }
         return token.get(uint8Array, 0);
     }
     /**
@@ -41,10 +61,13 @@ export class AbstractTokenizer {
      * @returns Promise with token data
      */
     async peekToken(token, position = this.position) {
+        const peekPosition = position ?? this.position;
+        this.validateTokenRead(token.len, peekPosition, true);
         const uint8Array = new Uint8Array(token.len);
-        const len = await this.peekBuffer(uint8Array, { position });
-        if (len < token.len)
+        const len = await this.peekBuffer(uint8Array, { position: peekPosition });
+        if (len < token.len) {
             throw new EndOfStreamError();
+        }
         return token.get(uint8Array, 0);
     }
     /**
@@ -54,8 +77,9 @@ export class AbstractTokenizer {
      */
     async readNumber(token) {
         const len = await this.readBuffer(this.numBuffer, { length: token.len });
-        if (len < token.len)
+        if (len < token.len) {
             throw new EndOfStreamError();
+        }
         return token.get(this.numBuffer, 0);
     }
     /**
@@ -65,16 +89,22 @@ export class AbstractTokenizer {
      */
     async peekNumber(token) {
         const len = await this.peekBuffer(this.numBuffer, { length: token.len });
-        if (len < token.len)
+        if (len < token.len) {
             throw new EndOfStreamError();
+        }
         return token.get(this.numBuffer, 0);
     }
     /**
      * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to ignore
+     * @param length - Number of bytes to ignore.  Must be ≥ 0.
      * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
      */
     async ignore(length) {
+        if (length < 0) {
+            throw new RangeError('ignore length must be ≥ 0 bytes');
+        }
+        this.validateLength(length, 'ignore length');
+        this.validateLength(this.position + length, 'end position');
         if (this.fileInfo.size !== undefined) {
             const bytesLeft = this.fileInfo.size - this.position;
             if (length > bytesLeft) {
@@ -86,23 +116,81 @@ export class AbstractTokenizer {
         return length;
     }
     async close() {
+        this.removeAbortListener();
         await this.abort();
         await this.onClose?.();
     }
+    /**
+     * Validate a byte length or position before using it in buffer calculations.
+     * @param value - Value that must be a non-negative safe integer
+     * @param name - Label used in the validation error
+     * @throws RangeError if the value is negative or not a safe integer
+     */
+    validateLength(value, name) {
+        if (!Number.isSafeInteger(value) || value < 0) {
+            throw new RangeError(`${name} must be a non-negative safe integer`);
+        }
+    }
+    /**
+     * Validate a requested buffer size against the configured allocation limit.
+     * @param length - Requested buffer size in bytes
+     * @throws RangeError if the length is negative or not a safe integer
+     * @throws BufferSizeError if the length exceeds maxBufferSize
+     */
+    validateBufferSize(length) {
+        this.validateLength(length, 'buffer length');
+        if (length > this.maxBufferSize) {
+            throw new BufferSizeError(length, this.maxBufferSize);
+        }
+    }
+    /**
+     * Validate a token request before allocating a buffer or consuming input.
+     * @param length - Token length in bytes
+     * @param position - Byte offset where the token starts
+     * @param peek - Whether to include skipped stream bytes in the lookahead limit
+     * @throws RangeError if a length or position is negative or not a safe integer
+     * @throws BufferSizeError if the token buffer or stream lookahead exceeds maxBufferSize
+     * @throws Error if a stream request starts before the current position
+     * @throws EndOfStreamError if the token extends beyond the known input size
+     */
+    validateTokenRead(length, position, peek = false) {
+        this.validateBufferSize(length);
+        this.validateLength(position, 'position');
+        this.validateLength(position + length, 'end position');
+        if (!this.supportsRandomAccess() && position < this.position) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+        }
+        if (peek && !this.supportsRandomAccess()) {
+            this.validateBufferSize(length + position - this.position);
+        }
+        if (this.fileInfo.size !== undefined && length > this.fileInfo.size - position) {
+            throw new EndOfStreamError();
+        }
+    }
+    /**
+     * Apply defaults and validate buffer read or peek options.
+     * @param uint8Array - Target buffer whose length is the default read length
+     * @param options - Requested length, position and partial-read behavior
+     * @returns Validated options with an explicit length, position and mayBeLess flag
+     * @throws RangeError if a length or position is invalid, or the length exceeds the target buffer
+     * @throws Error if a stream request starts before the current position
+     */
     normalizeOptions(uint8Array, options) {
-        if (!this.supportsRandomAccess() && options && options.position !== undefined && options.position < this.position) {
+        const length = options?.length ?? uint8Array.length;
+        const position = options?.position ?? this.position;
+        this.validateLength(length, 'length');
+        this.validateLength(position, 'position');
+        this.validateLength(position + length, 'end position');
+        if (length > uint8Array.length) {
+            throw new RangeError('length must not exceed the target buffer length');
+        }
+        if (!this.supportsRandomAccess() && position < this.position) {
             throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
         }
-        return {
-            ...{
-                mayBeLess: false,
-                offset: 0,
-                length: uint8Array.length,
-                position: this.position
-            }, ...options
-        };
+        return { length, position, mayBeLess: options?.mayBeLess ?? false };
     }
     abort() {
+        this.removeAbortListener();
         return Promise.resolve(); // Ignore abort signal
     }
 }

package/lib/BlobTokenizer.d.ts

@@ -1,5 +1,5 @@
-import type { ITokenizerOptions, IReadChunkOptions, IRandomAccessFileInfo, IRandomAccessTokenizer } from './types.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
+import type { IRandomAccessFileInfo, IRandomAccessTokenizer, IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare class BlobTokenizer extends AbstractTokenizer implements IRandomAccessTokenizer {
     private blob;
     fileInfo: IRandomAccessFileInfo;

package/lib/BlobTokenizer.js

@@ -1,5 +1,5 @@
-import { EndOfStreamError } from './stream/index.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { EndOfStreamError } from './stream/index.js';
 export class BlobTokenizer extends AbstractTokenizer {
     /**
      * Construct BufferTokenizer
@@ -9,7 +9,7 @@ export class BlobTokenizer extends AbstractTokenizer {
     constructor(blob, options) {
         super(options);
         this.blob = blob;
-        this.fileInfo = { ...options?.fileInfo ?? {}, ...{ size: blob.size, mimeType: blob.type } };
+        this.fileInfo = { ...(options?.fileInfo ?? {}), ...{ size: blob.size, mimeType: blob.type } };
     }
     /**
      * Read buffer from tokenizer
@@ -18,11 +18,9 @@ export class BlobTokenizer extends AbstractTokenizer {
      * @returns {Promise<number>}
      */
     async readBuffer(uint8Array, options) {
-        if (options?.position) {
-            this.position = options.position;
-        }
-        const bytesRead = await this.peekBuffer(uint8Array, options);
-        this.position += bytesRead;
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const bytesRead = await this.peekBuffer(uint8Array, normOptions);
+        this.position = normOptions.position + bytesRead;
         return bytesRead;
     }
     /**
@@ -33,12 +31,17 @@ export class BlobTokenizer extends AbstractTokenizer {
      */
     async peekBuffer(buffer, options) {
         const normOptions = this.normalizeOptions(buffer, options);
-        const bytes2read = Math.min(this.blob.size - normOptions.position, normOptions.length);
-        if ((!normOptions.mayBeLess) && bytes2read < normOptions.length) {
+        const bytes2read = Math.max(0, Math.min(this.blob.size - normOptions.position, normOptions.length));
+        if (!normOptions.mayBeLess && bytes2read < normOptions.length) {
             throw new EndOfStreamError();
         }
-        const arrayBuffer = await this.blob.slice(normOptions.position, normOptions.position + bytes2read).arrayBuffer();
-        buffer.set(new Uint8Array(arrayBuffer));
+        for (let offset = 0; offset < bytes2read;) {
+            const chunkLength = Math.min(this.maxBufferSize, bytes2read - offset);
+            const position = normOptions.position + offset;
+            const arrayBuffer = await this.blob.slice(position, position + chunkLength).arrayBuffer();
+            buffer.set(new Uint8Array(arrayBuffer), offset);
+            offset += chunkLength;
+        }
         return bytes2read;
     }
     close() {

package/lib/BufferSizeError.d.ts

@@ -0,0 +1,19 @@
+import type { ITokenizerOptions } from './types.js';
+/** Thrown before a dynamically sized tokenizer buffer exceeds its configured limit. */
+export declare class BufferSizeError extends RangeError {
+    readonly requestedSize: number;
+    readonly maxBufferSize: number;
+    /**
+     * Create an error describing an allocation that exceeds the configured limit.
+     * @param requestedSize - Requested buffer size in bytes
+     * @param maxBufferSize - Configured maximum buffer size in bytes
+     */
+    constructor(requestedSize: number, maxBufferSize: number);
+}
+/**
+ * Validate options before acquiring file handles or stream readers.
+ * @param options - Tokenizer options containing the optional allocation limit
+ * @returns Configured buffer limit in bytes, or Infinity when the limit is disabled
+ * @throws RangeError if maxBufferSize is neither a positive safe integer nor Infinity
+ */
+export declare function getMaxBufferSize(options?: ITokenizerOptions): number;

package/lib/BufferSizeError.js

@@ -0,0 +1,27 @@
+/** Thrown before a dynamically sized tokenizer buffer exceeds its configured limit. */
+export class BufferSizeError extends RangeError {
+    /**
+     * Create an error describing an allocation that exceeds the configured limit.
+     * @param requestedSize - Requested buffer size in bytes
+     * @param maxBufferSize - Configured maximum buffer size in bytes
+     */
+    constructor(requestedSize, maxBufferSize) {
+        super(`Requested buffer size ${requestedSize} exceeds maxBufferSize ${maxBufferSize}`);
+        this.requestedSize = requestedSize;
+        this.maxBufferSize = maxBufferSize;
+        this.name = 'BufferSizeError';
+    }
+}
+/**
+ * Validate options before acquiring file handles or stream readers.
+ * @param options - Tokenizer options containing the optional allocation limit
+ * @returns Configured buffer limit in bytes, or Infinity when the limit is disabled
+ * @throws RangeError if maxBufferSize is neither a positive safe integer nor Infinity
+ */
+export function getMaxBufferSize(options) {
+    const limit = options?.maxBufferSize ?? Number.POSITIVE_INFINITY;
+    if (limit !== Number.POSITIVE_INFINITY && (!Number.isSafeInteger(limit) || limit <= 0)) {
+        throw new RangeError('maxBufferSize must be a positive safe integer or Infinity');
+    }
+    return limit;
+}

package/lib/BufferTokenizer.d.ts

@@ -1,5 +1,5 @@
-import type { ITokenizerOptions, IReadChunkOptions, IRandomAccessFileInfo, IRandomAccessTokenizer } from './types.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
+import type { IRandomAccessFileInfo, IRandomAccessTokenizer, IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare class BufferTokenizer extends AbstractTokenizer implements IRandomAccessTokenizer {
     private uint8Array;
     fileInfo: IRandomAccessFileInfo;

package/lib/BufferTokenizer.js

@@ -1,5 +1,5 @@
-import { EndOfStreamError } from './stream/index.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { EndOfStreamError } from './stream/index.js';
 export class BufferTokenizer extends AbstractTokenizer {
     /**
      * Construct BufferTokenizer
@@ -9,7 +9,7 @@ export class BufferTokenizer extends AbstractTokenizer {
     constructor(uint8Array, options) {
         super(options);
         this.uint8Array = uint8Array;
-        this.fileInfo = { ...options?.fileInfo ?? {}, ...{ size: uint8Array.length } };
+        this.fileInfo = { ...(options?.fileInfo ?? {}), ...{ size: uint8Array.length } };
     }
     /**
      * Read buffer from tokenizer
@@ -18,11 +18,9 @@ export class BufferTokenizer extends AbstractTokenizer {
      * @returns {Promise<number>}
      */
     async readBuffer(uint8Array, options) {
-        if (options?.position) {
-            this.position = options.position;
-        }
-        const bytesRead = await this.peekBuffer(uint8Array, options);
-        this.position += bytesRead;
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const bytesRead = await this.peekBuffer(uint8Array, normOptions);
+        this.position = normOptions.position + bytesRead;
         return bytesRead;
     }
     /**
@@ -34,7 +32,7 @@ export class BufferTokenizer extends AbstractTokenizer {
     async peekBuffer(uint8Array, options) {
         const normOptions = this.normalizeOptions(uint8Array, options);
         const bytes2read = Math.min(this.uint8Array.length - normOptions.position, normOptions.length);
-        if ((!normOptions.mayBeLess) && bytes2read < normOptions.length) {
+        if (!normOptions.mayBeLess && bytes2read < normOptions.length) {
             throw new EndOfStreamError();
         }
         uint8Array.set(this.uint8Array.subarray(normOptions.position, normOptions.position + bytes2read));

package/lib/FileTokenizer.d.ts

@@ -1,6 +1,6 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import type { IRandomAccessTokenizer, IRandomAccessFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 import { type FileHandle } from 'node:fs/promises';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import type { IRandomAccessFileInfo, IRandomAccessTokenizer, IReadChunkOptions, ITokenizerOptions } from './types.js';
 interface IFileTokenizerOptions extends ITokenizerOptions {
     /**
      * Pass additional file information to the tokenizer
@@ -13,8 +13,9 @@ export declare class FileTokenizer extends AbstractTokenizer implements IRandomA
     /**
      * Create tokenizer from provided file path
      * @param sourceFilePath File path
+     * @param options Tokenizer options
      */
-    static fromFile(sourceFilePath: string): Promise<FileTokenizer>;
+    static fromFile(sourceFilePath: string, options?: ITokenizerOptions): Promise<FileTokenizer>;
     protected constructor(fileHandle: FileHandle, options: IFileTokenizerOptions);
     /**
      * Read buffer from file

package/lib/FileTokenizer.js

@@ -1,15 +1,29 @@
+import { open as fsOpen } from 'node:fs/promises';
+import { throwIfAborted } from './AbortSignal.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { getMaxBufferSize } from './BufferSizeError.js';
 import { EndOfStreamError } from './stream/index.js';
-import { open as fsOpen } from 'node:fs/promises';
 export class FileTokenizer extends AbstractTokenizer {
     /**
      * Create tokenizer from provided file path
      * @param sourceFilePath File path
+     * @param options Tokenizer options
      */
-    static async fromFile(sourceFilePath) {
+    static async fromFile(sourceFilePath, options) {
+        getMaxBufferSize(options);
+        throwIfAborted(options?.abortSignal);
         const fileHandle = await fsOpen(sourceFilePath, 'r');
-        const stat = await fileHandle.stat();
-        return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
+        try {
+            const stat = await fileHandle.stat();
+            return new FileTokenizer(fileHandle, {
+                ...options,
+                fileInfo: { ...options?.fileInfo, path: sourceFilePath, size: stat.size }
+            });
+        }
+        catch (error) {
+            await fileHandle.close();
+            throw error;
+        }
     }
     constructor(fileHandle, options) {
         super(options);
@@ -25,8 +39,9 @@ export class FileTokenizer extends AbstractTokenizer {
     async readBuffer(uint8Array, options) {
         const normOptions = this.normalizeOptions(uint8Array, options);
         this.position = normOptions.position;
-        if (normOptions.length === 0)
+        if (normOptions.length === 0) {
             return 0;
+        }
         const res = await this.fileHandle.read(uint8Array, 0, normOptions.length, normOptions.position);
         this.position += res.bytesRead;
         if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
@@ -43,12 +58,13 @@ export class FileTokenizer extends AbstractTokenizer {
     async peekBuffer(uint8Array, options) {
         const normOptions = this.normalizeOptions(uint8Array, options);
         const res = await this.fileHandle.read(uint8Array, 0, normOptions.length, normOptions.position);
-        if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
+        if (!normOptions.mayBeLess && res.bytesRead < normOptions.length) {
             throw new EndOfStreamError();
         }
         return res.bytesRead;
     }
     async close() {
+        this.removeAbortListener();
         await this.fileHandle.close();
         return super.close();
     }

package/lib/ReadStreamTokenizer.d.ts

@@ -24,6 +24,9 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
      * @returns Promise with number of bytes peeked
      */
     peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * @param length Number of bytes to ignore. Must be ≥ 0.
+     */
     ignore(length: number): Promise<number>;
     abort(): Promise<void>;
     close(): Promise<void>;

package/lib/ReadStreamTokenizer.js

@@ -1,6 +1,6 @@
 import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { EndOfStreamError } from './stream/index.js';
-const maxBufferSize = 256000;
+const ignoreChunkSize = 256000;
 export class ReadStreamTokenizer extends AbstractTokenizer {
     /**
      * Constructor
@@ -31,9 +31,17 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
         if (normOptions.length === 0) {
             return 0;
         }
-        const bytesRead = await this.streamReader.read(uint8Array.subarray(0, normOptions.length), normOptions.mayBeLess);
-        this.position += bytesRead;
-        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
+        let bytesRead = 0;
+        while (bytesRead < normOptions.length) {
+            const chunkLength = Math.min(this.maxBufferSize, normOptions.length - bytesRead);
+            const count = await this.streamReader.read(uint8Array.subarray(bytesRead, bytesRead + chunkLength), normOptions.mayBeLess);
+            bytesRead += count;
+            this.position += count;
+            if (count < chunkLength) {
+                break;
+            }
+        }
+        if (!normOptions.mayBeLess && bytesRead < normOptions.length) {
             throw new EndOfStreamError();
         }
         return bytesRead;
@@ -46,14 +54,20 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
      */
     async peekBuffer(uint8Array, options) {
         const normOptions = this.normalizeOptions(uint8Array, options);
+        if (normOptions.length === 0) {
+            return 0;
+        }
+        const lookaheadSize = normOptions.length + normOptions.position - this.position;
+        this.validateBufferSize(lookaheadSize);
         let bytesRead = 0;
         if (normOptions.position) {
             const skipBytes = normOptions.position - this.position;
             if (skipBytes > 0) {
                 const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
                 bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes));
-                return bytesRead - skipBytes;
+                const bytesPeeked = Math.max(0, bytesRead - skipBytes);
+                uint8Array.set(skipBuffer.subarray(skipBytes, skipBytes + bytesPeeked));
+                return bytesPeeked;
             }
             if (skipBytes < 0) {
                 throw new Error('Cannot peek from a negative offset in a stream');
@@ -69,15 +83,22 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
                 }
                 throw err;
             }
-            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
+            if (!normOptions.mayBeLess && bytesRead < normOptions.length) {
                 throw new EndOfStreamError();
             }
         }
         return bytesRead;
     }
+    /**
+     * @param length Number of bytes to ignore. Must be ≥ 0.
+     */
     async ignore(length) {
-        // debug(`ignore ${this.position}...${this.position + length - 1}`);
-        const bufSize = Math.min(maxBufferSize, length);
+        if (length < 0) {
+            throw new RangeError('ignore length must be ≥ 0 bytes');
+        }
+        this.validateLength(length, 'ignore length');
+        this.validateLength(this.position + length, 'end position');
+        const bufSize = Math.min(ignoreChunkSize, this.maxBufferSize, length);
         const buf = new Uint8Array(bufSize);
         let totBytesRead = 0;
         while (totBytesRead < length) {
@@ -91,9 +112,11 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
         return totBytesRead;
     }
     abort() {
+        this.removeAbortListener();
         return this.streamReader.abort();
     }
     async close() {
+        this.removeAbortListener();
         return this.streamReader.close();
     }
     supportsRandomAccess() {

package/lib/core.d.ts

@@ -1,13 +1,14 @@
 import type { Readable } from 'node:stream';
-import { type AnyWebByteStream } from './stream/index.js';
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { BlobTokenizer } from './BlobTokenizer.js';
 import { BufferTokenizer } from './BufferTokenizer.js';
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { type AnyWebByteStream } from './stream/index.js';
 import type { ITokenizerOptions } from './types.js';
-import { BlobTokenizer } from './BlobTokenizer.js';
-export { EndOfStreamError, AbortError, type AnyWebByteStream } from './stream/index.js';
-export type { ITokenizer, IRandomAccessTokenizer, IFileInfo, IRandomAccessFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
-export type { IToken, IGetToken } from '@tokenizer/token';
+export type { IGetToken, IToken } from '@tokenizer/token';
 export { AbstractTokenizer } from './AbstractTokenizer.js';
+export { BufferSizeError } from './BufferSizeError.js';
+export { AbortError, type AnyWebByteStream, EndOfStreamError } from './stream/index.js';
+export type { IFileInfo, IRandomAccessFileInfo, IRandomAccessTokenizer, IReadChunkOptions, ITokenizer, ITokenizerOptions, OnClose } from './types.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property/