strtok3 8.0.1 → 8.0.2

package/package.json

@@ -1,100 +1,89 @@
-{
-  "name": "strtok3",
-  "version": "8.0.1",
-  "description": "A promise based streaming tokenizer",
-  "author": {
-    "name": "Borewit",
-    "url": "https://github.com/Borewit"
-  },
-  "funding": {
-    "type": "github",
-    "url": "https://github.com/sponsors/Borewit"
-  },
-  "scripts": {
-    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
-    "compile-src": "tsc -p lib",
-    "compile-test": "tsc -p test",
-    "compile": "npm run compile-src && npm run compile-test",
-    "build": "npm run clean && npm run compile",
-    "eslint": "eslint lib test",
-    "lint-md": "remark -u preset-lint-recommended .",
-    "lint": "npm run lint-md && npm run eslint",
-    "fix": "eslint lib test --ext .ts --ignore-pattern *.d.ts --fix",
-    "test": "mocha",
-    "test-coverage": "c8 npm run test",
-    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
-    "start": "npm run compile && npm run lint && npm run cover-test"
-  },
-  "engines": {
-    "node": ">=16"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/Borewit/strtok3.git"
-  },
-  "license": "MIT",
-  "type": "module",
-  "exports": {
-    ".": {
-      "node": "./lib/index.js",
-      "default": "./lib/core.js"
-    },
-    "./core": "./lib/core.js"
-  },
-  "types": "lib/index.d.ts",
-  "files": [
-    "lib/**/*.js",
-    "lib/**/*.d.ts"
-  ],
-  "bugs": {
-    "url": "https://github.com/Borewit/strtok3/issues"
-  },
-  "devDependencies": {
-    "@eslint/compat": "^1.1.1",
-    "@eslint/eslintrc": "^3.1.0",
-    "@eslint/js": "^9.7.0",
-    "@types/chai": "^4.3.16",
-    "@types/debug": "^4.1.12",
-    "@types/mocha": "^10.0.7",
-    "@types/node": "^20.14.11",
-    "@typescript-eslint/eslint-plugin": "^7.16.1",
-    "@typescript-eslint/parser": "^7.16.1",
-    "c8": "^10.1.2",
-    "chai": "^5.1.1",
-    "del-cli": "^5.1.0",
-    "eslint": "^9.7.0",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-import-resolver-typescript": "^3.6.1",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jsdoc": "^48.8.3",
-    "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-unicorn": "^54.0.0",
-    "global": "^4.4.0",
-    "mocha": "^10.7.0",
-    "remark-cli": "^12.0.1",
-    "remark-preset-lint-recommended": "^7.0.0",
-    "token-types": "^6.0.0",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.5.3",
-    "uint8array-extras": "^1.4.0"
-  },
-  "dependencies": {
-    "@tokenizer/token": "^0.3.0",
-    "peek-readable": "^5.1.3"
-  },
-  "keywords": [
-    "tokenizer",
-    "reader",
-    "token",
-    "async",
-    "promise",
-    "parser",
-    "decoder",
-    "binary",
-    "endian",
-    "uint",
-    "stream",
-    "streaming"
-  ],
-  "packageManager": "yarn@4.3.1"
-}
+{
+  "name": "strtok3",
+  "version": "8.0.2",
+  "description": "A promise based streaming tokenizer",
+  "author": {
+    "name": "Borewit",
+    "url": "https://github.com/Borewit"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/Borewit"
+  },
+  "scripts": {
+    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
+    "compile-src": "tsc -p lib",
+    "compile-test": "tsc -p test",
+    "compile": "yarn run compile-src && yarn run compile-test",
+    "build": "yarn run clean && yarn run compile",
+    "eslint": "eslint lib test",
+    "lint-md": "remark -u preset-lint-recommended .",
+    "lint-ts": "biome check",
+    "lint": "yarn run lint-md && yarn run lint-ts",
+    "fix": "yarn run biome lint --write",
+    "test": "mocha",
+    "test-coverage": "c8 yarn run test",
+    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
+    "start": "yarn run compile && yarn run lint && yarn run cover-test"
+  },
+  "engines": {
+    "node": ">=16"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Borewit/strtok3.git"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": {
+      "node": "./lib/index.js",
+      "default": "./lib/core.js"
+    },
+    "./core": "./lib/core.js"
+  },
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib/**/*.js",
+    "lib/**/*.d.ts"
+  ],
+  "bugs": {
+    "url": "https://github.com/Borewit/strtok3/issues"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^1.8.3",
+    "@types/chai": "^4.3.17",
+    "@types/debug": "^4.1.12",
+    "@types/mocha": "^10.0.7",
+    "@types/node": "^22.1.0",
+    "c8": "^10.1.2",
+    "chai": "^5.1.1",
+    "del-cli": "^5.1.0",
+    "mocha": "^10.7.0",
+    "remark-cli": "^12.0.1",
+    "remark-preset-lint-recommended": "^7.0.0",
+    "token-types": "^6.0.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.4",
+    "uint8array-extras": "^1.4.0"
+  },
+  "dependencies": {
+    "@tokenizer/token": "^0.3.0",
+    "peek-readable": "^5.1.3"
+  },
+  "keywords": [
+    "tokenizer",
+    "reader",
+    "token",
+    "async",
+    "promise",
+    "parser",
+    "decoder",
+    "binary",
+    "endian",
+    "uint",
+    "stream",
+    "streaming"
+  ],
+  "packageManager": "yarn@4.3.1"
+}

package/lib/AbstractTokenizer.d.ts

@@ -1,75 +0,0 @@
-import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
-import type { IGetToken, IToken } from '@tokenizer/token';
-interface INormalizedReadChunkOptions extends IReadChunkOptions {
-    offset: number;
-    length: number;
-    position: number;
-    mayBeLess?: boolean;
-}
-/**
- * Core tokenizer
- */
-export declare abstract class AbstractTokenizer implements ITokenizer {
-    fileInfo: IFileInfo;
-    private onClose?;
-    /**
-     * Constructor
-     * @param options Tokenizer options
-     * @protected
-     */
-    protected constructor(options?: ITokenizerOptions);
-    /**
-     * Tokenizer-stream position
-     */
-    position: number;
-    private numBuffer;
-    /**
-     * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param options - Additional read options
-     * @returns Promise with number of bytes read
-     */
-    abstract readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Target buffer to fill with data peeked from the tokenizer-stream
-     * @param options - Peek behaviour options
-     * @returns Promise with number of bytes read
-     */
-    abstract peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Read a token from the tokenizer-stream
-     * @param token - The token to read
-     * @param position - If provided, the desired position in the tokenizer-stream
-     * @returns Promise with token data
-     */
-    readToken<Value>(token: IGetToken<Value>, position?: number): Promise<Value>;
-    /**
-     * Peek a token from the tokenizer-stream.
-     * @param token - Token to peek from the tokenizer-stream.
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @returns Promise with token data
-     */
-    peekToken<Value>(token: IGetToken<Value>, position?: number): Promise<Value>;
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    readNumber(token: IToken<number>): Promise<number>;
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    peekNumber(token: IToken<number>): Promise<number>;
-    /**
-     * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    ignore(length: number): Promise<number>;
-    close(): Promise<void>;
-    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): INormalizedReadChunkOptions;
-}
-export {};

package/lib/AbstractTokenizer.js

@@ -1,108 +0,0 @@
-import { EndOfStreamError } from 'peek-readable';
-/**
- * Core tokenizer
- */
-export class AbstractTokenizer {
-    /**
-     * Constructor
-     * @param options Tokenizer options
-     * @protected
-     */
-    constructor(options) {
-        var _a;
-        /**
-         * Tokenizer-stream position
-         */
-        this.position = 0;
-        this.numBuffer = new Uint8Array(8);
-        this.fileInfo = (_a = options === null || options === void 0 ? void 0 : options.fileInfo) !== null && _a !== void 0 ? _a : {};
-        this.onClose = options === null || options === void 0 ? void 0 : options.onClose;
-    }
-    /**
-     * Read a token from the tokenizer-stream
-     * @param token - The token to read
-     * @param position - If provided, the desired position in the tokenizer-stream
-     * @returns Promise with token data
-     */
-    async readToken(token, position = this.position) {
-        const uint8Array = new Uint8Array(token.len);
-        const len = await this.readBuffer(uint8Array, { position });
-        if (len < token.len)
-            throw new EndOfStreamError();
-        return token.get(uint8Array, 0);
-    }
-    /**
-     * Peek a token from the tokenizer-stream.
-     * @param token - Token to peek from the tokenizer-stream.
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @returns Promise with token data
-     */
-    async peekToken(token, position = this.position) {
-        const uint8Array = new Uint8Array(token.len);
-        const len = await this.peekBuffer(uint8Array, { position });
-        if (len < token.len)
-            throw new EndOfStreamError();
-        return token.get(uint8Array, 0);
-    }
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    async readNumber(token) {
-        const len = await this.readBuffer(this.numBuffer, { length: token.len });
-        if (len < token.len)
-            throw new EndOfStreamError();
-        return token.get(this.numBuffer, 0);
-    }
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    async peekNumber(token) {
-        const len = await this.peekBuffer(this.numBuffer, { length: token.len });
-        if (len < token.len)
-            throw new EndOfStreamError();
-        return token.get(this.numBuffer, 0);
-    }
-    /**
-     * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    async ignore(length) {
-        if (this.fileInfo.size !== undefined) {
-            const bytesLeft = this.fileInfo.size - this.position;
-            if (length > bytesLeft) {
-                this.position += bytesLeft;
-                return bytesLeft;
-            }
-        }
-        this.position += length;
-        return length;
-    }
-    async close() {
-        var _a;
-        await ((_a = this.onClose) === null || _a === void 0 ? void 0 : _a.call(this));
-    }
-    normalizeOptions(uint8Array, options) {
-        if (options && options.position !== undefined && options.position < this.position) {
-            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-        }
-        if (options) {
-            return {
-                mayBeLess: options.mayBeLess === true,
-                offset: options.offset ? options.offset : 0,
-                length: options.length ? options.length : (uint8Array.length - (options.offset ? options.offset : 0)),
-                position: options.position ? options.position : this.position
-            };
-        }
-        return {
-            mayBeLess: false,
-            offset: 0,
-            length: uint8Array.length,
-            position: this.position
-        };
-    }
-}

package/lib/BufferTokenizer.d.ts

@@ -1,26 +0,0 @@
-import { ITokenizerOptions, IReadChunkOptions } from './types.js';
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-export declare class BufferTokenizer extends AbstractTokenizer {
-    private uint8Array;
-    /**
-     * Construct BufferTokenizer
-     * @param uint8Array - Uint8Array to tokenize
-     * @param options Tokenizer options
-     */
-    constructor(uint8Array: Uint8Array, options?: ITokenizerOptions);
-    /**
-     * Read buffer from tokenizer
-     * @param uint8Array - Uint8Array to tokenize
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    close(): Promise<void>;
-}

package/lib/BufferTokenizer.js

@@ -1,51 +0,0 @@
-import { EndOfStreamError } from 'peek-readable';
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-export class BufferTokenizer extends AbstractTokenizer {
-    /**
-     * Construct BufferTokenizer
-     * @param uint8Array - Uint8Array to tokenize
-     * @param options Tokenizer options
-     */
-    constructor(uint8Array, options) {
-        super(options);
-        this.uint8Array = uint8Array;
-        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
-    }
-    /**
-     * Read buffer from tokenizer
-     * @param uint8Array - Uint8Array to tokenize
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    async readBuffer(uint8Array, options) {
-        if (options && options.position) {
-            if (options.position < this.position) {
-                throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-            }
-            this.position = options.position;
-        }
-        const bytesRead = await this.peekBuffer(uint8Array, options);
-        this.position += bytesRead;
-        return bytesRead;
-    }
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    async peekBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        const bytes2read = Math.min(this.uint8Array.length - normOptions.position, normOptions.length);
-        if ((!normOptions.mayBeLess) && bytes2read < normOptions.length) {
-            throw new EndOfStreamError();
-        }
-        else {
-            uint8Array.set(this.uint8Array.subarray(normOptions.position, normOptions.position + bytes2read), normOptions.offset);
-            return bytes2read;
-        }
-    }
-    close() {
-        return super.close();
-    }
-}

package/lib/FileTokenizer.d.ts

@@ -1,23 +0,0 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
-import { FileHandle } from 'node:fs/promises';
-export declare class FileTokenizer extends AbstractTokenizer {
-    private fileHandle;
-    constructor(fileHandle: FileHandle, options: ITokenizerOptions);
-    /**
-     * Read buffer from file
-     * @param uint8Array - Uint8Array to write result to
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek buffer from file
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    close(): Promise<void>;
-}
-export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;

package/lib/FileTokenizer.js

@@ -1,48 +0,0 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { EndOfStreamError } from 'peek-readable';
-import { open as fsOpen } from 'node:fs/promises';
-export class FileTokenizer extends AbstractTokenizer {
-    constructor(fileHandle, options) {
-        super(options);
-        this.fileHandle = fileHandle;
-    }
-    /**
-     * Read buffer from file
-     * @param uint8Array - Uint8Array to write result to
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    async readBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        this.position = normOptions.position;
-        const res = await this.fileHandle.read(uint8Array, normOptions.offset, normOptions.length, normOptions.position);
-        this.position += res.bytesRead;
-        if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
-            throw new EndOfStreamError();
-        }
-        return res.bytesRead;
-    }
-    /**
-     * Peek buffer from file
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    async peekBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        const res = await this.fileHandle.read(uint8Array, normOptions.offset, normOptions.length, normOptions.position);
-        if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
-            throw new EndOfStreamError();
-        }
-        return res.bytesRead;
-    }
-    async close() {
-        await this.fileHandle.close();
-        return super.close();
-    }
-}
-export async function fromFile(sourceFilePath) {
-    const fileHandle = await fsOpen(sourceFilePath, 'r');
-    const stat = await fileHandle.stat();
-    return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
-}

package/lib/ReadStreamTokenizer.d.ts

@@ -1,27 +0,0 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { type IStreamReader } from 'peek-readable';
-import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
-export declare class ReadStreamTokenizer extends AbstractTokenizer {
-    private streamReader;
-    /**
-     * Constructor
-     * @param streamReader stream-reader to read from
-     * @param options Tokenizer options
-     */
-    constructor(streamReader: IStreamReader, options?: ITokenizerOptions);
-    /**
-     * Read buffer from tokenizer
-     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes peeked
-     */
-    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    ignore(length: number): Promise<number>;
-}

package/lib/ReadStreamTokenizer.js

@@ -1,92 +0,0 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { EndOfStreamError } from 'peek-readable';
-const maxBufferSize = 256000;
-export class ReadStreamTokenizer extends AbstractTokenizer {
-    /**
-     * Constructor
-     * @param streamReader stream-reader to read from
-     * @param options Tokenizer options
-     */
-    constructor(streamReader, options) {
-        super(options);
-        this.streamReader = streamReader;
-    }
-    /**
-     * Read buffer from tokenizer
-     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    async readBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        const skipBytes = normOptions.position - this.position;
-        if (skipBytes > 0) {
-            await this.ignore(skipBytes);
-            return this.readBuffer(uint8Array, options);
-        }
-        else if (skipBytes < 0) {
-            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-        }
-        if (normOptions.length === 0) {
-            return 0;
-        }
-        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
-        this.position += bytesRead;
-        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
-            throw new EndOfStreamError();
-        }
-        return bytesRead;
-    }
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes peeked
-     */
-    async peekBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        let bytesRead = 0;
-        if (normOptions.position) {
-            const skipBytes = normOptions.position - this.position;
-            if (skipBytes > 0) {
-                const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
-                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
-                return bytesRead - skipBytes;
-            }
-            else if (skipBytes < 0) {
-                throw new Error('Cannot peek from a negative offset in a stream');
-            }
-        }
-        if (normOptions.length > 0) {
-            try {
-                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
-            }
-            catch (err) {
-                if (options && options.mayBeLess && err instanceof EndOfStreamError) {
-                    return 0;
-                }
-                throw err;
-            }
-            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
-                throw new EndOfStreamError();
-            }
-        }
-        return bytesRead;
-    }
-    async ignore(length) {
-        // debug(`ignore ${this.position}...${this.position + length - 1}`);
-        const bufSize = Math.min(maxBufferSize, length);
-        const buf = new Uint8Array(bufSize);
-        let totBytesRead = 0;
-        while (totBytesRead < length) {
-            const remaining = length - totBytesRead;
-            const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
-            if (bytesRead < 0) {
-                return bytesRead;
-            }
-            totBytesRead += bytesRead;
-        }
-        return totBytesRead;
-    }
-}