strtok3 9.0.1 → 9.1.1

package/lib/AbstractTokenizer.d.ts

@@ -10,19 +10,20 @@ interface INormalizedReadChunkOptions extends IReadChunkOptions {
  * Core tokenizer
  */
 export declare abstract class AbstractTokenizer implements ITokenizer {
-    fileInfo: IFileInfo;
     private onClose?;
+    private numBuffer;
+    abstract fileInfo: IFileInfo;
+    /**
+     * Tokenizer-stream position
+     */
+    position: number;
     /**
      * Constructor
      * @param options Tokenizer options
      * @protected
      */
     protected constructor(options?: ITokenizerOptions);
-    /**
-     * Tokenizer-stream position
-     */
-    position: number;
-    private numBuffer;
+    abstract supportsRandomAccess(): boolean;
     /**
      * Read buffer from tokenizer
      * @param buffer - Target buffer to fill with data read from the tokenizer-stream

package/lib/AbstractTokenizer.js

@@ -9,12 +9,11 @@ export class AbstractTokenizer {
      * @protected
      */
     constructor(options) {
+        this.numBuffer = new Uint8Array(8);
         /**
          * Tokenizer-stream position
          */
         this.position = 0;
-        this.numBuffer = new Uint8Array(8);
-        this.fileInfo = options?.fileInfo ?? {};
         this.onClose = options?.onClose;
         if (options?.abortSignal) {
             options.abortSignal.addEventListener('abort', () => {

package/lib/BufferTokenizer.d.ts

@@ -1,7 +1,8 @@
-import type { ITokenizerOptions, IReadChunkOptions } from './types.js';
+import type { ITokenizerOptions, IReadChunkOptions, IRandomAccessFileInfo, IRandomAccessTokenizer } from './types.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
-export declare class BufferTokenizer extends AbstractTokenizer {
+export declare class BufferTokenizer extends AbstractTokenizer implements IRandomAccessTokenizer {
     private uint8Array;
+    fileInfo: IRandomAccessFileInfo;
     /**
      * Construct BufferTokenizer
      * @param uint8Array - Uint8Array to tokenize
@@ -23,4 +24,6 @@ export declare class BufferTokenizer extends AbstractTokenizer {
      */
     peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     close(): Promise<void>;
+    supportsRandomAccess(): boolean;
+    setPosition(position: number): void;
 }

package/lib/BufferTokenizer.js

@@ -9,7 +9,7 @@ export class BufferTokenizer extends AbstractTokenizer {
     constructor(uint8Array, options) {
         super(options);
         this.uint8Array = uint8Array;
-        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
+        this.fileInfo = { ...options?.fileInfo ?? {}, ...{ size: uint8Array.length } };
     }
     /**
      * Read buffer from tokenizer
@@ -46,4 +46,10 @@ export class BufferTokenizer extends AbstractTokenizer {
     close() {
         return super.close();
     }
+    supportsRandomAccess() {
+        return true;
+    }
+    setPosition(position) {
+        this.position = position;
+    }
 }

package/lib/FileTokenizer.d.ts

@@ -1,9 +1,21 @@
 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
+import type { IRandomAccessTokenizer, IRandomAccessFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 import { type FileHandle } from 'node:fs/promises';
-export declare class FileTokenizer extends AbstractTokenizer {
+interface IFileTokenizerOptions extends ITokenizerOptions {
+    /**
+     * Pass additional file information to the tokenizer
+     */
+    fileInfo: IRandomAccessFileInfo;
+}
+export declare class FileTokenizer extends AbstractTokenizer implements IRandomAccessTokenizer {
     private fileHandle;
-    constructor(fileHandle: FileHandle, options: ITokenizerOptions);
+    fileInfo: IRandomAccessFileInfo;
+    /**
+     * Create tokenizer from provided file path
+     * @param sourceFilePath File path
+     */
+    static fromFile(sourceFilePath: string): Promise<FileTokenizer>;
+    protected constructor(fileHandle: FileHandle, options: IFileTokenizerOptions);
     /**
      * Read buffer from file
      * @param uint8Array - Uint8Array to write result to
@@ -19,5 +31,7 @@ export declare class FileTokenizer extends AbstractTokenizer {
      */
     peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     close(): Promise<void>;
+    setPosition(position: number): void;
+    supportsRandomAccess(): boolean;
 }
-export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;
+export {};

package/lib/FileTokenizer.js

@@ -2,9 +2,19 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { EndOfStreamError } from 'peek-readable';
 import { open as fsOpen } from 'node:fs/promises';
 export class FileTokenizer extends AbstractTokenizer {
+    /**
+     * Create tokenizer from provided file path
+     * @param sourceFilePath File path
+     */
+    static async fromFile(sourceFilePath) {
+        const fileHandle = await fsOpen(sourceFilePath, 'r');
+        const stat = await fileHandle.stat();
+        return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
+    }
     constructor(fileHandle, options) {
         super(options);
         this.fileHandle = fileHandle;
+        this.fileInfo = options.fileInfo;
     }
     /**
      * Read buffer from file
@@ -42,9 +52,10 @@ export class FileTokenizer extends AbstractTokenizer {
         await this.fileHandle.close();
         return super.close();
     }
-}
-export async function fromFile(sourceFilePath) {
-    const fileHandle = await fsOpen(sourceFilePath, 'r');
-    const stat = await fileHandle.stat();
-    return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
+    setPosition(position) {
+        this.position = position;
+    }
+    supportsRandomAccess() {
+        return true;
+    }
 }

package/lib/ReadStreamTokenizer.d.ts

@@ -1,8 +1,9 @@
 import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { type IStreamReader } from 'peek-readable';
-import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
+import type { IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
+    fileInfo: IFileInfo;
     /**
      * Constructor
      * @param streamReader stream-reader to read from
@@ -25,4 +26,5 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
     peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     ignore(length: number): Promise<number>;
     abort(): Promise<void>;
+    supportsRandomAccess(): boolean;
 }

package/lib/ReadStreamTokenizer.js

@@ -10,6 +10,7 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
     constructor(streamReader, options) {
         super(options);
         this.streamReader = streamReader;
+        this.fileInfo = options?.fileInfo ?? {};
     }
     /**
      * Read buffer from tokenizer
@@ -92,4 +93,7 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
     abort() {
         return this.streamReader.abort();
     }
+    supportsRandomAccess() {
+        return false;
+    }
 }

package/lib/core.d.ts

@@ -4,7 +4,7 @@ import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { BufferTokenizer } from './BufferTokenizer.js';
 import type { ITokenizerOptions } from './types.js';
 export { EndOfStreamError, type AnyWebByteStream } from 'peek-readable';
-export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
+export type { ITokenizer, IRandomAccessTokenizer, IFileInfo, IRandomAccessFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
 export type { IToken, IGetToken } from '@tokenizer/token';
 export { AbstractTokenizer } from './AbstractTokenizer.js';
 /**

package/lib/index.d.ts

@@ -1,7 +1,8 @@
 import type { Readable } from 'node:stream';
 import type { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { type ITokenizerOptions } from './core.js';
-export { fromFile } from './FileTokenizer.js';
+import { FileTokenizer } from "./FileTokenizer.js";
+export { FileTokenizer } from './FileTokenizer.js';
 export * from './core.js';
 export type { IToken, IGetToken } from '@tokenizer/token';
 /**
@@ -12,3 +13,4 @@ export type { IToken, IGetToken } from '@tokenizer/token';
  * @returns Tokenizer
  */
 export declare function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>;
+export declare const fromFile: typeof FileTokenizer.fromFile;

package/lib/index.js

@@ -1,6 +1,7 @@
 import { stat as fsStat } from 'node:fs/promises';
 import { fromStream as coreFromStream } from './core.js';
-export { fromFile } from './FileTokenizer.js';
+import { FileTokenizer } from "./FileTokenizer.js";
+export { FileTokenizer } from './FileTokenizer.js';
 export * from './core.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
@@ -10,12 +11,12 @@ export * from './core.js';
  * @returns Tokenizer
  */
 export async function fromStream(stream, options) {
-    const augmentedOptions = options ?? {};
-    augmentedOptions.fileInfo = augmentedOptions.fileInfo ?? {};
+    const rst = coreFromStream(stream, options);
     if (stream.path) {
         const stat = await fsStat(stream.path);
-        augmentedOptions.fileInfo.path = stream.path;
-        augmentedOptions.fileInfo.size = stat.size;
+        rst.fileInfo.path = stream.path;
+        rst.fileInfo.size = stat.size;
     }
-    return coreFromStream(stream, augmentedOptions);
+    return rst;
 }
+export const fromFile = FileTokenizer.fromFile;

package/lib/types.d.ts

@@ -17,6 +17,12 @@ export interface IFileInfo {
      */
     url?: string;
 }
+export interface IRandomAccessFileInfo extends IFileInfo {
+    /**
+     * File size in bytes
+     */
+    size: number;
+}
 export interface IReadChunkOptions {
     /**
      * The offset in the buffer to start writing at; default is 0
@@ -29,14 +35,25 @@ export interface IReadChunkOptions {
     /**
      * Position where to begin reading from the file.
      * Default it is `tokenizer.position`.
-     * Position may not be less then `tokenizer.position`.
+     * Position may not be less than `tokenizer.position`, unless `supportsRandomAccess()` returns `true`.
      */
     position?: number;
     /**
-     * If set, will not throw an EOF error if not all of the requested data could be read
+     * If set, will not throw an EOF error if not all off the requested data could be read
      */
     mayBeLess?: boolean;
 }
+export interface IRandomAccessTokenizer extends ITokenizer {
+    /**
+     * Provide access to information of the underlying information stream or file.
+     */
+    fileInfo: IRandomAccessFileInfo;
+    /**
+     * Change the position (offset) of the tokenizer
+     * @param position New position
+     */
+    setPosition(position: number): void;
+}
 /**
  * The tokenizer allows us to read or peek from the tokenizer-stream.
  * The tokenizer-stream is an abstraction of a stream, file or Buffer.
@@ -45,11 +62,11 @@ export interface ITokenizer {
     /**
      * Provide access to information of the underlying information stream or file.
      */
-    fileInfo: IFileInfo;
+    readonly fileInfo: IFileInfo;
     /**
      * Offset in bytes (= number of bytes read) since beginning of file or stream
      */
-    position: number;
+    readonly position: number;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
@@ -103,6 +120,10 @@ export interface ITokenizer {
      * Abort pending asynchronous operations
      */
     abort(): Promise<void>;
+    /**
+     * Returns true when the underlying file supports random access
+     */
+    supportsRandomAccess(): boolean;
 }
 export type OnClose = () => Promise<void>;
 export interface ITokenizerOptions {

package/package.json

@@ -1,91 +1,91 @@
-{
-  "name": "strtok3",
-  "version": "9.0.1",
-  "description": "A promise based streaming tokenizer",
-  "author": {
-    "name": "Borewit",
-    "url": "https://github.com/Borewit"
-  },
-  "funding": {
-    "type": "github",
-    "url": "https://github.com/sponsors/Borewit"
-  },
-  "scripts": {
-    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
-    "compile-src": "tsc -p lib",
-    "compile-test": "tsc -p test",
-    "compile": "yarn run compile-src && yarn run compile-test",
-    "build": "yarn run clean && yarn run compile",
-    "eslint": "eslint lib test",
-    "lint-md": "remark -u preset-lint-recommended .",
-    "lint-ts": "biome check",
-    "lint": "yarn run lint-md && yarn run lint-ts",
-    "fix": "yarn run biome lint --write",
-    "test": "mocha",
-    "test-coverage": "c8 yarn run test",
-    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
-    "start": "yarn run compile && yarn run lint && yarn run cover-test"
-  },
-  "engines": {
-    "node": ">=16"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/Borewit/strtok3.git"
-  },
-  "license": "MIT",
-  "type": "module",
-  "exports": {
-    ".": {
-      "node": "./lib/index.js",
-      "default": "./lib/core.js"
-    },
-    "./core": "./lib/core.js"
-  },
-  "types": "lib/index.d.ts",
-  "files": [
-    "lib/**/*.js",
-    "lib/**/*.d.ts"
-  ],
-  "bugs": {
-    "url": "https://github.com/Borewit/strtok3/issues"
-  },
-  "devDependencies": {
-    "@biomejs/biome": "^1.9.3",
-    "@types/chai": "^5.0.0",
-    "@types/chai-as-promised": "^8.0.1",
-    "@types/debug": "^4.1.12",
-    "@types/mocha": "^10.0.9",
-    "@types/node": "^22.7.5",
-    "c8": "^10.1.2",
-    "chai": "^5.1.1",
-    "chai-as-promised": "^8.0.0",
-    "del-cli": "^6.0.0",
-    "mocha": "^10.7.3",
-    "remark-cli": "^12.0.1",
-    "remark-preset-lint-recommended": "^7.0.0",
-    "token-types": "^6.0.0",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.6.3",
-    "uint8array-extras": "^1.4.0"
-  },
-  "dependencies": {
-    "@tokenizer/token": "^0.3.0",
-    "peek-readable": "^5.3.1"
-  },
-  "keywords": [
-    "tokenizer",
-    "reader",
-    "token",
-    "async",
-    "promise",
-    "parser",
-    "decoder",
-    "binary",
-    "endian",
-    "uint",
-    "stream",
-    "streaming"
-  ],
-  "packageManager": "yarn@4.3.1"
-}
+{
+  "name": "strtok3",
+  "version": "9.1.1",
+  "description": "A promise based streaming tokenizer",
+  "author": {
+    "name": "Borewit",
+    "url": "https://github.com/Borewit"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/Borewit"
+  },
+  "scripts": {
+    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
+    "compile-src": "tsc -p lib",
+    "compile-test": "tsc -p test",
+    "compile": "yarn run compile-src && yarn run compile-test",
+    "build": "yarn run clean && yarn run compile",
+    "eslint": "eslint lib test",
+    "lint-md": "remark -u preset-lint-recommended .",
+    "lint-ts": "biome check",
+    "lint": "yarn run lint-md && yarn run lint-ts",
+    "fix": "yarn run biome lint --write",
+    "test": "mocha",
+    "test-coverage": "c8 yarn run test",
+    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
+    "start": "yarn run compile && yarn run lint && yarn run cover-test"
+  },
+  "engines": {
+    "node": ">=16"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Borewit/strtok3.git"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": {
+      "node": "./lib/index.js",
+      "default": "./lib/core.js"
+    },
+    "./core": "./lib/core.js"
+  },
+  "types": "lib/index.d.ts",
+  "files": [
+    "lib/**/*.js",
+    "lib/**/*.d.ts"
+  ],
+  "bugs": {
+    "url": "https://github.com/Borewit/strtok3/issues"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^1.9.4",
+    "@types/chai": "^5.0.1",
+    "@types/chai-as-promised": "^8.0.1",
+    "@types/debug": "^4.1.12",
+    "@types/mocha": "^10.0.9",
+    "@types/node": "^22.9.0",
+    "c8": "^10.1.2",
+    "chai": "^5.1.2",
+    "chai-as-promised": "^8.0.1",
+    "del-cli": "^6.0.0",
+    "mocha": "^10.8.2",
+    "remark-cli": "^12.0.1",
+    "remark-preset-lint-recommended": "^7.0.0",
+    "token-types": "^6.0.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.7.2",
+    "uint8array-extras": "^1.4.0"
+  },
+  "dependencies": {
+    "@tokenizer/token": "^0.3.0",
+    "peek-readable": "^5.3.1"
+  },
+  "keywords": [
+    "tokenizer",
+    "reader",
+    "token",
+    "async",
+    "promise",
+    "parser",
+    "decoder",
+    "binary",
+    "endian",
+    "uint",
+    "stream",
+    "streaming"
+  ],
+  "packageManager": "yarn@4.3.1"
+}