strtok3 7.1.1 → 8.0.1

package/lib/AbstractTokenizer.d.ts

@@ -1,5 +1,5 @@
-import { ITokenizer, IFileInfo, IReadChunkOptions } from './types.js';
-import { IGetToken, IToken } from '@tokenizer/token';
+import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
+import type { IGetToken, IToken } from '@tokenizer/token';
 interface INormalizedReadChunkOptions extends IReadChunkOptions {
     offset: number;
     length: number;
@@ -11,7 +11,13 @@ interface INormalizedReadChunkOptions extends IReadChunkOptions {
  */
 export declare abstract class AbstractTokenizer implements ITokenizer {
     fileInfo: IFileInfo;
-    protected constructor(fileInfo?: IFileInfo);
+    private onClose?;
+    /**
+     * Constructor
+     * @param options Tokenizer options
+     * @protected
+     */
+    protected constructor(options?: ITokenizerOptions);
     /**
      * Tokenizer-stream position
      */
@@ -26,7 +32,7 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
     abstract readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array- Target buffer to fill with data peek from the tokenizer-stream
+     * @param uint8Array - Target buffer to fill with data peeked from the tokenizer-stream
      * @param options - Peek behaviour options
      * @returns Promise with number of bytes read
      */

package/lib/AbstractTokenizer.js

@@ -3,13 +3,20 @@ import { EndOfStreamError } from 'peek-readable';
  * Core tokenizer
  */
 export class AbstractTokenizer {
-    constructor(fileInfo) {
+    /**
+     * Constructor
+     * @param options Tokenizer options
+     * @protected
+     */
+    constructor(options) {
+        var _a;
         /**
          * Tokenizer-stream position
          */
         this.position = 0;
         this.numBuffer = new Uint8Array(8);
-        this.fileInfo = fileInfo ? fileInfo : {};
+        this.fileInfo = (_a = options === null || options === void 0 ? void 0 : options.fileInfo) !== null && _a !== void 0 ? _a : {};
+        this.onClose = options === null || options === void 0 ? void 0 : options.onClose;
     }
     /**
      * Read a token from the tokenizer-stream
@@ -76,7 +83,8 @@ export class AbstractTokenizer {
         return length;
     }
     async close() {
-        // empty
+        var _a;
+        await ((_a = this.onClose) === null || _a === void 0 ? void 0 : _a.call(this));
     }
     normalizeOptions(uint8Array, options) {
         if (options && options.position !== undefined && options.position < this.position) {

package/lib/BufferTokenizer.d.ts

@@ -1,13 +1,13 @@
-import { IFileInfo, IReadChunkOptions } from './types.js';
+import { ITokenizerOptions, IReadChunkOptions } from './types.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
 export declare class BufferTokenizer extends AbstractTokenizer {
     private uint8Array;
     /**
      * Construct BufferTokenizer
      * @param uint8Array - Uint8Array to tokenize
-     * @param fileInfo - Pass additional file information to the tokenizer
+     * @param options Tokenizer options
      */
-    constructor(uint8Array: Uint8Array, fileInfo?: IFileInfo);
+    constructor(uint8Array: Uint8Array, options?: ITokenizerOptions);
     /**
      * Read buffer from tokenizer
      * @param uint8Array - Uint8Array to tokenize

package/lib/BufferTokenizer.js

@@ -4,10 +4,10 @@ export class BufferTokenizer extends AbstractTokenizer {
     /**
      * Construct BufferTokenizer
      * @param uint8Array - Uint8Array to tokenize
-     * @param fileInfo - Pass additional file information to the tokenizer
+     * @param options Tokenizer options
      */
-    constructor(uint8Array, fileInfo) {
-        super(fileInfo);
+    constructor(uint8Array, options) {
+        super(options);
         this.uint8Array = uint8Array;
         this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
     }
@@ -45,7 +45,7 @@ export class BufferTokenizer extends AbstractTokenizer {
             return bytes2read;
         }
     }
-    async close() {
-        // empty
+    close() {
+        return super.close();
     }
 }

package/lib/FileTokenizer.d.ts

@@ -1,9 +1,9 @@
 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { IFileInfo, IReadChunkOptions } from './types.js';
+import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
 import { FileHandle } from 'node:fs/promises';
 export declare class FileTokenizer extends AbstractTokenizer {
     private fileHandle;
-    constructor(fileHandle: FileHandle, fileInfo: IFileInfo);
+    constructor(fileHandle: FileHandle, options: ITokenizerOptions);
     /**
      * Read buffer from file
      * @param uint8Array - Uint8Array to write result to

package/lib/FileTokenizer.js

@@ -2,8 +2,8 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { EndOfStreamError } from 'peek-readable';
 import { open as fsOpen } from 'node:fs/promises';
 export class FileTokenizer extends AbstractTokenizer {
-    constructor(fileHandle, fileInfo) {
-        super(fileInfo);
+    constructor(fileHandle, options) {
+        super(options);
         this.fileHandle = fileHandle;
     }
     /**
@@ -37,11 +37,12 @@ export class FileTokenizer extends AbstractTokenizer {
         return res.bytesRead;
     }
     async close() {
-        return this.fileHandle.close();
+        await this.fileHandle.close();
+        return super.close();
     }
 }
 export async function fromFile(sourceFilePath) {
     const fileHandle = await fsOpen(sourceFilePath, 'r');
     const stat = await fileHandle.stat();
-    return new FileTokenizer(fileHandle, { path: sourceFilePath, size: stat.size });
+    return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
 }

package/lib/ReadStreamTokenizer.d.ts

@@ -1,14 +1,14 @@
 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { IStreamReader } from 'peek-readable';
-import { IFileInfo, IReadChunkOptions } from './types.js';
+import { type IStreamReader } from 'peek-readable';
+import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
-    constructor(streamReader: IStreamReader, fileInfo?: IFileInfo);
     /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
+     * Constructor
+     * @param streamReader stream-reader to read from
+     * @param options Tokenizer options
      */
-    getFileInfo(): Promise<IFileInfo>;
+    constructor(streamReader: IStreamReader, options?: ITokenizerOptions);
     /**
      * Read buffer from tokenizer
      * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream

package/lib/ReadStreamTokenizer.js

@@ -2,16 +2,14 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { EndOfStreamError } from 'peek-readable';
 const maxBufferSize = 256000;
 export class ReadStreamTokenizer extends AbstractTokenizer {
-    constructor(streamReader, fileInfo) {
-        super(fileInfo);
-        this.streamReader = streamReader;
-    }
     /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
+     * Constructor
+     * @param streamReader stream-reader to read from
+     * @param options Tokenizer options
      */
-    async getFileInfo() {
-        return this.fileInfo;
+    constructor(streamReader, options) {
+        super(options);
+        this.streamReader = streamReader;
     }
     /**
      * Read buffer from tokenizer

package/lib/core.d.ts

@@ -1,31 +1,32 @@
 import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { BufferTokenizer } from './BufferTokenizer.js';
-import { IFileInfo } from './types.js';
+import type { ITokenizerOptions } from './types.js';
 import type { ReadableStream } from 'node:stream/web';
 import type { Readable } from 'node:stream';
 export { EndOfStreamError } from 'peek-readable';
-export { ITokenizer, IFileInfo } from './types.js';
-export { IToken, IGetToken } from '@tokenizer/token';
+export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
+export type { IToken, IGetToken } from '@tokenizer/token';
+export { AbstractTokenizer } from './AbstractTokenizer.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property/
  * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
+export declare function fromStream(stream: Readable, options?: ITokenizerOptions): ReadStreamTokenizer;
 /**
  * Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
  * Will set fileSize, if provided given Stream has set the .path property/
  * @param webStream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export declare function fromWebStream(webStream: ReadableStream<Uint8Array>, fileInfo?: IFileInfo): ReadStreamTokenizer;
+export declare function fromWebStream(webStream: ReadableStream<Uint8Array>, options?: ITokenizerOptions): ReadStreamTokenizer;
 /**
  * Construct ReadStreamTokenizer from given Buffer.
  * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Tokenizer options
  * @returns BufferTokenizer
  */
-export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;
+export declare function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer;

package/lib/core.js

@@ -2,34 +2,33 @@ import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { BufferTokenizer } from './BufferTokenizer.js';
 import { StreamReader, WebStreamReader } from 'peek-readable';
 export { EndOfStreamError } from 'peek-readable';
+export { AbstractTokenizer } from './AbstractTokenizer.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property/
  * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export function fromStream(stream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
-    return new ReadStreamTokenizer(new StreamReader(stream), fileInfo);
+export function fromStream(stream, options) {
+    return new ReadStreamTokenizer(new StreamReader(stream), options);
 }
 /**
  * Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
  * Will set fileSize, if provided given Stream has set the .path property/
  * @param webStream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export function fromWebStream(webStream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
-    return new ReadStreamTokenizer(new WebStreamReader(webStream), fileInfo);
+export function fromWebStream(webStream, options) {
+    return new ReadStreamTokenizer(new WebStreamReader(webStream), options);
 }
 /**
  * Construct ReadStreamTokenizer from given Buffer.
  * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Tokenizer options
  * @returns BufferTokenizer
  */
-export function fromBuffer(uint8Array, fileInfo) {
-    return new BufferTokenizer(uint8Array, fileInfo);
+export function fromBuffer(uint8Array, options) {
+    return new BufferTokenizer(uint8Array, options);
 }

package/lib/index.d.ts

@@ -1,14 +1,15 @@
 import { Readable } from 'node:stream';
 import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import * as core from './core.js';
+import { type ITokenizerOptions } from './core.js';
 export { fromFile } from './FileTokenizer.js';
-export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo } from './core.js';
-export { IToken, IGetToken } from '@tokenizer/token';
+export { EndOfStreamError, fromBuffer, fromWebStream, AbstractTokenizer } from './core.js';
+export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './core.js';
+export type { IToken, IGetToken } from '@tokenizer/token';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.
  * @param stream - Node.js Stream.Readable
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Pass additional file information to the tokenizer
  * @returns Tokenizer
  */
-export declare function fromStream(stream: Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
+export declare function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>;

package/lib/index.js

@@ -1,20 +1,22 @@
 import { stat as fsStat } from 'node:fs/promises';
-import * as core from './core.js';
+import { fromStream as coreFromStream } from './core.js';
 export { fromFile } from './FileTokenizer.js';
-export { EndOfStreamError, fromBuffer, fromWebStream } from './core.js';
+export { EndOfStreamError, fromBuffer, fromWebStream, AbstractTokenizer } from './core.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.
  * @param stream - Node.js Stream.Readable
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Pass additional file information to the tokenizer
  * @returns Tokenizer
  */
-export async function fromStream(stream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
+export async function fromStream(stream, options) {
+    var _a;
+    const augmentedOptions = options !== null && options !== void 0 ? options : {};
+    augmentedOptions.fileInfo = (_a = augmentedOptions.fileInfo) !== null && _a !== void 0 ? _a : {};
     if (stream.path) {
         const stat = await fsStat(stream.path);
-        fileInfo.path = stream.path;
-        fileInfo.size = stat.size;
+        augmentedOptions.fileInfo.path = stream.path;
+        augmentedOptions.fileInfo.size = stat.size;
     }
-    return core.fromStream(stream, fileInfo);
+    return coreFromStream(stream, augmentedOptions);
 }

package/lib/types.d.ts

@@ -100,3 +100,14 @@ export interface ITokenizer {
      */
     close(): Promise<void>;
 }
+export type OnClose = () => Promise<void>;
+export interface ITokenizerOptions {
+    /**
+     * Pass additional file information to the tokenizer
+     */
+    fileInfo?: IFileInfo;
+    /**
+     * On tokenizer close handler
+     */
+    onClose?: OnClose;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "7.1.1",
+  "version": "8.0.1",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -11,7 +11,7 @@
     "url": "https://github.com/sponsors/Borewit"
   },
   "scripts": {
-    "clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map",
+    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
     "compile-src": "tsc -p lib",
     "compile-test": "tsc -p test",
     "compile": "npm run compile-src && npm run compile-test",
@@ -56,7 +56,7 @@
     "@types/chai": "^4.3.16",
     "@types/debug": "^4.1.12",
     "@types/mocha": "^10.0.7",
-    "@types/node": "^20.14.10",
+    "@types/node": "^20.14.11",
     "@typescript-eslint/eslint-plugin": "^7.16.1",
     "@typescript-eslint/parser": "^7.16.1",
     "c8": "^10.1.2",
@@ -66,17 +66,17 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-jsdoc": "^48.7.0",
+    "eslint-plugin-jsdoc": "^48.8.3",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-unicorn": "^54.0.0",
     "global": "^4.4.0",
-    "mocha": "^10.6.0",
+    "mocha": "^10.7.0",
     "remark-cli": "^12.0.1",
     "remark-preset-lint-recommended": "^7.0.0",
     "token-types": "^6.0.0",
     "ts-node": "^10.9.2",
     "typescript": "^5.5.3",
-    "uint8array-extras": "^1.3.0"
+    "uint8array-extras": "^1.4.0"
   },
   "dependencies": {
     "@tokenizer/token": "^0.3.0",