strtok3 6.2.1 → 6.2.2

package/lib/AbstractTokenizer.d.ts

@@ -20,11 +20,11 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
     abstract readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
+     * @param uint8Array- Target buffer to fill with data peek from the tokenizer-stream
      * @param options - Peek behaviour options
      * @returns Promise with number of bytes read
      */
-    abstract peekBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    abstract peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Read a token from the tokenizer-stream
      * @param token - The token to read
@@ -52,10 +52,11 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      */
     peekNumber(token: IToken<number>): Promise<number>;
     /**
-     * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to skip (ignore)
-     * @return actual number of bytes ignored
+     *  Ignore number of bytes, advances the pointer in under tokenizer-stream.
+     * @param length - Number of bytes to ignore
+     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
      */
-    abstract ignore(length: number): Promise<number>;
+    ignore(length: number): Promise<number>;
     close(): Promise<void>;
+    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): IReadChunkOptions;
 }

package/lib/AbstractTokenizer.js

@@ -20,7 +20,7 @@ class AbstractTokenizer {
      * @param position - If provided, the desired position in the tokenizer-stream
      * @returns Promise with token data
      */
-    async readToken(token, position) {
+    async readToken(token, position = this.position) {
         const uint8Array = Buffer.alloc(token.len);
         const len = await this.readBuffer(uint8Array, { position });
         if (len < token.len)
@@ -62,8 +62,28 @@ class AbstractTokenizer {
             throw new peek_readable_1.EndOfStreamError();
         return token.get(this.numBuffer, 0);
     }
+    /**
+     *  Ignore number of bytes, advances the pointer in under tokenizer-stream.
+     * @param length - Number of bytes to ignore
+     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
+     */
+    async ignore(length) {
+        const bytesLeft = this.fileInfo.size - this.position;
+        if (length <= bytesLeft) {
+            this.position += length;
+            return length;
+        }
+        else {
+            this.position += bytesLeft;
+            return bytesLeft;
+        }
+    }
     async close() {
         // empty
     }
+    normalizeOptions(uint8Array, options) {
+        options = Object.assign({ offset: 0, length: uint8Array.length - ((options && options.offset) ? options.offset : 0), position: this.position }, options);
+        return options;
+    }
 }
 exports.AbstractTokenizer = AbstractTokenizer;

package/lib/BufferTokenizer.d.ts

@@ -1,22 +1,20 @@
-import { IFileInfo, IReadChunkOptions, ITokenizer } from './types';
-import { IGetToken, IToken } from '@tokenizer/token';
-export declare class BufferTokenizer implements ITokenizer {
+import { IFileInfo, IReadChunkOptions } from './types';
+import { AbstractTokenizer } from './AbstractTokenizer';
+export declare class BufferTokenizer extends AbstractTokenizer {
     private uint8Array;
-    fileInfo: IFileInfo;
-    position: number;
     /**
      * Construct BufferTokenizer
-     * @param buffer - Buffer to tokenize
+     * @param uint8Array - Uint8Array to tokenize
      * @param fileInfo - Pass additional file information to the tokenizer
      */
     constructor(uint8Array: Uint8Array, fileInfo?: IFileInfo);
     /**
      * Read buffer from tokenizer
-     * @param buffer
+     * @param uint8Array - Uint8Array to tokenize
      * @param options - Read behaviour options
      * @returns {Promise<number>}
      */
-    readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param uint8Array
@@ -24,13 +22,5 @@ export declare class BufferTokenizer implements ITokenizer {
      * @returns {Promise<number>}
      */
     peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    readNumber(token: IToken<number>): Promise<number>;
-    peekNumber(token: IToken<number>): Promise<number>;
-    /**
-     * @return actual number of bytes ignored
-     */
-    ignore(length: number): Promise<number>;
     close(): Promise<void>;
 }

package/lib/BufferTokenizer.js

@@ -2,35 +2,34 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BufferTokenizer = void 0;
 const peek_readable_1 = require("peek-readable");
-class BufferTokenizer {
+const AbstractTokenizer_1 = require("./AbstractTokenizer");
+class BufferTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     /**
      * Construct BufferTokenizer
-     * @param buffer - Buffer to tokenize
+     * @param uint8Array - Uint8Array to tokenize
      * @param fileInfo - Pass additional file information to the tokenizer
      */
     constructor(uint8Array, fileInfo) {
+        super(fileInfo);
         this.uint8Array = uint8Array;
-        this.position = 0;
-        this.fileInfo = fileInfo ? fileInfo : {};
         this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
     }
     /**
      * Read buffer from tokenizer
-     * @param buffer
+     * @param uint8Array - Uint8Array to tokenize
      * @param options - Read behaviour options
      * @returns {Promise<number>}
      */
-    async readBuffer(buffer, options) {
+    async readBuffer(uint8Array, options) {
         if (options && options.position) {
             if (options.position < this.position) {
                 throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
             }
             this.position = options.position;
         }
-        return this.peekBuffer(buffer, options).then(bytesRead => {
-            this.position += bytesRead;
-            return bytesRead;
-        });
+        const bytesRead = await this.peekBuffer(uint8Array, options);
+        this.position += bytesRead;
+        return bytesRead;
     }
     /**
      * Peek (read ahead) buffer from tokenizer
@@ -39,74 +38,16 @@ class BufferTokenizer {
      * @returns {Promise<number>}
      */
     async peekBuffer(uint8Array, options) {
-        let offset = 0;
-        let length = uint8Array.length;
-        let position = this.position;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` can be less than `tokenizer.position`');
-                }
-                position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        position = position || this.position;
-        if (!length) {
-            length = uint8Array.length;
-        }
-        const bytes2read = Math.min(this.uint8Array.length - position, length);
-        if ((!options || !options.mayBeLess) && bytes2read < length) {
+        options = this.normalizeOptions(uint8Array, options);
+        const bytes2read = Math.min(this.uint8Array.length - options.position, options.length);
+        if ((!options.mayBeLess) && bytes2read < options.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         else {
-            uint8Array.set(this.uint8Array.subarray(position, position + bytes2read), offset);
+            uint8Array.set(this.uint8Array.subarray(options.position, options.position + bytes2read), options.offset);
             return bytes2read;
         }
     }
-    async readToken(token, position) {
-        this.position = position || this.position;
-        try {
-            const tv = this.peekToken(token, this.position);
-            this.position += token.len;
-            return tv;
-        }
-        catch (err) {
-            this.position += this.uint8Array.length - position;
-            throw err;
-        }
-    }
-    async peekToken(token, position = this.position) {
-        if (this.uint8Array.length - position < token.len) {
-            throw new peek_readable_1.EndOfStreamError();
-        }
-        return token.get(this.uint8Array, position);
-    }
-    async readNumber(token) {
-        return this.readToken(token);
-    }
-    async peekNumber(token) {
-        return this.peekToken(token);
-    }
-    /**
-     * @return actual number of bytes ignored
-     */
-    async ignore(length) {
-        const bytesIgnored = Math.min(this.uint8Array.length - this.position, length);
-        this.position += bytesIgnored;
-        return bytesIgnored;
-    }
     async close() {
         // empty
     }

package/lib/FileTokenizer.d.ts

@@ -13,16 +13,11 @@ export declare class FileTokenizer extends AbstractTokenizer {
     readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    peekBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    ignore(length: number): Promise<number>;
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     close(): Promise<void>;
 }
 export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;

package/lib/FileTokenizer.js

@@ -47,55 +47,18 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     }
     /**
      * Peek buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    async peekBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        let position = this.position;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-                position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        const res = await fs.read(this.fd, buffer, offset, length, position);
-        if ((!options || !options.mayBeLess) && res.bytesRead < length) {
+    async peekBuffer(uint8Array, options) {
+        options = this.normalizeOptions(uint8Array, options);
+        const res = await fs.read(this.fd, uint8Array, options.offset, options.length, options.position);
+        if ((!options.mayBeLess) && res.bytesRead < options.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return res.bytesRead;
     }
-    /**
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    async ignore(length) {
-        const bytesLeft = this.fileInfo.size - this.position;
-        if (length <= bytesLeft) {
-            this.position += length;
-            return length;
-        }
-        else {
-            this.position += bytesLeft;
-            return bytesLeft;
-        }
-    }
     async close() {
         return fs.close(this.fd);
     }

package/lib/ReadStreamTokenizer.d.ts

@@ -19,10 +19,10 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
     readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to write the data read to
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise with number of bytes peeked
      */
-    peekBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     ignore(length: number): Promise<number>;
 }

package/lib/ReadStreamTokenizer.js

@@ -60,50 +60,38 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     }
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to write the data read to
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise with number of bytes peeked
      */
-    async peekBuffer(buffer, options) {
-        // const _offset = position ? position : this.position;
-        // debug(`peek ${_offset}...${_offset + length - 1}`);
-        let offset = 0;
-        let bytesRead;
-        let length = buffer.length;
-        if (options) {
-            if (options.offset) {
-                offset = options.offset;
+    async peekBuffer(uint8Array, options) {
+        options = this.normalizeOptions(uint8Array, options);
+        let bytesRead = 0;
+        if (options.position) {
+            const skipBytes = options.position - this.position;
+            if (skipBytes > 0) {
+                const skipBuffer = new Uint8Array(options.length + skipBytes);
+                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
+                uint8Array.set(skipBuffer.subarray(skipBytes), options.offset);
+                return bytesRead - skipBytes;
             }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
+            else if (skipBytes < 0) {
+                throw new Error('Cannot peek from a negative offset in a stream');
             }
-            else {
-                length -= options.offset || 0;
+        }
+        if (options.length > 0) {
+            try {
+                bytesRead = await this.streamReader.peek(uint8Array, options.offset, options.length);
             }
-            if (options.position) {
-                const skipBytes = options.position - this.position;
-                if (skipBytes > 0) {
-                    const skipBuffer = new Uint8Array(length + skipBytes);
-                    bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
-                    buffer.set(skipBuffer.subarray(skipBytes), offset);
-                    return bytesRead - skipBytes;
-                }
-                else if (skipBytes < 0) {
-                    throw new Error('Cannot peek from a negative offset in a stream');
+            catch (err) {
+                if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
+                    return 0;
                 }
+                throw err;
             }
-        }
-        try {
-            bytesRead = await this.streamReader.peek(buffer, offset, length);
-        }
-        catch (err) {
-            if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
-                return 0;
+            if ((!options.mayBeLess) && bytesRead < options.length) {
+                throw new peek_readable_1.EndOfStreamError();
             }
-            throw err;
-        }
-        if ((!options || !options.mayBeLess) && bytesRead < length) {
-            throw new peek_readable_1.EndOfStreamError();
         }
         return bytesRead;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "6.2.1",
+  "version": "6.2.2",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -47,7 +47,7 @@
     "@types/chai": "^4.2.21",
     "@types/debug": "^4.1.6",
     "@types/mocha": "^8.2.3",
-    "@types/node": "^16.3.2",
+    "@types/node": "^16.3.3",
     "@typescript-eslint/eslint-plugin": "^2.34.0",
     "@typescript-eslint/eslint-plugin-tslint": "^4.28.3",
     "@typescript-eslint/parser": "^2.34.0",
@@ -62,7 +62,7 @@
     "remark-cli": "^9.0.0",
     "remark-preset-lint-recommended": "^5.0.0",
     "source-map-support": "^0.5.16",
-    "token-types": "^4.0.1",
+    "token-types": "^4.1.0",
     "ts-node": "^10.1.0",
     "tslint": "^6.1.3",
     "typescript": "^4.3.5"