strtok3 6.2.3 → 6.2.4

package/lib/AbstractTokenizer.d.ts

@@ -1,5 +1,11 @@
 import { ITokenizer, IFileInfo, IReadChunkOptions } from './types';
 import { IGetToken, IToken } from '@tokenizer/token';
+interface INormalizedReadChunkOptions extends IReadChunkOptions {
+    offset: number;
+    length: number;
+    position: number;
+    mayBeLess?: boolean;
+}
 /**
  * Core tokenizer
  */
@@ -58,5 +64,6 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      */
     ignore(length: number): Promise<number>;
     close(): Promise<void>;
-    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): IReadChunkOptions;
+    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): INormalizedReadChunkOptions;
 }
+export {};

package/lib/AbstractTokenizer.js

@@ -68,22 +68,37 @@ class AbstractTokenizer {
      * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
      */
     async ignore(length) {
-        const bytesLeft = this.fileInfo.size - this.position;
-        if (length <= bytesLeft) {
-            this.position += length;
-            return length;
-        }
-        else {
-            this.position += bytesLeft;
-            return bytesLeft;
+        if (this.fileInfo.size !== undefined) {
+            const bytesLeft = this.fileInfo.size - this.position;
+            if (length > bytesLeft) {
+                this.position += bytesLeft;
+                return bytesLeft;
+            }
         }
+        this.position += length;
+        return length;
     }
     async close() {
         // empty
     }
     normalizeOptions(uint8Array, options) {
-        options = Object.assign({ offset: 0, length: uint8Array.length - ((options && options.offset) ? options.offset : 0), position: this.position }, options);
-        return options;
+        if (options && options.position !== undefined && options.position < this.position) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+        }
+        if (options) {
+            return {
+                mayBeLess: options.mayBeLess === true,
+                offset: options.offset ? options.offset : 0,
+                length: options.length ? options.length : (uint8Array.length - (options.offset ? options.offset : 0)),
+                position: options.position ? options.position : this.position
+            };
+        }
+        return {
+            mayBeLess: false,
+            offset: 0,
+            length: uint8Array.length,
+            position: this.position
+        };
     }
 }
 exports.AbstractTokenizer = AbstractTokenizer;

package/lib/BufferTokenizer.js

@@ -38,13 +38,13 @@ class BufferTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
      * @returns {Promise<number>}
      */
     async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
-        const bytes2read = Math.min(this.uint8Array.length - options.position, options.length);
-        if ((!options.mayBeLess) && bytes2read < options.length) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const bytes2read = Math.min(this.uint8Array.length - normOptions.position, normOptions.length);
+        if ((!normOptions.mayBeLess) && bytes2read < normOptions.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         else {
-            uint8Array.set(this.uint8Array.subarray(options.position, options.position + bytes2read), options.offset);
+            uint8Array.set(this.uint8Array.subarray(normOptions.position, normOptions.position + bytes2read), normOptions.offset);
             return bytes2read;
         }
     }

package/lib/FileTokenizer.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { AbstractTokenizer } from './AbstractTokenizer';
 import { IFileInfo, IReadChunkOptions } from './types';
 export declare class FileTokenizer extends AbstractTokenizer {
@@ -6,11 +5,11 @@ export declare class FileTokenizer extends AbstractTokenizer {
     constructor(fd: number, fileInfo: IFileInfo);
     /**
      * Read buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array to write result to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek buffer from file
      * @param uint8Array - Uint8Array (or Buffer) to write data to

package/lib/FileTokenizer.js

@@ -11,36 +11,16 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     }
     /**
      * Read buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array to write result to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    async readBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-                this.position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        const res = await fs.read(this.fd, buffer, offset, length, this.position);
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        this.position = normOptions.position;
+        const res = await fs.read(this.fd, uint8Array, normOptions.offset, normOptions.length, normOptions.position);
         this.position += res.bytesRead;
-        if (res.bytesRead < length && (!options || !options.mayBeLess)) {
+        if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return res.bytesRead;
@@ -52,9 +32,9 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
      * @returns Promise number of bytes read
      */
     async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
-        const res = await fs.read(this.fd, uint8Array, options.offset, options.length, options.position);
-        if ((!options.mayBeLess) && res.bytesRead < options.length) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const res = await fs.read(this.fd, uint8Array, normOptions.offset, normOptions.length, normOptions.position);
+        if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return res.bytesRead;

package/lib/FsPromise.d.ts

@@ -11,7 +11,7 @@ export declare const pathExists: typeof fs.existsSync;
 export declare const createReadStream: typeof fs.createReadStream;
 export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
 export declare function close(fd: number): Promise<void>;
-export declare function open(path: fs.PathLike, mode?: string): Promise<number>;
+export declare function open(path: fs.PathLike, mode: fs.Mode): Promise<number>;
 export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
 export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
 export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;

package/lib/ReadStreamTokenizer.d.ts

@@ -1,10 +1,10 @@
 /// <reference types="node" />
 import { AbstractTokenizer } from './AbstractTokenizer';
-import * as Stream from 'stream';
+import { Readable } from 'stream';
 import { IFileInfo, IReadChunkOptions } from './types';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
-    constructor(stream: Stream.Readable, fileInfo?: IFileInfo);
+    constructor(stream: Readable, fileInfo?: IFileInfo);
     /**
      * Get file information, an HTTP-client may implement this doing a HEAD request
      * @return Promise with file information
@@ -12,11 +12,11 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
     getFileInfo(): Promise<IFileInfo>;
     /**
      * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
      * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param uint8Array - Uint8Array (or Buffer) to write data to

package/lib/ReadStreamTokenizer.js

@@ -18,42 +18,26 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     }
     /**
      * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
      * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    async readBuffer(buffer, options) {
-        // const _offset = position ? position : this.position;
-        // debug(`readBuffer ${_offset}...${_offset + length - 1}`);
-        let offset = 0;
-        let length = buffer.length;
-        if (options) {
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.position) {
-                const skipBytes = options.position - this.position;
-                if (skipBytes > 0) {
-                    await this.ignore(skipBytes);
-                    return this.readBuffer(buffer, options);
-                }
-                else if (skipBytes < 0) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const skipBytes = normOptions.position - this.position;
+        if (skipBytes > 0) {
+            await this.ignore(skipBytes);
+            return this.readBuffer(uint8Array, options);
+        }
+        else if (skipBytes < 0) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
         }
-        if (length === 0) {
+        if (normOptions.length === 0) {
             return 0;
         }
-        const bytesRead = await this.streamReader.read(buffer, offset, length);
+        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
         this.position += bytesRead;
-        if ((!options || !options.mayBeLess) && bytesRead < length) {
+        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return bytesRead;
@@ -65,23 +49,23 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
      * @returns Promise with number of bytes peeked
      */
     async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
+        const normOptions = this.normalizeOptions(uint8Array, options);
         let bytesRead = 0;
-        if (options.position) {
-            const skipBytes = options.position - this.position;
+        if (normOptions.position) {
+            const skipBytes = normOptions.position - this.position;
             if (skipBytes > 0) {
-                const skipBuffer = new Uint8Array(options.length + skipBytes);
-                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes), options.offset);
+                const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
+                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
+                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
                 return bytesRead - skipBytes;
             }
             else if (skipBytes < 0) {
                 throw new Error('Cannot peek from a negative offset in a stream');
             }
         }
-        if (options.length > 0) {
+        if (normOptions.length > 0) {
             try {
-                bytesRead = await this.streamReader.peek(uint8Array, options.offset, options.length);
+                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
             }
             catch (err) {
                 if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
@@ -89,7 +73,7 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
                 }
                 throw err;
             }
-            if ((!options.mayBeLess) && bytesRead < options.length) {
+            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
                 throw new peek_readable_1.EndOfStreamError();
             }
         }

package/lib/core.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 import { ReadStreamTokenizer } from './ReadStreamTokenizer';
-import * as Stream from 'stream';
+import { Readable } from 'stream';
 import { BufferTokenizer } from './BufferTokenizer';
 import { IFileInfo } from './types';
 export { EndOfStreamError } from 'peek-readable';
@@ -13,7 +13,7 @@ export { IToken, IGetToken } from '@tokenizer/token';
  * @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
  * @returns ReadStreamTokenizer
  */
-export declare function fromStream(stream: Stream.Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
+export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
 /**
  * Construct ReadStreamTokenizer from given Buffer.
  * @param uint8Array - Uint8Array to tokenize

package/lib/index.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import * as Stream from 'stream';
+import { Readable } from 'stream';
 import { ReadStreamTokenizer } from './ReadStreamTokenizer';
 import * as core from './core';
 export { fromFile } from './FileTokenizer';
@@ -12,4 +12,4 @@ export { IToken, IGetToken } from '@tokenizer/token';
  * @param fileInfo - Pass additional file information to the tokenizer
  * @returns Tokenizer
  */
-export declare function fromStream(stream: Stream.Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
+export declare function fromStream(stream: Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "6.2.3",
+  "version": "6.2.4",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -19,6 +19,7 @@
     "eslint": "eslint lib test --ext .ts --ignore-pattern *.d.ts",
     "lint-md": "remark -u preset-lint-recommended .",
     "lint": "npm run lint-md && npm run eslint",
+    "fix": "eslint lib test --ext .ts --ignore-pattern *.d.ts --fix",
     "test": "mocha --require ts-node/register --require source-map-support/register --full-trace test/test.ts",
     "test-coverage": "nyc npm run test",
     "send-coveralls": "nyc report --reporter=text-lcov | coveralls",
@@ -44,9 +45,9 @@
   },
   "devDependencies": {
     "@types/chai": "^4.2.21",
-    "@types/debug": "^4.1.6",
+    "@types/debug": "^4.1.7",
     "@types/mocha": "^9.0.0",
-    "@types/node": "^16.4.6",
+    "@types/node": "^16.4.7",
     "@typescript-eslint/eslint-plugin": "^2.34.0",
     "@typescript-eslint/eslint-plugin-tslint": "^4.28.5",
     "@typescript-eslint/parser": "^2.34.0",
@@ -68,7 +69,7 @@
   },
   "dependencies": {
     "@tokenizer/token": "^0.3.0",
-    "peek-readable": "^4.0.0"
+    "peek-readable": "^4.0.1"
   },
   "keywords": [
     "tokenizer",