strtok3 5.0.2 → 6.0.3

package/README.md

@@ -48,7 +48,7 @@ All of the strtok3 methods return a [*tokenizer*](#tokenizer), either directly o
 |-----------|-----------------------|----------------------------|
 | path      | Path to file (string) | Path to file to read from  |
 
-Note that [file-information](#IFileInfo) is automatically added.
+Note that [file-information](#file-information) is automatically added.
 
 Returns, via a promise, a [*tokenizer*](#tokenizer) which can be used to parse a file.
 
@@ -118,7 +118,7 @@ What is the difference with Nodejs.js stream?
 *   The *tokenizer-stream* supports jumping / seeking in a the *tokenizer-stream* using [`tokenizer.ignore()`](#method-tokenizerignore)
 *   In addition to *read* methods, it has *peek* methods, to read a ahead and check what is coming.
 
-The [tokenizer.position](#attribute-tokenizerposition) keeps tracks of
+The [tokenizer.position](#attribute-tokenizerposition) keeps tracks of the read position.
 
 ### strtok3 attributes
 
@@ -139,62 +139,54 @@ There are to groups of methods
 #### Method `tokenizer.readBuffer()`
 
 Read buffer from stream.
-`readBuffer(buffer, offset?, length?, position?, maybeless?)`
+`readBuffer(buffer, options?)`
 
-| Parameter  | Type                                                           | Description                                                                                                                           |
-|------------|----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
-| buffer     | [Buffer](https://nodejs.org/api/buffer.html) | Uint8Array | Target buffer to write the data read to                                                                                               |
-| offset?    | number                                                         | The offset in the buffer to start writing at; if not provided, start at 0                                                             |
-| length?    | number                                                         | An integer specifying the number of bytes to read                                                                                     |
-| position?  | number                                                         | An integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position. |
-| maybeless? | boolean                                                        | If set, will not throw an EOF error if the less then the requested length could be read                                               |
+| Parameter  | Type                                                           | Description                                                                                                                                                                                                                            |
+|------------|----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| buffer     | [Buffer](https://nodejs.org/api/buffer.html) | Uint8Array | Target buffer to write the data read to                                                                                                                                                                                                |
+| options    | [IReadChunkOptions](#ireadchunkoptions)                        | An integer specifying the number of bytes to read                                                                                                                                                                                      |
 
-Return value `Promise<number>` Promise with number of bytes read
+Return value `Promise<number>` Promise with number of bytes read. The number of bytes read maybe if less, *mayBeLess* flag was set.
 
 #### Method `tokenizer.peekBuffer()`
 
 Peek (read ahead) buffer from [*tokenizer*](#tokenizer)
-`peekBuffer(buffer, offset?, length?, position?, maybeless?)`
+`peekBuffer(buffer,  options?)`
 
-| Parameter  | Type                     | Description                                                                                                           |
-|------------|--------------------------|-----------------------------------------------------------------------------------------------------------------------|
-| buffer     | Buffer &#124; Uint8Array | Target buffer to write the data read (peeked) to                                                                      |
-| offset?    | number                   | The offset in the buffer to start writing at; if not provided, start at 0                                             |
-| length?    | number                   | The number of bytes to read.                                                                                          |
-| position?  | number                   | Offset where to begin reading within the file. If position is null, data will be read from the current file position. |
-| maybeless? | boolean                  | If set, will not throw an EOF error if the less then the requested length could be read                               |
+| Parameter  | Type                                    | Description                                         |
+|------------|-----------------------------------------|-----------------------------------------------------|
+| buffer     | Buffer &#124; Uint8Array                | Target buffer to write the data read (peeked) to.   |
+| options    | [IReadChunkOptions](#ireadchunkoptions) | An integer specifying the number of bytes to read.  |                                                                                                                           |
 
-Return value `Promise<number>` Promise with number of bytes read
+Return value `Promise<number>` Promise with number of bytes read. The number of bytes read maybe if less, *mayBeLess* flag was set.
 
 #### Method `tokenizer.readToken()`
 
 Read a *token* from the tokenizer-stream.
-`readToken(token, position?, maybeless?)`
+`readToken(token, position?)`
 
-| Parameter  | Type                    | Description                                                                                                                           |
-|------------|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
-| token      | [IGetToken](#IGetToken) | Token to read from the tokenizer-stream.                                                                                              |
-| position?  | number                  | Offset where to begin reading within the file. If position is null, data will be read from the current file position.                 |
-| maybeless? | boolean                 | If set, will not throw an EOF error if the less then the requested length could be read.                                              |
+| Parameter  | Type                    | Description                                                                                                           |
+|------------|-------------------------|---------------------------------------------------------------------------------------------------------------------- |
+| token      | [IGetToken](#IGetToken) | Token to read from the tokenizer-stream.                                                                              |
+| position?  | number                  | Offset where to begin reading within the file. If position is null, data will be read from the current file position. |
 
-Return value `Promise<T>`. Promise with token value read from the *tokenizer-stream*.
+Return value `Promise<number>`. Promise with number of bytes read. The number of bytes read maybe if less, *mayBeLess* flag was set.
 
 #### Method `tokenizer.peekToken()`
 
-Peek a *token* from the tokenizer-stream.
-`peekToken(token, position?, maybeless?)`
+Peek a *token* from the [*tokenizer*](#tokenizer).
+`peekToken(token, position?)`
 
 | Parameter  | Type                       | Description                                                                                                             |
 |------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------|
 | token      | [IGetToken<T>](#IGetToken) | Token to read from the tokenizer-stream.                                                                                |
 | position?  | number                     | Offset where to begin reading within the file. If position is null, data will be read from the current file position.   |
-| maybeless? | boolean                    | If set, will not throw an EOF error if the less then the requested length could be read                                 |
 
-Return value `Promise<T>` Promise with token value peeked from the *tokenizer-stream*.
+Return value `Promise<T>` Promise with token value peeked from the [*tokenizer*](#tokenizer).
 
 #### Method `tokenizer.readNumber()`
 
-Peek a numeric [*token*](#token) from the *tokenizer-stream*.
+Peek a numeric [*token*](#token) from the [*tokenizer*](#tokenizer).
 `readNumber(token)`
 
 | Parameter  | Type                            | Description                                        |
@@ -217,33 +209,33 @@ Return value `Promise<number>` Promise with number peeked from the *tokenizer-st
 #### Method `tokenizer.close()`
 Clean up resources, such as closing a file pointer if applicable.
 
-## IFileInfo
+### IReadChunkOptions
 
-File information interface which describes the underlying file:
+Each attribute is optional:
 
-```ts
-export interface IFileInfo {
-  /**
-   * File size in bytes
-   */
-  size?: number;
-  /**
-   * MIME-type of file
-   */
-  mimeType?: string;
-
-  /**
-   * File path
-   */
-  path?: string;
+| Attribute | Type    | Description                                                                                                                                                                                                                   |
+|-----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| offset    | number  | The offset in the buffer to start writing at; if not provided, start at 0                                                                                                                                                     |
+| length    | number  | Requested number of bytes to read.                                                                                                                                                                                            |
+| position  | number  | Position where to peek from the file. If position is null, data will be read from the [current file position](#attribute-tokenizerposition). Position may not be less then [tokenizer.position](#attribute-tokenizerposition) |
+| mayBeLess | boolean | If and only if set, will not throw an EOF error if less then the requested *mayBeLess* could be read.                                                                                                                         |
 
-  /**
-   * File URL
-   */
-  url?: string;
-}
+Example:
+```js
+  tokenizer.peekBuffer(buffer, {mayBeLess: true});
 ```
 
+## IFileInfo
+
+File information interface which describes the underlying file, each attribute is optional.
+
+| Attribute | Type    | Description                                                                                       |
+|-----------|---------|---------------------------------------------------------------------------------------------------|
+| size      | number  | File size in bytes                                                                                | 
+| mimeType  | number  | [MIME-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types) of file. |
+| path      | number  | File path                                                                                         |
+| url       | boolean | File URL                                                                                          |
+
 ## Token
 
 The *token* is basically a description what to read form the [*tokenizer-stream*](#tokenizer). 

package/lib/AbstractTokenizer.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { ITokenizer, IFileInfo } from './types';
+import { ITokenizer, IFileInfo, IReadChunkOptions } from './types';
 import { IGetToken, IToken } from '@tokenizer/token';
 /**
  * Core tokenizer
@@ -15,39 +15,31 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
     /**
      * Read buffer from tokenizer
      * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param offset - Offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
+     * @param options - Additional read options
      * @returns Promise with number of bytes read
      */
-    abstract readBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    abstract readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
-     * @param offset - The offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
+     * @param options - Peek behaviour options
      * @returns Promise with number of bytes read
      */
-    abstract peekBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    abstract peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Read a token from the tokenizer-stream
      * @param token - The token to read
      * @param position - If provided, the desired position in the tokenizer-stream
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
      * @returns Promise with token data
      */
-    readToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
+    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
     /**
      * Peek a token from the tokenizer-stream.
      * @param token - Token to peek from the tokenizer-stream.
      * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
      * @returns Promise with token data
      */
-    peekToken<T>(token: IGetToken<T>, position?: number, maybeless?: boolean): Promise<T>;
+    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
     /**
      * Read a numeric token from the stream
      * @param token - Numeric token

package/lib/AbstractTokenizer.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractTokenizer = void 0;
 const peek_readable_1 = require("peek-readable");
 /**
  * Core tokenizer
@@ -17,13 +18,12 @@ class AbstractTokenizer {
      * Read a token from the tokenizer-stream
      * @param token - The token to read
      * @param position - If provided, the desired position in the tokenizer-stream
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
      * @returns Promise with token data
      */
-    async readToken(token, position = null, maybeless) {
+    async readToken(token, position) {
         const buffer = Buffer.alloc(token.len);
-        const len = await this.readBuffer(buffer, 0, token.len, position);
-        if (!maybeless && len < token.len)
+        const len = await this.readBuffer(buffer, { position });
+        if (len < token.len)
             throw new peek_readable_1.EndOfStreamError();
         return token.get(buffer, 0);
     }
@@ -31,13 +31,12 @@ class AbstractTokenizer {
      * Peek a token from the tokenizer-stream.
      * @param token - Token to peek from the tokenizer-stream.
      * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
      * @returns Promise with token data
      */
-    async peekToken(token, position = this.position, maybeless) {
+    async peekToken(token, position = this.position) {
         const buffer = Buffer.alloc(token.len);
-        const len = await this.peekBuffer(buffer, 0, token.len, position);
-        if (!maybeless && len < token.len)
+        const len = await this.peekBuffer(buffer, { position });
+        if (len < token.len)
             throw new peek_readable_1.EndOfStreamError();
         return token.get(buffer, 0);
     }
@@ -47,7 +46,7 @@ class AbstractTokenizer {
      * @returns Promise with number
      */
     async readNumber(token) {
-        const len = await this.readBuffer(this.numBuffer, 0, token.len, null);
+        const len = await this.readBuffer(this.numBuffer, { length: token.len });
         if (len < token.len)
             throw new peek_readable_1.EndOfStreamError();
         return token.get(this.numBuffer, 0);
@@ -58,7 +57,7 @@ class AbstractTokenizer {
      * @returns Promise with number
      */
     async peekNumber(token) {
-        const len = await this.peekBuffer(this.numBuffer, 0, token.len);
+        const len = await this.peekBuffer(this.numBuffer, { length: token.len });
         if (len < token.len)
             throw new peek_readable_1.EndOfStreamError();
         return token.get(this.numBuffer, 0);

package/lib/BufferTokenizer.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { IFileInfo, ITokenizer } from './types';
+import { IFileInfo, IReadChunkOptions, ITokenizer } from './types';
 import { IGetToken, IToken } from '@tokenizer/token';
 export declare class BufferTokenizer implements ITokenizer {
     private buffer;
@@ -14,22 +14,17 @@ export declare class BufferTokenizer implements ITokenizer {
     /**
      * Read buffer from tokenizer
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @returns {Promise<TResult|number>}
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
      */
-    readBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number, position?: number): Promise<number>;
+    readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeLess If true, will return the bytes available if available bytes is less then length.
-     * @returns {Promise<TResult|number>}
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
      */
-    peekBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number, position?: number, maybeLess?: boolean): Promise<number>;
+    peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
     readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
     peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
     readNumber(token: IToken<number>): Promise<number>;

package/lib/BufferTokenizer.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferTokenizer = void 0;
 const peek_readable_1 = require("peek-readable");
 class BufferTokenizer {
     /**
@@ -16,14 +17,17 @@ class BufferTokenizer {
     /**
      * Read buffer from tokenizer
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @returns {Promise<TResult|number>}
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
      */
-    async readBuffer(buffer, offset, length, position) {
-        this.position = position || this.position;
-        return this.peekBuffer(buffer, offset, length, this.position).then(bytesRead => {
+    async readBuffer(buffer, options) {
+        if (options && options.position) {
+            if (options.position < this.position) {
+                throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+            }
+            this.position = options.position;
+        }
+        return this.peekBuffer(buffer, options).then(bytesRead => {
             this.position += bytesRead;
             return bytesRead;
         });
@@ -31,19 +35,39 @@ class BufferTokenizer {
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeLess If true, will return the bytes available if available bytes is less then length.
-     * @returns {Promise<TResult|number>}
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
      */
-    async peekBuffer(buffer, offset, length, position, maybeLess = false) {
+    async peekBuffer(buffer, options) {
+        let offset = 0;
+        let length = buffer.length;
+        let position = this.position;
+        if (options) {
+            if (options.position) {
+                if (options.position < this.position) {
+                    throw new Error('`options.position` can be less than `tokenizer.position`');
+                }
+                position = options.position;
+            }
+            if (Number.isInteger(options.length)) {
+                length = options.length;
+            }
+            else {
+                length -= options.offset || 0;
+            }
+            if (options.offset) {
+                offset = options.offset;
+            }
+        }
+        if (length === 0) {
+            return Promise.resolve(0);
+        }
         position = position || this.position;
         if (!length) {
             length = buffer.length;
         }
         const bytes2read = Math.min(this.buffer.length - position, length);
-        if (!maybeLess && bytes2read < length) {
+        if ((!options || !options.mayBeLess) && bytes2read < length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         else {

package/lib/FileTokenizer.d.ts

@@ -1,28 +1,23 @@
 /// <reference types="node" />
 import { AbstractTokenizer } from './AbstractTokenizer';
-import { IFileInfo } from './types';
+import { IFileInfo, IReadChunkOptions } from './types';
 export declare class FileTokenizer extends AbstractTokenizer {
     private fd;
     constructor(fd: number, fileInfo: IFileInfo);
     /**
      * Read buffer from file
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read, of not provided the buffer length will be used
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
+     * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    readBuffer(buffer: Buffer, offset?: number, length?: number, position?: number): Promise<number>;
+    readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek buffer from file
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read, of not provided the buffer length will be used
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeless If set, will not throw an EOF error if the less then the requested length could be read
+     * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    peekBuffer(buffer: Buffer, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
     /**
      * @param length - Number of bytes to ignore
      * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available

package/lib/FileTokenizer.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromFile = exports.FileTokenizer = void 0;
 const AbstractTokenizer_1 = require("./AbstractTokenizer");
 const peek_readable_1 = require("peek-readable");
 const fs = require("./FsPromise");
@@ -11,26 +12,35 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     /**
      * Read buffer from file
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read, of not provided the buffer length will be used
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
+     * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    async readBuffer(buffer, offset = 0, length = buffer.length, position) {
-        if (position) {
-            this.position = position;
+    async readBuffer(buffer, options) {
+        let offset = 0;
+        let length = buffer.length;
+        if (options) {
+            if (options.position) {
+                if (options.position < this.position) {
+                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+                }
+                this.position = options.position;
+            }
+            if (Number.isInteger(options.length)) {
+                length = options.length;
+            }
+            else {
+                length -= options.offset || 0;
+            }
+            if (options.offset) {
+                offset = options.offset;
+            }
         }
         if (length === 0) {
             return Promise.resolve(0);
         }
-        if (!length) {
-            length = buffer.length;
-        }
         const res = await fs.read(this.fd, buffer, offset, length, this.position);
-        if (res.bytesRead < length)
-            throw new peek_readable_1.EndOfStreamError();
         this.position += res.bytesRead;
-        if (res.bytesRead < length) {
+        if (res.bytesRead < length && (!options || !options.mayBeLess)) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return res.bytesRead;
@@ -38,15 +48,35 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     /**
      * Peek buffer from file
      * @param buffer
-     * @param offset is the offset in the buffer to start writing at; if not provided, start at 0
-     * @param length is an integer specifying the number of bytes to read, of not provided the buffer length will be used
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeless If set, will not throw an EOF error if the less then the requested length could be read
+     * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    async peekBuffer(buffer, offset = 0, length = buffer.length, position = this.position, maybeless = false) {
+    async peekBuffer(buffer, options) {
+        let offset = 0;
+        let length = buffer.length;
+        let position = this.position;
+        if (options) {
+            if (options.position) {
+                if (options.position < this.position) {
+                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+                }
+                position = options.position;
+            }
+            if (Number.isInteger(options.length)) {
+                length = options.length;
+            }
+            else {
+                length -= options.offset || 0;
+            }
+            if (options.offset) {
+                offset = options.offset;
+            }
+        }
+        if (length === 0) {
+            return Promise.resolve(0);
+        }
         const res = await fs.read(this.fd, buffer, offset, length, position);
-        if (!maybeless && res.bytesRead < length) {
+        if ((!options || !options.mayBeLess) && res.bytesRead < length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return res.bytesRead;

package/lib/FsPromise.js

@@ -3,6 +3,7 @@
  * Module convert fs functions to promise based functions
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.readFile = exports.writeFileSync = exports.writeFile = exports.read = exports.open = exports.close = exports.stat = exports.createReadStream = exports.pathExists = void 0;
 const fs = require("fs");
 exports.pathExists = fs.existsSync;
 exports.createReadStream = fs.createReadStream;

package/lib/ReadStreamTokenizer.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 import { AbstractTokenizer } from './AbstractTokenizer';
 import * as Stream from 'stream';
-import { IFileInfo } from './types';
+import { IFileInfo, IReadChunkOptions } from './types';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
     constructor(stream: Stream.Readable, fileInfo?: IFileInfo);
@@ -13,22 +13,16 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
     /**
      * Read buffer from tokenizer
      * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param offset - Offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
+     * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    readBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer - Target buffer to write the data read to
-     * @param offset - The offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read
+     * @param options - Read behaviour options
      * @returns Promise with number of bytes peeked
      */
-    peekBuffer(buffer: Buffer | Uint8Array, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
     ignore(length: number): Promise<number>;
 }

package/lib/ReadStreamTokenizer.js

@@ -1,10 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReadStreamTokenizer = void 0;
 const AbstractTokenizer_1 = require("./AbstractTokenizer");
 const peek_readable_1 = require("peek-readable");
 const _debug = require("debug");
 const debug = _debug('strtok3:ReadStreamTokenizer');
-const maxBufferSize = 1 * 1000 * 1000;
+const maxBufferSize = 256000;
 class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     constructor(stream, fileInfo) {
         super(fileInfo);
@@ -20,31 +21,41 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     /**
      * Read buffer from tokenizer
      * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param offset - Offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
+     * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    async readBuffer(buffer, offset = 0, length = buffer.length, position, maybeless) {
+    async readBuffer(buffer, options) {
         // const _offset = position ? position : this.position;
         // debug(`readBuffer ${_offset}...${_offset + length - 1}`);
-        if (length === 0) {
-            return 0;
-        }
-        if (position) {
-            const skipBytes = position - this.position;
-            if (skipBytes > 0) {
-                await this.ignore(position - this.position);
-                return this.readBuffer(buffer, offset, length);
+        let offset = 0;
+        let length = buffer.length;
+        if (options) {
+            if (Number.isInteger(options.length)) {
+                length = options.length;
+            }
+            else {
+                length -= options.offset || 0;
             }
-            else if (skipBytes < 0) {
-                throw new Error('Cannot read from a negative offset in a stream');
+            if (options.position) {
+                const skipBytes = options.position - this.position;
+                if (skipBytes > 0) {
+                    await this.ignore(skipBytes);
+                    return this.readBuffer(buffer, options);
+                }
+                else if (skipBytes < 0) {
+                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+                }
+            }
+            if (options.offset) {
+                offset = options.offset;
             }
         }
+        if (length === 0) {
+            return 0;
+        }
         const bytesRead = await this.streamReader.read(buffer, offset, length);
         this.position += bytesRead;
-        if (!maybeless && bytesRead < length) {
+        if ((!options || !options.mayBeLess) && bytesRead < length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return bytesRead;
@@ -52,30 +63,48 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer - Target buffer to write the data read to
-     * @param offset - The offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read
+     * @param options - Read behaviour options
      * @returns Promise with number of bytes peeked
      */
-    async peekBuffer(buffer, offset = 0, length = buffer.length, position, maybeless) {
+    async peekBuffer(buffer, options) {
         // const _offset = position ? position : this.position;
         // debug(`peek ${_offset}...${_offset + length - 1}`);
+        let offset = 0;
         let bytesRead;
-        if (position) {
-            const skipBytes = position - this.position;
-            if (skipBytes > 0) {
-                const skipBuffer = Buffer.alloc(length + skipBytes);
-                bytesRead = await this.peekBuffer(skipBuffer, 0, skipBytes + length, undefined, maybeless);
-                skipBuffer.copy(buffer, offset, skipBytes);
-                return bytesRead - skipBytes;
+        let length = buffer.length;
+        if (options) {
+            if (options.offset) {
+                offset = options.offset;
+            }
+            if (Number.isInteger(options.length)) {
+                length = options.length;
+            }
+            else {
+                length -= options.offset || 0;
             }
-            else if (skipBytes < 0) {
-                throw new Error('Cannot peek from a negative offset in a stream');
+            if (options.position) {
+                const skipBytes = options.position - this.position;
+                if (skipBytes > 0) {
+                    const skipBuffer = Buffer.alloc(length + skipBytes);
+                    bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
+                    skipBuffer.copy(buffer, offset, skipBytes);
+                    return bytesRead - skipBytes;
+                }
+                else if (skipBytes < 0) {
+                    throw new Error('Cannot peek from a negative offset in a stream');
+                }
+            }
+        }
+        try {
+            bytesRead = await this.streamReader.peek(buffer, offset, length);
+        }
+        catch (err) {
+            if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
+                return 0;
             }
+            throw err;
         }
-        bytesRead = await this.streamReader.peek(buffer, offset, length);
-        if (!maybeless && bytesRead < length) {
+        if ((!options || !options.mayBeLess) && bytesRead < length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return bytesRead;
@@ -87,7 +116,7 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
         let totBytesRead = 0;
         while (totBytesRead < length) {
             const remaining = length - totBytesRead;
-            const bytesRead = await this.readBuffer(buf, 0, Math.min(bufSize, remaining));
+            const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
             if (bytesRead < 0) {
                 return bytesRead;
             }

package/lib/core.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromBuffer = exports.fromStream = void 0;
 const ReadStreamTokenizer_1 = require("./ReadStreamTokenizer");
 const BufferTokenizer_1 = require("./BufferTokenizer");
 var peek_readable_1 = require("peek-readable");
-exports.EndOfStreamError = peek_readable_1.EndOfStreamError;
+Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return peek_readable_1.EndOfStreamError; } });
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property/

package/lib/index.js

@@ -1,12 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromStream = void 0;
 const fs = require("./FsPromise");
 const core = require("./core");
 var FileTokenizer_1 = require("./FileTokenizer");
-exports.fromFile = FileTokenizer_1.fromFile;
+Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return FileTokenizer_1.fromFile; } });
 var core_1 = require("./core");
-exports.EndOfStreamError = core_1.EndOfStreamError;
-exports.fromBuffer = core_1.fromBuffer;
+Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return core_1.EndOfStreamError; } });
+Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return core_1.fromBuffer; } });
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.

package/lib/types.d.ts

@@ -18,6 +18,26 @@ export interface IFileInfo {
      */
     url?: string;
 }
+export interface IReadChunkOptions {
+    /**
+     * The offset in the buffer to start writing at; default is 0
+     */
+    offset?: number;
+    /**
+     * Number of bytes to read.
+     */
+    length?: number;
+    /**
+     * Position where to begin reading from the file.
+     * Default it is `tokenizer.position`.
+     * Position may not be less then `tokenizer.position`.
+     */
+    position?: number;
+    /**
+     * If set, will not throw an EOF error if not all of the requested data could be read
+     */
+    mayBeLess?: boolean;
+}
 /**
  * The tokenizer allows us to read or peek from the tokenizer-stream.
  * The tokenizer-stream is an abstraction of a stream, file or Buffer.
@@ -34,23 +54,17 @@ export interface ITokenizer {
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
-     * @param offset - The offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - is an integer specifying the number of bytes to read
-     * @param position is an integer specifying where to begin reading from in the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
+     * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    peekBuffer(buffer: Buffer, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param buffer - Target buffer to fill with data peeked from the tokenizer-stream
-     * @param offset - Offset in the buffer to start writing at; if not provided, start at 0
-     * @param length - The number of bytes to read
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if not all of the requested data could be read
+     * @param options - Additional read options
      * @returns Promise with number of bytes read
      */
-    readBuffer(buffer: Buffer, offset?: number, length?: number, position?: number, maybeless?: boolean): Promise<number>;
+    readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek a token from the tokenizer-stream.
      * @param token - Token to peek from the tokenizer-stream.
@@ -62,9 +76,8 @@ export interface ITokenizer {
      * Read a token from the tokenizer-stream.
      * @param token - Token to peek from the tokenizer-stream.
      * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
      */
-    readToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
+    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
     /**
      * Peek a numeric token from the stream
      * @param token - Numeric token
@@ -79,9 +92,9 @@ export interface ITokenizer {
     readNumber(token: IGetToken<number>): Promise<number>;
     /**
      * Ignore given number of bytes
-     * @param actual number of bytes ignored
+     * @param length - Number of bytes ignored
      */
-    ignore(length: number): any;
+    ignore(length: number): Promise<number>;
     /**
      * Clean up resources.
      * It does not close the stream for StreamReader, but is does close the file-descriptor.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "5.0.2",
+  "version": "6.0.3",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -15,7 +15,6 @@
     "eslint": "eslint lib test --ext .ts --ignore-pattern *.d.ts",
     "lint-md": "remark -u preset-lint-recommended .",
     "lint": "npm run lint-md && npm run eslint",
-    "lint-fix": "tslint --fix 'lib/**/*.ts' 'test/**/*.ts'",
     "test": "mocha --require ts-node/register --require source-map-support/register --full-trace test/test.ts",
     "test-coverage": "nyc npm run test",
     "coveralls": "npm run test-coverage && nyc report --reporter=text-lcov | coveralls",
@@ -39,26 +38,31 @@
     "url": "https://github.com/Borewit/strtok3/issues"
   },
   "devDependencies": {
-    "@types/chai": "^4.2.5",
-    "@types/mocha": "^5.2.7",
-    "@types/node": "^13.1.7",
-    "@typescript-eslint/eslint-plugin": "^2.16.0",
-    "@typescript-eslint/parser": "^2.16.0",
+    "@types/chai": "^4.2.11",
+    "@types/mocha": "^7.0.2",
+    "@types/node": "^14.0.13",
+    "@typescript-eslint/eslint-plugin": "^2.31.0",
+    "@typescript-eslint/eslint-plugin-tslint": "^2.31.0",
+    "@typescript-eslint/parser": "^2.31.0",
     "chai": "^4.2.0",
-    "coveralls": "^3.0.7",
+    "coveralls": "^3.1.0",
     "del-cli": "^3.0.0",
     "eslint": "^6.8.0",
-    "mocha": "^7.0.0",
-    "nyc": "^15.0.0",
-    "remark-cli": "^7.0.1",
-    "remark-preset-lint-recommended": "^3.0.3",
+    "eslint-plugin-import": "^2.21.2",
+    "eslint-plugin-prefer-arrow": "^1.2.1",
+    "mocha": "^7.2.0",
+    "nyc": "^15.1.0",
+    "remark-cli": "^8.0.0",
+    "remark-preset-lint-recommended": "^4.0.0",
     "source-map-support": "^0.5.16",
     "token-types": "^2.0.0",
-    "ts-node": "^8.6.2",
-    "typescript": "^3.7.5"
+    "ts-node": "^8.10.2",
+    "tslint": "^6.1.2",
+    "typescript": "^3.9.5"
   },
   "dependencies": {
     "@tokenizer/token": "^0.1.1",
+    "@types/debug": "^4.1.5",
     "debug": "^4.1.1",
     "peek-readable": "^3.1.0"
   },