strtok3 6.0.8 → 6.1.3

package/lib/AbstractTokenizer.d.ts

@@ -1,62 +1,62 @@
-/// <reference types="node" />
-import { ITokenizer, IFileInfo, IReadChunkOptions } from './types';
-import { IGetToken, IToken } from '@tokenizer/token';
-/**
- * Core tokenizer
- */
-export declare abstract class AbstractTokenizer implements ITokenizer {
-    fileInfo: IFileInfo;
-    protected constructor(fileInfo?: IFileInfo);
-    /**
-     * Tokenizer-stream position
-     */
-    position: number;
-    private numBuffer;
-    /**
-     * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param options - Additional read options
-     * @returns Promise with number of bytes read
-     */
-    abstract readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
-     * @param options - Peek behaviour options
-     * @returns Promise with number of bytes read
-     */
-    abstract peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Read a token from the tokenizer-stream
-     * @param token - The token to read
-     * @param position - If provided, the desired position in the tokenizer-stream
-     * @returns Promise with token data
-     */
-    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    /**
-     * Peek a token from the tokenizer-stream.
-     * @param token - Token to peek from the tokenizer-stream.
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @returns Promise with token data
-     */
-    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    readNumber(token: IToken<number>): Promise<number>;
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    peekNumber(token: IToken<number>): Promise<number>;
-    /**
-     * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to skip (ignore)
-     * @return actual number of bytes ignored
-     */
-    abstract ignore(length: number): Promise<number>;
-    close(): Promise<void>;
-}
+/// <reference types="node" />
+import { ITokenizer, IFileInfo, IReadChunkOptions } from './types';
+import { IGetToken, IToken } from '@tokenizer/token';
+/**
+ * Core tokenizer
+ */
+export declare abstract class AbstractTokenizer implements ITokenizer {
+    fileInfo: IFileInfo;
+    protected constructor(fileInfo?: IFileInfo);
+    /**
+     * Tokenizer-stream position
+     */
+    position: number;
+    private numBuffer;
+    /**
+     * Read buffer from tokenizer
+     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
+     * @param options - Additional read options
+     * @returns Promise with number of bytes read
+     */
+    abstract readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
+     * @param options - Peek behaviour options
+     * @returns Promise with number of bytes read
+     */
+    abstract peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * Read a token from the tokenizer-stream
+     * @param token - The token to read
+     * @param position - If provided, the desired position in the tokenizer-stream
+     * @returns Promise with token data
+     */
+    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
+    /**
+     * Peek a token from the tokenizer-stream.
+     * @param token - Token to peek from the tokenizer-stream.
+     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
+     * @returns Promise with token data
+     */
+    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
+    /**
+     * Read a numeric token from the stream
+     * @param token - Numeric token
+     * @returns Promise with number
+     */
+    readNumber(token: IToken<number>): Promise<number>;
+    /**
+     * Read a numeric token from the stream
+     * @param token - Numeric token
+     * @returns Promise with number
+     */
+    peekNumber(token: IToken<number>): Promise<number>;
+    /**
+     * Ignore number of bytes, advances the pointer in under tokenizer-stream.
+     * @param length - Number of bytes to skip (ignore)
+     * @return actual number of bytes ignored
+     */
+    abstract ignore(length: number): Promise<number>;
+    close(): Promise<void>;
+}

package/lib/AbstractTokenizer.js

@@ -1,69 +1,69 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AbstractTokenizer = void 0;
-const peek_readable_1 = require("peek-readable");
-/**
- * Core tokenizer
- */
-class AbstractTokenizer {
-    constructor(fileInfo) {
-        /**
-         * Tokenizer-stream position
-         */
-        this.position = 0;
-        this.numBuffer = Buffer.alloc(10);
-        this.fileInfo = fileInfo ? fileInfo : {};
-    }
-    /**
-     * Read a token from the tokenizer-stream
-     * @param token - The token to read
-     * @param position - If provided, the desired position in the tokenizer-stream
-     * @returns Promise with token data
-     */
-    async readToken(token, position) {
-        const buffer = Buffer.alloc(token.len);
-        const len = await this.readBuffer(buffer, { position });
-        if (len < token.len)
-            throw new peek_readable_1.EndOfStreamError();
-        return token.get(buffer, 0);
-    }
-    /**
-     * Peek a token from the tokenizer-stream.
-     * @param token - Token to peek from the tokenizer-stream.
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @returns Promise with token data
-     */
-    async peekToken(token, position = this.position) {
-        const buffer = Buffer.alloc(token.len);
-        const len = await this.peekBuffer(buffer, { position });
-        if (len < token.len)
-            throw new peek_readable_1.EndOfStreamError();
-        return token.get(buffer, 0);
-    }
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    async readNumber(token) {
-        const len = await this.readBuffer(this.numBuffer, { length: token.len });
-        if (len < token.len)
-            throw new peek_readable_1.EndOfStreamError();
-        return token.get(this.numBuffer, 0);
-    }
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    async peekNumber(token) {
-        const len = await this.peekBuffer(this.numBuffer, { length: token.len });
-        if (len < token.len)
-            throw new peek_readable_1.EndOfStreamError();
-        return token.get(this.numBuffer, 0);
-    }
-    async close() {
-        // empty
-    }
-}
-exports.AbstractTokenizer = AbstractTokenizer;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractTokenizer = void 0;
+const peek_readable_1 = require("peek-readable");
+/**
+ * Core tokenizer
+ */
+class AbstractTokenizer {
+    constructor(fileInfo) {
+        /**
+         * Tokenizer-stream position
+         */
+        this.position = 0;
+        this.numBuffer = Buffer.alloc(10);
+        this.fileInfo = fileInfo ? fileInfo : {};
+    }
+    /**
+     * Read a token from the tokenizer-stream
+     * @param token - The token to read
+     * @param position - If provided, the desired position in the tokenizer-stream
+     * @returns Promise with token data
+     */
+    async readToken(token, position) {
+        const buffer = Buffer.alloc(token.len);
+        const len = await this.readBuffer(buffer, { position });
+        if (len < token.len)
+            throw new peek_readable_1.EndOfStreamError();
+        return token.get(buffer, 0);
+    }
+    /**
+     * Peek a token from the tokenizer-stream.
+     * @param token - Token to peek from the tokenizer-stream.
+     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
+     * @returns Promise with token data
+     */
+    async peekToken(token, position = this.position) {
+        const buffer = Buffer.alloc(token.len);
+        const len = await this.peekBuffer(buffer, { position });
+        if (len < token.len)
+            throw new peek_readable_1.EndOfStreamError();
+        return token.get(buffer, 0);
+    }
+    /**
+     * Read a numeric token from the stream
+     * @param token - Numeric token
+     * @returns Promise with number
+     */
+    async readNumber(token) {
+        const len = await this.readBuffer(this.numBuffer, { length: token.len });
+        if (len < token.len)
+            throw new peek_readable_1.EndOfStreamError();
+        return token.get(this.numBuffer, 0);
+    }
+    /**
+     * Read a numeric token from the stream
+     * @param token - Numeric token
+     * @returns Promise with number
+     */
+    async peekNumber(token) {
+        const len = await this.peekBuffer(this.numBuffer, { length: token.len });
+        if (len < token.len)
+            throw new peek_readable_1.EndOfStreamError();
+        return token.get(this.numBuffer, 0);
+    }
+    async close() {
+        // empty
+    }
+}
+exports.AbstractTokenizer = AbstractTokenizer;

package/lib/BufferTokenizer.d.ts

@@ -1,37 +1,37 @@
-/// <reference types="node" />
-import { IFileInfo, IReadChunkOptions, ITokenizer } from './types';
-import { IGetToken, IToken } from '@tokenizer/token';
-export declare class BufferTokenizer implements ITokenizer {
-    private buffer;
-    fileInfo: IFileInfo;
-    position: number;
-    /**
-     * Construct BufferTokenizer
-     * @param buffer - Buffer to tokenize
-     * @param fileInfo - Pass additional file information to the tokenizer
-     */
-    constructor(buffer: Buffer, fileInfo?: IFileInfo);
-    /**
-     * Read buffer from tokenizer
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    readNumber(token: IToken<number>): Promise<number>;
-    peekNumber(token: IToken<number>): Promise<number>;
-    /**
-     * @return actual number of bytes ignored
-     */
-    ignore(length: number): Promise<number>;
-    close(): Promise<void>;
-}
+/// <reference types="node" />
+import { IFileInfo, IReadChunkOptions, ITokenizer } from './types';
+import { IGetToken, IToken } from '@tokenizer/token';
+export declare class BufferTokenizer implements ITokenizer {
+    private buffer;
+    fileInfo: IFileInfo;
+    position: number;
+    /**
+     * Construct BufferTokenizer
+     * @param buffer - Buffer to tokenize
+     * @param fileInfo - Pass additional file information to the tokenizer
+     */
+    constructor(buffer: Buffer, fileInfo?: IFileInfo);
+    /**
+     * Read buffer from tokenizer
+     * @param buffer
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
+     */
+    readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param buffer
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
+     */
+    peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
+    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
+    readNumber(token: IToken<number>): Promise<number>;
+    peekNumber(token: IToken<number>): Promise<number>;
+    /**
+     * @return actual number of bytes ignored
+     */
+    ignore(length: number): Promise<number>;
+    close(): Promise<void>;
+}

package/lib/BufferTokenizer.js

@@ -1,114 +1,114 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BufferTokenizer = void 0;
-const peek_readable_1 = require("peek-readable");
-class BufferTokenizer {
-    /**
-     * Construct BufferTokenizer
-     * @param buffer - Buffer to tokenize
-     * @param fileInfo - Pass additional file information to the tokenizer
-     */
-    constructor(buffer, fileInfo) {
-        this.buffer = buffer;
-        this.position = 0;
-        this.fileInfo = fileInfo ? fileInfo : {};
-        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : buffer.length;
-    }
-    /**
-     * Read buffer from tokenizer
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    async readBuffer(buffer, options) {
-        if (options && options.position) {
-            if (options.position < this.position) {
-                throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-            }
-            this.position = options.position;
-        }
-        return this.peekBuffer(buffer, options).then(bytesRead => {
-            this.position += bytesRead;
-            return bytesRead;
-        });
-    }
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns {Promise<number>}
-     */
-    async peekBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        let position = this.position;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` can be less than `tokenizer.position`');
-                }
-                position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        position = position || this.position;
-        if (!length) {
-            length = buffer.length;
-        }
-        const bytes2read = Math.min(this.buffer.length - position, length);
-        if ((!options || !options.mayBeLess) && bytes2read < length) {
-            throw new peek_readable_1.EndOfStreamError();
-        }
-        else {
-            this.buffer.copy(buffer, offset, position, position + bytes2read);
-            return bytes2read;
-        }
-    }
-    async readToken(token, position) {
-        this.position = position || this.position;
-        try {
-            const tv = this.peekToken(token, this.position);
-            this.position += token.len;
-            return tv;
-        }
-        catch (err) {
-            this.position += this.buffer.length - position;
-            throw err;
-        }
-    }
-    async peekToken(token, position = this.position) {
-        if (this.buffer.length - position < token.len) {
-            throw new peek_readable_1.EndOfStreamError();
-        }
-        return token.get(this.buffer, position);
-    }
-    async readNumber(token) {
-        return this.readToken(token);
-    }
-    async peekNumber(token) {
-        return this.peekToken(token);
-    }
-    /**
-     * @return actual number of bytes ignored
-     */
-    async ignore(length) {
-        const bytesIgnored = Math.min(this.buffer.length - this.position, length);
-        this.position += bytesIgnored;
-        return bytesIgnored;
-    }
-    async close() {
-        // empty
-    }
-}
-exports.BufferTokenizer = BufferTokenizer;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferTokenizer = void 0;
+const peek_readable_1 = require("peek-readable");
+class BufferTokenizer {
+    /**
+     * Construct BufferTokenizer
+     * @param buffer - Buffer to tokenize
+     * @param fileInfo - Pass additional file information to the tokenizer
+     */
+    constructor(buffer, fileInfo) {
+        this.buffer = buffer;
+        this.position = 0;
+        this.fileInfo = fileInfo ? fileInfo : {};
+        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : buffer.length;
+    }
+    /**
+     * Read buffer from tokenizer
+     * @param buffer
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
+     */
+    async readBuffer(buffer, options) {
+        if (options && options.position) {
+            if (options.position < this.position) {
+                throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+            }
+            this.position = options.position;
+        }
+        return this.peekBuffer(buffer, options).then(bytesRead => {
+            this.position += bytesRead;
+            return bytesRead;
+        });
+    }
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param buffer
+     * @param options - Read behaviour options
+     * @returns {Promise<number>}
+     */
+    async peekBuffer(buffer, options) {
+        let offset = 0;
+        let length = buffer.length;
+        let position = this.position;
+        if (options) {
+            if (options.position) {
+                if (options.position < this.position) {
+                    throw new Error('`options.position` can be less than `tokenizer.position`');
+                }
+                position = options.position;
+            }
+            if (Number.isInteger(options.length)) {
+                length = options.length;
+            }
+            else {
+                length -= options.offset || 0;
+            }
+            if (options.offset) {
+                offset = options.offset;
+            }
+        }
+        if (length === 0) {
+            return Promise.resolve(0);
+        }
+        position = position || this.position;
+        if (!length) {
+            length = buffer.length;
+        }
+        const bytes2read = Math.min(this.buffer.length - position, length);
+        if ((!options || !options.mayBeLess) && bytes2read < length) {
+            throw new peek_readable_1.EndOfStreamError();
+        }
+        else {
+            this.buffer.copy(buffer, offset, position, position + bytes2read);
+            return bytes2read;
+        }
+    }
+    async readToken(token, position) {
+        this.position = position || this.position;
+        try {
+            const tv = this.peekToken(token, this.position);
+            this.position += token.len;
+            return tv;
+        }
+        catch (err) {
+            this.position += this.buffer.length - position;
+            throw err;
+        }
+    }
+    async peekToken(token, position = this.position) {
+        if (this.buffer.length - position < token.len) {
+            throw new peek_readable_1.EndOfStreamError();
+        }
+        return token.get(this.buffer, position);
+    }
+    async readNumber(token) {
+        return this.readToken(token);
+    }
+    async peekNumber(token) {
+        return this.peekToken(token);
+    }
+    /**
+     * @return actual number of bytes ignored
+     */
+    async ignore(length) {
+        const bytesIgnored = Math.min(this.buffer.length - this.position, length);
+        this.position += bytesIgnored;
+        return bytesIgnored;
+    }
+    async close() {
+        // empty
+    }
+}
+exports.BufferTokenizer = BufferTokenizer;

package/lib/FileTokenizer.d.ts

@@ -1,28 +1,28 @@
-/// <reference types="node" />
-import { AbstractTokenizer } from './AbstractTokenizer';
-import { IFileInfo, IReadChunkOptions } from './types';
-export declare class FileTokenizer extends AbstractTokenizer {
-    private fd;
-    constructor(fd: number, fileInfo: IFileInfo);
-    /**
-     * Read buffer from file
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek buffer from file
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    ignore(length: number): Promise<number>;
-    close(): Promise<void>;
-}
-export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;
+/// <reference types="node" />
+import { AbstractTokenizer } from './AbstractTokenizer';
+import { IFileInfo, IReadChunkOptions } from './types';
+export declare class FileTokenizer extends AbstractTokenizer {
+    private fd;
+    constructor(fd: number, fileInfo: IFileInfo);
+    /**
+     * Read buffer from file
+     * @param buffer
+     * @param options - Read behaviour options
+     * @returns Promise number of bytes read
+     */
+    readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * Peek buffer from file
+     * @param buffer
+     * @param options - Read behaviour options
+     * @returns Promise number of bytes read
+     */
+    peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * @param length - Number of bytes to ignore
+     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
+     */
+    ignore(length: number): Promise<number>;
+    close(): Promise<void>;
+}
+export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;