strtok3 7.0.0-alpha.0 → 7.0.0-alpha.4

package/README.md

@@ -48,7 +48,7 @@ All of the strtok3 methods return a [*tokenizer*](#tokenizer), either directly o
 |-----------|-----------------------|----------------------------|
 | path      | Path to file (string) | Path to file to read from  |
 
-Note that [file-information](#file-information) is automatically added.
+> __Note__: that [file-information](#file-information) is automatically added.
 
 Returns, via a promise, a [*tokenizer*](#tokenizer) which can be used to parse a file.
 
@@ -131,10 +131,9 @@ If a *position* is provided to a *read* or *peek* method, is should be, at least
 
 ### Tokenizer methods 
 
-There are to groups of methods
-
-*   *read* methods: used to read a *token* of [Buffer](https://nodejs.org/api/buffer.html) from the [*tokenizer*](#tokenizer). The position of the *tokenizer-stream* will advance with the size of the token.
-*   *peek* methods: same as the read, but it will *not* advance the pointer. It allows to read (peek) ahead.
+There are two kind of methods:
+1.   *read* methods: used to read a *token* of [Buffer](https://nodejs.org/api/buffer.html) from the [*tokenizer*](#tokenizer). The position of the *tokenizer-stream* will advance with the size of the token.
+2.   *peek* methods: same as the read, but it will *not* advance the pointer. It allows to read (peek) ahead.
 
 #### Method `tokenizer.readBuffer()`
 

package/lib/AbstractTokenizer.d.ts

@@ -1,5 +1,11 @@
 import { ITokenizer, IFileInfo, IReadChunkOptions } from './types';
 import { IGetToken, IToken } from '@tokenizer/token';
+interface INormalizedReadChunkOptions extends IReadChunkOptions {
+    offset: number;
+    length: number;
+    position: number;
+    mayBeLess?: boolean;
+}
 /**
  * Core tokenizer
  */
@@ -58,5 +64,6 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      */
     ignore(length: number): Promise<number>;
     close(): Promise<void>;
-    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): IReadChunkOptions;
+    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): INormalizedReadChunkOptions;
 }
+export {};

package/lib/AbstractTokenizer.js

@@ -65,21 +65,36 @@ export class AbstractTokenizer {
      * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
      */
     async ignore(length) {
-        const bytesLeft = this.fileInfo.size - this.position;
-        if (length <= bytesLeft) {
-            this.position += length;
-            return length;
-        }
-        else {
-            this.position += bytesLeft;
-            return bytesLeft;
+        if (this.fileInfo.size !== undefined) {
+            const bytesLeft = this.fileInfo.size - this.position;
+            if (length > bytesLeft) {
+                this.position += bytesLeft;
+                return bytesLeft;
+            }
         }
+        this.position += length;
+        return length;
     }
     async close() {
         // empty
     }
     normalizeOptions(uint8Array, options) {
-        options = Object.assign({ offset: 0, length: uint8Array.length - ((options && options.offset) ? options.offset : 0), position: this.position }, options);
-        return options;
+        if (options && options.position !== undefined && options.position < this.position) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+        }
+        if (options) {
+            return {
+                mayBeLess: options.mayBeLess === true,
+                offset: options.offset ? options.offset : 0,
+                length: options.length ? options.length : (uint8Array.length - (options.offset ? options.offset : 0)),
+                position: options.position ? options.position : this.position
+            };
+        }
+        return {
+            mayBeLess: false,
+            offset: 0,
+            length: uint8Array.length,
+            position: this.position
+        };
     }
 }

package/lib/BufferTokenizer.d.ts

@@ -1,5 +1,5 @@
 import { IFileInfo, IReadChunkOptions } from './types';
-import { AbstractTokenizer } from './AbstractTokenizer';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
 export declare class BufferTokenizer extends AbstractTokenizer {
     private uint8Array;
     /**

package/lib/BufferTokenizer.js

@@ -1,5 +1,5 @@
 import { EndOfStreamError } from 'peek-readable';
-import { AbstractTokenizer } from './AbstractTokenizer';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
 export class BufferTokenizer extends AbstractTokenizer {
     /**
      * Construct BufferTokenizer
@@ -35,13 +35,13 @@ export class BufferTokenizer extends AbstractTokenizer {
      * @returns {Promise<number>}
      */
     async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
-        const bytes2read = Math.min(this.uint8Array.length - options.position, options.length);
-        if ((!options.mayBeLess) && bytes2read < options.length) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const bytes2read = Math.min(this.uint8Array.length - normOptions.position, normOptions.length);
+        if ((!normOptions.mayBeLess) && bytes2read < normOptions.length) {
             throw new EndOfStreamError();
         }
         else {
-            uint8Array.set(this.uint8Array.subarray(options.position, options.position + bytes2read), options.offset);
+            uint8Array.set(this.uint8Array.subarray(normOptions.position, normOptions.position + bytes2read), normOptions.offset);
             return bytes2read;
         }
     }

package/lib/FileTokenizer.d.ts

@@ -1,16 +1,15 @@
-/// <reference types="node" />
-import { AbstractTokenizer } from './AbstractTokenizer';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { IFileInfo, IReadChunkOptions } from './types';
 export declare class FileTokenizer extends AbstractTokenizer {
     private fd;
     constructor(fd: number, fileInfo: IFileInfo);
     /**
      * Read buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array to write result to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek buffer from file
      * @param uint8Array - Uint8Array (or Buffer) to write data to

package/lib/FileTokenizer.js

@@ -1,6 +1,6 @@
-import { AbstractTokenizer } from './AbstractTokenizer';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { EndOfStreamError } from 'peek-readable';
-import * as fs from './FsPromise';
+import * as fs from './FsPromise.js';
 export class FileTokenizer extends AbstractTokenizer {
     constructor(fd, fileInfo) {
         super(fileInfo);
@@ -8,36 +8,16 @@ export class FileTokenizer extends AbstractTokenizer {
     }
     /**
      * Read buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array to write result to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    async readBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-                this.position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        const res = await fs.read(this.fd, buffer, offset, length, this.position);
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        this.position = normOptions.position;
+        const res = await fs.read(this.fd, uint8Array, normOptions.offset, normOptions.length, normOptions.position);
         this.position += res.bytesRead;
-        if (res.bytesRead < length && (!options || !options.mayBeLess)) {
+        if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
             throw new EndOfStreamError();
         }
         return res.bytesRead;
@@ -49,9 +29,9 @@ export class FileTokenizer extends AbstractTokenizer {
      * @returns Promise number of bytes read
      */
     async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
-        const res = await fs.read(this.fd, uint8Array, options.offset, options.length, options.position);
-        if ((!options.mayBeLess) && res.bytesRead < options.length) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const res = await fs.read(this.fd, uint8Array, normOptions.offset, normOptions.length, normOptions.position);
+        if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
             throw new EndOfStreamError();
         }
         return res.bytesRead;

package/lib/FsPromise.d.ts

@@ -2,7 +2,7 @@
  * Module convert fs functions to promise based functions
  */
 /// <reference types="node" />
-import * as fs from 'fs';
+import fs from 'node:fs';
 export interface IReadResult {
     bytesRead: number;
     buffer: Uint8Array;
@@ -11,7 +11,7 @@ export declare const pathExists: typeof fs.existsSync;
 export declare const createReadStream: typeof fs.createReadStream;
 export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
 export declare function close(fd: number): Promise<void>;
-export declare function open(path: fs.PathLike, mode?: string): Promise<number>;
+export declare function open(path: fs.PathLike, mode: fs.Mode): Promise<number>;
 export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
 export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
 export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;

package/lib/FsPromise.js

@@ -1,7 +1,7 @@
 /**
  * Module convert fs functions to promise based functions
  */
-import * as fs from 'fs';
+import fs from 'node:fs';
 export const pathExists = fs.existsSync;
 export const createReadStream = fs.createReadStream;
 export async function stat(path) {

package/lib/ReadStreamTokenizer.d.ts

@@ -1,10 +1,10 @@
 /// <reference types="node" />
-import { AbstractTokenizer } from './AbstractTokenizer';
-import * as Stream from 'stream';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { Readable } from 'node:stream';
 import { IFileInfo, IReadChunkOptions } from './types';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
-    constructor(stream: Stream.Readable, fileInfo?: IFileInfo);
+    constructor(stream: Readable, fileInfo?: IFileInfo);
     /**
      * Get file information, an HTTP-client may implement this doing a HEAD request
      * @return Promise with file information
@@ -12,11 +12,11 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
     getFileInfo(): Promise<IFileInfo>;
     /**
      * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
      * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
      * @param uint8Array - Uint8Array (or Buffer) to write data to

package/lib/ReadStreamTokenizer.js

@@ -1,4 +1,4 @@
-import { AbstractTokenizer } from './AbstractTokenizer';
+import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { EndOfStreamError, StreamReader } from 'peek-readable';
 const maxBufferSize = 256000;
 export class ReadStreamTokenizer extends AbstractTokenizer {
@@ -15,42 +15,26 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
     }
     /**
      * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
      * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    async readBuffer(buffer, options) {
-        // const _offset = position ? position : this.position;
-        // debug(`readBuffer ${_offset}...${_offset + length - 1}`);
-        let offset = 0;
-        let length = buffer.length;
-        if (options) {
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.position) {
-                const skipBytes = options.position - this.position;
-                if (skipBytes > 0) {
-                    await this.ignore(skipBytes);
-                    return this.readBuffer(buffer, options);
-                }
-                else if (skipBytes < 0) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const skipBytes = normOptions.position - this.position;
+        if (skipBytes > 0) {
+            await this.ignore(skipBytes);
+            return this.readBuffer(uint8Array, options);
+        }
+        else if (skipBytes < 0) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
         }
-        if (length === 0) {
+        if (normOptions.length === 0) {
             return 0;
         }
-        const bytesRead = await this.streamReader.read(buffer, offset, length);
+        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
         this.position += bytesRead;
-        if ((!options || !options.mayBeLess) && bytesRead < length) {
+        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
             throw new EndOfStreamError();
         }
         return bytesRead;
@@ -62,23 +46,23 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
      * @returns Promise with number of bytes peeked
      */
     async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
+        const normOptions = this.normalizeOptions(uint8Array, options);
         let bytesRead = 0;
-        if (options.position) {
-            const skipBytes = options.position - this.position;
+        if (normOptions.position) {
+            const skipBytes = normOptions.position - this.position;
             if (skipBytes > 0) {
-                const skipBuffer = new Uint8Array(options.length + skipBytes);
-                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes), options.offset);
+                const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
+                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
+                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
                 return bytesRead - skipBytes;
             }
             else if (skipBytes < 0) {
                 throw new Error('Cannot peek from a negative offset in a stream');
             }
         }
-        if (options.length > 0) {
+        if (normOptions.length > 0) {
             try {
-                bytesRead = await this.streamReader.peek(uint8Array, options.offset, options.length);
+                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
             }
             catch (err) {
                 if (options && options.mayBeLess && err instanceof EndOfStreamError) {
@@ -86,7 +70,7 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
                 }
                 throw err;
             }
-            if ((!options.mayBeLess) && bytesRead < options.length) {
+            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
                 throw new EndOfStreamError();
             }
         }

package/lib/core.d.ts

@@ -1,10 +1,10 @@
 /// <reference types="node" />
-import { ReadStreamTokenizer } from './ReadStreamTokenizer';
-import * as Stream from 'stream';
-import { BufferTokenizer } from './BufferTokenizer';
-import { IFileInfo } from './types';
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { Readable } from 'node:stream';
+import { BufferTokenizer } from './BufferTokenizer.js';
+import { IFileInfo } from './types.js';
 export { EndOfStreamError } from 'peek-readable';
-export { ITokenizer, IFileInfo } from './types';
+export { ITokenizer, IFileInfo } from './types.js';
 export { IToken, IGetToken } from '@tokenizer/token';
 /**
  * Construct ReadStreamTokenizer from given Stream.
@@ -13,7 +13,7 @@ export { IToken, IGetToken } from '@tokenizer/token';
  * @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
  * @returns ReadStreamTokenizer
  */
-export declare function fromStream(stream: Stream.Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
+export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
 /**
  * Construct ReadStreamTokenizer from given Buffer.
  * @param uint8Array - Uint8Array to tokenize

package/lib/core.js

@@ -1,5 +1,5 @@
-import { ReadStreamTokenizer } from './ReadStreamTokenizer';
-import { BufferTokenizer } from './BufferTokenizer';
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { BufferTokenizer } from './BufferTokenizer.js';
 export { EndOfStreamError } from 'peek-readable';
 /**
  * Construct ReadStreamTokenizer from given Stream.

package/lib/index.d.ts

@@ -1,9 +1,9 @@
 /// <reference types="node" />
-import * as Stream from 'stream';
-import { ReadStreamTokenizer } from './ReadStreamTokenizer';
-import * as core from './core';
-export { fromFile } from './FileTokenizer';
-export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core';
+import { Readable } from 'node:stream';
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import * as core from './core.js';
+export { fromFile } from './FileTokenizer.js';
+export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core.js';
 export { IToken, IGetToken } from '@tokenizer/token';
 /**
  * Construct ReadStreamTokenizer from given Stream.
@@ -12,4 +12,4 @@ export { IToken, IGetToken } from '@tokenizer/token';
  * @param fileInfo - Pass additional file information to the tokenizer
  * @returns Tokenizer
  */
-export declare function fromStream(stream: Stream.Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
+export declare function fromStream(stream: Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;

package/lib/index.js

@@ -1,7 +1,7 @@
-import * as fs from './FsPromise';
-import * as core from './core';
-export { fromFile } from './FileTokenizer';
-export { EndOfStreamError, fromBuffer } from './core';
+import * as fs from './FsPromise.js';
+import * as core from './core.js';
+export { fromFile } from './FileTokenizer.js';
+export { EndOfStreamError, fromBuffer } from './core.js';
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "7.0.0-alpha.0",
+  "version": "7.0.0-alpha.4",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -19,14 +19,15 @@
     "eslint": "eslint lib test --ext .ts --ignore-pattern *.d.ts",
     "lint-md": "remark -u preset-lint-recommended .",
     "lint": "npm run lint-md && npm run eslint",
-    "test": "mocha --require ts-node/register --require source-map-support/register --full-trace test/test.ts",
-    "test-coverage": "nyc npm run test",
-    "send-coveralls": "nyc report --reporter=text-lcov | coveralls",
-    "send-codacy": "nyc report --reporter=text-lcov | codacy-coverage",
+    "fix": "eslint lib test --ext .ts --ignore-pattern *.d.ts --fix",
+    "test": "mocha",
+    "test-coverage": "c8 npm run test",
+    "send-coveralls": "c8 report --reporter=text-lcov | coveralls",
+    "send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
     "start": "npm run compile && npm run lint && npm run cover-test"
   },
   "engines": {
-    "node": ">=12.20"
+    "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
   },
   "repository": {
     "type": "git",
@@ -34,8 +35,12 @@
   },
   "license": "MIT",
   "main": "lib/index.js",
+  "type": "module",
   "exports": {
-    ".": "./lib/index.js",
+    ".": {
+      "node": "./lib/index.js",
+      "default": "./lib/core.js"
+    },
     "./core": "./lib/core.js"
   },
   "types": "lib/index.d.ts",
@@ -49,30 +54,30 @@
   "devDependencies": {
     "@tokenizer/token": "^0.3.0",
     "@types/chai": "^4.2.21",
-    "@types/debug": "^4.1.6",
-    "@types/mocha": "^8.2.3",
-    "@types/node": "^16.3.3",
+    "@types/debug": "^4.1.7",
+    "@types/mocha": "^9.0.0",
+    "@types/node": "^16.4.7",
     "@typescript-eslint/eslint-plugin": "^2.34.0",
-    "@typescript-eslint/eslint-plugin-tslint": "^4.28.3",
+    "@typescript-eslint/eslint-plugin-tslint": "^4.28.5",
     "@typescript-eslint/parser": "^2.34.0",
+    "c8": "^7.8.0",
     "chai": "^4.3.4",
     "coveralls": "^3.1.1",
     "del-cli": "^4.0.1",
     "eslint": "^6.8.0",
     "eslint-plugin-import": "^2.23.4",
     "eslint-plugin-prefer-arrow": "^1.2.3",
-    "mocha": "^9.0.2",
-    "nyc": "^15.1.0",
+    "mocha": "^9.0.3",
     "remark-cli": "^9.0.0",
     "remark-preset-lint-recommended": "^5.0.0",
-    "source-map-support": "^0.5.16",
     "token-types": "^4.1.0",
     "ts-node": "^10.1.0",
     "tslint": "^6.1.3",
     "typescript": "^4.3.5"
   },
   "dependencies": {
-    "peek-readable": "^5.0.0-alpha.0"
+    "@tokenizer/token": "^0.3.0",
+    "peek-readable": "^5.0.0-alpha.4"
   },
   "keywords": [
     "tokenizer",