strtok3 6.1.2 → 6.2.2

package/README.md

@@ -20,7 +20,7 @@ The `strtok3` contains a few methods to turn different input into a [*tokenizer*
 It can read from:
 *   A file (taking a file path as an input)
 *   A Node.js [stream](https://nodejs.org/api/stream.html).
-*   A [Buffer](https://nodejs.org/api/buffer.html)
+*   A [Buffer](https://nodejs.org/api/buffer.html) or [Uint8Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array)
 *   HTTP chunked transfer provided by [@tokenizer/http](https://github.com/Borewit/tokenizer-http).
 *   Chunked [Amazon S3](https://aws.amazon.com/s3) access provided by [@tokenizer/s3](https://github.com/Borewit/tokenizer-s3).
 
@@ -35,7 +35,8 @@ npm install strtok3
 Use one of the methods to instantiate an [*abstract tokenizer*](#tokenizer):
 *   [strtok3.fromFile](#method-strtok3fromfile)
 *   [strtok3.fromStream](#method-strtok3fromstream)
-*   [strtok3.fromBuffer](#method-strtok3frombuffer)
+*   [strtok3.fromBuffer](#method-strtok3fromBuffer)
+*   [strtok3.fromUint8Array](#method-strtok3fromUint8Array)
 
 ### strtok3 methods
 
@@ -92,10 +93,10 @@ strtok3.fromStream(stream).then(tokenizer => {
 
 #### Method `strtok3.fromBuffer()`
 
-| Parameter | Optional | Type                                         | Description              |
-|-----------|----------|----------------------------------------------|--------------------------|
-| buffer    | no       | [Buffer](https://nodejs.org/api/buffer.html) | Buffer to read from      |
-| fileInfo  | yes      | [IFileInfo](#IFileInfo)                      | Provide file information |
+| Parameter  | Optional | Type                                             | Description                            |
+|------------|----------|--------------------------------------------------|----------------------------------------|
+| uint8Array | no       | [Uint8Array](https://nodejs.org/api/buffer.html) | Uint8Array or Buffer to read from      |
+| fileInfo   | yes      | [IFileInfo](#IFileInfo)                          | Provide file information               |
 
 Returns a [*tokenizer*](#tokenizer) which can be used to parse the provided buffer.
 

package/lib/AbstractTokenizer.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { ITokenizer, IFileInfo, IReadChunkOptions } from './types';
 import { IGetToken, IToken } from '@tokenizer/token';
 /**
@@ -18,28 +17,28 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      * @param options - Additional read options
      * @returns Promise with number of bytes read
      */
-    abstract readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    abstract readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
+     * @param uint8Array- Target buffer to fill with data peek from the tokenizer-stream
      * @param options - Peek behaviour options
      * @returns Promise with number of bytes read
      */
-    abstract peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    abstract peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Read a token from the tokenizer-stream
      * @param token - The token to read
      * @param position - If provided, the desired position in the tokenizer-stream
      * @returns Promise with token data
      */
-    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
+    readToken<Value>(token: IGetToken<Value>, position?: number): Promise<Value>;
     /**
      * Peek a token from the tokenizer-stream.
      * @param token - Token to peek from the tokenizer-stream.
      * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
      * @returns Promise with token data
      */
-    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
+    peekToken<Value>(token: IGetToken<Value>, position?: number): Promise<Value>;
     /**
      * Read a numeric token from the stream
      * @param token - Numeric token
@@ -53,10 +52,11 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
      */
     peekNumber(token: IToken<number>): Promise<number>;
     /**
-     * Ignore number of bytes, advances the pointer in under tokenizer-stream.
-     * @param length - Number of bytes to skip (ignore)
-     * @return actual number of bytes ignored
+     *  Ignore number of bytes, advances the pointer in under tokenizer-stream.
+     * @param length - Number of bytes to ignore
+     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
      */
-    abstract ignore(length: number): Promise<number>;
+    ignore(length: number): Promise<number>;
     close(): Promise<void>;
+    protected normalizeOptions(uint8Array: Uint8Array, options?: IReadChunkOptions): IReadChunkOptions;
 }

package/lib/AbstractTokenizer.js

@@ -11,7 +11,7 @@ class AbstractTokenizer {
          * Tokenizer-stream position
          */
         this.position = 0;
-        this.numBuffer = Buffer.alloc(10);
+        this.numBuffer = new Uint8Array(8);
         this.fileInfo = fileInfo ? fileInfo : {};
     }
     /**
@@ -20,12 +20,12 @@ class AbstractTokenizer {
      * @param position - If provided, the desired position in the tokenizer-stream
      * @returns Promise with token data
      */
-    async readToken(token, position) {
-        const buffer = Buffer.alloc(token.len);
-        const len = await this.readBuffer(buffer, { position });
+    async readToken(token, position = this.position) {
+        const uint8Array = Buffer.alloc(token.len);
+        const len = await this.readBuffer(uint8Array, { position });
         if (len < token.len)
             throw new peek_readable_1.EndOfStreamError();
-        return token.get(buffer, 0);
+        return token.get(uint8Array, 0);
     }
     /**
      * Peek a token from the tokenizer-stream.
@@ -34,11 +34,11 @@ class AbstractTokenizer {
      * @returns Promise with token data
      */
     async peekToken(token, position = this.position) {
-        const buffer = Buffer.alloc(token.len);
-        const len = await this.peekBuffer(buffer, { position });
+        const uint8Array = Buffer.alloc(token.len);
+        const len = await this.peekBuffer(uint8Array, { position });
         if (len < token.len)
             throw new peek_readable_1.EndOfStreamError();
-        return token.get(buffer, 0);
+        return token.get(uint8Array, 0);
     }
     /**
      * Read a numeric token from the stream
@@ -62,8 +62,28 @@ class AbstractTokenizer {
             throw new peek_readable_1.EndOfStreamError();
         return token.get(this.numBuffer, 0);
     }
+    /**
+     *  Ignore number of bytes, advances the pointer in under tokenizer-stream.
+     * @param length - Number of bytes to ignore
+     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
+     */
+    async ignore(length) {
+        const bytesLeft = this.fileInfo.size - this.position;
+        if (length <= bytesLeft) {
+            this.position += length;
+            return length;
+        }
+        else {
+            this.position += bytesLeft;
+            return bytesLeft;
+        }
+    }
     async close() {
         // empty
     }
+    normalizeOptions(uint8Array, options) {
+        options = Object.assign({ offset: 0, length: uint8Array.length - ((options && options.offset) ? options.offset : 0), position: this.position }, options);
+        return options;
+    }
 }
 exports.AbstractTokenizer = AbstractTokenizer;

package/lib/BufferTokenizer.d.ts

@@ -1,37 +1,26 @@
-/// <reference types="node" />
-import { IFileInfo, IReadChunkOptions, ITokenizer } from './types';
-import { IGetToken, IToken } from '@tokenizer/token';
-export declare class BufferTokenizer implements ITokenizer {
-    private buffer;
-    fileInfo: IFileInfo;
-    position: number;
+import { IFileInfo, IReadChunkOptions } from './types';
+import { AbstractTokenizer } from './AbstractTokenizer';
+export declare class BufferTokenizer extends AbstractTokenizer {
+    private uint8Array;
     /**
      * Construct BufferTokenizer
-     * @param buffer - Buffer to tokenize
+     * @param uint8Array - Uint8Array to tokenize
      * @param fileInfo - Pass additional file information to the tokenizer
      */
-    constructor(buffer: Buffer, fileInfo?: IFileInfo);
+    constructor(uint8Array: Uint8Array, fileInfo?: IFileInfo);
     /**
      * Read buffer from tokenizer
-     * @param buffer
+     * @param uint8Array - Uint8Array to tokenize
      * @param options - Read behaviour options
      * @returns {Promise<number>}
      */
-    readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer
+     * @param uint8Array
      * @param options - Read behaviour options
      * @returns {Promise<number>}
      */
-    peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    peekToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    readNumber(token: IToken<number>): Promise<number>;
-    peekNumber(token: IToken<number>): Promise<number>;
-    /**
-     * @return actual number of bytes ignored
-     */
-    ignore(length: number): Promise<number>;
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     close(): Promise<void>;
 }

package/lib/BufferTokenizer.js

@@ -2,111 +2,52 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BufferTokenizer = void 0;
 const peek_readable_1 = require("peek-readable");
-class BufferTokenizer {
+const AbstractTokenizer_1 = require("./AbstractTokenizer");
+class BufferTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     /**
      * Construct BufferTokenizer
-     * @param buffer - Buffer to tokenize
+     * @param uint8Array - Uint8Array to tokenize
      * @param fileInfo - Pass additional file information to the tokenizer
      */
-    constructor(buffer, fileInfo) {
-        this.buffer = buffer;
-        this.position = 0;
-        this.fileInfo = fileInfo ? fileInfo : {};
-        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : buffer.length;
+    constructor(uint8Array, fileInfo) {
+        super(fileInfo);
+        this.uint8Array = uint8Array;
+        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
     }
     /**
      * Read buffer from tokenizer
-     * @param buffer
+     * @param uint8Array - Uint8Array to tokenize
      * @param options - Read behaviour options
      * @returns {Promise<number>}
      */
-    async readBuffer(buffer, options) {
+    async readBuffer(uint8Array, options) {
         if (options && options.position) {
             if (options.position < this.position) {
                 throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
             }
             this.position = options.position;
         }
-        return this.peekBuffer(buffer, options).then(bytesRead => {
-            this.position += bytesRead;
-            return bytesRead;
-        });
+        const bytesRead = await this.peekBuffer(uint8Array, options);
+        this.position += bytesRead;
+        return bytesRead;
     }
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer
+     * @param uint8Array
      * @param options - Read behaviour options
      * @returns {Promise<number>}
      */
-    async peekBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        let position = this.position;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` can be less than `tokenizer.position`');
-                }
-                position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        position = position || this.position;
-        if (!length) {
-            length = buffer.length;
-        }
-        const bytes2read = Math.min(this.buffer.length - position, length);
-        if ((!options || !options.mayBeLess) && bytes2read < length) {
+    async peekBuffer(uint8Array, options) {
+        options = this.normalizeOptions(uint8Array, options);
+        const bytes2read = Math.min(this.uint8Array.length - options.position, options.length);
+        if ((!options.mayBeLess) && bytes2read < options.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         else {
-            this.buffer.copy(buffer, offset, position, position + bytes2read);
+            uint8Array.set(this.uint8Array.subarray(options.position, options.position + bytes2read), options.offset);
             return bytes2read;
         }
     }
-    async readToken(token, position) {
-        this.position = position || this.position;
-        try {
-            const tv = this.peekToken(token, this.position);
-            this.position += token.len;
-            return tv;
-        }
-        catch (err) {
-            this.position += this.buffer.length - position;
-            throw err;
-        }
-    }
-    async peekToken(token, position = this.position) {
-        if (this.buffer.length - position < token.len) {
-            throw new peek_readable_1.EndOfStreamError();
-        }
-        return token.get(this.buffer, position);
-    }
-    async readNumber(token) {
-        return this.readToken(token);
-    }
-    async peekNumber(token) {
-        return this.peekToken(token);
-    }
-    /**
-     * @return actual number of bytes ignored
-     */
-    async ignore(length) {
-        const bytesIgnored = Math.min(this.buffer.length - this.position, length);
-        this.position += bytesIgnored;
-        return bytesIgnored;
-    }
     async close() {
         // empty
     }

package/lib/FileTokenizer.d.ts

@@ -13,16 +13,11 @@ export declare class FileTokenizer extends AbstractTokenizer {
     readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    ignore(length: number): Promise<number>;
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     close(): Promise<void>;
 }
 export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;

package/lib/FileTokenizer.js

@@ -47,55 +47,18 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     }
     /**
      * Peek buffer from file
-     * @param buffer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise number of bytes read
      */
-    async peekBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        let position = this.position;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-                position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        const res = await fs.read(this.fd, buffer, offset, length, position);
-        if ((!options || !options.mayBeLess) && res.bytesRead < length) {
+    async peekBuffer(uint8Array, options) {
+        options = this.normalizeOptions(uint8Array, options);
+        const res = await fs.read(this.fd, uint8Array, options.offset, options.length, options.position);
+        if ((!options.mayBeLess) && res.bytesRead < options.length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return res.bytesRead;
     }
-    /**
-     * @param length - Number of bytes to ignore
-     * @return resolves the number of bytes ignored, equals length if this available, otherwise the number of bytes available
-     */
-    async ignore(length) {
-        const bytesLeft = this.fileInfo.size - this.position;
-        if (length <= bytesLeft) {
-            this.position += length;
-            return length;
-        }
-        else {
-            this.position += bytesLeft;
-            return bytesLeft;
-        }
-    }
     async close() {
         return fs.close(this.fd);
     }

package/lib/FsPromise.d.ts

@@ -5,14 +5,14 @@
 import * as fs from 'fs';
 export interface IReadResult {
     bytesRead: number;
-    buffer: Buffer;
+    buffer: Uint8Array;
 }
 export declare const pathExists: typeof fs.existsSync;
 export declare const createReadStream: typeof fs.createReadStream;
 export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
 export declare function close(fd: number): Promise<void>;
 export declare function open(path: fs.PathLike, mode?: string): Promise<number>;
-export declare function read(fd: number, buffer: Buffer, offset: number, length: number, position: number): Promise<IReadResult>;
+export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
 export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
 export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;
 export declare function readFile(path: fs.PathLike): Promise<Buffer>;

package/lib/ReadStreamTokenizer.d.ts

@@ -16,13 +16,13 @@ export declare class ReadStreamTokenizer extends AbstractTokenizer {
      * @param options - Read behaviour options
      * @returns Promise with number of bytes read
      */
-    readBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to write the data read to
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise with number of bytes peeked
      */
-    peekBuffer(buffer: Buffer | Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
     ignore(length: number): Promise<number>;
 }

package/lib/ReadStreamTokenizer.js

@@ -3,8 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ReadStreamTokenizer = void 0;
 const AbstractTokenizer_1 = require("./AbstractTokenizer");
 const peek_readable_1 = require("peek-readable");
-// import * as _debug from 'debug';
-// const debug = _debug('strtok3:ReadStreamTokenizer');
 const maxBufferSize = 256000;
 class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     constructor(stream, fileInfo) {
@@ -62,57 +60,45 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     }
     /**
      * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to write the data read to
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
      * @param options - Read behaviour options
      * @returns Promise with number of bytes peeked
      */
-    async peekBuffer(buffer, options) {
-        // const _offset = position ? position : this.position;
-        // debug(`peek ${_offset}...${_offset + length - 1}`);
-        let offset = 0;
-        let bytesRead;
-        let length = buffer.length;
-        if (options) {
-            if (options.offset) {
-                offset = options.offset;
+    async peekBuffer(uint8Array, options) {
+        options = this.normalizeOptions(uint8Array, options);
+        let bytesRead = 0;
+        if (options.position) {
+            const skipBytes = options.position - this.position;
+            if (skipBytes > 0) {
+                const skipBuffer = new Uint8Array(options.length + skipBytes);
+                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
+                uint8Array.set(skipBuffer.subarray(skipBytes), options.offset);
+                return bytesRead - skipBytes;
             }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
+            else if (skipBytes < 0) {
+                throw new Error('Cannot peek from a negative offset in a stream');
             }
-            else {
-                length -= options.offset || 0;
+        }
+        if (options.length > 0) {
+            try {
+                bytesRead = await this.streamReader.peek(uint8Array, options.offset, options.length);
             }
-            if (options.position) {
-                const skipBytes = options.position - this.position;
-                if (skipBytes > 0) {
-                    const skipBuffer = Buffer.alloc(length + skipBytes);
-                    bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
-                    skipBuffer.copy(buffer, offset, skipBytes);
-                    return bytesRead - skipBytes;
-                }
-                else if (skipBytes < 0) {
-                    throw new Error('Cannot peek from a negative offset in a stream');
+            catch (err) {
+                if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
+                    return 0;
                 }
+                throw err;
             }
-        }
-        try {
-            bytesRead = await this.streamReader.peek(buffer, offset, length);
-        }
-        catch (err) {
-            if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
-                return 0;
+            if ((!options.mayBeLess) && bytesRead < options.length) {
+                throw new peek_readable_1.EndOfStreamError();
             }
-            throw err;
-        }
-        if ((!options || !options.mayBeLess) && bytesRead < length) {
-            throw new peek_readable_1.EndOfStreamError();
         }
         return bytesRead;
     }
     async ignore(length) {
         // debug(`ignore ${this.position}...${this.position + length - 1}`);
         const bufSize = Math.min(maxBufferSize, length);
-        const buf = Buffer.alloc(bufSize);
+        const buf = new Uint8Array(bufSize);
         let totBytesRead = 0;
         while (totBytesRead < length) {
             const remaining = length - totBytesRead;

package/lib/core.d.ts

@@ -16,8 +16,8 @@ export { IToken, IGetToken } from '@tokenizer/token';
 export declare function fromStream(stream: Stream.Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
 /**
  * Construct ReadStreamTokenizer from given Buffer.
- * @param buffer - Buffer to tokenize
+ * @param uint8Array - Uint8Array to tokenize
  * @param fileInfo - Pass additional file information to the tokenizer
  * @returns BufferTokenizer
  */
-export declare function fromBuffer(buffer: Buffer, fileInfo?: IFileInfo): BufferTokenizer;
+export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;

package/lib/core.js

@@ -19,11 +19,11 @@ function fromStream(stream, fileInfo) {
 exports.fromStream = fromStream;
 /**
  * Construct ReadStreamTokenizer from given Buffer.
- * @param buffer - Buffer to tokenize
+ * @param uint8Array - Uint8Array to tokenize
  * @param fileInfo - Pass additional file information to the tokenizer
  * @returns BufferTokenizer
  */
-function fromBuffer(buffer, fileInfo) {
-    return new BufferTokenizer_1.BufferTokenizer(buffer, fileInfo);
+function fromBuffer(uint8Array, fileInfo) {
+    return new BufferTokenizer_1.BufferTokenizer(uint8Array, fileInfo);
 }
 exports.fromBuffer = fromBuffer;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "6.1.2",
+  "version": "6.2.2",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -43,15 +43,17 @@
     "url": "https://github.com/Borewit/strtok3/issues"
   },
   "devDependencies": {
+    "@tokenizer/token": "^0.3.0",
     "@types/chai": "^4.2.21",
+    "@types/debug": "^4.1.6",
     "@types/mocha": "^8.2.3",
-    "@types/node": "^16.3.1",
+    "@types/node": "^16.3.3",
     "@typescript-eslint/eslint-plugin": "^2.34.0",
-    "@typescript-eslint/eslint-plugin-tslint": "^4.28.2",
+    "@typescript-eslint/eslint-plugin-tslint": "^4.28.3",
     "@typescript-eslint/parser": "^2.34.0",
     "chai": "^4.3.4",
     "coveralls": "^3.1.1",
-    "del-cli": "^4.0.0",
+    "del-cli": "^4.0.1",
     "eslint": "^6.8.0",
     "eslint-plugin-import": "^2.23.4",
     "eslint-plugin-prefer-arrow": "^1.2.3",
@@ -60,15 +62,13 @@
     "remark-cli": "^9.0.0",
     "remark-preset-lint-recommended": "^5.0.0",
     "source-map-support": "^0.5.16",
-    "token-types": "^3.0.0",
+    "token-types": "^4.1.0",
     "ts-node": "^10.1.0",
     "tslint": "^6.1.3",
     "typescript": "^4.3.5"
   },
   "dependencies": {
-    "@tokenizer/token": "^0.1.1",
-    "@types/debug": "^4.1.6",
-    "peek-readable": "^3.1.4"
+    "peek-readable": "^4.0.0"
   },
   "keywords": [
     "tokenizer",