strtok3 9.1.1 → 10.0.0

package/README.md

@@ -293,7 +293,6 @@ Each attribute is optional:
 
 | Attribute | Type    | Description                                                                                                                                                                                                                   |
 |-----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| offset    | number  | The offset in the buffer to start writing at; if not provided, start at 0                                                                                                                                                     |
 | length    | number  | Requested number of bytes to read.                                                                                                                                                                                            |
 | position  | number  | Position where to peek from the file. If position is null, data will be read from the [current file position](#attribute-tokenizerposition). Position may not be less then [tokenizer.position](#attribute-tokenizerposition) |
 | mayBeLess | boolean | If and only if set, will not throw an EOF error if less then the requested *mayBeLess* could be read.                                                                                                                         |

package/lib/AbstractTokenizer.d.ts

@@ -1,7 +1,6 @@
 import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 import type { IGetToken, IToken } from '@tokenizer/token';
 interface INormalizedReadChunkOptions extends IReadChunkOptions {
-    offset: number;
     length: number;
     position: number;
     mayBeLess?: boolean;

package/lib/AbstractTokenizer.js

@@ -90,22 +90,16 @@ export class AbstractTokenizer {
         await this.onClose?.();
     }
     normalizeOptions(uint8Array, options) {
-        if (options && options.position !== undefined && options.position < this.position) {
+        if (!this.supportsRandomAccess() && options && options.position !== undefined && options.position < this.position) {
             throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
         }
-        if (options) {
-            return {
-                mayBeLess: options.mayBeLess === true,
-                offset: options.offset ? options.offset : 0,
-                length: options.length ? options.length : (uint8Array.length - (options.offset ? options.offset : 0)),
-                position: options.position ? options.position : this.position
-            };
-        }
         return {
-            mayBeLess: false,
-            offset: 0,
-            length: uint8Array.length,
-            position: this.position
+            ...{
+                mayBeLess: false,
+                offset: 0,
+                length: uint8Array.length,
+                position: this.position
+            }, ...options
         };
     }
     abort() {

package/lib/BufferTokenizer.js

@@ -40,7 +40,7 @@ export class BufferTokenizer extends AbstractTokenizer {
         if ((!normOptions.mayBeLess) && bytes2read < normOptions.length) {
             throw new EndOfStreamError();
         }
-        uint8Array.set(this.uint8Array.subarray(normOptions.position, normOptions.position + bytes2read), normOptions.offset);
+        uint8Array.set(this.uint8Array.subarray(normOptions.position, normOptions.position + bytes2read));
         return bytes2read;
     }
     close() {

package/lib/FileTokenizer.js

@@ -27,7 +27,7 @@ export class FileTokenizer extends AbstractTokenizer {
         this.position = normOptions.position;
         if (normOptions.length === 0)
             return 0;
-        const res = await this.fileHandle.read(uint8Array, normOptions.offset, normOptions.length, normOptions.position);
+        const res = await this.fileHandle.read(uint8Array, 0, normOptions.length, normOptions.position);
         this.position += res.bytesRead;
         if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
             throw new EndOfStreamError();
@@ -42,7 +42,7 @@ export class FileTokenizer extends AbstractTokenizer {
      */
     async peekBuffer(uint8Array, options) {
         const normOptions = this.normalizeOptions(uint8Array, options);
-        const res = await this.fileHandle.read(uint8Array, normOptions.offset, normOptions.length, normOptions.position);
+        const res = await this.fileHandle.read(uint8Array, 0, normOptions.length, normOptions.position);
         if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
             throw new EndOfStreamError();
         }

package/lib/ReadStreamTokenizer.js

@@ -31,7 +31,7 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
         if (normOptions.length === 0) {
             return 0;
         }
-        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
+        const bytesRead = await this.streamReader.read(uint8Array, 0, normOptions.length);
         this.position += bytesRead;
         if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
             throw new EndOfStreamError();
@@ -52,7 +52,7 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
             if (skipBytes > 0) {
                 const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
                 bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
+                uint8Array.set(skipBuffer.subarray(skipBytes));
                 return bytesRead - skipBytes;
             }
             if (skipBytes < 0) {
@@ -61,7 +61,7 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
         }
         if (normOptions.length > 0) {
             try {
-                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
+                bytesRead = await this.streamReader.peek(uint8Array, 0, normOptions.length);
             }
             catch (err) {
                 if (options?.mayBeLess && err instanceof EndOfStreamError) {

package/lib/types.d.ts

@@ -24,10 +24,6 @@ export interface IRandomAccessFileInfo extends IFileInfo {
     size: number;
 }
 export interface IReadChunkOptions {
-    /**
-     * The offset in the buffer to start writing at; default is 0
-     */
-    offset?: number;
     /**
      * Number of bytes to read.
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "9.1.1",
+  "version": "10.0.0",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",