strtok3 10.3.0 → 10.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +106 -84
  2. package/lib/stream/AbstractStreamReader.js +1 -1
  3. package/lib/stream/WebStreamDefaultReader.js +1 -1
  4. package/package.json +7 -7
package/README.md CHANGED
@@ -57,15 +57,79 @@ If you find this project useful and would like to support its development, consi
57
57
  ### strtok3 methods
58
58
 
59
59
  Use one of the methods to instantiate an [*abstract tokenizer*](#tokenizer-object):
60
+ - [fromBlob](#fromblob-function)
61
+ - [fromBuffer](#frombuffer-function)
60
62
  - [fromFile](#fromfile-function)*
61
63
  - [fromStream](#fromstream-function)*
62
64
  - [fromWebStream](#fromwebstream-function)
63
- - [fromBuffer](#frombuffer-function)
64
65
 
65
- > **_NOTE:_** * `fromFile` and `fromStream` only available when importing this module with Node.js
66
+ > [!NOTE]
67
+ > `fromFile` and `fromStream` only available when importing this module with Node.js
66
68
 
67
69
  All methods return a [`Tokenizer`](#tokenizer-object), either directly or via a promise.
68
70
 
71
+ #### `fromBlob()` function
72
+
73
+ Create a tokenizer from a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob).
74
+
75
+ ```ts
76
+ function fromBlob(blob: Blob, options?: ITokenizerOptions): BlobTokenizer
77
+ ```
78
+
79
+ | Parameter | Optional | Type | Description |
80
+ |-----------|-----------|---------------------------------------------------|----------------------------------------------------------------------------------------|
81
+ | blob | no | [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) | [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) or [File](https://developer.mozilla.org/en-US/docs/Web/API/File) to read from |
82
+ | options | yes | [ITokenizerOptions](#ITokenizerOptions) | Tokenizer options |
83
+
84
+ Returns a [*tokenizer*](#tokenizer-object).
85
+
86
+ ```js
87
+ import { fromBlob } from 'strtok3';
88
+ import { openAsBlob } from 'node:fs';
89
+ import * as Token from 'token-types';
90
+
91
+ async function parse() {
92
+ const blob = await openAsBlob('somefile.bin');
93
+
94
+ const tokenizer = fromBlob(blob);
95
+
96
+ const myUint8Number = await tokenizer.readToken(Token.UINT8);
97
+ console.log(`My number: ${myUint8Number}`);
98
+ }
99
+
100
+ parse();
101
+
102
+ ```
103
+
104
+ #### `fromBuffer()` function
105
+
106
+ Create a tokenizer from memory ([Uint8Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array) or Node.js [Buffer](https://nodejs.org/api/buffer.html)).
107
+
108
+ ```ts
109
+ function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer
110
+ ```
111
+
112
+ | Parameter | Optional | Type | Description |
113
+ |------------|----------|--------------------------------------------------|-----------------------------------|
114
+ | uint8Array | no | [Uint8Array](https://nodejs.org/api/buffer.html) | Buffer or Uint8Array to read from |
115
+ | options | yes | [ITokenizerOptions](#ITokenizerOptions) | Tokenizer options |
116
+
117
+ Returns a [*tokenizer*](#tokenizer-object).
118
+
119
+ ```js
120
+ import { fromBuffer } from 'strtok3';
121
+ import * as Token from 'token-types';
122
+
123
+ const tokenizer = fromBuffer(buffer);
124
+
125
+ async function parse() {
126
+ const myUint8Number = await tokenizer.readToken(Token.UINT8);
127
+ console.log(`My number: ${myUint8Number}`);
128
+ }
129
+
130
+ parse();
131
+ ```
132
+
69
133
  #### `fromFile` function
70
134
 
71
135
  Creates a [*tokenizer*](#tokenizer-object) from a local file.
@@ -82,91 +146,57 @@ function fromFile(sourceFilePath: string): Promise<FileTokenizer>
82
146
  > - Only available for Node.js engines
83
147
  > - `fromFile` automatically embeds [file-information](#file-information)
84
148
 
85
- Returns, via a promise, a [*tokenizer*](#tokenizer-object) which can be used to parse a file.
149
+ A Promise resolving to a [*tokenizer*](#tokenizer-object) which can be used to parse a file.
86
150
 
87
151
  ```js
88
- import * as strtok3 from 'strtok3';
152
+ import { fromFile } from 'strtok3';
89
153
  import * as Token from 'token-types';
90
154
 
91
- (async () => {
92
-
93
- const tokenizer = await strtok3.fromFile("somefile.bin");
94
- try {
155
+ async function parse() {
156
+ const tokenizer = await fromFile('somefile.bin');
157
+ try {
95
158
  const myNumber = await tokenizer.readToken(Token.UINT8);
96
159
  console.log(`My number: ${myNumber}`);
97
160
  } finally {
98
161
  tokenizer.close(); // Close the file
99
162
  }
100
- })();
101
- ```
163
+ }
102
164
 
103
- #### `fromStream` function
165
+ parse();
104
166
 
105
- Creates a [*tokenizer*](#tokenizer-object) from a Node.js [readable stream](https://nodejs.org/api/stream.html#stream_class_stream_readable).
106
167
 
107
- ```ts
108
- function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>
109
168
  ```
110
169
 
111
- | Parameter | Optional | Type | Description |
112
- |-----------|-----------|-------------------------|--------------------------|
113
- | stream | no | [Readable](https://nodejs.org/api/stream.html#stream_class_stream_readable) | Stream to read from |
114
- | fileInfo | yes | [IFileInfo](#IFileInfo) | Provide file information |
115
-
116
- Returns a Promise providing a [*tokenizer*](#tokenizer-object).
117
-
118
- > [!NOTE]
119
- > - Only available for Node.js engines
120
-
121
- #### `fromWebStream` function
170
+ #### `fromWebStream()` function
122
171
 
123
- Creates [*tokenizer*](#tokenizer-object) from a [WHATWG ReadableStream](https://nodejs.org/api/webstreams.html#web-streams-api).
172
+ Create a tokenizer from a [WHATWG ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream).
124
173
 
125
174
  ```ts
126
175
  function fromWebStream(webStream: AnyWebByteStream, options?: ITokenizerOptions): ReadStreamTokenizer
127
176
  ```
128
177
 
129
- | Parameter | Optional | Type | Description |
130
- |----------------|-----------|--------------------------------------------------------------------------|------------------------------------|
131
- | readableStream | no | [ReadableStream](https://nodejs.org/api/webstreams.html#web-streams-api) | WHATWG ReadableStream to read from |
132
- | fileInfo | yes | [IFileInfo](#IFileInfo) | Provide file information |
178
+ | Parameter | Optional | Type | Description |
179
+ |----------------|----------|--------------------------------------------------------------------------|------------------------------------|
180
+ | webStream | no | [ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) | WHATWG ReadableStream to read from |
181
+ | options | yes | [ITokenizerOptions](#ITokenizerOptions) | Tokenizer options |
133
182
 
134
- Returns a Promise providing a [*tokenizer*](#tokenizer-object)
183
+ Returns a [*tokenizer*](#tokenizer-object).
135
184
 
136
185
  ```js
137
- import strtok3 from 'strtok3';
186
+ import { fromWebStream } from 'strtok3';
138
187
  import * as Token from 'token-types';
139
188
 
140
- strtok3.fromWebStream(readableStream).then(tokenizer => {
141
- return tokenizer.readToken(Token.UINT8).then(myUint8Number => {
189
+ async function parse() {
190
+ const tokenizer = fromWebStream(readableStream);
191
+ try {
192
+ const myUint8Number = await tokenizer.readToken(Token.UINT8);
142
193
  console.log(`My number: ${myUint8Number}`);
143
- });
144
- });
145
- ```
146
-
147
- #### `fromBuffer()` function
148
-
149
- Create a tokenizer from memory ([Uint8Array](https://nodejs.org/api/buffer.html)).
150
-
151
- ```ts
152
- function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer
153
- ```
154
-
155
- | Parameter | Optional | Type | Description |
156
- |------------|----------|--------------------------------------------------|----------------------------------------|
157
- | uint8Array | no | [Uint8Array](https://nodejs.org/api/buffer.html) | Uint8Array or Buffer to read from |
158
- | fileInfo | yes | [IFileInfo](#IFileInfo) | Provide file information |
159
-
160
- Returns a Promise providing a [*tokenizer*](#tokenizer-object).
161
-
162
- ```js
163
- import * as strtok3 from 'strtok3';
164
-
165
- const tokenizer = strtok3.fromBuffer(buffer);
194
+ } finally {
195
+ await tokenizer.close();
196
+ }
197
+ }
166
198
 
167
- tokenizer.readToken(Token.UINT8).then(myUint8Number => {
168
- console.log(`My number: ${myUint8Number}`);
169
- });
199
+ parse();
170
200
  ```
171
201
 
172
202
  ### `Tokenizer` object
@@ -202,7 +232,7 @@ readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
202
232
  | options | [IReadChunkOptions](#ireadchunkoptions) | An integer specifying the number of bytes to read |
203
233
 
204
234
  Return promise with number of bytes read.
205
- The number of bytes read maybe if less, *mayBeLess* flag was set.
235
+ The number of bytes read may be less than requested if the `mayBeLess` flag is set.
206
236
 
207
237
  #### `peekBuffer` function
208
238
 
@@ -217,7 +247,7 @@ peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>
217
247
  | buffer | Buffer &#124; Uint8Array | Target buffer to write the data read (peeked) to. |
218
248
  | options | [IReadChunkOptions](#ireadchunkoptions) | An integer specifying the number of bytes to read. | |
219
249
 
220
- Return value `Promise<number>` Promise with number of bytes read. The number of bytes read maybe if less, *mayBeLess* flag was set.
250
+ Return value `Promise<number>` Promise with number of bytes read. The number of bytes read may be less if the `mayBeLess` flag was set.
221
251
 
222
252
  #### `readToken` function
223
253
 
@@ -232,7 +262,7 @@ readToken<Value>(token: IGetToken<Value>, position: number = this.position): Pro
232
262
  | token | [IGetToken](#IGetToken) | Token to read from the tokenizer-stream. |
233
263
  | position? | number | Offset where to begin reading within the file. If position is null, data will be read from the current file position. |
234
264
 
235
- Return value `Promise<number>`. Promise with number of bytes read. The number of bytes read maybe if less, *mayBeLess* flag was set.
265
+ Return value `Promise<number>`. Promise with number of bytes read. The number of bytes read maybe if less, `mayBeLess` flag was set.
236
266
 
237
267
  #### `peek` function
238
268
 
@@ -251,7 +281,7 @@ Return a promise with the token value peeked from the [*tokenizer*](#tokenizer-o
251
281
 
252
282
  #### `readNumber` function
253
283
 
254
- Peek a numeric [*token*](#token) from the [*tokenizer*](#tokenizer-object).
284
+ Read a numeric [*token*](#token) from the [*tokenizer*](#tokenizer-object).
255
285
 
256
286
  ```ts
257
287
  readNumber(token: IToken<number>): Promise<number>
@@ -261,7 +291,7 @@ readNumber(token: IToken<number>): Promise<number>
261
291
  |------------|---------------------------------|----------------------------------------------------|
262
292
  | token | [IGetToken<number>](#IGetToken) | Numeric token to read from the tokenizer-stream. |
263
293
 
264
- Returns a promise with the decoded numeric value from the *tokenizer-stream*.
294
+ A promise resolving to a numeric value read and decoded from the *tokenizer-stream*.
265
295
 
266
296
  #### `ignore` function
267
297
 
@@ -271,11 +301,11 @@ Advance the offset pointer with the token number of bytes provided.
271
301
  ignore(length: number): Promise<number>
272
302
  ```
273
303
 
274
- | Parameter | Type | Description |
275
- |------------|--------|----------------------------------------------------------------------|
276
- | ignore | number | Numeric of bytes to ignore. Will advance the `tokenizer.position` |
304
+ | Parameter | Type | Description |
305
+ |------------|--------|------------------------------------------------------------------|
306
+ | length | number | Number of bytes to ignore. Will advance the `tokenizer.position` |
277
307
 
278
- Returns a promise with the decoded numeric value from the *tokenizer-stream*.
308
+ A promise resolving to the number of bytes ignored from the *tokenizer-stream*.
279
309
 
280
310
  #### `close` function
281
311
  Clean up resources, such as closing a file pointer if applicable.
@@ -299,7 +329,7 @@ Each attribute is optional:
299
329
  |-----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
300
330
  | length | number | Requested number of bytes to read. |
301
331
  | position | number | Position where to peek from the file. If position is null, data will be read from the [current file position](#attribute-tokenizerposition). Position may not be less then [tokenizer.position](#attribute-tokenizerposition) |
302
- | mayBeLess | boolean | If and only if set, will not throw an EOF error if less then the requested *mayBeLess* could be read. |
332
+ | mayBeLess | boolean | If and only if set, will not throw an EOF error if less than the requested `mayBeLess` could be read. |
303
333
 
304
334
  Example usage:
305
335
  ```js
@@ -313,13 +343,13 @@ Provides optional metadata about the file being tokenized.
313
343
  | Attribute | Type | Description |
314
344
  |-----------|---------|---------------------------------------------------------------------------------------------------|
315
345
  | size | number | File size in bytes |
316
- | mimeType | number | [MIME-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types) of file. |
317
- | path | number | File path |
318
- | url | boolean | File URL |
346
+ | mimeType | string | [MIME-type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types) of file. |
347
+ | path | string | File path |
348
+ | url | string | File URL |
319
349
 
320
350
  ### `Token` object
321
351
 
322
- The *token* is basically a description what to read form the [*tokenizer-stream*](#tokenizer-object).
352
+ The *token* is basically a description of what to read from the [*tokenizer-stream*](#tokenizer-object).
323
353
  A basic set of *token types* can be found here: [*token-types*](https://github.com/Borewit/token-types).
324
354
 
325
355
  A token is something which implements the following interface:
@@ -346,7 +376,7 @@ The `token.get` will be called with the Buffer. `token.get` is responsible for c
346
376
  To convert a [Web-API readable stream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStreamDefaultReader) into a [Node.js readable stream]((https://nodejs.org/api/stream.html#stream_readable_streams)), you can use [readable-web-to-node-stream](https://github.com/Borewit/readable-web-to-node-stream) to convert one in another.
347
377
 
348
378
  ```js
349
- import { fromWebStream } strtok3 from 'strtok3';
379
+ import { fromWebStream } from 'strtok3';
350
380
  import { ReadableWebToNodeStream } from 'readable-web-to-node-stream';
351
381
 
352
382
  (async () => {
@@ -361,15 +391,7 @@ import { ReadableWebToNodeStream } from 'readable-web-to-node-stream';
361
391
 
362
392
  ## Dependencies
363
393
 
364
- The diagram below illustrates the primary dependencies of `strtok3`:
365
-
366
- ```mermaid
367
- graph TD;
368
- S(strtok3)-->P(peek-readable)
369
- S(strtok3)-->TO("@tokenizer/token")
370
- ```
371
-
372
- - [peek-readable](https://github.com/Borewit/peek-readable): Manages reading operations with peeking capabilities, allowing data to be previewed without advancing the read pointer.
394
+ Dependencies:
373
395
  - [@tokenizer/token](https://github.com/Borewit/tokenizer-token): Provides token definitions and utilities used by `strtok3` for interpreting binary data.
374
396
 
375
397
  ## Licence
package/lib/stream/AbstractStreamReader.js CHANGED
@@ -22,7 +22,7 @@ export class AbstractStreamReader {
22
22
  if (!this.endOfStream) {
23
23
  bytesRead += await this.readRemainderFromStream(buffer.subarray(bytesRead), mayBeLess);
24
24
  }
25
- if (bytesRead === 0) {
25
+ if (bytesRead === 0 && !mayBeLess) {
26
26
  throw new EndOfStreamError();
27
27
  }
28
28
  return bytesRead;
package/lib/stream/WebStreamDefaultReader.js CHANGED
@@ -46,7 +46,7 @@ export class WebStreamDefaultReader extends AbstractStreamReader {
46
46
  totalBytesRead += this.writeChunk(buffer.subarray(totalBytesRead), result.value);
47
47
  }
48
48
  }
49
- if (totalBytesRead === 0 && this.endOfStream) {
49
+ if (!mayBeLess && totalBytesRead === 0 && this.endOfStream) {
50
50
  throw new EndOfStreamError();
51
51
  }
52
52
  return totalBytesRead;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "strtok3",
3
- "version": "10.3.0",
3
+ "version": "10.3.2",
4
4
  "description": "A promise based streaming tokenizer",
5
5
  "author": {
6
6
  "name": "Borewit",
@@ -55,21 +55,21 @@
55
55
  "@tokenizer/token": "^0.3.0"
56
56
  },
57
57
  "devDependencies": {
58
- "@biomejs/biome": "^1.9.4",
58
+ "@biomejs/biome": "2.1.1",
59
59
  "@types/chai": "^5.2.2",
60
60
  "@types/chai-as-promised": "^8.0.2",
61
61
  "@types/debug": "^4.1.12",
62
62
  "@types/mocha": "^10.0.10",
63
- "@types/node": "^22.15.21",
63
+ "@types/node": "^24.0.7",
64
64
  "c8": "^10.1.3",
65
- "chai": "^5.2.0",
65
+ "chai": "^5.2.1",
66
66
  "chai-as-promised": "^8.0.1",
67
67
  "del-cli": "^6.0.0",
68
- "mocha": "^11.5.0",
68
+ "mocha": "^11.7.1",
69
69
  "node-readable-to-web-readable-stream": "^0.4.2",
70
70
  "remark-cli": "^12.0.1",
71
71
  "remark-preset-lint-recommended": "^7.0.1",
72
- "token-types": "^6.0.0",
72
+ "token-types": "^6.0.3",
73
73
  "ts-node": "^10.9.2",
74
74
  "typescript": "^5.8.3",
75
75
  "uint8array-extras": "^1.4.0"
@@ -88,5 +88,5 @@
88
88
  "stream",
89
89
  "streaming"
90
90
  ],
91
- "packageManager": "yarn@4.9.1"
91
+ "packageManager": "yarn@4.9.2"
92
92
  }