strtok3 8.0.3 → 8.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/core.d.ts +4 -3
- package/lib/core.js +1 -1
- package/lib/index.d.ts +1 -1
- package/package.json +1 -1
package/lib/core.d.ts
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
|
|
2
2
|
import { BufferTokenizer } from './BufferTokenizer.js';
|
|
3
3
|
import type { ITokenizerOptions } from './types.js';
|
|
4
|
-
import type { ReadableStream } from 'node:stream/web';
|
|
4
|
+
import type { ReadableStream as NodeReadableStream, ReadableStream } from 'node:stream/web';
|
|
5
5
|
import type { Readable } from 'node:stream';
|
|
6
6
|
export { EndOfStreamError } from 'peek-readable';
|
|
7
7
|
export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
|
|
8
8
|
export type { IToken, IGetToken } from '@tokenizer/token';
|
|
9
9
|
export { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
10
|
+
export type AnyWebByteStream = NodeReadableStream<Uint8Array> | ReadableStream<Uint8Array>;
|
|
10
11
|
/**
|
|
11
12
|
* Construct ReadStreamTokenizer from given Stream.
|
|
12
13
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
@@ -18,11 +19,11 @@ export declare function fromStream(stream: Readable, options?: ITokenizerOptions
|
|
|
18
19
|
/**
|
|
19
20
|
* Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
|
|
20
21
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
21
|
-
* @param webStream - Read from Node.js Stream.Readable
|
|
22
|
+
* @param webStream - Read from Node.js Stream.Readable (must be a byte stream)
|
|
22
23
|
* @param options - Tokenizer options
|
|
23
24
|
* @returns ReadStreamTokenizer
|
|
24
25
|
*/
|
|
25
|
-
export declare function fromWebStream(webStream:
|
|
26
|
+
export declare function fromWebStream(webStream: AnyWebByteStream, options?: ITokenizerOptions): ReadStreamTokenizer;
|
|
26
27
|
/**
|
|
27
28
|
* Construct ReadStreamTokenizer from given Buffer.
|
|
28
29
|
* @param uint8Array - Uint8Array to tokenize
|
package/lib/core.js
CHANGED
|
@@ -16,7 +16,7 @@ export function fromStream(stream, options) {
|
|
|
16
16
|
/**
|
|
17
17
|
* Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
|
|
18
18
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
19
|
-
* @param webStream - Read from Node.js Stream.Readable
|
|
19
|
+
* @param webStream - Read from Node.js Stream.Readable (must be a byte stream)
|
|
20
20
|
* @param options - Tokenizer options
|
|
21
21
|
* @returns ReadStreamTokenizer
|
|
22
22
|
*/
|
package/lib/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
|
|
|
3
3
|
import { type ITokenizerOptions } from './core.js';
|
|
4
4
|
export { fromFile } from './FileTokenizer.js';
|
|
5
5
|
export { EndOfStreamError, fromBuffer, fromWebStream, AbstractTokenizer } from './core.js';
|
|
6
|
-
export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './core.js';
|
|
6
|
+
export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose, AnyWebByteStream } from './core.js';
|
|
7
7
|
export type { IToken, IGetToken } from '@tokenizer/token';
|
|
8
8
|
/**
|
|
9
9
|
* Construct ReadStreamTokenizer from given Stream.
|