strtok3 7.1.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/lib/AbstractTokenizer.d.ts +10 -4
- package/lib/AbstractTokenizer.js +11 -3
- package/lib/BufferTokenizer.d.ts +3 -3
- package/lib/BufferTokenizer.js +5 -5
- package/lib/FileTokenizer.d.ts +4 -3
- package/lib/FileTokenizer.js +11 -13
- package/lib/ReadStreamTokenizer.d.ts +6 -6
- package/lib/ReadStreamTokenizer.js +6 -8
- package/lib/core.d.ts +10 -10
- package/lib/core.js +10 -12
- package/lib/index.d.ts +4 -4
- package/lib/index.js +11 -9
- package/lib/types.d.ts +11 -0
- package/package.json +18 -13
- package/lib/FsPromise.d.ts +0 -17
- package/lib/FsPromise.js +0 -69
package/README.md
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
](https://github.com/Borewit/strtok3/actions/workflows/nodejs-ci.yml?query=branch%3Amaster)
|
|
2
|
+
[](https://github.com/Borewit/strtok3/actions/workflows/codeql.yml)
|
|
2
3
|
[](https://npmjs.org/package/strtok3)
|
|
3
4
|
[](https://npmcharts.com/compare/strtok3,token-types?start=1200&interval=30)
|
|
4
5
|
[](https://deepscan.io/dashboard#view=project&tid=5165&pid=8526&bid=103329)
|
|
@@ -32,7 +33,7 @@ npm install strtok3
|
|
|
32
33
|
|
|
33
34
|
Module: version 7 migrated from [CommonJS](https://en.wikipedia.org/wiki/CommonJS) to [pure ECMAScript Module (ESM)](https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c).
|
|
34
35
|
JavaScript is compliant with [ECMAScript 2019 (ES10)](https://en.wikipedia.org/wiki/ECMAScript#10th_Edition_%E2%80%93_ECMAScript_2019).
|
|
35
|
-
Requires Node.js ≥
|
|
36
|
+
Requires Node.js ≥ 16 engine.
|
|
36
37
|
|
|
37
38
|
## API
|
|
38
39
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ITokenizer, IFileInfo, IReadChunkOptions } from './types.js';
|
|
2
|
-
import { IGetToken, IToken } from '@tokenizer/token';
|
|
1
|
+
import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
|
|
2
|
+
import type { IGetToken, IToken } from '@tokenizer/token';
|
|
3
3
|
interface INormalizedReadChunkOptions extends IReadChunkOptions {
|
|
4
4
|
offset: number;
|
|
5
5
|
length: number;
|
|
@@ -11,7 +11,13 @@ interface INormalizedReadChunkOptions extends IReadChunkOptions {
|
|
|
11
11
|
*/
|
|
12
12
|
export declare abstract class AbstractTokenizer implements ITokenizer {
|
|
13
13
|
fileInfo: IFileInfo;
|
|
14
|
-
|
|
14
|
+
private onClose?;
|
|
15
|
+
/**
|
|
16
|
+
* Constructor
|
|
17
|
+
* @param options Tokenizer options
|
|
18
|
+
* @protected
|
|
19
|
+
*/
|
|
20
|
+
protected constructor(options?: ITokenizerOptions);
|
|
15
21
|
/**
|
|
16
22
|
* Tokenizer-stream position
|
|
17
23
|
*/
|
|
@@ -26,7 +32,7 @@ export declare abstract class AbstractTokenizer implements ITokenizer {
|
|
|
26
32
|
abstract readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
|
|
27
33
|
/**
|
|
28
34
|
* Peek (read ahead) buffer from tokenizer
|
|
29
|
-
* @param uint8Array- Target buffer to fill with data
|
|
35
|
+
* @param uint8Array - Target buffer to fill with data peeked from the tokenizer-stream
|
|
30
36
|
* @param options - Peek behaviour options
|
|
31
37
|
* @returns Promise with number of bytes read
|
|
32
38
|
*/
|
package/lib/AbstractTokenizer.js
CHANGED
|
@@ -3,13 +3,20 @@ import { EndOfStreamError } from 'peek-readable';
|
|
|
3
3
|
* Core tokenizer
|
|
4
4
|
*/
|
|
5
5
|
export class AbstractTokenizer {
|
|
6
|
-
|
|
6
|
+
/**
|
|
7
|
+
* Constructor
|
|
8
|
+
* @param options Tokenizer options
|
|
9
|
+
* @protected
|
|
10
|
+
*/
|
|
11
|
+
constructor(options) {
|
|
12
|
+
var _a;
|
|
7
13
|
/**
|
|
8
14
|
* Tokenizer-stream position
|
|
9
15
|
*/
|
|
10
16
|
this.position = 0;
|
|
11
17
|
this.numBuffer = new Uint8Array(8);
|
|
12
|
-
this.fileInfo =
|
|
18
|
+
this.fileInfo = (_a = options === null || options === void 0 ? void 0 : options.fileInfo) !== null && _a !== void 0 ? _a : {};
|
|
19
|
+
this.onClose = options === null || options === void 0 ? void 0 : options.onClose;
|
|
13
20
|
}
|
|
14
21
|
/**
|
|
15
22
|
* Read a token from the tokenizer-stream
|
|
@@ -76,7 +83,8 @@ export class AbstractTokenizer {
|
|
|
76
83
|
return length;
|
|
77
84
|
}
|
|
78
85
|
async close() {
|
|
79
|
-
|
|
86
|
+
var _a;
|
|
87
|
+
await ((_a = this.onClose) === null || _a === void 0 ? void 0 : _a.call(this));
|
|
80
88
|
}
|
|
81
89
|
normalizeOptions(uint8Array, options) {
|
|
82
90
|
if (options && options.position !== undefined && options.position < this.position) {
|
package/lib/BufferTokenizer.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ITokenizerOptions, IReadChunkOptions } from './types.js';
|
|
2
2
|
import { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
3
3
|
export declare class BufferTokenizer extends AbstractTokenizer {
|
|
4
4
|
private uint8Array;
|
|
5
5
|
/**
|
|
6
6
|
* Construct BufferTokenizer
|
|
7
7
|
* @param uint8Array - Uint8Array to tokenize
|
|
8
|
-
* @param
|
|
8
|
+
* @param options Tokenizer options
|
|
9
9
|
*/
|
|
10
|
-
constructor(uint8Array: Uint8Array,
|
|
10
|
+
constructor(uint8Array: Uint8Array, options?: ITokenizerOptions);
|
|
11
11
|
/**
|
|
12
12
|
* Read buffer from tokenizer
|
|
13
13
|
* @param uint8Array - Uint8Array to tokenize
|
package/lib/BufferTokenizer.js
CHANGED
|
@@ -4,10 +4,10 @@ export class BufferTokenizer extends AbstractTokenizer {
|
|
|
4
4
|
/**
|
|
5
5
|
* Construct BufferTokenizer
|
|
6
6
|
* @param uint8Array - Uint8Array to tokenize
|
|
7
|
-
* @param
|
|
7
|
+
* @param options Tokenizer options
|
|
8
8
|
*/
|
|
9
|
-
constructor(uint8Array,
|
|
10
|
-
super(
|
|
9
|
+
constructor(uint8Array, options) {
|
|
10
|
+
super(options);
|
|
11
11
|
this.uint8Array = uint8Array;
|
|
12
12
|
this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
|
|
13
13
|
}
|
|
@@ -45,7 +45,7 @@ export class BufferTokenizer extends AbstractTokenizer {
|
|
|
45
45
|
return bytes2read;
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
close() {
|
|
49
|
+
return super.close();
|
|
50
50
|
}
|
|
51
51
|
}
|
package/lib/FileTokenizer.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
2
|
-
import {
|
|
2
|
+
import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
|
|
3
|
+
import { FileHandle } from 'node:fs/promises';
|
|
3
4
|
export declare class FileTokenizer extends AbstractTokenizer {
|
|
4
|
-
private
|
|
5
|
-
constructor(
|
|
5
|
+
private fileHandle;
|
|
6
|
+
constructor(fileHandle: FileHandle, options: ITokenizerOptions);
|
|
6
7
|
/**
|
|
7
8
|
* Read buffer from file
|
|
8
9
|
* @param uint8Array - Uint8Array to write result to
|
package/lib/FileTokenizer.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
2
2
|
import { EndOfStreamError } from 'peek-readable';
|
|
3
|
-
import
|
|
3
|
+
import { open as fsOpen } from 'node:fs/promises';
|
|
4
4
|
export class FileTokenizer extends AbstractTokenizer {
|
|
5
|
-
constructor(
|
|
6
|
-
super(
|
|
7
|
-
this.
|
|
5
|
+
constructor(fileHandle, options) {
|
|
6
|
+
super(options);
|
|
7
|
+
this.fileHandle = fileHandle;
|
|
8
8
|
}
|
|
9
9
|
/**
|
|
10
10
|
* Read buffer from file
|
|
@@ -15,7 +15,7 @@ export class FileTokenizer extends AbstractTokenizer {
|
|
|
15
15
|
async readBuffer(uint8Array, options) {
|
|
16
16
|
const normOptions = this.normalizeOptions(uint8Array, options);
|
|
17
17
|
this.position = normOptions.position;
|
|
18
|
-
const res = await
|
|
18
|
+
const res = await this.fileHandle.read(uint8Array, normOptions.offset, normOptions.length, normOptions.position);
|
|
19
19
|
this.position += res.bytesRead;
|
|
20
20
|
if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
|
|
21
21
|
throw new EndOfStreamError();
|
|
@@ -30,21 +30,19 @@ export class FileTokenizer extends AbstractTokenizer {
|
|
|
30
30
|
*/
|
|
31
31
|
async peekBuffer(uint8Array, options) {
|
|
32
32
|
const normOptions = this.normalizeOptions(uint8Array, options);
|
|
33
|
-
const res = await
|
|
33
|
+
const res = await this.fileHandle.read(uint8Array, normOptions.offset, normOptions.length, normOptions.position);
|
|
34
34
|
if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
|
|
35
35
|
throw new EndOfStreamError();
|
|
36
36
|
}
|
|
37
37
|
return res.bytesRead;
|
|
38
38
|
}
|
|
39
39
|
async close() {
|
|
40
|
-
|
|
40
|
+
await this.fileHandle.close();
|
|
41
|
+
return super.close();
|
|
41
42
|
}
|
|
42
43
|
}
|
|
43
44
|
export async function fromFile(sourceFilePath) {
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
const fd = await fs.open(sourceFilePath, 'r');
|
|
49
|
-
return new FileTokenizer(fd, { path: sourceFilePath, size: stat.size });
|
|
45
|
+
const fileHandle = await fsOpen(sourceFilePath, 'r');
|
|
46
|
+
const stat = await fileHandle.stat();
|
|
47
|
+
return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
|
|
50
48
|
}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
2
|
-
import { IStreamReader } from 'peek-readable';
|
|
3
|
-
import {
|
|
2
|
+
import { type IStreamReader } from 'peek-readable';
|
|
3
|
+
import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
|
|
4
4
|
export declare class ReadStreamTokenizer extends AbstractTokenizer {
|
|
5
5
|
private streamReader;
|
|
6
|
-
constructor(streamReader: IStreamReader, fileInfo?: IFileInfo);
|
|
7
6
|
/**
|
|
8
|
-
*
|
|
9
|
-
* @
|
|
7
|
+
* Constructor
|
|
8
|
+
* @param streamReader stream-reader to read from
|
|
9
|
+
* @param options Tokenizer options
|
|
10
10
|
*/
|
|
11
|
-
|
|
11
|
+
constructor(streamReader: IStreamReader, options?: ITokenizerOptions);
|
|
12
12
|
/**
|
|
13
13
|
* Read buffer from tokenizer
|
|
14
14
|
* @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
|
|
@@ -2,16 +2,14 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
|
2
2
|
import { EndOfStreamError } from 'peek-readable';
|
|
3
3
|
const maxBufferSize = 256000;
|
|
4
4
|
export class ReadStreamTokenizer extends AbstractTokenizer {
|
|
5
|
-
constructor(streamReader, fileInfo) {
|
|
6
|
-
super(fileInfo);
|
|
7
|
-
this.streamReader = streamReader;
|
|
8
|
-
}
|
|
9
5
|
/**
|
|
10
|
-
*
|
|
11
|
-
* @
|
|
6
|
+
* Constructor
|
|
7
|
+
* @param streamReader stream-reader to read from
|
|
8
|
+
* @param options Tokenizer options
|
|
12
9
|
*/
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
constructor(streamReader, options) {
|
|
11
|
+
super(options);
|
|
12
|
+
this.streamReader = streamReader;
|
|
15
13
|
}
|
|
16
14
|
/**
|
|
17
15
|
* Read buffer from tokenizer
|
package/lib/core.d.ts
CHANGED
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
|
|
2
|
-
import { Readable } from 'node:stream';
|
|
3
2
|
import { BufferTokenizer } from './BufferTokenizer.js';
|
|
4
|
-
import {
|
|
3
|
+
import type { ITokenizerOptions } from './types.js';
|
|
5
4
|
import type { ReadableStream } from 'node:stream/web';
|
|
5
|
+
import type { Readable } from 'node:stream';
|
|
6
6
|
export { EndOfStreamError } from 'peek-readable';
|
|
7
|
-
export { ITokenizer, IFileInfo } from './types.js';
|
|
8
|
-
export { IToken, IGetToken } from '@tokenizer/token';
|
|
7
|
+
export type { ITokenizer, IFileInfo, ITokenizerOptions, OnClose } from './types.js';
|
|
8
|
+
export type { IToken, IGetToken } from '@tokenizer/token';
|
|
9
9
|
/**
|
|
10
10
|
* Construct ReadStreamTokenizer from given Stream.
|
|
11
11
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
12
12
|
* @param stream - Read from Node.js Stream.Readable
|
|
13
|
-
* @param
|
|
13
|
+
* @param options - Tokenizer options
|
|
14
14
|
* @returns ReadStreamTokenizer
|
|
15
15
|
*/
|
|
16
|
-
export declare function fromStream(stream: Readable,
|
|
16
|
+
export declare function fromStream(stream: Readable, options?: ITokenizerOptions): ReadStreamTokenizer;
|
|
17
17
|
/**
|
|
18
18
|
* Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
|
|
19
19
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
20
20
|
* @param webStream - Read from Node.js Stream.Readable
|
|
21
|
-
* @param
|
|
21
|
+
* @param options - Tokenizer options
|
|
22
22
|
* @returns ReadStreamTokenizer
|
|
23
23
|
*/
|
|
24
|
-
export declare function fromWebStream(webStream: ReadableStream<Uint8Array>,
|
|
24
|
+
export declare function fromWebStream(webStream: ReadableStream<Uint8Array>, options?: ITokenizerOptions): ReadStreamTokenizer;
|
|
25
25
|
/**
|
|
26
26
|
* Construct ReadStreamTokenizer from given Buffer.
|
|
27
27
|
* @param uint8Array - Uint8Array to tokenize
|
|
28
|
-
* @param
|
|
28
|
+
* @param options - Tokenizer options
|
|
29
29
|
* @returns BufferTokenizer
|
|
30
30
|
*/
|
|
31
|
-
export declare function fromBuffer(uint8Array: Uint8Array,
|
|
31
|
+
export declare function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer;
|
package/lib/core.js
CHANGED
|
@@ -1,35 +1,33 @@
|
|
|
1
1
|
import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
|
|
2
2
|
import { BufferTokenizer } from './BufferTokenizer.js';
|
|
3
|
-
import { StreamReader, WebStreamReader } from
|
|
3
|
+
import { StreamReader, WebStreamReader } from 'peek-readable';
|
|
4
4
|
export { EndOfStreamError } from 'peek-readable';
|
|
5
5
|
/**
|
|
6
6
|
* Construct ReadStreamTokenizer from given Stream.
|
|
7
7
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
8
8
|
* @param stream - Read from Node.js Stream.Readable
|
|
9
|
-
* @param
|
|
9
|
+
* @param options - Tokenizer options
|
|
10
10
|
* @returns ReadStreamTokenizer
|
|
11
11
|
*/
|
|
12
|
-
export function fromStream(stream,
|
|
13
|
-
|
|
14
|
-
return new ReadStreamTokenizer(new StreamReader(stream), fileInfo);
|
|
12
|
+
export function fromStream(stream, options) {
|
|
13
|
+
return new ReadStreamTokenizer(new StreamReader(stream), options);
|
|
15
14
|
}
|
|
16
15
|
/**
|
|
17
16
|
* Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
|
|
18
17
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
19
18
|
* @param webStream - Read from Node.js Stream.Readable
|
|
20
|
-
* @param
|
|
19
|
+
* @param options - Tokenizer options
|
|
21
20
|
* @returns ReadStreamTokenizer
|
|
22
21
|
*/
|
|
23
|
-
export function fromWebStream(webStream,
|
|
24
|
-
|
|
25
|
-
return new ReadStreamTokenizer(new WebStreamReader(webStream), fileInfo);
|
|
22
|
+
export function fromWebStream(webStream, options) {
|
|
23
|
+
return new ReadStreamTokenizer(new WebStreamReader(webStream), options);
|
|
26
24
|
}
|
|
27
25
|
/**
|
|
28
26
|
* Construct ReadStreamTokenizer from given Buffer.
|
|
29
27
|
* @param uint8Array - Uint8Array to tokenize
|
|
30
|
-
* @param
|
|
28
|
+
* @param options - Tokenizer options
|
|
31
29
|
* @returns BufferTokenizer
|
|
32
30
|
*/
|
|
33
|
-
export function fromBuffer(uint8Array,
|
|
34
|
-
return new BufferTokenizer(uint8Array,
|
|
31
|
+
export function fromBuffer(uint8Array, options) {
|
|
32
|
+
return new BufferTokenizer(uint8Array, options);
|
|
35
33
|
}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
|
|
3
|
-
import
|
|
3
|
+
import { type ITokenizerOptions } from './core.js';
|
|
4
4
|
export { fromFile } from './FileTokenizer.js';
|
|
5
|
-
export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo } from './core.js';
|
|
5
|
+
export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo, ITokenizerOptions, OnClose } from './core.js';
|
|
6
6
|
export { IToken, IGetToken } from '@tokenizer/token';
|
|
7
7
|
/**
|
|
8
8
|
* Construct ReadStreamTokenizer from given Stream.
|
|
9
9
|
* Will set fileSize, if provided given Stream has set the .path property.
|
|
10
10
|
* @param stream - Node.js Stream.Readable
|
|
11
|
-
* @param
|
|
11
|
+
* @param options - Pass additional file information to the tokenizer
|
|
12
12
|
* @returns Tokenizer
|
|
13
13
|
*/
|
|
14
|
-
export declare function fromStream(stream: Readable,
|
|
14
|
+
export declare function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>;
|
package/lib/index.js
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
1
|
+
import { stat as fsStat } from 'node:fs/promises';
|
|
2
|
+
import { fromStream as coreFromStream } from './core.js';
|
|
3
3
|
export { fromFile } from './FileTokenizer.js';
|
|
4
4
|
export { EndOfStreamError, fromBuffer, fromWebStream } from './core.js';
|
|
5
5
|
/**
|
|
6
6
|
* Construct ReadStreamTokenizer from given Stream.
|
|
7
7
|
* Will set fileSize, if provided given Stream has set the .path property.
|
|
8
8
|
* @param stream - Node.js Stream.Readable
|
|
9
|
-
* @param
|
|
9
|
+
* @param options - Pass additional file information to the tokenizer
|
|
10
10
|
* @returns Tokenizer
|
|
11
11
|
*/
|
|
12
|
-
export async function fromStream(stream,
|
|
13
|
-
|
|
12
|
+
export async function fromStream(stream, options) {
|
|
13
|
+
var _a;
|
|
14
|
+
const augmentedOptions = options !== null && options !== void 0 ? options : {};
|
|
15
|
+
augmentedOptions.fileInfo = (_a = augmentedOptions.fileInfo) !== null && _a !== void 0 ? _a : {};
|
|
14
16
|
if (stream.path) {
|
|
15
|
-
const stat = await
|
|
16
|
-
fileInfo.path = stream.path;
|
|
17
|
-
fileInfo.size = stat.size;
|
|
17
|
+
const stat = await fsStat(stream.path);
|
|
18
|
+
augmentedOptions.fileInfo.path = stream.path;
|
|
19
|
+
augmentedOptions.fileInfo.size = stat.size;
|
|
18
20
|
}
|
|
19
|
-
return
|
|
21
|
+
return coreFromStream(stream, augmentedOptions);
|
|
20
22
|
}
|
package/lib/types.d.ts
CHANGED
|
@@ -100,3 +100,14 @@ export interface ITokenizer {
|
|
|
100
100
|
*/
|
|
101
101
|
close(): Promise<void>;
|
|
102
102
|
}
|
|
103
|
+
export type OnClose = () => Promise<void>;
|
|
104
|
+
export interface ITokenizerOptions {
|
|
105
|
+
/**
|
|
106
|
+
* Pass additional file information to the tokenizer
|
|
107
|
+
*/
|
|
108
|
+
fileInfo?: IFileInfo;
|
|
109
|
+
/**
|
|
110
|
+
* On tokenizer close handler
|
|
111
|
+
*/
|
|
112
|
+
onClose?: OnClose;
|
|
113
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "strtok3",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "8.0.0",
|
|
4
4
|
"description": "A promise based streaming tokenizer",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Borewit",
|
|
@@ -11,12 +11,12 @@
|
|
|
11
11
|
"url": "https://github.com/sponsors/Borewit"
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
|
-
"clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map",
|
|
14
|
+
"clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
|
|
15
15
|
"compile-src": "tsc -p lib",
|
|
16
16
|
"compile-test": "tsc -p test",
|
|
17
17
|
"compile": "npm run compile-src && npm run compile-test",
|
|
18
18
|
"build": "npm run clean && npm run compile",
|
|
19
|
-
"eslint": "eslint lib test
|
|
19
|
+
"eslint": "eslint lib test",
|
|
20
20
|
"lint-md": "remark -u preset-lint-recommended .",
|
|
21
21
|
"lint": "npm run lint-md && npm run eslint",
|
|
22
22
|
"fix": "eslint lib test --ext .ts --ignore-pattern *.d.ts --fix",
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
"start": "npm run compile && npm run lint && npm run cover-test"
|
|
27
27
|
},
|
|
28
28
|
"engines": {
|
|
29
|
-
"node": ">=
|
|
29
|
+
"node": ">=16"
|
|
30
30
|
},
|
|
31
31
|
"repository": {
|
|
32
32
|
"type": "git",
|
|
@@ -50,22 +50,26 @@
|
|
|
50
50
|
"url": "https://github.com/Borewit/strtok3/issues"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
+
"@eslint/compat": "^1.1.1",
|
|
54
|
+
"@eslint/eslintrc": "^3.1.0",
|
|
55
|
+
"@eslint/js": "^9.7.0",
|
|
53
56
|
"@types/chai": "^4.3.16",
|
|
54
57
|
"@types/debug": "^4.1.12",
|
|
55
58
|
"@types/mocha": "^10.0.7",
|
|
56
|
-
"@types/node": "^20.14.
|
|
57
|
-
"@typescript-eslint/eslint-plugin": "^7.
|
|
58
|
-
"@typescript-eslint/parser": "^7.
|
|
59
|
+
"@types/node": "^20.14.10",
|
|
60
|
+
"@typescript-eslint/eslint-plugin": "^7.16.1",
|
|
61
|
+
"@typescript-eslint/parser": "^7.16.1",
|
|
59
62
|
"c8": "^10.1.2",
|
|
60
63
|
"chai": "^5.1.1",
|
|
61
64
|
"del-cli": "^5.1.0",
|
|
62
|
-
"eslint": "^
|
|
63
|
-
"eslint-config-prettier": "^9.
|
|
65
|
+
"eslint": "^9.7.0",
|
|
66
|
+
"eslint-config-prettier": "^9.1.0",
|
|
64
67
|
"eslint-import-resolver-typescript": "^3.6.1",
|
|
65
|
-
"eslint-plugin-import": "^2.29.
|
|
66
|
-
"eslint-plugin-jsdoc": "^
|
|
68
|
+
"eslint-plugin-import": "^2.29.1",
|
|
69
|
+
"eslint-plugin-jsdoc": "^48.7.0",
|
|
67
70
|
"eslint-plugin-node": "^11.1.0",
|
|
68
71
|
"eslint-plugin-unicorn": "^54.0.0",
|
|
72
|
+
"global": "^4.4.0",
|
|
69
73
|
"mocha": "^10.6.0",
|
|
70
74
|
"remark-cli": "^12.0.1",
|
|
71
75
|
"remark-preset-lint-recommended": "^7.0.0",
|
|
@@ -76,7 +80,7 @@
|
|
|
76
80
|
},
|
|
77
81
|
"dependencies": {
|
|
78
82
|
"@tokenizer/token": "^0.3.0",
|
|
79
|
-
"peek-readable": "^5.1.
|
|
83
|
+
"peek-readable": "^5.1.3"
|
|
80
84
|
},
|
|
81
85
|
"keywords": [
|
|
82
86
|
"tokenizer",
|
|
@@ -91,5 +95,6 @@
|
|
|
91
95
|
"uint",
|
|
92
96
|
"stream",
|
|
93
97
|
"streaming"
|
|
94
|
-
]
|
|
98
|
+
],
|
|
99
|
+
"packageManager": "yarn@4.3.1"
|
|
95
100
|
}
|
package/lib/FsPromise.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Module convert fs functions to promise based functions
|
|
3
|
-
*/
|
|
4
|
-
import fs from 'node:fs';
|
|
5
|
-
export interface IReadResult {
|
|
6
|
-
bytesRead: number;
|
|
7
|
-
buffer: Uint8Array;
|
|
8
|
-
}
|
|
9
|
-
export declare const pathExists: typeof fs.existsSync;
|
|
10
|
-
export declare const createReadStream: typeof fs.createReadStream;
|
|
11
|
-
export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
|
|
12
|
-
export declare function close(fd: number): Promise<void>;
|
|
13
|
-
export declare function open(path: fs.PathLike, mode: fs.Mode): Promise<number>;
|
|
14
|
-
export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
|
|
15
|
-
export declare function writeFile(path: fs.PathLike, data: Uint8Array | string, options?: fs.WriteFileOptions): Promise<void>;
|
|
16
|
-
export declare function writeFileSync(path: fs.PathLike, data: Uint8Array | string): void;
|
|
17
|
-
export declare function readFile(path: fs.PathLike): Promise<Uint8Array>;
|
package/lib/FsPromise.js
DELETED
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Module convert fs functions to promise based functions
|
|
3
|
-
*/
|
|
4
|
-
import fs from 'node:fs';
|
|
5
|
-
export const pathExists = fs.existsSync;
|
|
6
|
-
export const createReadStream = fs.createReadStream;
|
|
7
|
-
export async function stat(path) {
|
|
8
|
-
return new Promise((resolve, reject) => {
|
|
9
|
-
fs.stat(path, (err, stats) => {
|
|
10
|
-
if (err)
|
|
11
|
-
reject(err);
|
|
12
|
-
else
|
|
13
|
-
resolve(stats);
|
|
14
|
-
});
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
|
-
export async function close(fd) {
|
|
18
|
-
return new Promise((resolve, reject) => {
|
|
19
|
-
fs.close(fd, err => {
|
|
20
|
-
if (err)
|
|
21
|
-
reject(err);
|
|
22
|
-
else
|
|
23
|
-
resolve();
|
|
24
|
-
});
|
|
25
|
-
});
|
|
26
|
-
}
|
|
27
|
-
export async function open(path, mode) {
|
|
28
|
-
return new Promise((resolve, reject) => {
|
|
29
|
-
fs.open(path, mode, (err, fd) => {
|
|
30
|
-
if (err)
|
|
31
|
-
reject(err);
|
|
32
|
-
else
|
|
33
|
-
resolve(fd);
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
export async function read(fd, buffer, offset, length, position) {
|
|
38
|
-
return new Promise((resolve, reject) => {
|
|
39
|
-
fs.read(fd, buffer, offset, length, position, (err, bytesRead, _buffer) => {
|
|
40
|
-
if (err)
|
|
41
|
-
reject(err);
|
|
42
|
-
else
|
|
43
|
-
resolve({ bytesRead, buffer: _buffer });
|
|
44
|
-
});
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
export async function writeFile(path, data, options = null) {
|
|
48
|
-
return new Promise((resolve, reject) => {
|
|
49
|
-
fs.writeFile(path, data, options, err => {
|
|
50
|
-
if (err)
|
|
51
|
-
reject(err);
|
|
52
|
-
else
|
|
53
|
-
resolve();
|
|
54
|
-
});
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
export function writeFileSync(path, data) {
|
|
58
|
-
fs.writeFileSync(path, data);
|
|
59
|
-
}
|
|
60
|
-
export async function readFile(path) {
|
|
61
|
-
return new Promise((resolve, reject) => {
|
|
62
|
-
fs.readFile(path, (err, buffer) => {
|
|
63
|
-
if (err)
|
|
64
|
-
reject(err);
|
|
65
|
-
else
|
|
66
|
-
resolve(buffer);
|
|
67
|
-
});
|
|
68
|
-
});
|
|
69
|
-
}
|