strtok3 7.0.0-alpha.7 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -14
- package/README.md +306 -300
- package/lib/AbstractTokenizer.d.ts +69 -69
- package/lib/AbstractTokenizer.js +101 -101
- package/lib/BufferTokenizer.d.ts +26 -26
- package/lib/BufferTokenizer.js +51 -51
- package/lib/FileTokenizer.d.ts +22 -22
- package/lib/FileTokenizer.js +50 -50
- package/lib/FsPromise.d.ts +19 -18
- package/lib/FsPromise.js +69 -69
- package/lib/ReadStreamTokenizer.d.ts +28 -28
- package/lib/ReadStreamTokenizer.js +94 -94
- package/lib/core.d.ts +23 -23
- package/lib/core.js +23 -23
- package/lib/index.d.ts +15 -15
- package/lib/index.js +20 -20
- package/lib/types.d.ts +103 -103
- package/lib/types.js +1 -1
- package/package.json +94 -95
package/lib/types.d.ts
CHANGED
|
@@ -1,103 +1,103 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { IGetToken } from '@tokenizer/token';
|
|
3
|
-
export interface IFileInfo {
|
|
4
|
-
/**
|
|
5
|
-
* File size in bytes
|
|
6
|
-
*/
|
|
7
|
-
size?: number;
|
|
8
|
-
/**
|
|
9
|
-
* MIME-type of file
|
|
10
|
-
*/
|
|
11
|
-
mimeType?: string;
|
|
12
|
-
/**
|
|
13
|
-
* File path
|
|
14
|
-
*/
|
|
15
|
-
path?: string;
|
|
16
|
-
/**
|
|
17
|
-
* File URL
|
|
18
|
-
*/
|
|
19
|
-
url?: string;
|
|
20
|
-
}
|
|
21
|
-
export interface IReadChunkOptions {
|
|
22
|
-
/**
|
|
23
|
-
* The offset in the buffer to start writing at; default is 0
|
|
24
|
-
*/
|
|
25
|
-
offset?: number;
|
|
26
|
-
/**
|
|
27
|
-
* Number of bytes to read.
|
|
28
|
-
*/
|
|
29
|
-
length?: number;
|
|
30
|
-
/**
|
|
31
|
-
* Position where to begin reading from the file.
|
|
32
|
-
* Default it is `tokenizer.position`.
|
|
33
|
-
* Position may not be less then `tokenizer.position`.
|
|
34
|
-
*/
|
|
35
|
-
position?: number;
|
|
36
|
-
/**
|
|
37
|
-
* If set, will not throw an EOF error if not all of the requested data could be read
|
|
38
|
-
*/
|
|
39
|
-
mayBeLess?: boolean;
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* The tokenizer allows us to read or peek from the tokenizer-stream.
|
|
43
|
-
* The tokenizer-stream is an abstraction of a stream, file or Buffer.
|
|
44
|
-
*/
|
|
45
|
-
export interface ITokenizer {
|
|
46
|
-
/**
|
|
47
|
-
* Provide access to information of the underlying information stream or file.
|
|
48
|
-
*/
|
|
49
|
-
fileInfo: IFileInfo;
|
|
50
|
-
/**
|
|
51
|
-
* Offset in bytes (= number of bytes read) since beginning of file or stream
|
|
52
|
-
*/
|
|
53
|
-
position: number;
|
|
54
|
-
/**
|
|
55
|
-
* Peek (read ahead) buffer from tokenizer
|
|
56
|
-
* @param buffer - Target buffer to fill with data peek from the tokenizer-stream
|
|
57
|
-
* @param options - Read behaviour options
|
|
58
|
-
* @returns Promise with number of bytes read
|
|
59
|
-
*/
|
|
60
|
-
peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
61
|
-
/**
|
|
62
|
-
* Peek (read ahead) buffer from tokenizer
|
|
63
|
-
* @param buffer - Target buffer to fill with data peeked from the tokenizer-stream
|
|
64
|
-
* @param options - Additional read options
|
|
65
|
-
* @returns Promise with number of bytes read
|
|
66
|
-
*/
|
|
67
|
-
readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
68
|
-
/**
|
|
69
|
-
* Peek a token from the tokenizer-stream.
|
|
70
|
-
* @param token - Token to peek from the tokenizer-stream.
|
|
71
|
-
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
72
|
-
* @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
|
|
73
|
-
*/
|
|
74
|
-
peekToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
|
|
75
|
-
/**
|
|
76
|
-
* Read a token from the tokenizer-stream.
|
|
77
|
-
* @param token - Token to peek from the tokenizer-stream.
|
|
78
|
-
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
79
|
-
*/
|
|
80
|
-
readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
|
|
81
|
-
/**
|
|
82
|
-
* Peek a numeric token from the stream
|
|
83
|
-
* @param token - Numeric token
|
|
84
|
-
* @returns Promise with number
|
|
85
|
-
*/
|
|
86
|
-
peekNumber(token: IGetToken<number>): Promise<number>;
|
|
87
|
-
/**
|
|
88
|
-
* Read a numeric token from the stream
|
|
89
|
-
* @param token - Numeric token
|
|
90
|
-
* @returns Promise with number
|
|
91
|
-
*/
|
|
92
|
-
readNumber(token: IGetToken<number>): Promise<number>;
|
|
93
|
-
/**
|
|
94
|
-
* Ignore given number of bytes
|
|
95
|
-
* @param length - Number of bytes ignored
|
|
96
|
-
*/
|
|
97
|
-
ignore(length: number): Promise<number>;
|
|
98
|
-
/**
|
|
99
|
-
* Clean up resources.
|
|
100
|
-
* It does not close the stream for StreamReader, but is does close the file-descriptor.
|
|
101
|
-
*/
|
|
102
|
-
close(): Promise<void>;
|
|
103
|
-
}
|
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
+
import { IGetToken } from '@tokenizer/token';
|
|
3
|
+
export interface IFileInfo {
|
|
4
|
+
/**
|
|
5
|
+
* File size in bytes
|
|
6
|
+
*/
|
|
7
|
+
size?: number;
|
|
8
|
+
/**
|
|
9
|
+
* MIME-type of file
|
|
10
|
+
*/
|
|
11
|
+
mimeType?: string;
|
|
12
|
+
/**
|
|
13
|
+
* File path
|
|
14
|
+
*/
|
|
15
|
+
path?: string;
|
|
16
|
+
/**
|
|
17
|
+
* File URL
|
|
18
|
+
*/
|
|
19
|
+
url?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface IReadChunkOptions {
|
|
22
|
+
/**
|
|
23
|
+
* The offset in the buffer to start writing at; default is 0
|
|
24
|
+
*/
|
|
25
|
+
offset?: number;
|
|
26
|
+
/**
|
|
27
|
+
* Number of bytes to read.
|
|
28
|
+
*/
|
|
29
|
+
length?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Position where to begin reading from the file.
|
|
32
|
+
* Default it is `tokenizer.position`.
|
|
33
|
+
* Position may not be less then `tokenizer.position`.
|
|
34
|
+
*/
|
|
35
|
+
position?: number;
|
|
36
|
+
/**
|
|
37
|
+
* If set, will not throw an EOF error if not all of the requested data could be read
|
|
38
|
+
*/
|
|
39
|
+
mayBeLess?: boolean;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The tokenizer allows us to read or peek from the tokenizer-stream.
|
|
43
|
+
* The tokenizer-stream is an abstraction of a stream, file or Buffer.
|
|
44
|
+
*/
|
|
45
|
+
export interface ITokenizer {
|
|
46
|
+
/**
|
|
47
|
+
* Provide access to information of the underlying information stream or file.
|
|
48
|
+
*/
|
|
49
|
+
fileInfo: IFileInfo;
|
|
50
|
+
/**
|
|
51
|
+
* Offset in bytes (= number of bytes read) since beginning of file or stream
|
|
52
|
+
*/
|
|
53
|
+
position: number;
|
|
54
|
+
/**
|
|
55
|
+
* Peek (read ahead) buffer from tokenizer
|
|
56
|
+
* @param buffer - Target buffer to fill with data peek from the tokenizer-stream
|
|
57
|
+
* @param options - Read behaviour options
|
|
58
|
+
* @returns Promise with number of bytes read
|
|
59
|
+
*/
|
|
60
|
+
peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
61
|
+
/**
|
|
62
|
+
* Peek (read ahead) buffer from tokenizer
|
|
63
|
+
* @param buffer - Target buffer to fill with data peeked from the tokenizer-stream
|
|
64
|
+
* @param options - Additional read options
|
|
65
|
+
* @returns Promise with number of bytes read
|
|
66
|
+
*/
|
|
67
|
+
readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
68
|
+
/**
|
|
69
|
+
* Peek a token from the tokenizer-stream.
|
|
70
|
+
* @param token - Token to peek from the tokenizer-stream.
|
|
71
|
+
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
72
|
+
* @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
|
|
73
|
+
*/
|
|
74
|
+
peekToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
|
|
75
|
+
/**
|
|
76
|
+
* Read a token from the tokenizer-stream.
|
|
77
|
+
* @param token - Token to peek from the tokenizer-stream.
|
|
78
|
+
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
79
|
+
*/
|
|
80
|
+
readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
|
|
81
|
+
/**
|
|
82
|
+
* Peek a numeric token from the stream
|
|
83
|
+
* @param token - Numeric token
|
|
84
|
+
* @returns Promise with number
|
|
85
|
+
*/
|
|
86
|
+
peekNumber(token: IGetToken<number>): Promise<number>;
|
|
87
|
+
/**
|
|
88
|
+
* Read a numeric token from the stream
|
|
89
|
+
* @param token - Numeric token
|
|
90
|
+
* @returns Promise with number
|
|
91
|
+
*/
|
|
92
|
+
readNumber(token: IGetToken<number>): Promise<number>;
|
|
93
|
+
/**
|
|
94
|
+
* Ignore given number of bytes
|
|
95
|
+
* @param length - Number of bytes ignored
|
|
96
|
+
*/
|
|
97
|
+
ignore(length: number): Promise<number>;
|
|
98
|
+
/**
|
|
99
|
+
* Clean up resources.
|
|
100
|
+
* It does not close the stream for StreamReader, but is does close the file-descriptor.
|
|
101
|
+
*/
|
|
102
|
+
close(): Promise<void>;
|
|
103
|
+
}
|
package/lib/types.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {};
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,95 +1,94 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "strtok3",
|
|
3
|
-
"version": "7.0.0
|
|
4
|
-
"description": "A promise based streaming tokenizer",
|
|
5
|
-
"author": {
|
|
6
|
-
"name": "Borewit",
|
|
7
|
-
"url": "https://github.com/Borewit"
|
|
8
|
-
},
|
|
9
|
-
"funding": {
|
|
10
|
-
"type": "github",
|
|
11
|
-
"url": "https://github.com/sponsors/Borewit"
|
|
12
|
-
},
|
|
13
|
-
"scripts": {
|
|
14
|
-
"clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map",
|
|
15
|
-
"compile-src": "tsc -p lib",
|
|
16
|
-
"compile-test": "tsc -p test",
|
|
17
|
-
"compile": "npm run compile-src && npm run compile-test",
|
|
18
|
-
"build": "npm run clean && npm run compile",
|
|
19
|
-
"eslint": "eslint lib test --ext .ts --ignore-pattern *.d.ts",
|
|
20
|
-
"lint-md": "remark -u preset-lint-recommended .",
|
|
21
|
-
"lint": "npm run lint-md && npm run eslint",
|
|
22
|
-
"fix": "eslint lib test --ext .ts --ignore-pattern *.d.ts --fix",
|
|
23
|
-
"test": "mocha",
|
|
24
|
-
"test-coverage": "c8 npm run test",
|
|
25
|
-
"send-
|
|
26
|
-
"
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
"
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
"@types/
|
|
56
|
-
"@types/
|
|
57
|
-
"@
|
|
58
|
-
"@
|
|
59
|
-
"
|
|
60
|
-
"
|
|
61
|
-
"
|
|
62
|
-
"
|
|
63
|
-
"
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"eslint": "^
|
|
67
|
-
"eslint-plugin-
|
|
68
|
-
"eslint-plugin-
|
|
69
|
-
"mocha": "^
|
|
70
|
-
"remark-cli": "^
|
|
71
|
-
"remark-preset-lint-recommended": "^6.1.2",
|
|
72
|
-
"token-types": "^
|
|
73
|
-
"ts-node": "^10.
|
|
74
|
-
"
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
"
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
92
|
-
"
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "strtok3",
|
|
3
|
+
"version": "7.0.0",
|
|
4
|
+
"description": "A promise based streaming tokenizer",
|
|
5
|
+
"author": {
|
|
6
|
+
"name": "Borewit",
|
|
7
|
+
"url": "https://github.com/Borewit"
|
|
8
|
+
},
|
|
9
|
+
"funding": {
|
|
10
|
+
"type": "github",
|
|
11
|
+
"url": "https://github.com/sponsors/Borewit"
|
|
12
|
+
},
|
|
13
|
+
"scripts": {
|
|
14
|
+
"clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map",
|
|
15
|
+
"compile-src": "tsc -p lib",
|
|
16
|
+
"compile-test": "tsc -p test",
|
|
17
|
+
"compile": "npm run compile-src && npm run compile-test",
|
|
18
|
+
"build": "npm run clean && npm run compile",
|
|
19
|
+
"eslint": "eslint lib test --ext .ts --ignore-pattern *.d.ts",
|
|
20
|
+
"lint-md": "remark -u preset-lint-recommended .",
|
|
21
|
+
"lint": "npm run lint-md && npm run eslint",
|
|
22
|
+
"fix": "eslint lib test --ext .ts --ignore-pattern *.d.ts --fix",
|
|
23
|
+
"test": "mocha",
|
|
24
|
+
"test-coverage": "c8 npm run test",
|
|
25
|
+
"send-codacy": "c8 report --reporter=text-lcov | codacy-coverage",
|
|
26
|
+
"start": "npm run compile && npm run lint && npm run cover-test"
|
|
27
|
+
},
|
|
28
|
+
"engines": {
|
|
29
|
+
"node": ">=14.16"
|
|
30
|
+
},
|
|
31
|
+
"repository": {
|
|
32
|
+
"type": "git",
|
|
33
|
+
"url": "https://github.com/Borewit/strtok3.git"
|
|
34
|
+
},
|
|
35
|
+
"license": "MIT",
|
|
36
|
+
"type": "module",
|
|
37
|
+
"exports": {
|
|
38
|
+
".": {
|
|
39
|
+
"node": "./lib/index.js",
|
|
40
|
+
"default": "./lib/core.js"
|
|
41
|
+
},
|
|
42
|
+
"./core": "./lib/core.js"
|
|
43
|
+
},
|
|
44
|
+
"types": "lib/index.d.ts",
|
|
45
|
+
"files": [
|
|
46
|
+
"lib/**/*.js",
|
|
47
|
+
"lib/**/*.d.ts"
|
|
48
|
+
],
|
|
49
|
+
"bugs": {
|
|
50
|
+
"url": "https://github.com/Borewit/strtok3/issues"
|
|
51
|
+
},
|
|
52
|
+
"devDependencies": {
|
|
53
|
+
"@types/chai": "^4.3.1",
|
|
54
|
+
"@types/debug": "^4.1.7",
|
|
55
|
+
"@types/mocha": "^9.1.0",
|
|
56
|
+
"@types/node": "^18.6.3",
|
|
57
|
+
"@typescript-eslint/eslint-plugin": "^5.32.0",
|
|
58
|
+
"@typescript-eslint/parser": "^5.32.0",
|
|
59
|
+
"c8": "^7.12.0",
|
|
60
|
+
"chai": "^4.3.6",
|
|
61
|
+
"del-cli": "^5.0.0",
|
|
62
|
+
"eslint": "^8.21.0",
|
|
63
|
+
"eslint-config-prettier": "^8.5.0",
|
|
64
|
+
"eslint-import-resolver-typescript": "^3.4.0",
|
|
65
|
+
"eslint-plugin-import": "^2.26.0",
|
|
66
|
+
"eslint-plugin-jsdoc": "^39.3.4",
|
|
67
|
+
"eslint-plugin-node": "^11.1.0",
|
|
68
|
+
"eslint-plugin-unicorn": "^43.0.2",
|
|
69
|
+
"mocha": "^10.0.0",
|
|
70
|
+
"remark-cli": "^11.0.0",
|
|
71
|
+
"remark-preset-lint-recommended": "^6.1.2",
|
|
72
|
+
"token-types": "^5.0.0",
|
|
73
|
+
"ts-node": "^10.9.1",
|
|
74
|
+
"typescript": "^4.7.4"
|
|
75
|
+
},
|
|
76
|
+
"dependencies": {
|
|
77
|
+
"@tokenizer/token": "^0.3.0",
|
|
78
|
+
"peek-readable": "^5.0.0"
|
|
79
|
+
},
|
|
80
|
+
"keywords": [
|
|
81
|
+
"tokenizer",
|
|
82
|
+
"reader",
|
|
83
|
+
"token",
|
|
84
|
+
"async",
|
|
85
|
+
"promise",
|
|
86
|
+
"parser",
|
|
87
|
+
"decoder",
|
|
88
|
+
"binary",
|
|
89
|
+
"endian",
|
|
90
|
+
"uint",
|
|
91
|
+
"stream",
|
|
92
|
+
"streaming"
|
|
93
|
+
]
|
|
94
|
+
}
|