strtok3 6.0.8 → 6.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -14
- package/README.md +300 -301
- package/lib/AbstractTokenizer.d.ts +62 -62
- package/lib/AbstractTokenizer.js +69 -69
- package/lib/BufferTokenizer.d.ts +37 -37
- package/lib/BufferTokenizer.js +114 -114
- package/lib/FileTokenizer.d.ts +28 -28
- package/lib/FileTokenizer.js +112 -112
- package/lib/FsPromise.d.ts +18 -18
- package/lib/FsPromise.js +79 -79
- package/lib/ReadStreamTokenizer.d.ts +28 -28
- package/lib/ReadStreamTokenizer.js +128 -128
- package/lib/core.d.ts +23 -23
- package/lib/core.js +29 -29
- package/lib/index.d.ts +15 -15
- package/lib/index.js +27 -27
- package/lib/types.d.ts +103 -103
- package/lib/types.js +2 -2
- package/package.json +100 -100
|
@@ -1,128 +1,128 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ReadStreamTokenizer = void 0;
|
|
4
|
-
const AbstractTokenizer_1 = require("./AbstractTokenizer");
|
|
5
|
-
const peek_readable_1 = require("peek-readable");
|
|
6
|
-
// import * as _debug from 'debug';
|
|
7
|
-
// const debug = _debug('strtok3:ReadStreamTokenizer');
|
|
8
|
-
const maxBufferSize = 256000;
|
|
9
|
-
class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
|
|
10
|
-
constructor(stream, fileInfo) {
|
|
11
|
-
super(fileInfo);
|
|
12
|
-
this.streamReader = new peek_readable_1.StreamReader(stream);
|
|
13
|
-
}
|
|
14
|
-
/**
|
|
15
|
-
* Get file information, an HTTP-client may implement this doing a HEAD request
|
|
16
|
-
* @return Promise with file information
|
|
17
|
-
*/
|
|
18
|
-
async getFileInfo() {
|
|
19
|
-
return this.fileInfo;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Read buffer from tokenizer
|
|
23
|
-
* @param buffer - Target buffer to fill with data read from the tokenizer-stream
|
|
24
|
-
* @param options - Read behaviour options
|
|
25
|
-
* @returns Promise with number of bytes read
|
|
26
|
-
*/
|
|
27
|
-
async readBuffer(buffer, options) {
|
|
28
|
-
// const _offset = position ? position : this.position;
|
|
29
|
-
// debug(`readBuffer ${_offset}...${_offset + length - 1}`);
|
|
30
|
-
let offset = 0;
|
|
31
|
-
let length = buffer.length;
|
|
32
|
-
if (options) {
|
|
33
|
-
if (Number.isInteger(options.length)) {
|
|
34
|
-
length = options.length;
|
|
35
|
-
}
|
|
36
|
-
else {
|
|
37
|
-
length -= options.offset || 0;
|
|
38
|
-
}
|
|
39
|
-
if (options.position) {
|
|
40
|
-
const skipBytes = options.position - this.position;
|
|
41
|
-
if (skipBytes > 0) {
|
|
42
|
-
await this.ignore(skipBytes);
|
|
43
|
-
return this.readBuffer(buffer, options);
|
|
44
|
-
}
|
|
45
|
-
else if (skipBytes < 0) {
|
|
46
|
-
throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
if (options.offset) {
|
|
50
|
-
offset = options.offset;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
if (length === 0) {
|
|
54
|
-
return 0;
|
|
55
|
-
}
|
|
56
|
-
const bytesRead = await this.streamReader.read(buffer, offset, length);
|
|
57
|
-
this.position += bytesRead;
|
|
58
|
-
if ((!options || !options.mayBeLess) && bytesRead < length) {
|
|
59
|
-
throw new peek_readable_1.EndOfStreamError();
|
|
60
|
-
}
|
|
61
|
-
return bytesRead;
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Peek (read ahead) buffer from tokenizer
|
|
65
|
-
* @param buffer - Target buffer to write the data read to
|
|
66
|
-
* @param options - Read behaviour options
|
|
67
|
-
* @returns Promise with number of bytes peeked
|
|
68
|
-
*/
|
|
69
|
-
async peekBuffer(buffer, options) {
|
|
70
|
-
// const _offset = position ? position : this.position;
|
|
71
|
-
// debug(`peek ${_offset}...${_offset + length - 1}`);
|
|
72
|
-
let offset = 0;
|
|
73
|
-
let bytesRead;
|
|
74
|
-
let length = buffer.length;
|
|
75
|
-
if (options) {
|
|
76
|
-
if (options.offset) {
|
|
77
|
-
offset = options.offset;
|
|
78
|
-
}
|
|
79
|
-
if (Number.isInteger(options.length)) {
|
|
80
|
-
length = options.length;
|
|
81
|
-
}
|
|
82
|
-
else {
|
|
83
|
-
length -= options.offset || 0;
|
|
84
|
-
}
|
|
85
|
-
if (options.position) {
|
|
86
|
-
const skipBytes = options.position - this.position;
|
|
87
|
-
if (skipBytes > 0) {
|
|
88
|
-
const skipBuffer = Buffer.alloc(length + skipBytes);
|
|
89
|
-
bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
|
|
90
|
-
skipBuffer.copy(buffer, offset, skipBytes);
|
|
91
|
-
return bytesRead - skipBytes;
|
|
92
|
-
}
|
|
93
|
-
else if (skipBytes < 0) {
|
|
94
|
-
throw new Error('Cannot peek from a negative offset in a stream');
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
try {
|
|
99
|
-
bytesRead = await this.streamReader.peek(buffer, offset, length);
|
|
100
|
-
}
|
|
101
|
-
catch (err) {
|
|
102
|
-
if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
|
|
103
|
-
return 0;
|
|
104
|
-
}
|
|
105
|
-
throw err;
|
|
106
|
-
}
|
|
107
|
-
if ((!options || !options.mayBeLess) && bytesRead < length) {
|
|
108
|
-
throw new peek_readable_1.EndOfStreamError();
|
|
109
|
-
}
|
|
110
|
-
return bytesRead;
|
|
111
|
-
}
|
|
112
|
-
async ignore(length) {
|
|
113
|
-
// debug(`ignore ${this.position}...${this.position + length - 1}`);
|
|
114
|
-
const bufSize = Math.min(maxBufferSize, length);
|
|
115
|
-
const buf = Buffer.alloc(bufSize);
|
|
116
|
-
let totBytesRead = 0;
|
|
117
|
-
while (totBytesRead < length) {
|
|
118
|
-
const remaining = length - totBytesRead;
|
|
119
|
-
const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
|
|
120
|
-
if (bytesRead < 0) {
|
|
121
|
-
return bytesRead;
|
|
122
|
-
}
|
|
123
|
-
totBytesRead += bytesRead;
|
|
124
|
-
}
|
|
125
|
-
return totBytesRead;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
exports.ReadStreamTokenizer = ReadStreamTokenizer;
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ReadStreamTokenizer = void 0;
|
|
4
|
+
const AbstractTokenizer_1 = require("./AbstractTokenizer");
|
|
5
|
+
const peek_readable_1 = require("peek-readable");
|
|
6
|
+
// import * as _debug from 'debug';
|
|
7
|
+
// const debug = _debug('strtok3:ReadStreamTokenizer');
|
|
8
|
+
const maxBufferSize = 256000;
|
|
9
|
+
class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
|
|
10
|
+
constructor(stream, fileInfo) {
|
|
11
|
+
super(fileInfo);
|
|
12
|
+
this.streamReader = new peek_readable_1.StreamReader(stream);
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Get file information, an HTTP-client may implement this doing a HEAD request
|
|
16
|
+
* @return Promise with file information
|
|
17
|
+
*/
|
|
18
|
+
async getFileInfo() {
|
|
19
|
+
return this.fileInfo;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Read buffer from tokenizer
|
|
23
|
+
* @param buffer - Target buffer to fill with data read from the tokenizer-stream
|
|
24
|
+
* @param options - Read behaviour options
|
|
25
|
+
* @returns Promise with number of bytes read
|
|
26
|
+
*/
|
|
27
|
+
async readBuffer(buffer, options) {
|
|
28
|
+
// const _offset = position ? position : this.position;
|
|
29
|
+
// debug(`readBuffer ${_offset}...${_offset + length - 1}`);
|
|
30
|
+
let offset = 0;
|
|
31
|
+
let length = buffer.length;
|
|
32
|
+
if (options) {
|
|
33
|
+
if (Number.isInteger(options.length)) {
|
|
34
|
+
length = options.length;
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
length -= options.offset || 0;
|
|
38
|
+
}
|
|
39
|
+
if (options.position) {
|
|
40
|
+
const skipBytes = options.position - this.position;
|
|
41
|
+
if (skipBytes > 0) {
|
|
42
|
+
await this.ignore(skipBytes);
|
|
43
|
+
return this.readBuffer(buffer, options);
|
|
44
|
+
}
|
|
45
|
+
else if (skipBytes < 0) {
|
|
46
|
+
throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (options.offset) {
|
|
50
|
+
offset = options.offset;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (length === 0) {
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
const bytesRead = await this.streamReader.read(buffer, offset, length);
|
|
57
|
+
this.position += bytesRead;
|
|
58
|
+
if ((!options || !options.mayBeLess) && bytesRead < length) {
|
|
59
|
+
throw new peek_readable_1.EndOfStreamError();
|
|
60
|
+
}
|
|
61
|
+
return bytesRead;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Peek (read ahead) buffer from tokenizer
|
|
65
|
+
* @param buffer - Target buffer to write the data read to
|
|
66
|
+
* @param options - Read behaviour options
|
|
67
|
+
* @returns Promise with number of bytes peeked
|
|
68
|
+
*/
|
|
69
|
+
async peekBuffer(buffer, options) {
|
|
70
|
+
// const _offset = position ? position : this.position;
|
|
71
|
+
// debug(`peek ${_offset}...${_offset + length - 1}`);
|
|
72
|
+
let offset = 0;
|
|
73
|
+
let bytesRead;
|
|
74
|
+
let length = buffer.length;
|
|
75
|
+
if (options) {
|
|
76
|
+
if (options.offset) {
|
|
77
|
+
offset = options.offset;
|
|
78
|
+
}
|
|
79
|
+
if (Number.isInteger(options.length)) {
|
|
80
|
+
length = options.length;
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
length -= options.offset || 0;
|
|
84
|
+
}
|
|
85
|
+
if (options.position) {
|
|
86
|
+
const skipBytes = options.position - this.position;
|
|
87
|
+
if (skipBytes > 0) {
|
|
88
|
+
const skipBuffer = Buffer.alloc(length + skipBytes);
|
|
89
|
+
bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
|
|
90
|
+
skipBuffer.copy(buffer, offset, skipBytes);
|
|
91
|
+
return bytesRead - skipBytes;
|
|
92
|
+
}
|
|
93
|
+
else if (skipBytes < 0) {
|
|
94
|
+
throw new Error('Cannot peek from a negative offset in a stream');
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
bytesRead = await this.streamReader.peek(buffer, offset, length);
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
throw err;
|
|
106
|
+
}
|
|
107
|
+
if ((!options || !options.mayBeLess) && bytesRead < length) {
|
|
108
|
+
throw new peek_readable_1.EndOfStreamError();
|
|
109
|
+
}
|
|
110
|
+
return bytesRead;
|
|
111
|
+
}
|
|
112
|
+
async ignore(length) {
|
|
113
|
+
// debug(`ignore ${this.position}...${this.position + length - 1}`);
|
|
114
|
+
const bufSize = Math.min(maxBufferSize, length);
|
|
115
|
+
const buf = Buffer.alloc(bufSize);
|
|
116
|
+
let totBytesRead = 0;
|
|
117
|
+
while (totBytesRead < length) {
|
|
118
|
+
const remaining = length - totBytesRead;
|
|
119
|
+
const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
|
|
120
|
+
if (bytesRead < 0) {
|
|
121
|
+
return bytesRead;
|
|
122
|
+
}
|
|
123
|
+
totBytesRead += bytesRead;
|
|
124
|
+
}
|
|
125
|
+
return totBytesRead;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
exports.ReadStreamTokenizer = ReadStreamTokenizer;
|
package/lib/core.d.ts
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { ReadStreamTokenizer } from './ReadStreamTokenizer';
|
|
3
|
-
import * as Stream from 'stream';
|
|
4
|
-
import { BufferTokenizer } from './BufferTokenizer';
|
|
5
|
-
import { IFileInfo } from './types';
|
|
6
|
-
export { EndOfStreamError } from 'peek-readable';
|
|
7
|
-
export { ITokenizer, IFileInfo } from './types';
|
|
8
|
-
export { IToken, IGetToken } from '@tokenizer/token';
|
|
9
|
-
/**
|
|
10
|
-
* Construct ReadStreamTokenizer from given Stream.
|
|
11
|
-
* Will set fileSize, if provided given Stream has set the .path property/
|
|
12
|
-
* @param stream - Read from Node.js Stream.Readable
|
|
13
|
-
* @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
|
|
14
|
-
* @returns ReadStreamTokenizer
|
|
15
|
-
*/
|
|
16
|
-
export declare function fromStream(stream: Stream.Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
|
|
17
|
-
/**
|
|
18
|
-
* Construct ReadStreamTokenizer from given Buffer.
|
|
19
|
-
* @param buffer - Buffer to tokenize
|
|
20
|
-
* @param fileInfo - Pass additional file information to the tokenizer
|
|
21
|
-
* @returns BufferTokenizer
|
|
22
|
-
*/
|
|
23
|
-
export declare function fromBuffer(buffer: Buffer, fileInfo?: IFileInfo): BufferTokenizer;
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { ReadStreamTokenizer } from './ReadStreamTokenizer';
|
|
3
|
+
import * as Stream from 'stream';
|
|
4
|
+
import { BufferTokenizer } from './BufferTokenizer';
|
|
5
|
+
import { IFileInfo } from './types';
|
|
6
|
+
export { EndOfStreamError } from 'peek-readable';
|
|
7
|
+
export { ITokenizer, IFileInfo } from './types';
|
|
8
|
+
export { IToken, IGetToken } from '@tokenizer/token';
|
|
9
|
+
/**
|
|
10
|
+
* Construct ReadStreamTokenizer from given Stream.
|
|
11
|
+
* Will set fileSize, if provided given Stream has set the .path property/
|
|
12
|
+
* @param stream - Read from Node.js Stream.Readable
|
|
13
|
+
* @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
|
|
14
|
+
* @returns ReadStreamTokenizer
|
|
15
|
+
*/
|
|
16
|
+
export declare function fromStream(stream: Stream.Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
|
|
17
|
+
/**
|
|
18
|
+
* Construct ReadStreamTokenizer from given Buffer.
|
|
19
|
+
* @param buffer - Buffer to tokenize
|
|
20
|
+
* @param fileInfo - Pass additional file information to the tokenizer
|
|
21
|
+
* @returns BufferTokenizer
|
|
22
|
+
*/
|
|
23
|
+
export declare function fromBuffer(buffer: Buffer, fileInfo?: IFileInfo): BufferTokenizer;
|
package/lib/core.js
CHANGED
|
@@ -1,29 +1,29 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.fromBuffer = exports.fromStream = exports.EndOfStreamError = void 0;
|
|
4
|
-
const ReadStreamTokenizer_1 = require("./ReadStreamTokenizer");
|
|
5
|
-
const BufferTokenizer_1 = require("./BufferTokenizer");
|
|
6
|
-
var peek_readable_1 = require("peek-readable");
|
|
7
|
-
Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return peek_readable_1.EndOfStreamError; } });
|
|
8
|
-
/**
|
|
9
|
-
* Construct ReadStreamTokenizer from given Stream.
|
|
10
|
-
* Will set fileSize, if provided given Stream has set the .path property/
|
|
11
|
-
* @param stream - Read from Node.js Stream.Readable
|
|
12
|
-
* @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
|
|
13
|
-
* @returns ReadStreamTokenizer
|
|
14
|
-
*/
|
|
15
|
-
function fromStream(stream, fileInfo) {
|
|
16
|
-
fileInfo = fileInfo ? fileInfo : {};
|
|
17
|
-
return new ReadStreamTokenizer_1.ReadStreamTokenizer(stream, fileInfo);
|
|
18
|
-
}
|
|
19
|
-
exports.fromStream = fromStream;
|
|
20
|
-
/**
|
|
21
|
-
* Construct ReadStreamTokenizer from given Buffer.
|
|
22
|
-
* @param buffer - Buffer to tokenize
|
|
23
|
-
* @param fileInfo - Pass additional file information to the tokenizer
|
|
24
|
-
* @returns BufferTokenizer
|
|
25
|
-
*/
|
|
26
|
-
function fromBuffer(buffer, fileInfo) {
|
|
27
|
-
return new BufferTokenizer_1.BufferTokenizer(buffer, fileInfo);
|
|
28
|
-
}
|
|
29
|
-
exports.fromBuffer = fromBuffer;
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.fromBuffer = exports.fromStream = exports.EndOfStreamError = void 0;
|
|
4
|
+
const ReadStreamTokenizer_1 = require("./ReadStreamTokenizer");
|
|
5
|
+
const BufferTokenizer_1 = require("./BufferTokenizer");
|
|
6
|
+
var peek_readable_1 = require("peek-readable");
|
|
7
|
+
Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return peek_readable_1.EndOfStreamError; } });
|
|
8
|
+
/**
|
|
9
|
+
* Construct ReadStreamTokenizer from given Stream.
|
|
10
|
+
* Will set fileSize, if provided given Stream has set the .path property/
|
|
11
|
+
* @param stream - Read from Node.js Stream.Readable
|
|
12
|
+
* @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
|
|
13
|
+
* @returns ReadStreamTokenizer
|
|
14
|
+
*/
|
|
15
|
+
function fromStream(stream, fileInfo) {
|
|
16
|
+
fileInfo = fileInfo ? fileInfo : {};
|
|
17
|
+
return new ReadStreamTokenizer_1.ReadStreamTokenizer(stream, fileInfo);
|
|
18
|
+
}
|
|
19
|
+
exports.fromStream = fromStream;
|
|
20
|
+
/**
|
|
21
|
+
* Construct ReadStreamTokenizer from given Buffer.
|
|
22
|
+
* @param buffer - Buffer to tokenize
|
|
23
|
+
* @param fileInfo - Pass additional file information to the tokenizer
|
|
24
|
+
* @returns BufferTokenizer
|
|
25
|
+
*/
|
|
26
|
+
function fromBuffer(buffer, fileInfo) {
|
|
27
|
+
return new BufferTokenizer_1.BufferTokenizer(buffer, fileInfo);
|
|
28
|
+
}
|
|
29
|
+
exports.fromBuffer = fromBuffer;
|
package/lib/index.d.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import * as Stream from 'stream';
|
|
3
|
-
import { ReadStreamTokenizer } from './ReadStreamTokenizer';
|
|
4
|
-
import * as core from './core';
|
|
5
|
-
export { fromFile } from './FileTokenizer';
|
|
6
|
-
export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core';
|
|
7
|
-
export { IToken, IGetToken } from '@tokenizer/token';
|
|
8
|
-
/**
|
|
9
|
-
* Construct ReadStreamTokenizer from given Stream.
|
|
10
|
-
* Will set fileSize, if provided given Stream has set the .path property.
|
|
11
|
-
* @param stream - Node.js Stream.Readable
|
|
12
|
-
* @param fileInfo - Pass additional file information to the tokenizer
|
|
13
|
-
* @returns Tokenizer
|
|
14
|
-
*/
|
|
15
|
-
export declare function fromStream(stream: Stream.Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import * as Stream from 'stream';
|
|
3
|
+
import { ReadStreamTokenizer } from './ReadStreamTokenizer';
|
|
4
|
+
import * as core from './core';
|
|
5
|
+
export { fromFile } from './FileTokenizer';
|
|
6
|
+
export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core';
|
|
7
|
+
export { IToken, IGetToken } from '@tokenizer/token';
|
|
8
|
+
/**
|
|
9
|
+
* Construct ReadStreamTokenizer from given Stream.
|
|
10
|
+
* Will set fileSize, if provided given Stream has set the .path property.
|
|
11
|
+
* @param stream - Node.js Stream.Readable
|
|
12
|
+
* @param fileInfo - Pass additional file information to the tokenizer
|
|
13
|
+
* @returns Tokenizer
|
|
14
|
+
*/
|
|
15
|
+
export declare function fromStream(stream: Stream.Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
|
package/lib/index.js
CHANGED
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.fromStream = exports.fromBuffer = exports.EndOfStreamError = exports.fromFile = void 0;
|
|
4
|
-
const fs = require("./FsPromise");
|
|
5
|
-
const core = require("./core");
|
|
6
|
-
var FileTokenizer_1 = require("./FileTokenizer");
|
|
7
|
-
Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return FileTokenizer_1.fromFile; } });
|
|
8
|
-
var core_1 = require("./core");
|
|
9
|
-
Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return core_1.EndOfStreamError; } });
|
|
10
|
-
Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return core_1.fromBuffer; } });
|
|
11
|
-
/**
|
|
12
|
-
* Construct ReadStreamTokenizer from given Stream.
|
|
13
|
-
* Will set fileSize, if provided given Stream has set the .path property.
|
|
14
|
-
* @param stream - Node.js Stream.Readable
|
|
15
|
-
* @param fileInfo - Pass additional file information to the tokenizer
|
|
16
|
-
* @returns Tokenizer
|
|
17
|
-
*/
|
|
18
|
-
async function fromStream(stream, fileInfo) {
|
|
19
|
-
fileInfo = fileInfo ? fileInfo : {};
|
|
20
|
-
if (stream.path) {
|
|
21
|
-
const stat = await fs.stat(stream.path);
|
|
22
|
-
fileInfo.path = stream.path;
|
|
23
|
-
fileInfo.size = stat.size;
|
|
24
|
-
}
|
|
25
|
-
return core.fromStream(stream, fileInfo);
|
|
26
|
-
}
|
|
27
|
-
exports.fromStream = fromStream;
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.fromStream = exports.fromBuffer = exports.EndOfStreamError = exports.fromFile = void 0;
|
|
4
|
+
const fs = require("./FsPromise");
|
|
5
|
+
const core = require("./core");
|
|
6
|
+
var FileTokenizer_1 = require("./FileTokenizer");
|
|
7
|
+
Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return FileTokenizer_1.fromFile; } });
|
|
8
|
+
var core_1 = require("./core");
|
|
9
|
+
Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return core_1.EndOfStreamError; } });
|
|
10
|
+
Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return core_1.fromBuffer; } });
|
|
11
|
+
/**
|
|
12
|
+
* Construct ReadStreamTokenizer from given Stream.
|
|
13
|
+
* Will set fileSize, if provided given Stream has set the .path property.
|
|
14
|
+
* @param stream - Node.js Stream.Readable
|
|
15
|
+
* @param fileInfo - Pass additional file information to the tokenizer
|
|
16
|
+
* @returns Tokenizer
|
|
17
|
+
*/
|
|
18
|
+
async function fromStream(stream, fileInfo) {
|
|
19
|
+
fileInfo = fileInfo ? fileInfo : {};
|
|
20
|
+
if (stream.path) {
|
|
21
|
+
const stat = await fs.stat(stream.path);
|
|
22
|
+
fileInfo.path = stream.path;
|
|
23
|
+
fileInfo.size = stat.size;
|
|
24
|
+
}
|
|
25
|
+
return core.fromStream(stream, fileInfo);
|
|
26
|
+
}
|
|
27
|
+
exports.fromStream = fromStream;
|
package/lib/types.d.ts
CHANGED
|
@@ -1,103 +1,103 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { IGetToken } from '@tokenizer/token';
|
|
3
|
-
export interface IFileInfo {
|
|
4
|
-
/**
|
|
5
|
-
* File size in bytes
|
|
6
|
-
*/
|
|
7
|
-
size?: number;
|
|
8
|
-
/**
|
|
9
|
-
* MIME-type of file
|
|
10
|
-
*/
|
|
11
|
-
mimeType?: string;
|
|
12
|
-
/**
|
|
13
|
-
* File path
|
|
14
|
-
*/
|
|
15
|
-
path?: string;
|
|
16
|
-
/**
|
|
17
|
-
* File URL
|
|
18
|
-
*/
|
|
19
|
-
url?: string;
|
|
20
|
-
}
|
|
21
|
-
export interface IReadChunkOptions {
|
|
22
|
-
/**
|
|
23
|
-
* The offset in the buffer to start writing at; default is 0
|
|
24
|
-
*/
|
|
25
|
-
offset?: number;
|
|
26
|
-
/**
|
|
27
|
-
* Number of bytes to read.
|
|
28
|
-
*/
|
|
29
|
-
length?: number;
|
|
30
|
-
/**
|
|
31
|
-
* Position where to begin reading from the file.
|
|
32
|
-
* Default it is `tokenizer.position`.
|
|
33
|
-
* Position may not be less then `tokenizer.position`.
|
|
34
|
-
*/
|
|
35
|
-
position?: number;
|
|
36
|
-
/**
|
|
37
|
-
* If set, will not throw an EOF error if not all of the requested data could be read
|
|
38
|
-
*/
|
|
39
|
-
mayBeLess?: boolean;
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* The tokenizer allows us to read or peek from the tokenizer-stream.
|
|
43
|
-
* The tokenizer-stream is an abstraction of a stream, file or Buffer.
|
|
44
|
-
*/
|
|
45
|
-
export interface ITokenizer {
|
|
46
|
-
/**
|
|
47
|
-
* Provide access to information of the underlying information stream or file.
|
|
48
|
-
*/
|
|
49
|
-
fileInfo: IFileInfo;
|
|
50
|
-
/**
|
|
51
|
-
* Offset in bytes (= number of bytes read) since beginning of file or stream
|
|
52
|
-
*/
|
|
53
|
-
position: number;
|
|
54
|
-
/**
|
|
55
|
-
* Peek (read ahead) buffer from tokenizer
|
|
56
|
-
* @param buffer - Target buffer to fill with data peek from the tokenizer-stream
|
|
57
|
-
* @param options - Read behaviour options
|
|
58
|
-
* @returns Promise with number of bytes read
|
|
59
|
-
*/
|
|
60
|
-
peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
61
|
-
/**
|
|
62
|
-
* Peek (read ahead) buffer from tokenizer
|
|
63
|
-
* @param buffer - Target buffer to fill with data peeked from the tokenizer-stream
|
|
64
|
-
* @param options - Additional read options
|
|
65
|
-
* @returns Promise with number of bytes read
|
|
66
|
-
*/
|
|
67
|
-
readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
68
|
-
/**
|
|
69
|
-
* Peek a token from the tokenizer-stream.
|
|
70
|
-
* @param token - Token to peek from the tokenizer-stream.
|
|
71
|
-
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
72
|
-
* @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
|
|
73
|
-
*/
|
|
74
|
-
peekToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
|
|
75
|
-
/**
|
|
76
|
-
* Read a token from the tokenizer-stream.
|
|
77
|
-
* @param token - Token to peek from the tokenizer-stream.
|
|
78
|
-
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
79
|
-
*/
|
|
80
|
-
readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
|
|
81
|
-
/**
|
|
82
|
-
* Peek a numeric token from the stream
|
|
83
|
-
* @param token - Numeric token
|
|
84
|
-
* @returns Promise with number
|
|
85
|
-
*/
|
|
86
|
-
peekNumber(token: IGetToken<number>): Promise<number>;
|
|
87
|
-
/**
|
|
88
|
-
* Read a numeric token from the stream
|
|
89
|
-
* @param token - Numeric token
|
|
90
|
-
* @returns Promise with number
|
|
91
|
-
*/
|
|
92
|
-
readNumber(token: IGetToken<number>): Promise<number>;
|
|
93
|
-
/**
|
|
94
|
-
* Ignore given number of bytes
|
|
95
|
-
* @param length - Number of bytes ignored
|
|
96
|
-
*/
|
|
97
|
-
ignore(length: number): Promise<number>;
|
|
98
|
-
/**
|
|
99
|
-
* Clean up resources.
|
|
100
|
-
* It does not close the stream for StreamReader, but is does close the file-descriptor.
|
|
101
|
-
*/
|
|
102
|
-
close(): Promise<void>;
|
|
103
|
-
}
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { IGetToken } from '@tokenizer/token';
|
|
3
|
+
export interface IFileInfo {
|
|
4
|
+
/**
|
|
5
|
+
* File size in bytes
|
|
6
|
+
*/
|
|
7
|
+
size?: number;
|
|
8
|
+
/**
|
|
9
|
+
* MIME-type of file
|
|
10
|
+
*/
|
|
11
|
+
mimeType?: string;
|
|
12
|
+
/**
|
|
13
|
+
* File path
|
|
14
|
+
*/
|
|
15
|
+
path?: string;
|
|
16
|
+
/**
|
|
17
|
+
* File URL
|
|
18
|
+
*/
|
|
19
|
+
url?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface IReadChunkOptions {
|
|
22
|
+
/**
|
|
23
|
+
* The offset in the buffer to start writing at; default is 0
|
|
24
|
+
*/
|
|
25
|
+
offset?: number;
|
|
26
|
+
/**
|
|
27
|
+
* Number of bytes to read.
|
|
28
|
+
*/
|
|
29
|
+
length?: number;
|
|
30
|
+
/**
|
|
31
|
+
* Position where to begin reading from the file.
|
|
32
|
+
* Default it is `tokenizer.position`.
|
|
33
|
+
* Position may not be less then `tokenizer.position`.
|
|
34
|
+
*/
|
|
35
|
+
position?: number;
|
|
36
|
+
/**
|
|
37
|
+
* If set, will not throw an EOF error if not all of the requested data could be read
|
|
38
|
+
*/
|
|
39
|
+
mayBeLess?: boolean;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The tokenizer allows us to read or peek from the tokenizer-stream.
|
|
43
|
+
* The tokenizer-stream is an abstraction of a stream, file or Buffer.
|
|
44
|
+
*/
|
|
45
|
+
export interface ITokenizer {
|
|
46
|
+
/**
|
|
47
|
+
* Provide access to information of the underlying information stream or file.
|
|
48
|
+
*/
|
|
49
|
+
fileInfo: IFileInfo;
|
|
50
|
+
/**
|
|
51
|
+
* Offset in bytes (= number of bytes read) since beginning of file or stream
|
|
52
|
+
*/
|
|
53
|
+
position: number;
|
|
54
|
+
/**
|
|
55
|
+
* Peek (read ahead) buffer from tokenizer
|
|
56
|
+
* @param buffer - Target buffer to fill with data peek from the tokenizer-stream
|
|
57
|
+
* @param options - Read behaviour options
|
|
58
|
+
* @returns Promise with number of bytes read
|
|
59
|
+
*/
|
|
60
|
+
peekBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
61
|
+
/**
|
|
62
|
+
* Peek (read ahead) buffer from tokenizer
|
|
63
|
+
* @param buffer - Target buffer to fill with data peeked from the tokenizer-stream
|
|
64
|
+
* @param options - Additional read options
|
|
65
|
+
* @returns Promise with number of bytes read
|
|
66
|
+
*/
|
|
67
|
+
readBuffer(buffer: Buffer, options?: IReadChunkOptions): Promise<number>;
|
|
68
|
+
/**
|
|
69
|
+
* Peek a token from the tokenizer-stream.
|
|
70
|
+
* @param token - Token to peek from the tokenizer-stream.
|
|
71
|
+
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
72
|
+
* @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
|
|
73
|
+
*/
|
|
74
|
+
peekToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
|
|
75
|
+
/**
|
|
76
|
+
* Read a token from the tokenizer-stream.
|
|
77
|
+
* @param token - Token to peek from the tokenizer-stream.
|
|
78
|
+
* @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
|
|
79
|
+
*/
|
|
80
|
+
readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
|
|
81
|
+
/**
|
|
82
|
+
* Peek a numeric token from the stream
|
|
83
|
+
* @param token - Numeric token
|
|
84
|
+
* @returns Promise with number
|
|
85
|
+
*/
|
|
86
|
+
peekNumber(token: IGetToken<number>): Promise<number>;
|
|
87
|
+
/**
|
|
88
|
+
* Read a numeric token from the stream
|
|
89
|
+
* @param token - Numeric token
|
|
90
|
+
* @returns Promise with number
|
|
91
|
+
*/
|
|
92
|
+
readNumber(token: IGetToken<number>): Promise<number>;
|
|
93
|
+
/**
|
|
94
|
+
* Ignore given number of bytes
|
|
95
|
+
* @param length - Number of bytes ignored
|
|
96
|
+
*/
|
|
97
|
+
ignore(length: number): Promise<number>;
|
|
98
|
+
/**
|
|
99
|
+
* Clean up resources.
|
|
100
|
+
* It does not close the stream for StreamReader, but is does close the file-descriptor.
|
|
101
|
+
*/
|
|
102
|
+
close(): Promise<void>;
|
|
103
|
+
}
|