strtok3 7.0.0-alpha.7 → 7.0.0

package/lib/FsPromise.d.ts

@@ -1,18 +1,19 @@
-/**
- * Module convert fs functions to promise based functions
- */
-/// <reference types="node" />
-import fs from 'node:fs';
-export interface IReadResult {
-    bytesRead: number;
-    buffer: Uint8Array;
-}
-export declare const pathExists: typeof fs.existsSync;
-export declare const createReadStream: typeof fs.createReadStream;
-export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
-export declare function close(fd: number): Promise<void>;
-export declare function open(path: fs.PathLike, mode: fs.Mode): Promise<number>;
-export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
-export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
-export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;
-export declare function readFile(path: fs.PathLike): Promise<Buffer>;
+/**
+ * Module convert fs functions to promise based functions
+ */
+/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" resolution-mode="require"/>
+import fs from 'node:fs';
+export interface IReadResult {
+    bytesRead: number;
+    buffer: Uint8Array;
+}
+export declare const pathExists: typeof fs.existsSync;
+export declare const createReadStream: typeof fs.createReadStream;
+export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
+export declare function close(fd: number): Promise<void>;
+export declare function open(path: fs.PathLike, mode: fs.Mode): Promise<number>;
+export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
+export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
+export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;
+export declare function readFile(path: fs.PathLike): Promise<Buffer>;

package/lib/FsPromise.js

@@ -1,69 +1,69 @@
-/**
- * Module convert fs functions to promise based functions
- */
-import fs from 'node:fs';
-export const pathExists = fs.existsSync;
-export const createReadStream = fs.createReadStream;
-export async function stat(path) {
-    return new Promise((resolve, reject) => {
-        fs.stat(path, (err, stats) => {
-            if (err)
-                reject(err);
-            else
-                resolve(stats);
-        });
-    });
-}
-export async function close(fd) {
-    return new Promise((resolve, reject) => {
-        fs.close(fd, err => {
-            if (err)
-                reject(err);
-            else
-                resolve();
-        });
-    });
-}
-export async function open(path, mode) {
-    return new Promise((resolve, reject) => {
-        fs.open(path, mode, (err, fd) => {
-            if (err)
-                reject(err);
-            else
-                resolve(fd);
-        });
-    });
-}
-export async function read(fd, buffer, offset, length, position) {
-    return new Promise((resolve, reject) => {
-        fs.read(fd, buffer, offset, length, position, (err, bytesRead, _buffer) => {
-            if (err)
-                reject(err);
-            else
-                resolve({ bytesRead, buffer: _buffer });
-        });
-    });
-}
-export async function writeFile(path, data) {
-    return new Promise((resolve, reject) => {
-        fs.writeFile(path, data, err => {
-            if (err)
-                reject(err);
-            else
-                resolve();
-        });
-    });
-}
-export function writeFileSync(path, data) {
-    fs.writeFileSync(path, data);
-}
-export async function readFile(path) {
-    return new Promise((resolve, reject) => {
-        fs.readFile(path, (err, buffer) => {
-            if (err)
-                reject(err);
-            else
-                resolve(buffer);
-        });
-    });
-}
+/**
+ * Module convert fs functions to promise based functions
+ */
+import fs from 'node:fs';
+export const pathExists = fs.existsSync;
+export const createReadStream = fs.createReadStream;
+export async function stat(path) {
+    return new Promise((resolve, reject) => {
+        fs.stat(path, (err, stats) => {
+            if (err)
+                reject(err);
+            else
+                resolve(stats);
+        });
+    });
+}
+export async function close(fd) {
+    return new Promise((resolve, reject) => {
+        fs.close(fd, err => {
+            if (err)
+                reject(err);
+            else
+                resolve();
+        });
+    });
+}
+export async function open(path, mode) {
+    return new Promise((resolve, reject) => {
+        fs.open(path, mode, (err, fd) => {
+            if (err)
+                reject(err);
+            else
+                resolve(fd);
+        });
+    });
+}
+export async function read(fd, buffer, offset, length, position) {
+    return new Promise((resolve, reject) => {
+        fs.read(fd, buffer, offset, length, position, (err, bytesRead, _buffer) => {
+            if (err)
+                reject(err);
+            else
+                resolve({ bytesRead, buffer: _buffer });
+        });
+    });
+}
+export async function writeFile(path, data) {
+    return new Promise((resolve, reject) => {
+        fs.writeFile(path, data, err => {
+            if (err)
+                reject(err);
+            else
+                resolve();
+        });
+    });
+}
+export function writeFileSync(path, data) {
+    fs.writeFileSync(path, data);
+}
+export async function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (err, buffer) => {
+            if (err)
+                reject(err);
+            else
+                resolve(buffer);
+        });
+    });
+}

package/lib/ReadStreamTokenizer.d.ts

@@ -1,28 +1,28 @@
-/// <reference types="node" />
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { Readable } from 'node:stream';
-import { IFileInfo, IReadChunkOptions } from './types.js';
-export declare class ReadStreamTokenizer extends AbstractTokenizer {
-    private streamReader;
-    constructor(stream: Readable, fileInfo?: IFileInfo);
-    /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
-     */
-    getFileInfo(): Promise<IFileInfo>;
-    /**
-     * Read buffer from tokenizer
-     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes peeked
-     */
-    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    ignore(length: number): Promise<number>;
-}
+/// <reference types="node" resolution-mode="require"/>
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { Readable } from 'node:stream';
+import { IFileInfo, IReadChunkOptions } from './types.js';
+export declare class ReadStreamTokenizer extends AbstractTokenizer {
+    private streamReader;
+    constructor(stream: Readable, fileInfo?: IFileInfo);
+    /**
+     * Get file information, an HTTP-client may implement this doing a HEAD request
+     * @return Promise with file information
+     */
+    getFileInfo(): Promise<IFileInfo>;
+    /**
+     * Read buffer from tokenizer
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes read
+     */
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes peeked
+     */
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    ignore(length: number): Promise<number>;
+}

package/lib/ReadStreamTokenizer.js

@@ -1,94 +1,94 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { EndOfStreamError, StreamReader } from 'peek-readable';
-const maxBufferSize = 256000;
-export class ReadStreamTokenizer extends AbstractTokenizer {
-    constructor(stream, fileInfo) {
-        super(fileInfo);
-        this.streamReader = new StreamReader(stream);
-    }
-    /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
-     */
-    async getFileInfo() {
-        return this.fileInfo;
-    }
-    /**
-     * Read buffer from tokenizer
-     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    async readBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        const skipBytes = normOptions.position - this.position;
-        if (skipBytes > 0) {
-            await this.ignore(skipBytes);
-            return this.readBuffer(uint8Array, options);
-        }
-        else if (skipBytes < 0) {
-            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-        }
-        if (normOptions.length === 0) {
-            return 0;
-        }
-        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
-        this.position += bytesRead;
-        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
-            throw new EndOfStreamError();
-        }
-        return bytesRead;
-    }
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes peeked
-     */
-    async peekBuffer(uint8Array, options) {
-        const normOptions = this.normalizeOptions(uint8Array, options);
-        let bytesRead = 0;
-        if (normOptions.position) {
-            const skipBytes = normOptions.position - this.position;
-            if (skipBytes > 0) {
-                const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
-                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
-                return bytesRead - skipBytes;
-            }
-            else if (skipBytes < 0) {
-                throw new Error('Cannot peek from a negative offset in a stream');
-            }
-        }
-        if (normOptions.length > 0) {
-            try {
-                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
-            }
-            catch (err) {
-                if (options && options.mayBeLess && err instanceof EndOfStreamError) {
-                    return 0;
-                }
-                throw err;
-            }
-            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
-                throw new EndOfStreamError();
-            }
-        }
-        return bytesRead;
-    }
-    async ignore(length) {
-        // debug(`ignore ${this.position}...${this.position + length - 1}`);
-        const bufSize = Math.min(maxBufferSize, length);
-        const buf = new Uint8Array(bufSize);
-        let totBytesRead = 0;
-        while (totBytesRead < length) {
-            const remaining = length - totBytesRead;
-            const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
-            if (bytesRead < 0) {
-                return bytesRead;
-            }
-            totBytesRead += bytesRead;
-        }
-        return totBytesRead;
-    }
-}
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { EndOfStreamError, StreamReader } from 'peek-readable';
+const maxBufferSize = 256000;
+export class ReadStreamTokenizer extends AbstractTokenizer {
+    constructor(stream, fileInfo) {
+        super(fileInfo);
+        this.streamReader = new StreamReader(stream);
+    }
+    /**
+     * Get file information, an HTTP-client may implement this doing a HEAD request
+     * @return Promise with file information
+     */
+    async getFileInfo() {
+        return this.fileInfo;
+    }
+    /**
+     * Read buffer from tokenizer
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes read
+     */
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const skipBytes = normOptions.position - this.position;
+        if (skipBytes > 0) {
+            await this.ignore(skipBytes);
+            return this.readBuffer(uint8Array, options);
+        }
+        else if (skipBytes < 0) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+        }
+        if (normOptions.length === 0) {
+            return 0;
+        }
+        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
+        this.position += bytesRead;
+        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
+            throw new EndOfStreamError();
+        }
+        return bytesRead;
+    }
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes peeked
+     */
+    async peekBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        let bytesRead = 0;
+        if (normOptions.position) {
+            const skipBytes = normOptions.position - this.position;
+            if (skipBytes > 0) {
+                const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
+                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
+                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
+                return bytesRead - skipBytes;
+            }
+            else if (skipBytes < 0) {
+                throw new Error('Cannot peek from a negative offset in a stream');
+            }
+        }
+        if (normOptions.length > 0) {
+            try {
+                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
+            }
+            catch (err) {
+                if (options && options.mayBeLess && err instanceof EndOfStreamError) {
+                    return 0;
+                }
+                throw err;
+            }
+            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
+                throw new EndOfStreamError();
+            }
+        }
+        return bytesRead;
+    }
+    async ignore(length) {
+        // debug(`ignore ${this.position}...${this.position + length - 1}`);
+        const bufSize = Math.min(maxBufferSize, length);
+        const buf = new Uint8Array(bufSize);
+        let totBytesRead = 0;
+        while (totBytesRead < length) {
+            const remaining = length - totBytesRead;
+            const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
+            if (bytesRead < 0) {
+                return bytesRead;
+            }
+            totBytesRead += bytesRead;
+        }
+        return totBytesRead;
+    }
+}

package/lib/core.d.ts

@@ -1,23 +1,23 @@
-/// <reference types="node" />
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import { Readable } from 'node:stream';
-import { BufferTokenizer } from './BufferTokenizer.js';
-import { IFileInfo } from './types.js';
-export { EndOfStreamError } from 'peek-readable';
-export { ITokenizer, IFileInfo } from './types.js';
-export { IToken, IGetToken } from '@tokenizer/token';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
- * @returns ReadStreamTokenizer
- */
-export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
-/**
- * Construct ReadStreamTokenizer from given Buffer.
- * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
- * @returns BufferTokenizer
- */
-export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;
+/// <reference types="node" resolution-mode="require"/>
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { Readable } from 'node:stream';
+import { BufferTokenizer } from './BufferTokenizer.js';
+import { IFileInfo } from './types.js';
+export { EndOfStreamError } from 'peek-readable';
+export { ITokenizer, IFileInfo } from './types.js';
+export { IToken, IGetToken } from '@tokenizer/token';
+/**
+ * Construct ReadStreamTokenizer from given Stream.
+ * Will set fileSize, if provided given Stream has set the .path property/
+ * @param stream - Read from Node.js Stream.Readable
+ * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @returns ReadStreamTokenizer
+ */
+export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
+/**
+ * Construct ReadStreamTokenizer from given Buffer.
+ * @param uint8Array - Uint8Array to tokenize
+ * @param fileInfo - Pass additional file information to the tokenizer
+ * @returns BufferTokenizer
+ */
+export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;

package/lib/core.js

@@ -1,23 +1,23 @@
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import { BufferTokenizer } from './BufferTokenizer.js';
-export { EndOfStreamError } from 'peek-readable';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
- * @returns ReadStreamTokenizer
- */
-export function fromStream(stream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
-    return new ReadStreamTokenizer(stream, fileInfo);
-}
-/**
- * Construct ReadStreamTokenizer from given Buffer.
- * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
- * @returns BufferTokenizer
- */
-export function fromBuffer(uint8Array, fileInfo) {
-    return new BufferTokenizer(uint8Array, fileInfo);
-}
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { BufferTokenizer } from './BufferTokenizer.js';
+export { EndOfStreamError } from 'peek-readable';
+/**
+ * Construct ReadStreamTokenizer from given Stream.
+ * Will set fileSize, if provided given Stream has set the .path property/
+ * @param stream - Read from Node.js Stream.Readable
+ * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @returns ReadStreamTokenizer
+ */
+export function fromStream(stream, fileInfo) {
+    fileInfo = fileInfo ? fileInfo : {};
+    return new ReadStreamTokenizer(stream, fileInfo);
+}
+/**
+ * Construct ReadStreamTokenizer from given Buffer.
+ * @param uint8Array - Uint8Array to tokenize
+ * @param fileInfo - Pass additional file information to the tokenizer
+ * @returns BufferTokenizer
+ */
+export function fromBuffer(uint8Array, fileInfo) {
+    return new BufferTokenizer(uint8Array, fileInfo);
+}

package/lib/index.d.ts

@@ -1,15 +1,15 @@
-/// <reference types="node" />
-import { Readable } from 'node:stream';
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import * as core from './core.js';
-export { fromFile } from './FileTokenizer.js';
-export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core.js';
-export { IToken, IGetToken } from '@tokenizer/token';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property.
- * @param stream - Node.js Stream.Readable
- * @param fileInfo - Pass additional file information to the tokenizer
- * @returns Tokenizer
- */
-export declare function fromStream(stream: Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
+/// <reference types="node" resolution-mode="require"/>
+import { Readable } from 'node:stream';
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import * as core from './core.js';
+export { fromFile } from './FileTokenizer.js';
+export { ITokenizer, EndOfStreamError, fromBuffer, IFileInfo } from './core.js';
+export { IToken, IGetToken } from '@tokenizer/token';
+/**
+ * Construct ReadStreamTokenizer from given Stream.
+ * Will set fileSize, if provided given Stream has set the .path property.
+ * @param stream - Node.js Stream.Readable
+ * @param fileInfo - Pass additional file information to the tokenizer
+ * @returns Tokenizer
+ */
+export declare function fromStream(stream: Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;

package/lib/index.js

@@ -1,20 +1,20 @@
-import * as fs from './FsPromise.js';
-import * as core from './core.js';
-export { fromFile } from './FileTokenizer.js';
-export { EndOfStreamError, fromBuffer } from './core.js';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property.
- * @param stream - Node.js Stream.Readable
- * @param fileInfo - Pass additional file information to the tokenizer
- * @returns Tokenizer
- */
-export async function fromStream(stream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
-    if (stream.path) {
-        const stat = await fs.stat(stream.path);
-        fileInfo.path = stream.path;
-        fileInfo.size = stat.size;
-    }
-    return core.fromStream(stream, fileInfo);
-}
+import * as fs from './FsPromise.js';
+import * as core from './core.js';
+export { fromFile } from './FileTokenizer.js';
+export { EndOfStreamError, fromBuffer } from './core.js';
+/**
+ * Construct ReadStreamTokenizer from given Stream.
+ * Will set fileSize, if provided given Stream has set the .path property.
+ * @param stream - Node.js Stream.Readable
+ * @param fileInfo - Pass additional file information to the tokenizer
+ * @returns Tokenizer
+ */
+export async function fromStream(stream, fileInfo) {
+    fileInfo = fileInfo ? fileInfo : {};
+    if (stream.path) {
+        const stat = await fs.stat(stream.path);
+        fileInfo.path = stream.path;
+        fileInfo.size = stat.size;
+    }
+    return core.fromStream(stream, fileInfo);
+}