strtok3 7.0.0-alpha.2 → 7.0.0-alpha.6

package/lib/FileTokenizer.js

@@ -1,70 +1,50 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { EndOfStreamError } from 'peek-readable';
-import * as fs from './FsPromise.js';
-export class FileTokenizer extends AbstractTokenizer {
-    constructor(fd, fileInfo) {
-        super(fileInfo);
-        this.fd = fd;
-    }
-    /**
-     * Read buffer from file
-     * @param buffer
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    async readBuffer(buffer, options) {
-        let offset = 0;
-        let length = buffer.length;
-        if (options) {
-            if (options.position) {
-                if (options.position < this.position) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-                this.position = options.position;
-            }
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return Promise.resolve(0);
-        }
-        const res = await fs.read(this.fd, buffer, offset, length, this.position);
-        this.position += res.bytesRead;
-        if (res.bytesRead < length && (!options || !options.mayBeLess)) {
-            throw new EndOfStreamError();
-        }
-        return res.bytesRead;
-    }
-    /**
-     * Peek buffer from file
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise number of bytes read
-     */
-    async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
-        const res = await fs.read(this.fd, uint8Array, options.offset, options.length, options.position);
-        if ((!options.mayBeLess) && res.bytesRead < options.length) {
-            throw new EndOfStreamError();
-        }
-        return res.bytesRead;
-    }
-    async close() {
-        return fs.close(this.fd);
-    }
-}
-export async function fromFile(sourceFilePath) {
-    const stat = await fs.stat(sourceFilePath);
-    if (!stat.isFile) {
-        throw new Error(`File not a file: ${sourceFilePath}`);
-    }
-    const fd = await fs.open(sourceFilePath, 'r');
-    return new FileTokenizer(fd, { path: sourceFilePath, size: stat.size });
-}
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { EndOfStreamError } from 'peek-readable';
+import * as fs from './FsPromise.js';
+export class FileTokenizer extends AbstractTokenizer {
+    constructor(fd, fileInfo) {
+        super(fileInfo);
+        this.fd = fd;
+    }
+    /**
+     * Read buffer from file
+     * @param uint8Array - Uint8Array to write result to
+     * @param options - Read behaviour options
+     * @returns Promise number of bytes read
+     */
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        this.position = normOptions.position;
+        const res = await fs.read(this.fd, uint8Array, normOptions.offset, normOptions.length, normOptions.position);
+        this.position += res.bytesRead;
+        if (res.bytesRead < normOptions.length && (!options || !options.mayBeLess)) {
+            throw new EndOfStreamError();
+        }
+        return res.bytesRead;
+    }
+    /**
+     * Peek buffer from file
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
+     * @param options - Read behaviour options
+     * @returns Promise number of bytes read
+     */
+    async peekBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const res = await fs.read(this.fd, uint8Array, normOptions.offset, normOptions.length, normOptions.position);
+        if ((!normOptions.mayBeLess) && res.bytesRead < normOptions.length) {
+            throw new EndOfStreamError();
+        }
+        return res.bytesRead;
+    }
+    async close() {
+        return fs.close(this.fd);
+    }
+}
+export async function fromFile(sourceFilePath) {
+    const stat = await fs.stat(sourceFilePath);
+    if (!stat.isFile) {
+        throw new Error(`File not a file: ${sourceFilePath}`);
+    }
+    const fd = await fs.open(sourceFilePath, 'r');
+    return new FileTokenizer(fd, { path: sourceFilePath, size: stat.size });
+}

package/lib/FsPromise.d.ts

@@ -1,18 +1,18 @@
-/**
- * Module convert fs functions to promise based functions
- */
-/// <reference types="node" />
-import * as fs from 'node:fs';
-export interface IReadResult {
-    bytesRead: number;
-    buffer: Uint8Array;
-}
-export declare const pathExists: typeof fs.existsSync;
-export declare const createReadStream: typeof fs.createReadStream;
-export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
-export declare function close(fd: number): Promise<void>;
-export declare function open(path: fs.PathLike, mode?: string): Promise<number>;
-export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
-export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
-export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;
-export declare function readFile(path: fs.PathLike): Promise<Buffer>;
+/**
+ * Module convert fs functions to promise based functions
+ */
+/// <reference types="node" />
+import fs from 'node:fs';
+export interface IReadResult {
+    bytesRead: number;
+    buffer: Uint8Array;
+}
+export declare const pathExists: typeof fs.existsSync;
+export declare const createReadStream: typeof fs.createReadStream;
+export declare function stat(path: fs.PathLike): Promise<fs.Stats>;
+export declare function close(fd: number): Promise<void>;
+export declare function open(path: fs.PathLike, mode: fs.Mode): Promise<number>;
+export declare function read(fd: number, buffer: Uint8Array, offset: number, length: number, position: number): Promise<IReadResult>;
+export declare function writeFile(path: fs.PathLike, data: Buffer | string): Promise<void>;
+export declare function writeFileSync(path: fs.PathLike, data: Buffer | string): void;
+export declare function readFile(path: fs.PathLike): Promise<Buffer>;

package/lib/FsPromise.js

@@ -1,69 +1,69 @@
-/**
- * Module convert fs functions to promise based functions
- */
-import * as fs from 'node:fs';
-export const pathExists = fs.existsSync;
-export const createReadStream = fs.createReadStream;
-export async function stat(path) {
-    return new Promise((resolve, reject) => {
-        fs.stat(path, (err, stats) => {
-            if (err)
-                reject(err);
-            else
-                resolve(stats);
-        });
-    });
-}
-export async function close(fd) {
-    return new Promise((resolve, reject) => {
-        fs.close(fd, err => {
-            if (err)
-                reject(err);
-            else
-                resolve();
-        });
-    });
-}
-export async function open(path, mode) {
-    return new Promise((resolve, reject) => {
-        fs.open(path, mode, (err, fd) => {
-            if (err)
-                reject(err);
-            else
-                resolve(fd);
-        });
-    });
-}
-export async function read(fd, buffer, offset, length, position) {
-    return new Promise((resolve, reject) => {
-        fs.read(fd, buffer, offset, length, position, (err, bytesRead, _buffer) => {
-            if (err)
-                reject(err);
-            else
-                resolve({ bytesRead, buffer: _buffer });
-        });
-    });
-}
-export async function writeFile(path, data) {
-    return new Promise((resolve, reject) => {
-        fs.writeFile(path, data, err => {
-            if (err)
-                reject(err);
-            else
-                resolve();
-        });
-    });
-}
-export function writeFileSync(path, data) {
-    fs.writeFileSync(path, data);
-}
-export async function readFile(path) {
-    return new Promise((resolve, reject) => {
-        fs.readFile(path, (err, buffer) => {
-            if (err)
-                reject(err);
-            else
-                resolve(buffer);
-        });
-    });
-}
+/**
+ * Module convert fs functions to promise based functions
+ */
+import fs from 'node:fs';
+export const pathExists = fs.existsSync;
+export const createReadStream = fs.createReadStream;
+export async function stat(path) {
+    return new Promise((resolve, reject) => {
+        fs.stat(path, (err, stats) => {
+            if (err)
+                reject(err);
+            else
+                resolve(stats);
+        });
+    });
+}
+export async function close(fd) {
+    return new Promise((resolve, reject) => {
+        fs.close(fd, err => {
+            if (err)
+                reject(err);
+            else
+                resolve();
+        });
+    });
+}
+export async function open(path, mode) {
+    return new Promise((resolve, reject) => {
+        fs.open(path, mode, (err, fd) => {
+            if (err)
+                reject(err);
+            else
+                resolve(fd);
+        });
+    });
+}
+export async function read(fd, buffer, offset, length, position) {
+    return new Promise((resolve, reject) => {
+        fs.read(fd, buffer, offset, length, position, (err, bytesRead, _buffer) => {
+            if (err)
+                reject(err);
+            else
+                resolve({ bytesRead, buffer: _buffer });
+        });
+    });
+}
+export async function writeFile(path, data) {
+    return new Promise((resolve, reject) => {
+        fs.writeFile(path, data, err => {
+            if (err)
+                reject(err);
+            else
+                resolve();
+        });
+    });
+}
+export function writeFileSync(path, data) {
+    fs.writeFileSync(path, data);
+}
+export async function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (err, buffer) => {
+            if (err)
+                reject(err);
+            else
+                resolve(buffer);
+        });
+    });
+}

package/lib/ReadStreamTokenizer.d.ts

@@ -1,28 +1,28 @@
-/// <reference types="node" />
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import * as Stream from 'node:stream';
-import { IFileInfo, IReadChunkOptions } from './types';
-export declare class ReadStreamTokenizer extends AbstractTokenizer {
-    private streamReader;
-    constructor(stream: Stream.Readable, fileInfo?: IFileInfo);
-    /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
-     */
-    getFileInfo(): Promise<IFileInfo>;
-    /**
-     * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes peeked
-     */
-    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    ignore(length: number): Promise<number>;
-}
+/// <reference types="node" />
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { Readable } from 'node:stream';
+import { IFileInfo, IReadChunkOptions } from './types.js';
+export declare class ReadStreamTokenizer extends AbstractTokenizer {
+    private streamReader;
+    constructor(stream: Readable, fileInfo?: IFileInfo);
+    /**
+     * Get file information, an HTTP-client may implement this doing a HEAD request
+     * @return Promise with file information
+     */
+    getFileInfo(): Promise<IFileInfo>;
+    /**
+     * Read buffer from tokenizer
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes read
+     */
+    readBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes peeked
+     */
+    peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>;
+    ignore(length: number): Promise<number>;
+}

package/lib/ReadStreamTokenizer.js

@@ -1,110 +1,94 @@
-import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { EndOfStreamError, StreamReader } from 'peek-readable';
-const maxBufferSize = 256000;
-export class ReadStreamTokenizer extends AbstractTokenizer {
-    constructor(stream, fileInfo) {
-        super(fileInfo);
-        this.streamReader = new StreamReader(stream);
-    }
-    /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
-     */
-    async getFileInfo() {
-        return this.fileInfo;
-    }
-    /**
-     * Read buffer from tokenizer
-     * @param buffer - Target buffer to fill with data read from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    async readBuffer(buffer, options) {
-        // const _offset = position ? position : this.position;
-        // debug(`readBuffer ${_offset}...${_offset + length - 1}`);
-        let offset = 0;
-        let length = buffer.length;
-        if (options) {
-            if (Number.isInteger(options.length)) {
-                length = options.length;
-            }
-            else {
-                length -= options.offset || 0;
-            }
-            if (options.position) {
-                const skipBytes = options.position - this.position;
-                if (skipBytes > 0) {
-                    await this.ignore(skipBytes);
-                    return this.readBuffer(buffer, options);
-                }
-                else if (skipBytes < 0) {
-                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
-                }
-            }
-            if (options.offset) {
-                offset = options.offset;
-            }
-        }
-        if (length === 0) {
-            return 0;
-        }
-        const bytesRead = await this.streamReader.read(buffer, offset, length);
-        this.position += bytesRead;
-        if ((!options || !options.mayBeLess) && bytesRead < length) {
-            throw new EndOfStreamError();
-        }
-        return bytesRead;
-    }
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array - Uint8Array (or Buffer) to write data to
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes peeked
-     */
-    async peekBuffer(uint8Array, options) {
-        options = this.normalizeOptions(uint8Array, options);
-        let bytesRead = 0;
-        if (options.position) {
-            const skipBytes = options.position - this.position;
-            if (skipBytes > 0) {
-                const skipBuffer = new Uint8Array(options.length + skipBytes);
-                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: options.mayBeLess });
-                uint8Array.set(skipBuffer.subarray(skipBytes), options.offset);
-                return bytesRead - skipBytes;
-            }
-            else if (skipBytes < 0) {
-                throw new Error('Cannot peek from a negative offset in a stream');
-            }
-        }
-        if (options.length > 0) {
-            try {
-                bytesRead = await this.streamReader.peek(uint8Array, options.offset, options.length);
-            }
-            catch (err) {
-                if (options && options.mayBeLess && err instanceof EndOfStreamError) {
-                    return 0;
-                }
-                throw err;
-            }
-            if ((!options.mayBeLess) && bytesRead < options.length) {
-                throw new EndOfStreamError();
-            }
-        }
-        return bytesRead;
-    }
-    async ignore(length) {
-        // debug(`ignore ${this.position}...${this.position + length - 1}`);
-        const bufSize = Math.min(maxBufferSize, length);
-        const buf = new Uint8Array(bufSize);
-        let totBytesRead = 0;
-        while (totBytesRead < length) {
-            const remaining = length - totBytesRead;
-            const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
-            if (bytesRead < 0) {
-                return bytesRead;
-            }
-            totBytesRead += bytesRead;
-        }
-        return totBytesRead;
-    }
-}
+import { AbstractTokenizer } from './AbstractTokenizer.js';
+import { EndOfStreamError, StreamReader } from 'peek-readable';
+const maxBufferSize = 256000;
+export class ReadStreamTokenizer extends AbstractTokenizer {
+    constructor(stream, fileInfo) {
+        super(fileInfo);
+        this.streamReader = new StreamReader(stream);
+    }
+    /**
+     * Get file information, an HTTP-client may implement this doing a HEAD request
+     * @return Promise with file information
+     */
+    async getFileInfo() {
+        return this.fileInfo;
+    }
+    /**
+     * Read buffer from tokenizer
+     * @param uint8Array - Target Uint8Array to fill with data read from the tokenizer-stream
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes read
+     */
+    async readBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        const skipBytes = normOptions.position - this.position;
+        if (skipBytes > 0) {
+            await this.ignore(skipBytes);
+            return this.readBuffer(uint8Array, options);
+        }
+        else if (skipBytes < 0) {
+            throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
+        }
+        if (normOptions.length === 0) {
+            return 0;
+        }
+        const bytesRead = await this.streamReader.read(uint8Array, normOptions.offset, normOptions.length);
+        this.position += bytesRead;
+        if ((!options || !options.mayBeLess) && bytesRead < normOptions.length) {
+            throw new EndOfStreamError();
+        }
+        return bytesRead;
+    }
+    /**
+     * Peek (read ahead) buffer from tokenizer
+     * @param uint8Array - Uint8Array (or Buffer) to write data to
+     * @param options - Read behaviour options
+     * @returns Promise with number of bytes peeked
+     */
+    async peekBuffer(uint8Array, options) {
+        const normOptions = this.normalizeOptions(uint8Array, options);
+        let bytesRead = 0;
+        if (normOptions.position) {
+            const skipBytes = normOptions.position - this.position;
+            if (skipBytes > 0) {
+                const skipBuffer = new Uint8Array(normOptions.length + skipBytes);
+                bytesRead = await this.peekBuffer(skipBuffer, { mayBeLess: normOptions.mayBeLess });
+                uint8Array.set(skipBuffer.subarray(skipBytes), normOptions.offset);
+                return bytesRead - skipBytes;
+            }
+            else if (skipBytes < 0) {
+                throw new Error('Cannot peek from a negative offset in a stream');
+            }
+        }
+        if (normOptions.length > 0) {
+            try {
+                bytesRead = await this.streamReader.peek(uint8Array, normOptions.offset, normOptions.length);
+            }
+            catch (err) {
+                if (options && options.mayBeLess && err instanceof EndOfStreamError) {
+                    return 0;
+                }
+                throw err;
+            }
+            if ((!normOptions.mayBeLess) && bytesRead < normOptions.length) {
+                throw new EndOfStreamError();
+            }
+        }
+        return bytesRead;
+    }
+    async ignore(length) {
+        // debug(`ignore ${this.position}...${this.position + length - 1}`);
+        const bufSize = Math.min(maxBufferSize, length);
+        const buf = new Uint8Array(bufSize);
+        let totBytesRead = 0;
+        while (totBytesRead < length) {
+            const remaining = length - totBytesRead;
+            const bytesRead = await this.readBuffer(buf, { length: Math.min(bufSize, remaining) });
+            if (bytesRead < 0) {
+                return bytesRead;
+            }
+            totBytesRead += bytesRead;
+        }
+        return totBytesRead;
+    }
+}

package/lib/core.d.ts

@@ -1,23 +1,23 @@
-/// <reference types="node" />
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import * as Stream from 'node:stream';
-import { BufferTokenizer } from './BufferTokenizer.js';
-import { IFileInfo } from './types.js';
-export { EndOfStreamError } from 'peek-readable';
-export { ITokenizer, IFileInfo } from './types';
-export { IToken, IGetToken } from '@tokenizer/token';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the correspnding stream.
- * @returns ReadStreamTokenizer
- */
-export declare function fromStream(stream: Stream.Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
-/**
- * Construct ReadStreamTokenizer from given Buffer.
- * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
- * @returns BufferTokenizer
- */
-export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;
+/// <reference types="node" />
+import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
+import { Readable } from 'node:stream';
+import { BufferTokenizer } from './BufferTokenizer.js';
+import { IFileInfo } from './types.js';
+export { EndOfStreamError } from 'peek-readable';
+export { ITokenizer, IFileInfo } from './types.js';
+export { IToken, IGetToken } from '@tokenizer/token';
+/**
+ * Construct ReadStreamTokenizer from given Stream.
+ * Will set fileSize, if provided given Stream has set the .path property/
+ * @param stream - Read from Node.js Stream.Readable
+ * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @returns ReadStreamTokenizer
+ */
+export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
+/**
+ * Construct ReadStreamTokenizer from given Buffer.
+ * @param uint8Array - Uint8Array to tokenize
+ * @param fileInfo - Pass additional file information to the tokenizer
+ * @returns BufferTokenizer
+ */
+export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;