strtok3 8.0.0 → 8.0.2

package/lib/core.d.ts

@@ -1,31 +0,0 @@
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import { BufferTokenizer } from './BufferTokenizer.js';
-import type { ITokenizerOptions } from './types.js';
-import type { ReadableStream } from 'node:stream/web';
-import type { Readable } from 'node:stream';
-export { EndOfStreamError } from 'peek-readable';
-export type { ITokenizer, IFileInfo, ITokenizerOptions, OnClose } from './types.js';
-export type { IToken, IGetToken } from '@tokenizer/token';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param stream - Read from Node.js Stream.Readable
- * @param options - Tokenizer options
- * @returns ReadStreamTokenizer
- */
-export declare function fromStream(stream: Readable, options?: ITokenizerOptions): ReadStreamTokenizer;
-/**
- * Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param webStream - Read from Node.js Stream.Readable
- * @param options - Tokenizer options
- * @returns ReadStreamTokenizer
- */
-export declare function fromWebStream(webStream: ReadableStream<Uint8Array>, options?: ITokenizerOptions): ReadStreamTokenizer;
-/**
- * Construct ReadStreamTokenizer from given Buffer.
- * @param uint8Array - Uint8Array to tokenize
- * @param options - Tokenizer options
- * @returns BufferTokenizer
- */
-export declare function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer;

package/lib/core.js

@@ -1,33 +0,0 @@
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import { BufferTokenizer } from './BufferTokenizer.js';
-import { StreamReader, WebStreamReader } from 'peek-readable';
-export { EndOfStreamError } from 'peek-readable';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param stream - Read from Node.js Stream.Readable
- * @param options - Tokenizer options
- * @returns ReadStreamTokenizer
- */
-export function fromStream(stream, options) {
-    return new ReadStreamTokenizer(new StreamReader(stream), options);
-}
-/**
- * Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
- * Will set fileSize, if provided given Stream has set the .path property/
- * @param webStream - Read from Node.js Stream.Readable
- * @param options - Tokenizer options
- * @returns ReadStreamTokenizer
- */
-export function fromWebStream(webStream, options) {
-    return new ReadStreamTokenizer(new WebStreamReader(webStream), options);
-}
-/**
- * Construct ReadStreamTokenizer from given Buffer.
- * @param uint8Array - Uint8Array to tokenize
- * @param options - Tokenizer options
- * @returns BufferTokenizer
- */
-export function fromBuffer(uint8Array, options) {
-    return new BufferTokenizer(uint8Array, options);
-}

package/lib/index.d.ts

@@ -1,14 +0,0 @@
-import { Readable } from 'node:stream';
-import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import { type ITokenizerOptions } from './core.js';
-export { fromFile } from './FileTokenizer.js';
-export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo, ITokenizerOptions, OnClose } from './core.js';
-export { IToken, IGetToken } from '@tokenizer/token';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property.
- * @param stream - Node.js Stream.Readable
- * @param options - Pass additional file information to the tokenizer
- * @returns Tokenizer
- */
-export declare function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>;

package/lib/index.js

@@ -1,22 +0,0 @@
-import { stat as fsStat } from 'node:fs/promises';
-import { fromStream as coreFromStream } from './core.js';
-export { fromFile } from './FileTokenizer.js';
-export { EndOfStreamError, fromBuffer, fromWebStream } from './core.js';
-/**
- * Construct ReadStreamTokenizer from given Stream.
- * Will set fileSize, if provided given Stream has set the .path property.
- * @param stream - Node.js Stream.Readable
- * @param options - Pass additional file information to the tokenizer
- * @returns Tokenizer
- */
-export async function fromStream(stream, options) {
-    var _a;
-    const augmentedOptions = options !== null && options !== void 0 ? options : {};
-    augmentedOptions.fileInfo = (_a = augmentedOptions.fileInfo) !== null && _a !== void 0 ? _a : {};
-    if (stream.path) {
-        const stat = await fsStat(stream.path);
-        augmentedOptions.fileInfo.path = stream.path;
-        augmentedOptions.fileInfo.size = stat.size;
-    }
-    return coreFromStream(stream, augmentedOptions);
-}

package/lib/types.d.ts

@@ -1,113 +0,0 @@
-import { IGetToken } from '@tokenizer/token';
-export interface IFileInfo {
-    /**
-     * File size in bytes
-     */
-    size?: number;
-    /**
-     * MIME-type of file
-     */
-    mimeType?: string;
-    /**
-     * File path
-     */
-    path?: string;
-    /**
-     * File URL
-     */
-    url?: string;
-}
-export interface IReadChunkOptions {
-    /**
-     * The offset in the buffer to start writing at; default is 0
-     */
-    offset?: number;
-    /**
-     * Number of bytes to read.
-     */
-    length?: number;
-    /**
-     * Position where to begin reading from the file.
-     * Default it is `tokenizer.position`.
-     * Position may not be less then `tokenizer.position`.
-     */
-    position?: number;
-    /**
-     * If set, will not throw an EOF error if not all of the requested data could be read
-     */
-    mayBeLess?: boolean;
-}
-/**
- * The tokenizer allows us to read or peek from the tokenizer-stream.
- * The tokenizer-stream is an abstraction of a stream, file or Buffer.
- */
-export interface ITokenizer {
-    /**
-     * Provide access to information of the underlying information stream or file.
-     */
-    fileInfo: IFileInfo;
-    /**
-     * Offset in bytes (= number of bytes read) since beginning of file or stream
-     */
-    position: number;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to fill with data peek from the tokenizer-stream
-     * @param options - Read behaviour options
-     * @returns Promise with number of bytes read
-     */
-    peekBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek (read ahead) buffer from tokenizer
-     * @param buffer - Target buffer to fill with data peeked from the tokenizer-stream
-     * @param options - Additional read options
-     * @returns Promise with number of bytes read
-     */
-    readBuffer(buffer: Uint8Array, options?: IReadChunkOptions): Promise<number>;
-    /**
-     * Peek a token from the tokenizer-stream.
-     * @param token - Token to peek from the tokenizer-stream.
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     * @param maybeless - If set, will not throw an EOF error if the less then the requested length could be read.
-     */
-    peekToken<T>(token: IGetToken<T>, position?: number | null, maybeless?: boolean): Promise<T>;
-    /**
-     * Read a token from the tokenizer-stream.
-     * @param token - Token to peek from the tokenizer-stream.
-     * @param position - Offset where to begin reading within the file. If position is null, data will be read from the current file position.
-     */
-    readToken<T>(token: IGetToken<T>, position?: number): Promise<T>;
-    /**
-     * Peek a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    peekNumber(token: IGetToken<number>): Promise<number>;
-    /**
-     * Read a numeric token from the stream
-     * @param token - Numeric token
-     * @returns Promise with number
-     */
-    readNumber(token: IGetToken<number>): Promise<number>;
-    /**
-     * Ignore given number of bytes
-     * @param length - Number of bytes ignored
-     */
-    ignore(length: number): Promise<number>;
-    /**
-     * Clean up resources.
-     * It does not close the stream for StreamReader, but is does close the file-descriptor.
-     */
-    close(): Promise<void>;
-}
-export type OnClose = () => Promise<void>;
-export interface ITokenizerOptions {
-    /**
-     * Pass additional file information to the tokenizer
-     */
-    fileInfo?: IFileInfo;
-    /**
-     * On tokenizer close handler
-     */
-    onClose?: OnClose;
-}

package/lib/types.js

@@ -1 +0,0 @@
-export {};