tokenfill 0.0.1

package/dist/corpus.d.ts

@@ -0,0 +1,2 @@
+export declare const CORPUS_ARTICLE_SEPARATOR = "\n\n";
+export declare const BUILT_IN_CORPUS_ARTICLES: string[];

package/dist/corpus.js

@@ -0,0 +1,19 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+export const CORPUS_ARTICLE_SEPARATOR = "\n\n";
+const corpusDirectoryPath = join(dirname(fileURLToPath(import.meta.url)), "corpus");
+function getCorpusFileNames() {
+    return readdirSync(corpusDirectoryPath, { withFileTypes: true })
+        .filter(entry => entry.isFile() && entry.name.endsWith(".md"))
+        .map(entry => entry.name)
+        .sort((left, right) => left.localeCompare(right));
+}
+function loadBuiltInCorpusArticles() {
+    const corpusFileNames = getCorpusFileNames();
+    if (corpusFileNames.length === 0) {
+        throw new Error(`No built-in corpus markdown files found in ${corpusDirectoryPath}`);
+    }
+    return corpusFileNames.map(fileName => readFileSync(join(corpusDirectoryPath, fileName), "utf8").trim());
+}
+export const BUILT_IN_CORPUS_ARTICLES = loadBuiltInCorpusArticles();

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { createTokenizer, DEFAULT_ENCODING } from "./tokenizer.js";
+export type { Tokenizer, TokenizerOptions } from "./tokenizer.js";
+export { tokenfill } from "./tokenfill.js";
+export type { TokenfillOptions, TokenfillResult } from "./tokenfill.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export { createTokenizer, DEFAULT_ENCODING } from "./tokenizer.js";
+export { tokenfill } from "./tokenfill.js";

package/dist/tokenfill.d.ts

@@ -0,0 +1,9 @@
+import type { TiktokenEncoding } from "tiktoken";
+export interface TokenfillOptions {
+    encoding?: TiktokenEncoding;
+}
+export interface TokenfillResult {
+    text: string;
+    actualTokens: number;
+}
+export declare function tokenfill(tokenCount: number, options?: TokenfillOptions): TokenfillResult;

package/dist/tokenfill.js

@@ -0,0 +1,34 @@
+import { BUILT_IN_CORPUS_ARTICLES, CORPUS_ARTICLE_SEPARATOR } from "./corpus.js";
+import { createTokenizer } from "./tokenizer.js";
+const builtInCorpusText = BUILT_IN_CORPUS_ARTICLES.join(CORPUS_ARTICLE_SEPARATOR);
+const corpusTokensByEncoding = new Map();
+function getCorpusTokens(encoding, encode) {
+    const cachedTokens = corpusTokensByEncoding.get(encoding);
+    if (cachedTokens) {
+        return cachedTokens;
+    }
+    const encodedCorpus = encode(builtInCorpusText);
+    corpusTokensByEncoding.set(encoding, encodedCorpus);
+    return encodedCorpus;
+}
+export function tokenfill(tokenCount, options = {}) {
+    if (!Number.isInteger(tokenCount) || tokenCount < 0) {
+        throw new TypeError(`tokenCount must be a non-negative integer, received ${tokenCount}`);
+    }
+    const tokenizer = createTokenizer({ encoding: options.encoding });
+    try {
+        const corpusTokens = getCorpusTokens(tokenizer.encoding, tokenizer.encode);
+        const maxCorpusTokens = corpusTokens.length;
+        if (tokenCount > maxCorpusTokens) {
+            throw new Error(`Requested token count ${tokenCount} exceeds built-in corpus size ${maxCorpusTokens} for encoding ${tokenizer.encoding}`);
+        }
+        const text = tokenizer.decode(corpusTokens.slice(0, tokenCount));
+        return {
+            text,
+            actualTokens: tokenCount
+        };
+    }
+    finally {
+        tokenizer.free();
+    }
+}

package/dist/tokenizer.d.ts

@@ -0,0 +1,14 @@
+import { type TiktokenEncoding } from "tiktoken";
+export declare const DEFAULT_ENCODING: TiktokenEncoding;
+export interface TokenizerOptions {
+    encoding?: TiktokenEncoding;
+}
+export interface Tokenizer {
+    readonly encoding: TiktokenEncoding;
+    encode(text: string): Uint32Array;
+    decode(tokens: Uint32Array | number[]): string;
+    count(text: string): number;
+    truncate(text: string, tokenCount: number): string;
+    free(): void;
+}
+export declare function createTokenizer(options?: TokenizerOptions): Tokenizer;

package/dist/tokenizer.js

@@ -0,0 +1,31 @@
+import { get_encoding } from "tiktoken";
+export const DEFAULT_ENCODING = "cl100k_base";
+export function createTokenizer(options = {}) {
+    const encoding = options.encoding ?? DEFAULT_ENCODING;
+    const tokenizer = get_encoding(encoding);
+    const utf8Decoder = new TextDecoder();
+    const encode = (text) => tokenizer.encode(text);
+    const decode = (tokens) => {
+        const tokenArray = tokens instanceof Uint32Array ? tokens : Uint32Array.from(tokens);
+        return utf8Decoder.decode(tokenizer.decode(tokenArray));
+    };
+    const count = (text) => encode(text).length;
+    const truncate = (text, tokenCount) => {
+        if (tokenCount <= 0) {
+            return "";
+        }
+        const tokens = encode(text);
+        if (tokens.length <= tokenCount) {
+            return text;
+        }
+        return decode(tokens.slice(0, tokenCount));
+    };
+    return {
+        encoding,
+        encode,
+        decode,
+        count,
+        truncate,
+        free: () => tokenizer.free()
+    };
+}

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "tokenfill",
+  "version": "0.0.1",
+  "private": false,
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "tokenfill": "dist/bin.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc && node ./scripts/copy-corpus.mjs"
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "commander": "^14.0.3",
+    "tiktoken": "^1.0.22"
+  }
+}