poe-code 3.0.273 → 3.0.275

package/packages/tokenfill/dist/cli.js

@@ -6,9 +6,18 @@ const defaultOutput = {
     stderr: process.stderr
 };
 function parseTokenCount(value) {
+    const isDecimalDigit = (character) => {
+        const code = character.charCodeAt(0);
+        return code >= 48 && code <= 57;
+    };
+    const isCanonicalDecimal = value === "0" ||
+        (value.length > 0 && value[0] !== "0" && Array.from(value).every(isDecimalDigit));
+    if (!isCanonicalDecimal) {
+        throw new InvalidArgumentError("count must be a non-negative decimal integer");
+    }
     const tokenCount = Number(value);
-    if (!Number.isInteger(tokenCount) || tokenCount < 0) {
-        throw new InvalidArgumentError("count must be a non-negative integer");
+    if (!Number.isSafeInteger(tokenCount) || tokenCount < 0) {
+        throw new InvalidArgumentError("count must be a non-negative decimal integer");
     }
     return tokenCount;
 }
@@ -38,10 +47,10 @@ function createTokenfillProgram(output) {
         output.stderr.write(`Generated ${result.actualTokens} tokens using ${encoding}\n`);
     });
     program.configureOutput({
-        writeOut: value => {
+        writeOut: (value) => {
             output.stdout.write(value);
         },
-        writeErr: value => {
+        writeErr: (value) => {
             output.stderr.write(value);
         }
     });

package/packages/tokenfill/dist/tokenizer.js

@@ -3,12 +3,29 @@ export const DEFAULT_ENCODING = "cl100k_base";
 export function createTokenizer(options = {}) {
     const encoding = options.encoding ?? DEFAULT_ENCODING;
     const tokenizer = get_encoding(encoding);
-    const utf8Decoder = new TextDecoder();
     const strictUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
     const encode = (text) => tokenizer.encode(text);
+    const normalizeDecodeTokens = (tokens) => {
+        if (tokens instanceof Uint32Array) {
+            return tokens;
+        }
+        const tokenArray = new Uint32Array(tokens.length);
+        tokens.forEach((token, index) => {
+            if (!Number.isFinite(token) || !Number.isInteger(token) || token < 0 || token > 0xffffffff) {
+                throw new TypeError(`token id at index ${index} must be a finite non-negative integer.`);
+            }
+            tokenArray[index] = token;
+        });
+        return tokenArray;
+    };
     const decode = (tokens) => {
-        const tokenArray = tokens instanceof Uint32Array ? tokens : Uint32Array.from(tokens);
-        return utf8Decoder.decode(tokenizer.decode(tokenArray));
+        const tokenArray = normalizeDecodeTokens(tokens);
+        try {
+            return strictUtf8Decoder.decode(tokenizer.decode(tokenArray));
+        }
+        catch {
+            throw new Error("Cannot decode tokens without corrupting UTF-8 text.");
+        }
     };
     const count = (text) => encode(text).length;
     const truncate = (text, tokenCount) => {