poe-code 3.0.274 → 3.0.275

package/dist/index.js

@@ -100945,12 +100945,28 @@ import { get_encoding } from "tiktoken";
 function createTokenizer(options = {}) {
   const encoding = options.encoding ?? DEFAULT_ENCODING;
   const tokenizer = get_encoding(encoding);
-  const utf8Decoder = new TextDecoder();
   const strictUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
   const encode = (text5) => tokenizer.encode(text5);
+  const normalizeDecodeTokens = (tokens) => {
+    if (tokens instanceof Uint32Array) {
+      return tokens;
+    }
+    const tokenArray = new Uint32Array(tokens.length);
+    tokens.forEach((token, index) => {
+      if (!Number.isFinite(token) || !Number.isInteger(token) || token < 0 || token > 4294967295) {
+        throw new TypeError(`token id at index ${index} must be a finite non-negative integer.`);
+      }
+      tokenArray[index] = token;
+    });
+    return tokenArray;
+  };
   const decode = (tokens) => {
-    const tokenArray = tokens instanceof Uint32Array ? tokens : Uint32Array.from(tokens);
-    return utf8Decoder.decode(tokenizer.decode(tokenArray));
+    const tokenArray = normalizeDecodeTokens(tokens);
+    try {
+      return strictUtf8Decoder.decode(tokenizer.decode(tokenArray));
+    } catch {
+      throw new Error("Cannot decode tokens without corrupting UTF-8 text.");
+    }
   };
   const count = (text5) => encode(text5).length;
   const truncate4 = (text5, tokenCount) => {
@@ -136474,7 +136490,7 @@ var init_package2 = __esm({
   "package.json"() {
     package_default2 = {
       name: "poe-code",
-      version: "3.0.274",
+      version: "3.0.275",
       description: "CLI tool to configure Poe API for developer workflows.",
       type: "module",
       main: "./dist/index.js",