poe-code 3.0.274 → 3.0.276

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "poe-code",
-  "version": "3.0.274",
+  "version": "3.0.276",
   "description": "CLI tool to configure Poe API for developer workflows.",
   "type": "module",
   "main": "./dist/index.js",

package/packages/memory/dist/index.js

@@ -14185,12 +14185,28 @@ var DEFAULT_ENCODING = "cl100k_base";
 function createTokenizer(options = {}) {
   const encoding = options.encoding ?? DEFAULT_ENCODING;
   const tokenizer = get_encoding(encoding);
-  const utf8Decoder = new TextDecoder();
   const strictUtf8Decoder = new TextDecoder("utf-8", { fatal: true });
   const encode = (text4) => tokenizer.encode(text4);
+  const normalizeDecodeTokens = (tokens) => {
+    if (tokens instanceof Uint32Array) {
+      return tokens;
+    }
+    const tokenArray = new Uint32Array(tokens.length);
+    tokens.forEach((token, index) => {
+      if (!Number.isFinite(token) || !Number.isInteger(token) || token < 0 || token > 4294967295) {
+        throw new TypeError(`token id at index ${index} must be a finite non-negative integer.`);
+      }
+      tokenArray[index] = token;
+    });
+    return tokenArray;
+  };
   const decode = (tokens) => {
-    const tokenArray = tokens instanceof Uint32Array ? tokens : Uint32Array.from(tokens);
-    return utf8Decoder.decode(tokenizer.decode(tokenArray));
+    const tokenArray = normalizeDecodeTokens(tokens);
+    try {
+      return strictUtf8Decoder.decode(tokenizer.decode(tokenArray));
+    } catch {
+      throw new Error("Cannot decode tokens without corrupting UTF-8 text.");
+    }
   };
   const count = (text4) => encode(text4).length;
   const truncate2 = (text4, tokenCount) => {