proskomma-core 0.11.2 → 0.11.3

package/dist/index.js

@@ -15992,7 +15992,7 @@ var bitset = { exports: {} };
 var bitsetExports = bitset.exports;
 const BitSet = /* @__PURE__ */ getDefaultExportFromCjs(bitsetExports);
 const name = "proskomma-core";
-const version = "0.11.2";
+const version = "0.11.3";
 const description$h = "A Scripture Runtime Engine";
 const files = [
   "dist"
@@ -16084,7 +16084,7 @@ const packageJson = {
   dependencies: dependencies$1,
   devDependencies
 };
-const stringToUtf8ByteArray = function(str) {
+const stringToUtf8ByteArray$1 = function(str) {
   var out = [], p = 0;
   for (var i = 0; i < str.length; i++) {
     var c = str.charCodeAt(i);
@@ -16133,7 +16133,7 @@ const utf8ByteArrayToString = function(bytes) {
 };
 var utf8StringBytes = {
   utf8ByteArrayToString,
-  stringToUtf8ByteArray
+  stringToUtf8ByteArray: stringToUtf8ByteArray$1
 };
 var base64Js = {};
 base64Js.byteLength = byteLength;
@@ -18917,19 +18917,36 @@ const recordPreEnum = (docSet, category, value) => {
     docSet.preEnums[category].get(value).frequency++;
   }
 };
+const stringToUtf8ByteArray = (s) => {
+  const encoder = new TextEncoder();
+  return Array.from(encoder.encode(s));
+};
 const buildEnum = (docSet, category, preEnumOb) => {
   const sortedPreEnums = new Map([...preEnumOb.entries()]);
   for (const enumText of sortedPreEnums.keys()) {
-    if (enumText.length > 255) {
+    let truncatedText = enumText;
+    const utf8Bytes = stringToUtf8ByteArray(enumText);
+    if (utf8Bytes.length > 255) {
       console.log(
         "enum text for",
         category,
-        "has length",
-        enumText.length,
-        "in buildEnum - truncating"
+        "has byte length",
+        utf8Bytes.length,
+        "in buildEnum - truncating to 255 bytes"
       );
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of enumText) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedText = enumText.substring(0, charCount);
     }
-    docSet.enums[category].pushCountedString(enumText.substring(0, 255));
+    docSet.enums[category].pushCountedString(truncatedText);
   }
   docSet.enums[category].trim();
 };
@@ -18949,7 +18966,22 @@ const enumForCategoryValue = (docSet, category, value, addUnknown) => {
       enum: docSet.preEnums[category].size,
       frequency: 1
     });
-    docSet.enums[category].pushCountedString(value);
+    let truncatedValue = value;
+    const utf8Bytes = stringToUtf8ByteArray(value);
+    if (utf8Bytes.length > 255) {
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of value) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedValue = value.substring(0, charCount);
+    }
+    docSet.enums[category].pushCountedString(truncatedValue);
     docSet.buildEnumIndex(category);
     return docSet.preEnums[category].get(value).enum;
   } else {

package/dist/index.mjs

@@ -15990,7 +15990,7 @@ var bitset = { exports: {} };
 var bitsetExports = bitset.exports;
 const BitSet = /* @__PURE__ */ getDefaultExportFromCjs(bitsetExports);
 const name = "proskomma-core";
-const version = "0.11.2";
+const version = "0.11.3";
 const description$h = "A Scripture Runtime Engine";
 const files = [
   "dist"
@@ -16082,7 +16082,7 @@ const packageJson = {
   dependencies: dependencies$1,
   devDependencies
 };
-const stringToUtf8ByteArray = function(str) {
+const stringToUtf8ByteArray$1 = function(str) {
   var out = [], p = 0;
   for (var i = 0; i < str.length; i++) {
     var c = str.charCodeAt(i);
@@ -16131,7 +16131,7 @@ const utf8ByteArrayToString = function(bytes) {
 };
 var utf8StringBytes = {
   utf8ByteArrayToString,
-  stringToUtf8ByteArray
+  stringToUtf8ByteArray: stringToUtf8ByteArray$1
 };
 var base64Js = {};
 base64Js.byteLength = byteLength;
@@ -18915,19 +18915,36 @@ const recordPreEnum = (docSet, category, value) => {
     docSet.preEnums[category].get(value).frequency++;
   }
 };
+const stringToUtf8ByteArray = (s) => {
+  const encoder = new TextEncoder();
+  return Array.from(encoder.encode(s));
+};
 const buildEnum = (docSet, category, preEnumOb) => {
   const sortedPreEnums = new Map([...preEnumOb.entries()]);
   for (const enumText of sortedPreEnums.keys()) {
-    if (enumText.length > 255) {
+    let truncatedText = enumText;
+    const utf8Bytes = stringToUtf8ByteArray(enumText);
+    if (utf8Bytes.length > 255) {
       console.log(
         "enum text for",
         category,
-        "has length",
-        enumText.length,
-        "in buildEnum - truncating"
+        "has byte length",
+        utf8Bytes.length,
+        "in buildEnum - truncating to 255 bytes"
       );
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of enumText) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedText = enumText.substring(0, charCount);
     }
-    docSet.enums[category].pushCountedString(enumText.substring(0, 255));
+    docSet.enums[category].pushCountedString(truncatedText);
   }
   docSet.enums[category].trim();
 };
@@ -18947,7 +18964,22 @@ const enumForCategoryValue = (docSet, category, value, addUnknown) => {
       enum: docSet.preEnums[category].size,
       frequency: 1
     });
-    docSet.enums[category].pushCountedString(value);
+    let truncatedValue = value;
+    const utf8Bytes = stringToUtf8ByteArray(value);
+    if (utf8Bytes.length > 255) {
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of value) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedValue = value.substring(0, charCount);
+    }
+    docSet.enums[category].pushCountedString(truncatedValue);
     docSet.buildEnumIndex(category);
     return docSet.preEnums[category].get(value).enum;
   } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "proskomma-core",
-  "version": "0.11.2",
+  "version": "0.11.3",
   "description": "A Scripture Runtime Engine",
   "files": [
     "dist"