proskomma-core 0.11.2 → 0.11.4

package/dist/index.js

@@ -15992,7 +15992,7 @@ var bitset = { exports: {} };
 var bitsetExports = bitset.exports;
 const BitSet = /* @__PURE__ */ getDefaultExportFromCjs(bitsetExports);
 const name = "proskomma-core";
-const version = "0.11.2";
+const version = "0.11.4";
 const description$h = "A Scripture Runtime Engine";
 const files = [
   "dist"
@@ -16084,7 +16084,7 @@ const packageJson = {
   dependencies: dependencies$1,
   devDependencies
 };
-const stringToUtf8ByteArray = function(str) {
+const stringToUtf8ByteArray$1 = function(str) {
   var out = [], p = 0;
   for (var i = 0; i < str.length; i++) {
     var c = str.charCodeAt(i);
@@ -16133,7 +16133,7 @@ const utf8ByteArrayToString = function(bytes) {
 };
 var utf8StringBytes = {
   utf8ByteArrayToString,
-  stringToUtf8ByteArray
+  stringToUtf8ByteArray: stringToUtf8ByteArray$1
 };
 var base64Js = {};
 base64Js.byteLength = byteLength;
@@ -18917,19 +18917,36 @@ const recordPreEnum = (docSet, category, value) => {
     docSet.preEnums[category].get(value).frequency++;
   }
 };
+const stringToUtf8ByteArray = (s) => {
+  const encoder = new TextEncoder();
+  return Array.from(encoder.encode(s));
+};
 const buildEnum = (docSet, category, preEnumOb) => {
   const sortedPreEnums = new Map([...preEnumOb.entries()]);
   for (const enumText of sortedPreEnums.keys()) {
-    if (enumText.length > 255) {
+    let truncatedText = enumText;
+    const utf8Bytes = stringToUtf8ByteArray(enumText);
+    if (utf8Bytes.length > 255) {
       console.log(
         "enum text for",
         category,
-        "has length",
-        enumText.length,
-        "in buildEnum - truncating"
+        "has byte length",
+        utf8Bytes.length,
+        "in buildEnum - truncating to 255 bytes"
       );
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of enumText) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedText = enumText.substring(0, charCount);
     }
-    docSet.enums[category].pushCountedString(enumText.substring(0, 255));
+    docSet.enums[category].pushCountedString(truncatedText);
   }
   docSet.enums[category].trim();
 };
@@ -18949,7 +18966,22 @@ const enumForCategoryValue = (docSet, category, value, addUnknown) => {
       enum: docSet.preEnums[category].size,
       frequency: 1
     });
-    docSet.enums[category].pushCountedString(value);
+    let truncatedValue = value;
+    const utf8Bytes = stringToUtf8ByteArray(value);
+    if (utf8Bytes.length > 255) {
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of value) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedValue = value.substring(0, charCount);
+    }
+    docSet.enums[category].pushCountedString(truncatedValue);
     docSet.buildEnumIndex(category);
     return docSet.preEnums[category].get(value).enum;
   } else {
@@ -43664,31 +43696,30 @@ const Block = class {
       }
     }
     for (const swaStart of swaStarts) {
+      const spanLabel = this.items[swaStart - 1].payload;
+      const attPrefix = `attribute/${spanLabel}/`;
       let pos = swaStart;
-      let tokens2 = [];
+      let others = [];
       let scopes = [];
-      while (true) {
-        if (pos >= this.items.length) {
-          break;
-        }
+      while (pos < this.items.length) {
         const item = this.items[pos];
-        if (item.type === "token") {
-          tokens2.push(item);
-        } else if (item.subType === "start" && item.payload.startsWith("attribute/spanWithAtts")) {
+        if (item.type === "scope" && item.payload === spanLabel) {
+          break;
+        } else if (item.subType === "start" && item.payload.startsWith(attPrefix)) {
           scopes.push(item);
         } else {
-          break;
+          others.push(item);
         }
         pos++;
       }
-      if (tokens2.length !== 0 && scopes.length !== 0) {
+      if (others.length !== 0 && scopes.length !== 0) {
         let pos2 = swaStart;
         for (const s of scopes) {
           this.items[pos2] = s;
           pos2++;
         }
-        for (const t of tokens2) {
-          this.items[pos2] = t;
+        for (const o of others) {
+          this.items[pos2] = o;
           pos2++;
         }
       }
@@ -44531,7 +44562,7 @@ const specs = (pt) => [
   {
     // END MILESTONE - close scope, clear attribute context
     contexts: [["endMilestoneMarker"]],
-    parser: { during: (parser) => parser.clearAttributeContext() }
+    parser: { during: (parser) => parser.clearAttributeContext("milestone") }
   },
   {
     // ATTRIBUTE - open scope based on attribute context
@@ -44689,7 +44720,9 @@ const Parser2 = class {
       parentSequence: null,
       baseSequenceType: "main",
       inlineSequenceType: null,
-      attributeContext: null
+      attributeContext: null,
+      spanAttributeContext: null,
+      milestoneAttributeContext: null
     };
   }
   parseItem(lexedItem) {
@@ -45098,11 +45131,20 @@ const Parser2 = class {
       payload: label
     });
   }
+  // Milestones may sit inside spans with attributes (eg \w ... \zx-s |a="b"\* ... \zx-e\* |lemma="c"\w*),
+  // so keep separate contexts for each, with the milestone context taking priority while it is open
   setAttributeContext(label) {
-    this.current.attributeContext = label;
+    const contextKey = label.startsWith("milestone") ? "milestoneAttributeContext" : "spanAttributeContext";
+    this.current[contextKey] = label;
+    this.updateAttributeContext();
+  }
+  clearAttributeContext(scopeType) {
+    const contextKey = scopeType === "milestone" ? "milestoneAttributeContext" : "spanAttributeContext";
+    this.current[contextKey] = null;
+    this.updateAttributeContext();
   }
-  clearAttributeContext() {
-    this.current.attributeContext = null;
+  updateAttributeContext() {
+    this.current.attributeContext = this.current.milestoneAttributeContext || this.current.spanAttributeContext || null;
   }
 };
 const ByteArray$2 = utils.ByteArray;

package/dist/index.mjs

@@ -15990,7 +15990,7 @@ var bitset = { exports: {} };
 var bitsetExports = bitset.exports;
 const BitSet = /* @__PURE__ */ getDefaultExportFromCjs(bitsetExports);
 const name = "proskomma-core";
-const version = "0.11.2";
+const version = "0.11.4";
 const description$h = "A Scripture Runtime Engine";
 const files = [
   "dist"
@@ -16082,7 +16082,7 @@ const packageJson = {
   dependencies: dependencies$1,
   devDependencies
 };
-const stringToUtf8ByteArray = function(str) {
+const stringToUtf8ByteArray$1 = function(str) {
   var out = [], p = 0;
   for (var i = 0; i < str.length; i++) {
     var c = str.charCodeAt(i);
@@ -16131,7 +16131,7 @@ const utf8ByteArrayToString = function(bytes) {
 };
 var utf8StringBytes = {
   utf8ByteArrayToString,
-  stringToUtf8ByteArray
+  stringToUtf8ByteArray: stringToUtf8ByteArray$1
 };
 var base64Js = {};
 base64Js.byteLength = byteLength;
@@ -18915,19 +18915,36 @@ const recordPreEnum = (docSet, category, value) => {
     docSet.preEnums[category].get(value).frequency++;
   }
 };
+const stringToUtf8ByteArray = (s) => {
+  const encoder = new TextEncoder();
+  return Array.from(encoder.encode(s));
+};
 const buildEnum = (docSet, category, preEnumOb) => {
   const sortedPreEnums = new Map([...preEnumOb.entries()]);
   for (const enumText of sortedPreEnums.keys()) {
-    if (enumText.length > 255) {
+    let truncatedText = enumText;
+    const utf8Bytes = stringToUtf8ByteArray(enumText);
+    if (utf8Bytes.length > 255) {
       console.log(
         "enum text for",
         category,
-        "has length",
-        enumText.length,
-        "in buildEnum - truncating"
+        "has byte length",
+        utf8Bytes.length,
+        "in buildEnum - truncating to 255 bytes"
       );
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of enumText) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedText = enumText.substring(0, charCount);
     }
-    docSet.enums[category].pushCountedString(enumText.substring(0, 255));
+    docSet.enums[category].pushCountedString(truncatedText);
   }
   docSet.enums[category].trim();
 };
@@ -18947,7 +18964,22 @@ const enumForCategoryValue = (docSet, category, value, addUnknown) => {
       enum: docSet.preEnums[category].size,
       frequency: 1
     });
-    docSet.enums[category].pushCountedString(value);
+    let truncatedValue = value;
+    const utf8Bytes = stringToUtf8ByteArray(value);
+    if (utf8Bytes.length > 255) {
+      let byteCount = 0;
+      let charCount = 0;
+      for (const char of value) {
+        const charBytes = stringToUtf8ByteArray(char).length;
+        if (byteCount + charBytes > 255) {
+          break;
+        }
+        byteCount += charBytes;
+        charCount++;
+      }
+      truncatedValue = value.substring(0, charCount);
+    }
+    docSet.enums[category].pushCountedString(truncatedValue);
     docSet.buildEnumIndex(category);
     return docSet.preEnums[category].get(value).enum;
   } else {
@@ -43662,31 +43694,30 @@ const Block = class {
       }
     }
     for (const swaStart of swaStarts) {
+      const spanLabel = this.items[swaStart - 1].payload;
+      const attPrefix = `attribute/${spanLabel}/`;
       let pos = swaStart;
-      let tokens2 = [];
+      let others = [];
       let scopes = [];
-      while (true) {
-        if (pos >= this.items.length) {
-          break;
-        }
+      while (pos < this.items.length) {
         const item = this.items[pos];
-        if (item.type === "token") {
-          tokens2.push(item);
-        } else if (item.subType === "start" && item.payload.startsWith("attribute/spanWithAtts")) {
+        if (item.type === "scope" && item.payload === spanLabel) {
+          break;
+        } else if (item.subType === "start" && item.payload.startsWith(attPrefix)) {
           scopes.push(item);
         } else {
-          break;
+          others.push(item);
         }
         pos++;
       }
-      if (tokens2.length !== 0 && scopes.length !== 0) {
+      if (others.length !== 0 && scopes.length !== 0) {
         let pos2 = swaStart;
         for (const s of scopes) {
           this.items[pos2] = s;
           pos2++;
         }
-        for (const t of tokens2) {
-          this.items[pos2] = t;
+        for (const o of others) {
+          this.items[pos2] = o;
           pos2++;
         }
       }
@@ -44529,7 +44560,7 @@ const specs = (pt) => [
   {
     // END MILESTONE - close scope, clear attribute context
     contexts: [["endMilestoneMarker"]],
-    parser: { during: (parser) => parser.clearAttributeContext() }
+    parser: { during: (parser) => parser.clearAttributeContext("milestone") }
   },
   {
     // ATTRIBUTE - open scope based on attribute context
@@ -44687,7 +44718,9 @@ const Parser2 = class {
       parentSequence: null,
       baseSequenceType: "main",
       inlineSequenceType: null,
-      attributeContext: null
+      attributeContext: null,
+      spanAttributeContext: null,
+      milestoneAttributeContext: null
     };
   }
   parseItem(lexedItem) {
@@ -45096,11 +45129,20 @@ const Parser2 = class {
       payload: label
     });
   }
+  // Milestones may sit inside spans with attributes (eg \w ... \zx-s |a="b"\* ... \zx-e\* |lemma="c"\w*),
+  // so keep separate contexts for each, with the milestone context taking priority while it is open
   setAttributeContext(label) {
-    this.current.attributeContext = label;
+    const contextKey = label.startsWith("milestone") ? "milestoneAttributeContext" : "spanAttributeContext";
+    this.current[contextKey] = label;
+    this.updateAttributeContext();
+  }
+  clearAttributeContext(scopeType) {
+    const contextKey = scopeType === "milestone" ? "milestoneAttributeContext" : "spanAttributeContext";
+    this.current[contextKey] = null;
+    this.updateAttributeContext();
   }
-  clearAttributeContext() {
-    this.current.attributeContext = null;
+  updateAttributeContext() {
+    this.current.attributeContext = this.current.milestoneAttributeContext || this.current.spanAttributeContext || null;
   }
 };
 const ByteArray$2 = utils.ByteArray;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "proskomma-core",
-  "version": "0.11.2",
+  "version": "0.11.4",
   "description": "A Scripture Runtime Engine",
   "files": [
     "dist"