twl-generator 1.4.14 → 1.4.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "twl-generator",
-  "version": "1.4.14",
+  "version": "1.4.16",
   "description": "Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments.",
   "main": "src/index.js",
   "bin": {
@@ -51,7 +51,7 @@
     "csv-stringify": "^6.5.0",
     "en-inflectors": "^1.0.12",
     "jszip": "^3.10.1",
-    "tsv-quote-converters": "^1.1.18",
+    "tsv-quote-converters": "^1.1.21",
     "usfm-alignment-remover": "^0.1.6"
   },
   "peerDependencies": {

package/src/index.js

@@ -903,8 +903,10 @@ export async function generateTwlByBook(bookCode, options = {}) {
   const chapterNums = Object.keys(versesByChapter).map(n => parseInt(n, 10)).sort((a, b) => a - b);
   for (const c of chapterNums) {
     const verses = versesByChapter[c] || {};
-    const verseNums = Object.keys(verses).map(n => parseInt(n, 10)).sort((a, b) => a - b);
-    for (const v of verseNums) {
+    const verseNums = Object.keys(verses).filter(k => k !== 'front').map(n => parseInt(n, 10)).sort((a, b) => a - b);
+    // Chapter front matter (\d) is emitted as `${c}:front`, ordered before verse 1.
+    const orderedKeys = verses.front ? ['front', ...verseNums] : verseNums;
+    for (const v of orderedKeys) {
       const text = verses[v] || '';
       const matches = scanVerseMatches(text, trie);
       // Count occurrences per exact matchedText (case-sensitive)
@@ -922,8 +924,10 @@ export async function generateTwlByBook(bookCode, options = {}) {
         else if (primaryArticle.startsWith('names/')) tag = 'name';
         const twLink = primaryArticle ? `rc://*/tw/dict/bible/${primaryArticle}` : '';
 
-        // Variant of: only if beyond plural/-ed/-ing differences
-        const variantOf = allowNoVariant(m.term, glq) ? '' : m.term;
+        // Variant of: only if beyond plural/-ed/-ing differences. Compare on the
+        // brace-free reading so a supplied-morpheme brace (e.g. "creature{s}")
+        // isn't itself counted as a difference from the term.
+        const variantOf = allowNoVariant(m.term, glq.replace(/[{}]/g, '')) ? '' : m.term;
         // Disambiguation: list all candidate articles for this match
         const disamb = (m.articles && m.articles.length > 1) ? `(${m.articles.join(', ')})` : '';
 

package/src/utils/twl-matcher.js

@@ -137,6 +137,15 @@ class PrefixTrie {
     while (currentPos < searchText.length) {
       const char = searchText[currentPos];
 
+      // Curly braces wrap "supplied" words/morphemes in the ULT (e.g.
+      // "creature{s}"). Match through them transparently (they never appear in
+      // a trie term) so "creature{s}" matches the term "creatures"; the braces
+      // are re-included when the matched span is extracted below.
+      if ((char === '{' || char === '}') && currentPos > startPos) {
+        currentPos++;
+        continue;
+      }
+
       if (!node[char]) {
         break; // No more matches possible
       }
@@ -192,14 +201,42 @@ class PrefixTrie {
           originalMatchedText = originalText.substring(extendedStartPos, extendedEndPos);
         }
 
-        // Check if this is a valid word boundary match (both start and end)
-        const isStartBoundary = extendedStartPos === 0 ||
-          /[\s\p{P}]/u.test(originalText[extendedStartPos - 1]) ||
-          !/[\w]/.test(originalText[extendedStartPos - 1]);
+        // Balance curly braces so a brace split across the match boundary keeps
+        // its partner. Matching "creature{s}" through the brace stops the span
+        // at "creature{s" (open '{' with no '}'); pull in the trailing '}' so
+        // OrigWords becomes "creature{s}". Symmetric for a leading '{'.
+        let open = 0, close = 0;
+        for (const ch of originalText.substring(extendedStartPos, extendedEndPos)) {
+          if (ch === '{') open++;
+          else if (ch === '}') close++;
+        }
+        while (open > close && extendedEndPos < originalText.length && originalText[extendedEndPos] === '}') {
+          extendedEndPos++; close++;
+        }
+        while (close > open && extendedStartPos > 0 && originalText[extendedStartPos - 1] === '{') {
+          extendedStartPos--; open++;
+        }
+        if (extendedStartPos < startPos || extendedEndPos > currentPos) {
+          originalMatchedText = originalText.substring(extendedStartPos, extendedEndPos);
+        }
 
-        const isEndBoundary = extendedEndPos >= originalText.length ||
-          /[\s\p{P}]/u.test(originalText[extendedEndPos]) ||
-          !/[\w]/.test(originalText[extendedEndPos]);
+        // Check if this is a valid word boundary match (both start and end).
+        // Skip past any braces when locating the neighbouring character so a
+        // supplied-word brace does not act as a false word boundary: the
+        // brace-free reading ("creatures") must be treated as one word, so a
+        // bare "creature" before the "{s}" is correctly rejected.
+        let beforePos = extendedStartPos - 1;
+        while (beforePos >= 0 && (originalText[beforePos] === '{' || originalText[beforePos] === '}')) beforePos--;
+        let afterPos = extendedEndPos;
+        while (afterPos < originalText.length && (originalText[afterPos] === '{' || originalText[afterPos] === '}')) afterPos++;
+
+        const isStartBoundary = beforePos < 0 ||
+          /[\s\p{P}]/u.test(originalText[beforePos]) ||
+          !/[\w]/.test(originalText[beforePos]);
+
+        const isEndBoundary = afterPos >= originalText.length ||
+          /[\s\p{P}]/u.test(originalText[afterPos]) ||
+          !/[\w]/.test(originalText[afterPos]);
 
         const isWordBoundary = isStartBoundary && isEndBoundary;
 

package/src/utils/usfm-alignment-remover.js

@@ -45,11 +45,18 @@ export const removeAllTagsExceptChapterVerse = (usfmContent) => {
   cleanContent = cleanContent.replace(/ +\\v +/g, '\n\\v ');
   cleanContent = cleanContent.replace(/ +\\c +/g, '\n\\c ');
   cleanContent = cleanContent.replace(/ *(\\q\d*|\\p|\\ts\\\*) */g, ' ');
-  cleanContent = cleanContent.replace(/\\[ds].*?(\\|\n)/g, '$1');
+  // Strip section headings (\s, \s1, \sr, \sp, etc.) but preserve \d (chapter
+  // descriptions / psalm superscriptions) so front-matter TWLs can be generated.
+  cleanContent = cleanContent.replace(/\\s.*?(\\|\n)/g, '$1');
   cleanContent = cleanContent.replace(/ +/g, ' ');
   cleanContent = cleanContent.replace(/^ +$/g, '');
   cleanContent = cleanContent.replace(/\\f .*?\\f\*/g, ' ');
-  cleanContent = cleanContent.replace(/[\{\}]/g, ''); // Remove any curly braces
+  // NOTE: Curly braces ({ }) wrap "supplied" words/morphemes in the ULT (e.g.
+  // "creature{s}") that the matcher must see through but must NOT discard. They
+  // are preserved here so the matcher can match the brace-free reading
+  // ("creatures") yet retain the braces in OrigWords ("creature{s}"), which is
+  // what tsv-quote-converters needs to align the quote back to the original
+  // language. Brace-transparency lives in twl-matcher.js (findMatches).
 
   // Remove all lines before the first \c marker, keeping the \c line
   const lines = cleanContent.split('\n');
@@ -107,6 +114,16 @@ export function parseUsfmToVerses(usfm) {
       if (!versesObj[currentChapter]) {
         versesObj[currentChapter] = {};
       }
+      // Capture chapter front matter (\d description / psalm superscription) so it
+      // can produce `<chapter>:front` TWL rows. Other pre-verse markers (\s, \q, \p)
+      // have already been stripped, leaving the \d text in the chapter head.
+      const frontMatch = text.match(/\\d\s+([^\\]*)/);
+      if (frontMatch) {
+        const frontText = frontMatch[1].replace(/\s+/g, ' ').trim();
+        if (frontText) {
+          versesObj[currentChapter].front = frontText;
+        }
+      }
     } else if (tag === 'v') {
       if (!versesObj[currentChapter]) {
         versesObj[currentChapter] = {};