twl-generator 1.4.13 → 1.4.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "twl-generator",
-  "version": "1.4.13",
+  "version": "1.4.15",
   "description": "Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments.",
   "main": "src/index.js",
   "bin": {
@@ -51,7 +51,7 @@
     "csv-stringify": "^6.5.0",
     "en-inflectors": "^1.0.12",
     "jszip": "^3.10.1",
-    "tsv-quote-converters": "^1.1.18",
+    "tsv-quote-converters": "^1.1.21",
     "usfm-alignment-remover": "^0.1.6"
   },
   "peerDependencies": {

package/src/index.js

@@ -903,8 +903,10 @@ export async function generateTwlByBook(bookCode, options = {}) {
   const chapterNums = Object.keys(versesByChapter).map(n => parseInt(n, 10)).sort((a, b) => a - b);
   for (const c of chapterNums) {
     const verses = versesByChapter[c] || {};
-    const verseNums = Object.keys(verses).map(n => parseInt(n, 10)).sort((a, b) => a - b);
-    for (const v of verseNums) {
+    const verseNums = Object.keys(verses).filter(k => k !== 'front').map(n => parseInt(n, 10)).sort((a, b) => a - b);
+    // Chapter front matter (\d) is emitted as `${c}:front`, ordered before verse 1.
+    const orderedKeys = verses.front ? ['front', ...verseNums] : verseNums;
+    for (const v of orderedKeys) {
       const text = verses[v] || '';
       const matches = scanVerseMatches(text, trie);
       // Count occurrences per exact matchedText (case-sensitive)

package/src/utils/twl-matcher.js

@@ -194,11 +194,11 @@ class PrefixTrie {
 
         // Check if this is a valid word boundary match (both start and end)
         const isStartBoundary = extendedStartPos === 0 ||
-          /[\s\p{P}]/.test(originalText[extendedStartPos - 1]) ||
+          /[\s\p{P}]/u.test(originalText[extendedStartPos - 1]) ||
           !/[\w]/.test(originalText[extendedStartPos - 1]);
 
         const isEndBoundary = extendedEndPos >= originalText.length ||
-          /[\s\p{P}]/.test(originalText[extendedEndPos]) ||
+          /[\s\p{P}]/u.test(originalText[extendedEndPos]) ||
           !/[\w]/.test(originalText[extendedEndPos]);
 
         const isWordBoundary = isStartBoundary && isEndBoundary;

package/src/utils/usfm-alignment-remover.js

@@ -45,7 +45,9 @@ export const removeAllTagsExceptChapterVerse = (usfmContent) => {
   cleanContent = cleanContent.replace(/ +\\v +/g, '\n\\v ');
   cleanContent = cleanContent.replace(/ +\\c +/g, '\n\\c ');
   cleanContent = cleanContent.replace(/ *(\\q\d*|\\p|\\ts\\\*) */g, ' ');
-  cleanContent = cleanContent.replace(/\\[ds].*?(\\|\n)/g, '$1');
+  // Strip section headings (\s, \s1, \sr, \sp, etc.) but preserve \d (chapter
+  // descriptions / psalm superscriptions) so front-matter TWLs can be generated.
+  cleanContent = cleanContent.replace(/\\s.*?(\\|\n)/g, '$1');
   cleanContent = cleanContent.replace(/ +/g, ' ');
   cleanContent = cleanContent.replace(/^ +$/g, '');
   cleanContent = cleanContent.replace(/\\f .*?\\f\*/g, ' ');
@@ -107,6 +109,16 @@ export function parseUsfmToVerses(usfm) {
       if (!versesObj[currentChapter]) {
         versesObj[currentChapter] = {};
       }
+      // Capture chapter front matter (\d description / psalm superscription) so it
+      // can produce `<chapter>:front` TWL rows. Other pre-verse markers (\s, \q, \p)
+      // have already been stripped, leaving the \d text in the chapter head.
+      const frontMatch = text.match(/\\d\s+([^\\]*)/);
+      if (frontMatch) {
+        const frontText = frontMatch[1].replace(/\s+/g, ' ').trim();
+        if (frontText) {
+          versesObj[currentChapter].front = frontText;
+        }
+      }
     } else if (tag === 'v') {
       if (!versesObj[currentChapter]) {
         versesObj[currentChapter] = {};