word-aligner 1.0.3 → 1.1.0

package/CLAUDE.md

@@ -0,0 +1,111 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Purpose
+
+**word-aligner** is an NPM library for handling word alignment logic between original-language Bible text (Greek, Hebrew, Aramaic) and target-language translations. It converts between USFM verse strings, verse object hierarchies, and structured alignment data used across unfoldingWord tools.
+
+## Commands
+
+```bash
+# Install (legacy-peer-deps required due to usfm-js peer dependency)
+npm i --legacy-peer-deps
+
+# Run linting + all tests
+npm test
+
+# Run tests only (skip lint)
+npx jest
+
+# Run a single test file
+npx jest __tests__/align.test.js
+
+# Run tests matching a pattern
+npx jest --testNamePattern "oneToOne"
+
+# Lint only
+npx eslint ./src
+
+# Lint with auto-fix
+npm run fix
+
+# Compile src/ → lib/ (required before publishing)
+npm run build
+
+# Rebuild test fixture JSON files
+npm run build:test-data
+```
+
+## Architecture
+
+### Entry point
+
+`src/index.js` exports four things:
+- `default` — the main aligner (`src/js/aligner.js`)
+- `VerseObjectUtils` — verse object parsing helpers (`src/js/utils/verseObjects.js`)
+- `MorphUtils` — morphological code parser for Greek/Hebrew/Aramaic (`src/js/utils/MorphUtils.js`)
+- `ArrayUtils` — low-level array helpers (`src/js/utils/array.js`)
+
+The compiled output lives in `lib/` (Babel 6, `babel-preset-es2015`). The `lib/` directory is what npm consumers receive; `src/` is the authoritative source.
+
+### Core module: `src/js/aligner.js`
+
+The heart of the library. Two inverse operations:
+
+- **`merge(alignments, wordBank, verseString, useVerseText)`** — Takes alignment data and a target verse, rebuilds a nested verse object hierarchy with alignment milestones.
+- **`unmerge(verseObjects, alignedVerse)`** — Extracts structured `alignments` and `wordBank` arrays from verse objects.
+
+Supporting functions `restoreVerseObjects()` and `restoreHierarchy()` handle the parent-child reconstruction after array manipulation. `combineConsecutiveText()` cleans up adjacent text nodes in the output. `wordMap` (an array of `{object, parent}` pairs) is used throughout to safely navigate and mutate deeply nested structures without losing position.
+
+### Key data shapes
+
+**Alignment:**
+```js
+{
+  topWords: [{ word, strong, lemma, morph, occurrence, occurrences }],  // original language
+  bottomWords: [{ word, occurrence, occurrences }]                       // target language
+}
+```
+
+**VerseObject:**
+```js
+{
+  tag: 'w' | 'k' | ...,
+  type: 'word' | 'text' | 'paragraph' | 'milestone',
+  text?: string,
+  children?: VerseObject[],
+  occurrence?: number,
+  occurrences?: number
+}
+```
+
+Milestones (`type: 'milestone'`) act as alignment containers; their `children` hold the target-language word objects that are aligned to the `topWords` stored on the milestone itself.
+
+### Tests and fixtures
+
+Test files live in `__tests__/`. Fixture JSON and USFM files are in `__tests__/fixtures/pivotAlignmentVerseObjects/` — each fixture represents a real Bible verse edge case (nested milestones, punctuation, non-contiguous alignments, etc.).
+
+`scripts/BuildTestData.js` regenerates the fixture JSON from USFM sources using `word-aligner-rcl`. Run `npm run build:test-data` after updating fixture source files.
+
+### Morphological parsing
+
+`MorphUtils.js` decodes morph code strings (e.g., `Gr,V,,,,,AAN,,`) into localization key arrays using language-specific maps in `src/js/utils/morphCodeLocalizationMap.js`. It branches on Greek vs. Hebrew/Aramaic and handles sub-type disambiguation within each language family.
+
+## Dependencies to know
+
+| Package | Role |
+|---|---|
+| `usfm-js` | Parses USFM Bible markup into verse object arrays |
+| `string-punctuation-tokenizer` | Splits verse text, preserving punctuation as separate tokens |
+| `lodash` | `cloneDeep` used when mutating verse object trees |
+| `word-aligner-rcl` | React component library (dev only); used by `BuildTestData.js` |
+
+## Publish workflow
+
+```bash
+npm i --legacy-peer-deps
+npm run build     # compiles src/ → lib/
+npm publish       # runs npm test + build automatically via prepublishOnly hook
+                  # then auto-tags git: v$npm_package_version
+```

package/README.md

@@ -1,2 +1,14 @@
 # word-aligner
 Logic for handling word alignment
+
+## Publish
+
+```bash
+npm i --legacy-peer-deps && npm run build && npm publish
+```
+
+## Generate test fixture data
+```bash
+# Rebuild test fixture JSON files
+npm run build:test-data
+```

package/lib/js/aligner.js

@@ -54,6 +54,134 @@ var hasAlignments = exports.hasAlignments = function hasAlignments(alignments) {
   return indexFirstAlignment >= 0;
 };
 
+/**
+ * Combines consecutive text objects in an array of verse objects recursively.
+ * When multiple text objects appear consecutively, they are merged into a single text object.
+ * Also processes nested children arrays recursively.
+ * @param {Array} objects - Array of verse objects to process
+ * @return {Array} - Array with consecutive text objects combined
+ */
+var combineConsecutiveText = function combineConsecutiveText(objects) {
+  var result = [];
+  for (var i = 0; i < objects.length; i++) {
+    var current = objects[i];
+
+    if (current.type === 'text' && result.length > 0 && result[result.length - 1].type === 'text') {
+      // combine with previous text object
+      result[result.length - 1].text += current.text;
+    } else {
+      // recursively process children if they exist
+      if (current.children && Array.isArray(current.children)) {
+        current.children = combineConsecutiveText(current.children);
+      }
+      result.push(current);
+    }
+  }
+  return result;
+};
+
+/**
+ * Restores verse objects from a flattened state by rebuilding their hierarchical structure,
+ * removing null/undefined objects, and combining consecutive text objects.
+ * @param {Array} verseObjects - Array of verse objects to restore
+ * @return {Array} - Cleaned and restored array of verse objects
+ */
+function restoreVerseObjects(verseObjects) {
+  restoreHierarchy(verseObjects);
+  // remove null objects
+  var filteredObjects = verseObjects.filter(function (item) {
+    return item !== null && item !== undefined;
+  });
+  // combine consecutive text objects in nested verseObjects
+  var cleanedVerseObjects = combineConsecutiveText(filteredObjects);
+  cleanChildReferences({ children: cleanedVerseObjects }, 'parentIndex');
+  return cleanedVerseObjects;
+}
+
+/**
+ * Recursively removes a specified property (default 'parentIndex') from all children
+ * in a verse object's hierarchy.
+ * @param {Object} verseObject - The verse object whose children should be cleaned
+ * @param {string} [key='parentIndex'] - The property key to remove from children
+ */
+function cleanChildReferences(verseObject) {
+  var key = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'parentIndex';
+
+  var children = verseObject.children || [];
+  for (var j = 0, cLen = children.length; j < cLen; j++) {
+    var child = children[j];
+    var childKeyValue = child[key];
+    if (childKeyValue >= 0) {
+      delete child[key];
+    }
+    if (child.children) {
+      cleanChildReferences(child, key);
+    }
+  }
+}
+
+/**
+ * Restores the hierarchical structure of flattened verse objects.
+ * Verse objects that have a parentIndex property are moved into their parent's children array
+ * and then removed from the top-level array by setting them to null.
+ *
+ * @param {Array} unalignedOrdered - Array of flattened verse objects that may contain parentIndex properties
+ */
+function restoreHierarchy(unalignedOrdered) {
+  var toRemove = [];
+
+  var _loop = function _loop(i, oLen) {
+    var verseObject = unalignedOrdered[i];
+    var parentIndex = verseObject.parentIndex;
+    if (parentIndex >= 0) {
+      var parent = unalignedOrdered.find(function (obj) {
+        return obj && obj.originalIndex === parentIndex;
+      });
+      if (parent && parent.children) {
+        parent.children.push(verseObject);
+        toRemove.push(i);
+      }
+      delete verseObject.parentIndex;
+      delete verseObject.originalIndex;
+      cleanChildReferences(verseObject, 'parentIndex');
+    }
+  };
+
+  for (var i = 0, oLen = unalignedOrdered.length; i < oLen; i++) {
+    _loop(i, oLen);
+  }
+
+  // remove from original location by nulling
+  for (var i = toRemove.length - 1; i >= 0; i--) {
+    var toRemoveElement = toRemove[i];
+    unalignedOrdered.splice(toRemoveElement, 1);
+  }
+
+  // clean up originalIndex property
+  for (var _i = 0, oLen = unalignedOrdered.length; _i < oLen; _i++) {
+    var _verseObject2 = unalignedOrdered[_i];
+    if (_verseObject2 && _verseObject2.originalIndex >= 0) {
+      delete _verseObject2.originalIndex;
+    }
+    cleanChildReferences(_verseObject2, 'originalIndex');
+  }
+}
+
+/**
+ * Saves the original position of each verse object in the array by adding an originalIndex property.
+ * This allows tracking of objects' positions before any modifications or deletions occur.
+ *
+ * @param {Array} unalignedOrdered - Array of verse objects whose positions need to be saved
+ */
+function savePosition(unalignedOrdered) {
+  for (var i = 0, dLen = unalignedOrdered.length; i < dLen; i++) {
+    var _verseObject3 = unalignedOrdered[i];
+    if (_verseObject3) {
+      _verseObject3.originalIndex = i; // so we can keep track of where the object was before deletions
+    }
+  }
+}
+
 /**
  * @description pivots alignments into bottomWords/targetLanguage verseObjectArray sorted by verseText
  * @param {Array} alignments - array of aligned word objects {bottomWords, topWords}
@@ -94,11 +222,11 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
   var wbLen = wordBank.length;
   for (var i = 0; i < wbLen; i++) {
     var bottomWord = wordBank[i];
-    var verseObject = VerseObjectUtils.wordVerseObjectFromBottomWord(bottomWord);
-    var index = VerseObjectUtils.indexOfVerseObject(wordMap, verseObject);
+    var _verseObject4 = VerseObjectUtils.wordVerseObjectFromBottomWord(bottomWord);
+    var index = VerseObjectUtils.indexOfVerseObject(wordMap, _verseObject4);
     if (index > -1) {
       var location = wordMap[index];
-      location.array[location.pos] = verseObject;
+      location.array[location.pos] = _verseObject4;
     } else if (hasAlignments(alignments)) {
       // if verse has some alignments
       throw { message: 'Word "' + bottomWord.word + '" is in wordBank, but missing from target language verse.', type: 'InvalidatedAlignments' };
@@ -109,8 +237,8 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
   }
   var indicesToDelete = [];
   // each alignment should result in one verseObject
-  for (var _i = 0, aLen = alignments.length; _i < aLen; _i++) {
-    var alignment = alignments[_i];
+  for (var _i2 = 0, aLen = alignments.length; _i2 < aLen; _i2++) {
+    var alignment = alignments[_i2];
     var topWords = alignment.topWords,
         bottomWords = alignment.bottomWords;
     // each bottomWord results in a nested verseObject of tag: w, type: word
@@ -119,12 +247,12 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
     var replacements = {};
     for (var j = 0, bwLen = bottomWords.length; j < bwLen; j++) {
       var _bottomWord = bottomWords[j];
-      var _verseObject2 = VerseObjectUtils.wordVerseObjectFromBottomWord(_bottomWord);
-      var _index = VerseObjectUtils.indexOfVerseObject(wordMap, _verseObject2);
+      var _verseObject5 = VerseObjectUtils.wordVerseObjectFromBottomWord(_bottomWord);
+      var _index = VerseObjectUtils.indexOfVerseObject(wordMap, _verseObject5);
       if (_index === -1) {
-        throw { message: 'VerseObject not found in verseText while merging:' + (0, _stringify2.default)(_verseObject2), type: 'InvalidatedAlignments' };
+        throw { message: 'VerseObject not found in verseText while merging:' + (0, _stringify2.default)(_verseObject5), type: 'InvalidatedAlignments' };
       }
-      replacements[_index] = _verseObject2;
+      replacements[_index] = _verseObject5;
     }
     // each topWord results in a nested verseObject of tag: k, type: milestone
     var milestones = topWords.map(function (topWord) {
@@ -156,12 +284,19 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
       var milestone = VerseObjectUtils.nestMilestones(milestones);
       // replace the original verseObject from the verse text with the aligned milestone verseObject
       var _location = wordMap[indexToReplace];
+      if (_location.parentIndex >= 0) {
+        milestone.parentIndex = _location.parentIndex; // preserve the parent index
+      }
       _location.array[_location.pos] = milestone;
     }
   }
+
+  savePosition(unalignedOrdered); // save original position of each verseObject to keep track even after deletions
+
   // deleteIndices that were queued due to consecutive bottomWords in alignments
   var verseObjects = ArrayUtils.deleteIndices(unalignedOrdered, indicesToDelete, wordMap);
-  return verseObjects;
+  var restoredObjects = restoreVerseObjects(verseObjects);
+  return restoredObjects;
 };
 
 /**
@@ -413,13 +548,13 @@ var unmerge = exports.unmerge = function unmerge(verseObjects, alignedVerse) {
   }
   var len = verseObjects.length;
   for (var i = 0; i < len; i++) {
-    var verseObject = verseObjects[i];
-    addAlignment(baseMilestones, verseObject, alignments);
+    var _verseObject6 = verseObjects[i];
+    addAlignment(baseMilestones, _verseObject6, alignments);
   }
   var alignmentUnOrdered = [];
   len = alignments.length;
-  for (var _i2 = 0; _i2 < len; _i2++) {
-    var _alignment = alignments[_i2];
+  for (var _i3 = 0; _i3 < len; _i3++) {
+    var _alignment = alignments[_i3];
     if (_alignment.topWords.length > 0) {
       alignmentUnOrdered.push(_alignment);
     } else {

package/lib/js/utils/verseObjects.js

@@ -185,21 +185,29 @@ var getVerseObjectsText = function getVerseObjectsText(verseObjects) {
 };
 
 /**
- * make sure we pick up white space between tokens
- * @param {string} text - string to tokenize
- * @param {Number} lastPos - position of end of last token
- * @param {Number} pos - position to grab up to
- * @param {Array} newVerseObjects - nested verse objects
- * @param {Boolean} end - if true, then at end of line
- * @return {{lastPos: *, verseObject: *}} - new verse object and updated position
+ * Fills gaps (whitespace and text) between tokens in the verse object array.
+ * Ensures whitespace between tokens is preserved by creating text verse objects.
+ * If possible, appends to the previous text object if it exists at the same nesting level;
+ * otherwise creates a new text verse object.
+ *
+ * @param {string} text - The complete string being tokenized
+ * @param {Number} lastPos - Position of the end of the last processed token
+ * @param {Number} pos - Position to process up to (start of next token or end of string)
+ * @param {Array} newVerseObjects - Array of verse objects being populated
+ * @param {Boolean} [end=false] - If true, forces creation of text object even if gap is empty (for end of line)
+ * @param {Number} [parentIndex=-1] - Index of parent verse object if nested, -1 if at root level
+ * @return {Number} Updated position after processing the gap (lastPos + gap.length)
  */
 var fillGap = function fillGap(text, lastPos, pos, newVerseObjects) {
   var end = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : false;
+  var parentIndex = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : -1;
 
   var verseObject = null;
   var gap = text.substring(lastPos, pos);
   var lastVerseObject = newVerseObjects.length && newVerseObjects[newVerseObjects.length - 1];
-  if (lastVerseObject && lastVerseObject.type === 'text') {
+  var lastParentIndex = typeof lastVerseObject.parentIndex === 'number' ? lastVerseObject.parentIndex : -1;
+  var canAppendToPreviousText = lastVerseObject && lastVerseObject.type === 'text' && lastParentIndex === parentIndex;
+  if (canAppendToPreviousText) {
     // append to previous text
     lastVerseObject.text += gap;
   } else if (end || gap) {
@@ -208,6 +216,11 @@ var fillGap = function fillGap(text, lastPos, pos, newVerseObjects) {
       type: 'text',
       text: gap
     };
+
+    if (parentIndex >= 0) {
+      verseObject.parentIndex = parentIndex;
+    }
+
     newVerseObjects.push(verseObject);
   }
   lastPos += gap.length;
@@ -215,15 +228,22 @@ var fillGap = function fillGap(text, lastPos, pos, newVerseObjects) {
 };
 
 /**
- * parse text into tokens
- * @param {string} text - string to tokenize
- * @param {Array} newVerseObjects - nested verse objects
- * @param {Array} wordMap - ordered map of word locations in verseObjects
- * @param {Number} nonWordVerseObjectCount - keeps count of entries that are not actually words
- * @param {String} verseText - text of the entire verse
- * @return {Number} new nonWordVerseObjectCount
+ * Parses text into tokens and creates word or text verse objects.
+ * Tokenizes the input text and identifies words (containing word/number characters)
+ * versus punctuation/text. For words, creates word objects with occurrence tracking.
+ * For non-word tokens, creates text objects. Preserves whitespace between tokens.
+ *
+ * @param {string} text - The string to tokenize
+ * @param {Array} newVerseObjects - Array to populate with newly created verse objects
+ * @param {Array} wordMap - Ordered map tracking word locations in verseObjects for occurrence counting
+ * @param {Number} nonWordVerseObjectCount - Counter for entries that are not words (text/punctuation)
+ * @param {String} verseText - Complete text of the entire verse for occurrence calculation
+ * @param {Number} [parentIndex=-1] - Index of parent verse object if this text is nested, -1 if at root level
+ * @return {Number} Updated nonWordVerseObjectCount after processing
  */
 var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText) {
+  var parentIndex = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : -1;
+
   if (text) {
     var tokens = tokenizer.tokenize({ text: text, includePunctuation: true });
     var tokenLength = tokens.length;
@@ -234,7 +254,7 @@ var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWord
       var pos = text.indexOf(word, lastPos);
       if (pos > lastPos) {
         // make sure we are not dropping white space
-        lastPos = fillGap(text, lastPos, pos, newVerseObjects);
+        lastPos = fillGap(text, lastPos, pos, newVerseObjects, false, parentIndex);
       }
       if (tokenizer.word.test(word) || tokenizer.number.test(word)) {
         // if the text has word or number characters, its a word object
@@ -249,7 +269,8 @@ var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWord
           occurrence: occurrence,
           occurrences: occurrences
         };
-        wordMap.push({ array: newVerseObjects, pos: newVerseObjects.length });
+        var _pos = newVerseObjects.length;
+        wordMap.push({ array: newVerseObjects, pos: _pos, parentIndex: parentIndex });
       } else {
         // the text does not have word characters
         nonWordVerseObjectCount++;
@@ -259,28 +280,46 @@ var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWord
         };
       }
       lastPos += word.length;
+
+      if (parentIndex >= 0) {
+        verseObject.parentIndex = parentIndex;
+      }
+
       newVerseObjects.push(verseObject);
     }
     if (lastPos < text.length) {
-      lastPos = fillGap(text, lastPos, text.length, newVerseObjects, true);
+      lastPos = fillGap(text, lastPos, text.length, newVerseObjects, true, parentIndex);
     }
   }
   return nonWordVerseObjectCount;
 };
 
 /**
- * step through verse objects extracting words
- * @param {Array} verseObjects - original array of verse objects with words split
- * @param {Array} newVerseObjects - new array of verse objects with words split
- * @param {Array} wordMap - ordered map of word locations in verseObjects
- * @param {String} verseText - text of the entire verse
- * @param {Number} nonWordVerseObjectCount - keeps count of entries that are not actually words
- * @return {Number} updated nonWordVerseObjectCount
+ * Recursively processes nested verse objects to extract and tokenize words.
+ * Traverses through verse objects, preserving non-text objects (like milestones) while
+ * extracting and tokenizing any text content. Handles nested children recursively.
+ * Maintains parent-child relationships through parentIndex tracking.
+ *
+ * @param {Array} verseObjects - Original array of verse objects to process (may contain nested structures)
+ * @param {Array} newVerseObjects - Output array to populate with processed verse objects with words split
+ * @param {Array} wordMap - Ordered map tracking word locations in verseObjects for occurrence counting
+ * @param {String} verseText - Complete text of the entire verse for occurrence calculation
+ * @param {Number} nonWordVerseObjectCount - Counter for entries that are not words (text/punctuation)
+ * @param {Number} [parentIndex=-1] - Index of parent verse object for nested elements, -1 if at root level
+ * @return {Number} Updated nonWordVerseObjectCount after processing all verse objects
  */
 var getWordsFromNestedVerseObjects = function getWordsFromNestedVerseObjects(verseObjects, newVerseObjects, wordMap, verseText, nonWordVerseObjectCount) {
+  var parentIndex = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : -1;
+
   var voLength = verseObjects.length;
   for (var i = 0; i < voLength; i++) {
     var verseObject = verseObjects[i];
+
+    if (parentIndex >= 0) {
+      // keep track of where the parent is
+      verseObject.parentIndex = parentIndex;
+    }
+
     var vsObjText = verseObject.text;
     if (verseObject.type !== 'text') {
       // preseserve non-text verseObject except for text part which will be split into words
@@ -292,16 +331,17 @@ var getWordsFromNestedVerseObjects = function getWordsFromNestedVerseObjects(ver
         verseObject.nextChar = ' '; // preserve space before text
       }
       newVerseObjects.push(verseObject);
+      var indexOfThisObject = newVerseObjects.length - 1;
       if (verseObject.children) {
         var newChildVerseObjects = [];
-        nonWordVerseObjectCount = tokenizeText(vsObjText, newChildVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
-        nonWordVerseObjectCount = getWordsFromNestedVerseObjects(verseObject.children, newChildVerseObjects, wordMap, verseText, nonWordVerseObjectCount);
+        nonWordVerseObjectCount = tokenizeText(vsObjText, newChildVerseObjects, wordMap, nonWordVerseObjectCount, verseText, indexOfThisObject);
+        nonWordVerseObjectCount = getWordsFromNestedVerseObjects(verseObject.children, newChildVerseObjects, wordMap, verseText, nonWordVerseObjectCount, indexOfThisObject);
         verseObject.children = newChildVerseObjects;
       } else {
-        nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+        nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, indexOfThisObject);
       }
     } else {
-      nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+      nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, parentIndex);
     }
   }
   return nonWordVerseObjectCount;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "word-aligner",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "description": "A library for handling word alignment",
   "main": "lib/index.js",
   "scripts": {
@@ -9,6 +9,8 @@
     "compile": "rimraf lib && babel src/ -d lib/",
     "prebuild": "rm -rf ./lib",
     "build": "babel ./src -d ./lib",
+    "build:test-data": "node scripts/BuildTestData.js",
+    "debug:test-data": "node --inspect-brk scripts/BuildTestData.js",
     "prepare": "if [ ! -d './lib/' ]; then npm run build; fi",
     "prepublishOnly": "npm test && npm run build",
     "postpublish": "git tag v$npm_package_version && git push origin v$npm_package_version"
@@ -38,7 +40,7 @@
   },
   "homepage": "https://github.com/unfoldingWord/word-aligner#readme",
   "peerDependencies": {
-    "usfm-js": "^2.1.0"
+    "usfm-js": "^3.5.0"
   },
   "devDependencies": {
     "babel-cli": "^6.26.0",
@@ -48,6 +50,7 @@
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-preset-env": "^1.7.0",
     "babel-preset-es2015": "^6.24.1",
+    "deep-equal": "1.1.2",
     "eslint": "^5.12.1",
     "eslint-config-google": "^0.12.0",
     "eslint-plugin-jest": "^22.1.3",
@@ -58,7 +61,8 @@
     "path": "0.12.7",
     "path-extra": "^4.2.1",
     "rimraf": "^2.6.2",
-    "usfm-js": "2.1.0"
+    "usfm-js": "3.5.0",
+    "word-aligner-rcl": "1.3.7-beta.6"
   },
   "dependencies": {
     "babel-runtime": "^6.26.0",

package/scripts/BuildTestData.js

@@ -0,0 +1,117 @@
+const path = require('path');
+const fs = require('fs-extra');
+const {AlignmentHelpers, UsfmFileConversionHelpers, usfmHelpers} = require('word-aligner-rcl');
+
+const RESOURCES = path.join('__tests__', 'fixtures', 'pivotAlignmentVerseObjects');
+// const folder = fs.readdirSync(RESOURCES);
+// console.log(folder);
+
+/**
+ * Reads a json file from the resources dir
+ * @param {string} filename relative path to usfm file
+ * @return {Object} - The read JSON object
+ */
+const readJSON = filename => {
+  const fullPath = path.join(RESOURCES, filename);
+  if (fs.existsSync(fullPath)) {
+    const json = fs.readJsonSync(fullPath);
+    return json;
+  }
+  console.log('File not found.');
+  return false;
+};
+
+/**
+ * Writes a JSON object to a file in the resources dir
+ * @param {string} filename relative path to json file
+ * @param {Object} json - The JSON object to write
+ */
+const writeJSON = (filename, json) => {
+  const fullPath = path.join(RESOURCES, filename);
+  fs.writeJsonSync(fullPath, json, {spaces: 2});
+};
+
+/**
+ * Recursively normalizes verse objects by converting occurrence properties to a standardized format.
+ * Traverses the verse objects array and their nested children, applying occurrence conversions where needed.
+ *
+ * @param {Array} verseObjects - Array of verse objects to normalize. Each object may contain occurrence
+ *                                properties and/or children arrays that will be processed recursively.
+ */
+function normalizeVerseObjects(verseObjects) {
+  if (Array.isArray(verseObjects)) {
+    for (let i = 0; i < verseObjects.length; i++) {
+      const item = verseObjects[i];
+      if (item.occurrence) {
+        const newItem = AlignmentHelpers.convertOccurrencesInWord(item);
+        if (newItem) {
+          verseObjects[i] = newItem;
+        }
+      }
+      if (item.children) {
+        normalizeVerseObjects(item.children);
+      }
+    }
+  }
+}
+
+/**
+ * Converts an aligned verse string to verse objects
+ * @param {string} alignedVerseString - The USFM aligned verse string to convert
+ * @return {Array} - Array of verse objects
+ */
+function getVerseObjects(alignedVerseString) {
+  const verseObjects = usfmHelpers.usfmVerseToJson(alignedVerseString);
+  normalizeVerseObjects(verseObjects);
+  return verseObjects;
+}
+
+/**
+ * Builds test data by reading a template JSON file and populating missing fields
+ * Generates verseObjects from alignedVerseString if not present
+ * Generates alignment and wordBank from alignedVerseString if alignment is empty
+ */
+function buildTestData() {
+  const testData = readJSON('template.json');
+  console.log(testData.comment);
+
+  if (testData.alignedVerseString) {
+    console.log(`found alignedVerseString: ${testData.alignedVerseString.length} chars`);
+
+    if (!testData.verseObjects.length) {
+      console.log('Generating verseObjects from alignedVerseString');
+      const verseObjects = getVerseObjects(testData.alignedVerseString);
+
+      if (verseObjects) {
+        testData.verseObjects = verseObjects;
+      }
+    }
+  }
+
+  if (testData.alignedVerseString && testData.verseObjects.length) {
+    if (!testData.alignment.length) {
+      console.log('Generating alignment and wordBank from alignedVerseString');
+      AlignmentHelpers.extractAlignmentsFromTargetVerse(testData.alignedVerseString);
+      const {
+        targetWords: wordBank,
+        verseAlignments: alignments,
+      } = AlignmentHelpers.parseUsfmToWordAlignerData(testData.alignedVerseString, null);
+      const cleanedAlignments = AlignmentHelpers.getCleanedAlignments(wordBank, alignments);
+      testData.alignment = cleanedAlignments.alignments;
+      testData.wordBank = cleanedAlignments.wordBank;
+    }
+
+    if (!testData.verseString) {
+      console.log('Generating verseString from verseObjects');
+      const verseString = UsfmFileConversionHelpers.getUsfmForVerseContent(testData.verseObjects);
+      console.log(`verseString: ${verseString}`);
+      testData.verseString = verseString;
+    }
+
+  }
+
+  delete testData.comment;
+  console.log('New test data:', testData);
+  writeJSON('new-test-data.json', testData);
+}
+buildTestData();

package/src/js/aligner.js

@@ -20,6 +20,124 @@ export const hasAlignments = (alignments) => {
   return indexFirstAlignment >= 0;
 };
 
+/**
+ * Combines consecutive text objects in an array of verse objects recursively.
+ * When multiple text objects appear consecutively, they are merged into a single text object.
+ * Also processes nested children arrays recursively.
+ * @param {Array} objects - Array of verse objects to process
+ * @return {Array} - Array with consecutive text objects combined
+ */
+const combineConsecutiveText = (objects) => {
+  const result = [];
+  for (let i = 0; i < objects.length; i++) {
+    const current = objects[i];
+
+    if (current.type === 'text' && result.length > 0 && result[result.length - 1].type === 'text') {
+      // combine with previous text object
+      result[result.length - 1].text += current.text;
+    } else {
+      // recursively process children if they exist
+      if (current.children && Array.isArray(current.children)) {
+        current.children = combineConsecutiveText(current.children);
+      }
+      result.push(current);
+    }
+  }
+  return result;
+};
+
+/**
+ * Restores verse objects from a flattened state by rebuilding their hierarchical structure,
+ * removing null/undefined objects, and combining consecutive text objects.
+ * @param {Array} verseObjects - Array of verse objects to restore
+ * @return {Array} - Cleaned and restored array of verse objects
+ */
+function restoreVerseObjects(verseObjects) {
+  restoreHierarchy(verseObjects);
+  // remove null objects
+  const filteredObjects = verseObjects.filter(item => item !== null && item !== undefined);
+  // combine consecutive text objects in nested verseObjects
+  const cleanedVerseObjects = combineConsecutiveText(filteredObjects);
+  cleanChildReferences({children: cleanedVerseObjects}, 'parentIndex');
+  return cleanedVerseObjects;
+}
+
+/**
+ * Recursively removes a specified property (default 'parentIndex') from all children
+ * in a verse object's hierarchy.
+ * @param {Object} verseObject - The verse object whose children should be cleaned
+ * @param {string} [key='parentIndex'] - The property key to remove from children
+ */
+function cleanChildReferences(verseObject, key = 'parentIndex') {
+  const children = verseObject.children || [];
+  for (let j = 0, cLen = children.length; j < cLen; j++) {
+    const child = children[j];
+    const childKeyValue = child[key];
+    if (childKeyValue >= 0) {
+      delete child[key];
+    }
+    if (child.children) {
+      cleanChildReferences(child, key);
+    }
+  }
+}
+
+/**
+ * Restores the hierarchical structure of flattened verse objects.
+ * Verse objects that have a parentIndex property are moved into their parent's children array
+ * and then removed from the top-level array by setting them to null.
+ *
+ * @param {Array} unalignedOrdered - Array of flattened verse objects that may contain parentIndex properties
+ */
+function restoreHierarchy(unalignedOrdered) {
+  const toRemove = [];
+
+  for (let i = 0, oLen = unalignedOrdered.length; i < oLen; i++) {
+    const verseObject = unalignedOrdered[i];
+    const parentIndex = verseObject.parentIndex;
+    if (parentIndex >= 0) {
+      const parent = unalignedOrdered.find(obj => obj && obj.originalIndex === parentIndex);
+      if (parent && parent.children) {
+        parent.children.push(verseObject);
+        toRemove.push(i);
+      }
+      delete verseObject.parentIndex;
+      delete verseObject.originalIndex;
+      cleanChildReferences(verseObject, 'parentIndex');
+    }
+  }
+
+  // remove from original location by nulling
+  for (let i = toRemove.length - 1; i >= 0; i--) {
+    const toRemoveElement = toRemove[i];
+    unalignedOrdered.splice(toRemoveElement, 1);
+  }
+
+  // clean up originalIndex property
+  for (let i = 0, oLen = unalignedOrdered.length; i < oLen; i++) {
+    const verseObject = unalignedOrdered[i];
+    if (verseObject && (verseObject.originalIndex >= 0)) {
+      delete verseObject.originalIndex;
+    }
+    cleanChildReferences(verseObject, 'originalIndex');
+  }
+}
+
+/**
+ * Saves the original position of each verse object in the array by adding an originalIndex property.
+ * This allows tracking of objects' positions before any modifications or deletions occur.
+ *
+ * @param {Array} unalignedOrdered - Array of verse objects whose positions need to be saved
+ */
+function savePosition(unalignedOrdered) {
+  for (let i = 0, dLen = unalignedOrdered.length; i < dLen; i++) {
+    const verseObject = unalignedOrdered[i];
+    if (verseObject) {
+      verseObject.originalIndex = i; // so we can keep track of where the object was before deletions
+    }
+  }
+}
+
 /**
  * @description pivots alignments into bottomWords/targetLanguage verseObjectArray sorted by verseText
  * @param {Array} alignments - array of aligned word objects {bottomWords, topWords}
@@ -113,12 +231,19 @@ export const merge = (alignments, wordBank, verseString,
       const milestone = VerseObjectUtils.nestMilestones(milestones);
       // replace the original verseObject from the verse text with the aligned milestone verseObject
       const location = wordMap[indexToReplace];
+      if (location.parentIndex >= 0) {
+        milestone.parentIndex = location.parentIndex; // preserve the parent index
+      }
       location.array[location.pos] = milestone;
     }
   }
+
+  savePosition(unalignedOrdered); // save original position of each verseObject to keep track even after deletions
+
   // deleteIndices that were queued due to consecutive bottomWords in alignments
   const verseObjects = ArrayUtils.deleteIndices(unalignedOrdered, indicesToDelete, wordMap);
-  return verseObjects;
+  const restoredObjects = restoreVerseObjects(verseObjects);
+  return restoredObjects;
 };
 
 /**

package/src/js/utils/verseObjects.js

@@ -129,25 +129,38 @@ const getVerseObjectsText = (verseObjects) => {
 };
 
 /**
- * make sure we pick up white space between tokens
- * @param {string} text - string to tokenize
- * @param {Number} lastPos - position of end of last token
- * @param {Number} pos - position to grab up to
- * @param {Array} newVerseObjects - nested verse objects
- * @param {Boolean} end - if true, then at end of line
- * @return {{lastPos: *, verseObject: *}} - new verse object and updated position
+ * Fills gaps (whitespace and text) between tokens in the verse object array.
+ * Ensures whitespace between tokens is preserved by creating text verse objects.
+ * If possible, appends to the previous text object if it exists at the same nesting level;
+ * otherwise creates a new text verse object.
+ *
+ * @param {string} text - The complete string being tokenized
+ * @param {Number} lastPos - Position of the end of the last processed token
+ * @param {Number} pos - Position to process up to (start of next token or end of string)
+ * @param {Array} newVerseObjects - Array of verse objects being populated
+ * @param {Boolean} [end=false] - If true, forces creation of text object even if gap is empty (for end of line)
+ * @param {Number} [parentIndex=-1] - Index of parent verse object if nested, -1 if at root level
+ * @return {Number} Updated position after processing the gap (lastPos + gap.length)
  */
-const fillGap = (text, lastPos, pos, newVerseObjects, end = false) => {
+const fillGap = (text, lastPos, pos, newVerseObjects, end = false, parentIndex = -1) => {
   let verseObject = null;
   const gap = text.substring(lastPos, pos);
   const lastVerseObject = newVerseObjects.length && newVerseObjects[newVerseObjects.length - 1];
-  if (lastVerseObject && (lastVerseObject.type === 'text')) { // append to previous text
+  const lastParentIndex = (typeof lastVerseObject.parentIndex === 'number') ? lastVerseObject.parentIndex : -1;
+  const canAppendToPreviousText = lastVerseObject && (lastVerseObject.type === 'text')
+    && (lastParentIndex === parentIndex);
+  if (canAppendToPreviousText) { // append to previous text
     lastVerseObject.text += gap;
   } else if (end || gap) { // save gap
     verseObject = {
       type: 'text',
       text: gap,
     };
+
+    if (parentIndex >= 0) {
+      verseObject.parentIndex = parentIndex;
+    }
+
     newVerseObjects.push(verseObject);
   }
   lastPos += gap.length;
@@ -155,15 +168,20 @@ const fillGap = (text, lastPos, pos, newVerseObjects, end = false) => {
 };
 
 /**
- * parse text into tokens
- * @param {string} text - string to tokenize
- * @param {Array} newVerseObjects - nested verse objects
- * @param {Array} wordMap - ordered map of word locations in verseObjects
- * @param {Number} nonWordVerseObjectCount - keeps count of entries that are not actually words
- * @param {String} verseText - text of the entire verse
- * @return {Number} new nonWordVerseObjectCount
+ * Parses text into tokens and creates word or text verse objects.
+ * Tokenizes the input text and identifies words (containing word/number characters)
+ * versus punctuation/text. For words, creates word objects with occurrence tracking.
+ * For non-word tokens, creates text objects. Preserves whitespace between tokens.
+ *
+ * @param {string} text - The string to tokenize
+ * @param {Array} newVerseObjects - Array to populate with newly created verse objects
+ * @param {Array} wordMap - Ordered map tracking word locations in verseObjects for occurrence counting
+ * @param {Number} nonWordVerseObjectCount - Counter for entries that are not words (text/punctuation)
+ * @param {String} verseText - Complete text of the entire verse for occurrence calculation
+ * @param {Number} [parentIndex=-1] - Index of parent verse object if this text is nested, -1 if at root level
+ * @return {Number} Updated nonWordVerseObjectCount after processing
  */
-const tokenizeText = (text, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText) => {
+const tokenizeText = (text, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, parentIndex = -1) => {
   if (text) {
     const tokens = tokenizer.tokenize({text, includePunctuation: true});
     const tokenLength = tokens.length;
@@ -173,7 +191,7 @@ const tokenizeText = (text, newVerseObjects, wordMap, nonWordVerseObjectCount, v
       const word = tokens[j];
       const pos = text.indexOf(word, lastPos);
       if (pos > lastPos) { // make sure we are not dropping white space
-        lastPos = fillGap(text, lastPos, pos, newVerseObjects);
+        lastPos = fillGap(text, lastPos, pos, newVerseObjects, false, parentIndex);
       }
       if (tokenizer.word.test(word) || tokenizer.number.test(word)) { // if the text has word or number characters, its a word object
         const wordIndex = wordMap.length;
@@ -192,7 +210,8 @@ const tokenizeText = (text, newVerseObjects, wordMap, nonWordVerseObjectCount, v
           occurrence,
           occurrences,
         };
-        wordMap.push({array: newVerseObjects, pos: newVerseObjects.length});
+        const pos = newVerseObjects.length;
+        wordMap.push({array: newVerseObjects, pos, parentIndex});
       } else { // the text does not have word characters
         nonWordVerseObjectCount++;
         verseObject = {
@@ -201,28 +220,50 @@ const tokenizeText = (text, newVerseObjects, wordMap, nonWordVerseObjectCount, v
         };
       }
       lastPos += word.length;
+
+      if (parentIndex >= 0) {
+        verseObject.parentIndex = parentIndex;
+      }
+
       newVerseObjects.push(verseObject);
     }
     if (lastPos < text.length) {
-      lastPos = fillGap(text, lastPos, text.length, newVerseObjects, true);
+      lastPos = fillGap(text, lastPos, text.length, newVerseObjects, true, parentIndex);
     }
   }
   return nonWordVerseObjectCount;
 };
 
 /**
- * step through verse objects extracting words
- * @param {Array} verseObjects - original array of verse objects with words split
- * @param {Array} newVerseObjects - new array of verse objects with words split
- * @param {Array} wordMap - ordered map of word locations in verseObjects
- * @param {String} verseText - text of the entire verse
- * @param {Number} nonWordVerseObjectCount - keeps count of entries that are not actually words
- * @return {Number} updated nonWordVerseObjectCount
+ * Recursively processes nested verse objects to extract and tokenize words.
+ * Traverses through verse objects, preserving non-text objects (like milestones) while
+ * extracting and tokenizing any text content. Handles nested children recursively.
+ * Maintains parent-child relationships through parentIndex tracking.
+ *
+ * @param {Array} verseObjects - Original array of verse objects to process (may contain nested structures)
+ * @param {Array} newVerseObjects - Output array to populate with processed verse objects with words split
+ * @param {Array} wordMap - Ordered map tracking word locations in verseObjects for occurrence counting
+ * @param {String} verseText - Complete text of the entire verse for occurrence calculation
+ * @param {Number} nonWordVerseObjectCount - Counter for entries that are not words (text/punctuation)
+ * @param {Number} [parentIndex=-1] - Index of parent verse object for nested elements, -1 if at root level
+ * @return {Number} Updated nonWordVerseObjectCount after processing all verse objects
  */
-const getWordsFromNestedVerseObjects = (verseObjects, newVerseObjects, wordMap, verseText, nonWordVerseObjectCount) => {
+const getWordsFromNestedVerseObjects = (
+  verseObjects,
+  newVerseObjects,
+  wordMap,
+  verseText,
+  nonWordVerseObjectCount,
+  parentIndex = -1
+) => {
   const voLength = verseObjects.length;
   for (let i = 0; i < voLength; i++) {
     const verseObject = verseObjects[i];
+
+    if (parentIndex >= 0) { // keep track of where the parent is
+      verseObject.parentIndex = parentIndex;
+    }
+
     let vsObjText = verseObject.text;
     if ((verseObject.type !== 'text')) {
       // preseserve non-text verseObject except for text part which will be split into words
@@ -234,17 +275,19 @@ const getWordsFromNestedVerseObjects = (verseObjects, newVerseObjects, wordMap,
         verseObject.nextChar = ' '; // preserve space before text
       }
       newVerseObjects.push(verseObject);
+      const indexOfThisObject = newVerseObjects.length - 1;
       if (verseObject.children) {
         const newChildVerseObjects = [];
-        nonWordVerseObjectCount = tokenizeText(vsObjText, newChildVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+        nonWordVerseObjectCount = tokenizeText(vsObjText, newChildVerseObjects, wordMap, nonWordVerseObjectCount, verseText, indexOfThisObject);
         nonWordVerseObjectCount = getWordsFromNestedVerseObjects(verseObject.children, newChildVerseObjects,
-                                                                 wordMap, verseText, nonWordVerseObjectCount);
+          wordMap, verseText, nonWordVerseObjectCount,
+          indexOfThisObject);
         verseObject.children = newChildVerseObjects;
       } else {
-        nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+        nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, indexOfThisObject);
       }
     } else {
-      nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+      nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, parentIndex);
     }
   }
   return nonWordVerseObjectCount;