word-aligner 1.0.3 → 1.1.0-beta.2

package/lib/js/aligner.js

@@ -54,6 +54,134 @@ var hasAlignments = exports.hasAlignments = function hasAlignments(alignments) {
   return indexFirstAlignment >= 0;
 };
 
+/**
+ * Combines consecutive text objects in an array of verse objects recursively.
+ * When multiple text objects appear consecutively, they are merged into a single text object.
+ * Also processes nested children arrays recursively.
+ * @param {Array} objects - Array of verse objects to process
+ * @return {Array} - Array with consecutive text objects combined
+ */
+var combineConsecutiveText = function combineConsecutiveText(objects) {
+  var result = [];
+  for (var i = 0; i < objects.length; i++) {
+    var current = objects[i];
+
+    if (current.type === 'text' && result.length > 0 && result[result.length - 1].type === 'text') {
+      // combine with previous text object
+      result[result.length - 1].text += current.text;
+    } else {
+      // recursively process children if they exist
+      if (current.children && Array.isArray(current.children)) {
+        current.children = combineConsecutiveText(current.children);
+      }
+      result.push(current);
+    }
+  }
+  return result;
+};
+
+/**
+ * Restores verse objects from a flattened state by rebuilding their hierarchical structure,
+ * removing null/undefined objects, and combining consecutive text objects.
+ * @param {Array} verseObjects - Array of verse objects to restore
+ * @return {Array} - Cleaned and restored array of verse objects
+ */
+function restoreVerseObjects(verseObjects) {
+  restoreHierarchy(verseObjects);
+  // remove null objects
+  var filteredObjects = verseObjects.filter(function (item) {
+    return item !== null && item !== undefined;
+  });
+  // combine consecutive text objects in nested verseObjects
+  var cleanedVerseObjects = combineConsecutiveText(filteredObjects);
+  cleanChildReferences({ children: cleanedVerseObjects }, 'parentIndex');
+  return cleanedVerseObjects;
+}
+
+/**
+ * Recursively removes a specified property (default 'parentIndex') from all children
+ * in a verse object's hierarchy.
+ * @param {Object} verseObject - The verse object whose children should be cleaned
+ * @param {string} [key='parentIndex'] - The property key to remove from children
+ */
+function cleanChildReferences(verseObject) {
+  var key = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'parentIndex';
+
+  var children = verseObject.children || [];
+  for (var j = 0, cLen = children.length; j < cLen; j++) {
+    var child = children[j];
+    var childKeyValue = child[key];
+    if (childKeyValue >= 0) {
+      delete child[key];
+    }
+    if (child.children) {
+      cleanChildReferences(child, key);
+    }
+  }
+}
+
+/**
+ * Restores the hierarchical structure of flattened verse objects.
+ * Verse objects that have a parentIndex property are moved into their parent's children array
+ * and then removed from the top-level array by setting them to null.
+ *
+ * @param {Array} unalignedOrdered - Array of flattened verse objects that may contain parentIndex properties
+ */
+function restoreHierarchy(unalignedOrdered) {
+  var toRemove = [];
+
+  var _loop = function _loop(i, oLen) {
+    var verseObject = unalignedOrdered[i];
+    var parentIndex = verseObject.parentIndex;
+    if (parentIndex >= 0) {
+      var parent = unalignedOrdered.find(function (obj) {
+        return obj && obj.originalIndex === parentIndex;
+      });
+      if (parent && parent.children) {
+        parent.children.push(verseObject);
+        toRemove.push(i);
+      }
+      delete verseObject.parentIndex;
+      delete verseObject.originalIndex;
+      cleanChildReferences(verseObject, 'parentIndex');
+    }
+  };
+
+  for (var i = 0, oLen = unalignedOrdered.length; i < oLen; i++) {
+    _loop(i, oLen);
+  }
+
+  // remove from original location by nulling
+  for (var i = toRemove.length - 1; i >= 0; i--) {
+    var toRemoveElement = toRemove[i];
+    unalignedOrdered.splice(toRemoveElement, 1);
+  }
+
+  // clean up originalIndex property
+  for (var _i = 0, oLen = unalignedOrdered.length; _i < oLen; _i++) {
+    var _verseObject2 = unalignedOrdered[_i];
+    if (_verseObject2 && _verseObject2.originalIndex >= 0) {
+      delete _verseObject2.originalIndex;
+    }
+    cleanChildReferences(_verseObject2, 'originalIndex');
+  }
+}
+
+/**
+ * Saves the original position of each verse object in the array by adding an originalIndex property.
+ * This allows tracking of objects' positions before any modifications or deletions occur.
+ *
+ * @param {Array} unalignedOrdered - Array of verse objects whose positions need to be saved
+ */
+function savePosition(unalignedOrdered) {
+  for (var i = 0, dLen = unalignedOrdered.length; i < dLen; i++) {
+    var _verseObject3 = unalignedOrdered[i];
+    if (_verseObject3) {
+      _verseObject3.originalIndex = i; // so we can keep track of where the object was before deletions
+    }
+  }
+}
+
 /**
  * @description pivots alignments into bottomWords/targetLanguage verseObjectArray sorted by verseText
  * @param {Array} alignments - array of aligned word objects {bottomWords, topWords}
@@ -94,11 +222,11 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
   var wbLen = wordBank.length;
   for (var i = 0; i < wbLen; i++) {
     var bottomWord = wordBank[i];
-    var verseObject = VerseObjectUtils.wordVerseObjectFromBottomWord(bottomWord);
-    var index = VerseObjectUtils.indexOfVerseObject(wordMap, verseObject);
+    var _verseObject4 = VerseObjectUtils.wordVerseObjectFromBottomWord(bottomWord);
+    var index = VerseObjectUtils.indexOfVerseObject(wordMap, _verseObject4);
     if (index > -1) {
       var location = wordMap[index];
-      location.array[location.pos] = verseObject;
+      location.array[location.pos] = _verseObject4;
     } else if (hasAlignments(alignments)) {
       // if verse has some alignments
       throw { message: 'Word "' + bottomWord.word + '" is in wordBank, but missing from target language verse.', type: 'InvalidatedAlignments' };
@@ -109,8 +237,8 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
   }
   var indicesToDelete = [];
   // each alignment should result in one verseObject
-  for (var _i = 0, aLen = alignments.length; _i < aLen; _i++) {
-    var alignment = alignments[_i];
+  for (var _i2 = 0, aLen = alignments.length; _i2 < aLen; _i2++) {
+    var alignment = alignments[_i2];
     var topWords = alignment.topWords,
         bottomWords = alignment.bottomWords;
     // each bottomWord results in a nested verseObject of tag: w, type: word
@@ -119,12 +247,12 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
     var replacements = {};
     for (var j = 0, bwLen = bottomWords.length; j < bwLen; j++) {
       var _bottomWord = bottomWords[j];
-      var _verseObject2 = VerseObjectUtils.wordVerseObjectFromBottomWord(_bottomWord);
-      var _index = VerseObjectUtils.indexOfVerseObject(wordMap, _verseObject2);
+      var _verseObject5 = VerseObjectUtils.wordVerseObjectFromBottomWord(_bottomWord);
+      var _index = VerseObjectUtils.indexOfVerseObject(wordMap, _verseObject5);
       if (_index === -1) {
-        throw { message: 'VerseObject not found in verseText while merging:' + (0, _stringify2.default)(_verseObject2), type: 'InvalidatedAlignments' };
+        throw { message: 'VerseObject not found in verseText while merging:' + (0, _stringify2.default)(_verseObject5), type: 'InvalidatedAlignments' };
       }
-      replacements[_index] = _verseObject2;
+      replacements[_index] = _verseObject5;
     }
     // each topWord results in a nested verseObject of tag: k, type: milestone
     var milestones = topWords.map(function (topWord) {
@@ -156,12 +284,19 @@ var merge = exports.merge = function merge(alignments, wordBank, verseString) {
       var milestone = VerseObjectUtils.nestMilestones(milestones);
       // replace the original verseObject from the verse text with the aligned milestone verseObject
       var _location = wordMap[indexToReplace];
+      if (_location.parentIndex >= 0) {
+        milestone.parentIndex = _location.parentIndex; // preserve the parent index
+      }
       _location.array[_location.pos] = milestone;
     }
   }
+
+  savePosition(unalignedOrdered); // save original position of each verseObject to keep track even after deletions
+
   // deleteIndices that were queued due to consecutive bottomWords in alignments
   var verseObjects = ArrayUtils.deleteIndices(unalignedOrdered, indicesToDelete, wordMap);
-  return verseObjects;
+  var restoredObjects = restoreVerseObjects(verseObjects);
+  return restoredObjects;
 };
 
 /**
@@ -413,13 +548,13 @@ var unmerge = exports.unmerge = function unmerge(verseObjects, alignedVerse) {
   }
   var len = verseObjects.length;
   for (var i = 0; i < len; i++) {
-    var verseObject = verseObjects[i];
-    addAlignment(baseMilestones, verseObject, alignments);
+    var _verseObject6 = verseObjects[i];
+    addAlignment(baseMilestones, _verseObject6, alignments);
   }
   var alignmentUnOrdered = [];
   len = alignments.length;
-  for (var _i2 = 0; _i2 < len; _i2++) {
-    var _alignment = alignments[_i2];
+  for (var _i3 = 0; _i3 < len; _i3++) {
+    var _alignment = alignments[_i3];
     if (_alignment.topWords.length > 0) {
       alignmentUnOrdered.push(_alignment);
     } else {

package/lib/js/utils/verseObjects.js

@@ -185,21 +185,29 @@ var getVerseObjectsText = function getVerseObjectsText(verseObjects) {
 };
 
 /**
- * make sure we pick up white space between tokens
- * @param {string} text - string to tokenize
- * @param {Number} lastPos - position of end of last token
- * @param {Number} pos - position to grab up to
- * @param {Array} newVerseObjects - nested verse objects
- * @param {Boolean} end - if true, then at end of line
- * @return {{lastPos: *, verseObject: *}} - new verse object and updated position
+ * Fills gaps (whitespace and text) between tokens in the verse object array.
+ * Ensures whitespace between tokens is preserved by creating text verse objects.
+ * If possible, appends to the previous text object if it exists at the same nesting level;
+ * otherwise creates a new text verse object.
+ *
+ * @param {string} text - The complete string being tokenized
+ * @param {Number} lastPos - Position of the end of the last processed token
+ * @param {Number} pos - Position to process up to (start of next token or end of string)
+ * @param {Array} newVerseObjects - Array of verse objects being populated
+ * @param {Boolean} [end=false] - If true, forces creation of text object even if gap is empty (for end of line)
+ * @param {Number} [parentIndex=-1] - Index of parent verse object if nested, -1 if at root level
+ * @return {Number} Updated position after processing the gap (lastPos + gap.length)
  */
 var fillGap = function fillGap(text, lastPos, pos, newVerseObjects) {
   var end = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : false;
+  var parentIndex = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : -1;
 
   var verseObject = null;
   var gap = text.substring(lastPos, pos);
   var lastVerseObject = newVerseObjects.length && newVerseObjects[newVerseObjects.length - 1];
-  if (lastVerseObject && lastVerseObject.type === 'text') {
+  var lastParentIndex = typeof lastVerseObject.parentIndex === 'number' ? lastVerseObject.parentIndex : -1;
+  var canAppendToPreviousText = lastVerseObject && lastVerseObject.type === 'text' && lastParentIndex === parentIndex;
+  if (canAppendToPreviousText) {
     // append to previous text
     lastVerseObject.text += gap;
   } else if (end || gap) {
@@ -208,6 +216,11 @@ var fillGap = function fillGap(text, lastPos, pos, newVerseObjects) {
       type: 'text',
       text: gap
     };
+
+    if (parentIndex >= 0) {
+      verseObject.parentIndex = parentIndex;
+    }
+
     newVerseObjects.push(verseObject);
   }
   lastPos += gap.length;
@@ -215,15 +228,22 @@ var fillGap = function fillGap(text, lastPos, pos, newVerseObjects) {
 };
 
 /**
- * parse text into tokens
- * @param {string} text - string to tokenize
- * @param {Array} newVerseObjects - nested verse objects
- * @param {Array} wordMap - ordered map of word locations in verseObjects
- * @param {Number} nonWordVerseObjectCount - keeps count of entries that are not actually words
- * @param {String} verseText - text of the entire verse
- * @return {Number} new nonWordVerseObjectCount
+ * Parses text into tokens and creates word or text verse objects.
+ * Tokenizes the input text and identifies words (containing word/number characters)
+ * versus punctuation/text. For words, creates word objects with occurrence tracking.
+ * For non-word tokens, creates text objects. Preserves whitespace between tokens.
+ *
+ * @param {string} text - The string to tokenize
+ * @param {Array} newVerseObjects - Array to populate with newly created verse objects
+ * @param {Array} wordMap - Ordered map tracking word locations in verseObjects for occurrence counting
+ * @param {Number} nonWordVerseObjectCount - Counter for entries that are not words (text/punctuation)
+ * @param {String} verseText - Complete text of the entire verse for occurrence calculation
+ * @param {Number} [parentIndex=-1] - Index of parent verse object if this text is nested, -1 if at root level
+ * @return {Number} Updated nonWordVerseObjectCount after processing
  */
 var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText) {
+  var parentIndex = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : -1;
+
   if (text) {
     var tokens = tokenizer.tokenize({ text: text, includePunctuation: true });
     var tokenLength = tokens.length;
@@ -234,7 +254,7 @@ var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWord
       var pos = text.indexOf(word, lastPos);
       if (pos > lastPos) {
         // make sure we are not dropping white space
-        lastPos = fillGap(text, lastPos, pos, newVerseObjects);
+        lastPos = fillGap(text, lastPos, pos, newVerseObjects, false, parentIndex);
       }
       if (tokenizer.word.test(word) || tokenizer.number.test(word)) {
         // if the text has word or number characters, its a word object
@@ -249,7 +269,8 @@ var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWord
           occurrence: occurrence,
           occurrences: occurrences
         };
-        wordMap.push({ array: newVerseObjects, pos: newVerseObjects.length });
+        var _pos = newVerseObjects.length;
+        wordMap.push({ array: newVerseObjects, pos: _pos, parentIndex: parentIndex });
       } else {
         // the text does not have word characters
         nonWordVerseObjectCount++;
@@ -259,28 +280,46 @@ var tokenizeText = function tokenizeText(text, newVerseObjects, wordMap, nonWord
         };
       }
       lastPos += word.length;
+
+      if (parentIndex >= 0) {
+        verseObject.parentIndex = parentIndex;
+      }
+
       newVerseObjects.push(verseObject);
     }
     if (lastPos < text.length) {
-      lastPos = fillGap(text, lastPos, text.length, newVerseObjects, true);
+      lastPos = fillGap(text, lastPos, text.length, newVerseObjects, true, parentIndex);
     }
   }
   return nonWordVerseObjectCount;
 };
 
 /**
- * step through verse objects extracting words
- * @param {Array} verseObjects - original array of verse objects with words split
- * @param {Array} newVerseObjects - new array of verse objects with words split
- * @param {Array} wordMap - ordered map of word locations in verseObjects
- * @param {String} verseText - text of the entire verse
- * @param {Number} nonWordVerseObjectCount - keeps count of entries that are not actually words
- * @return {Number} updated nonWordVerseObjectCount
+ * Recursively processes nested verse objects to extract and tokenize words.
+ * Traverses through verse objects, preserving non-text objects (like milestones) while
+ * extracting and tokenizing any text content. Handles nested children recursively.
+ * Maintains parent-child relationships through parentIndex tracking.
+ *
+ * @param {Array} verseObjects - Original array of verse objects to process (may contain nested structures)
+ * @param {Array} newVerseObjects - Output array to populate with processed verse objects with words split
+ * @param {Array} wordMap - Ordered map tracking word locations in verseObjects for occurrence counting
+ * @param {String} verseText - Complete text of the entire verse for occurrence calculation
+ * @param {Number} nonWordVerseObjectCount - Counter for entries that are not words (text/punctuation)
+ * @param {Number} [parentIndex=-1] - Index of parent verse object for nested elements, -1 if at root level
+ * @return {Number} Updated nonWordVerseObjectCount after processing all verse objects
  */
 var getWordsFromNestedVerseObjects = function getWordsFromNestedVerseObjects(verseObjects, newVerseObjects, wordMap, verseText, nonWordVerseObjectCount) {
+  var parentIndex = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : -1;
+
   var voLength = verseObjects.length;
   for (var i = 0; i < voLength; i++) {
     var verseObject = verseObjects[i];
+
+    if (parentIndex >= 0) {
+      // keep track of where the parent is
+      verseObject.parentIndex = parentIndex;
+    }
+
     var vsObjText = verseObject.text;
     if (verseObject.type !== 'text') {
       // preseserve non-text verseObject except for text part which will be split into words
@@ -292,16 +331,17 @@ var getWordsFromNestedVerseObjects = function getWordsFromNestedVerseObjects(ver
         verseObject.nextChar = ' '; // preserve space before text
       }
       newVerseObjects.push(verseObject);
+      var indexOfThisObject = newVerseObjects.length - 1;
       if (verseObject.children) {
         var newChildVerseObjects = [];
-        nonWordVerseObjectCount = tokenizeText(vsObjText, newChildVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
-        nonWordVerseObjectCount = getWordsFromNestedVerseObjects(verseObject.children, newChildVerseObjects, wordMap, verseText, nonWordVerseObjectCount);
+        nonWordVerseObjectCount = tokenizeText(vsObjText, newChildVerseObjects, wordMap, nonWordVerseObjectCount, verseText, indexOfThisObject);
+        nonWordVerseObjectCount = getWordsFromNestedVerseObjects(verseObject.children, newChildVerseObjects, wordMap, verseText, nonWordVerseObjectCount, indexOfThisObject);
         verseObject.children = newChildVerseObjects;
       } else {
-        nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+        nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, indexOfThisObject);
       }
     } else {
-      nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText);
+      nonWordVerseObjectCount = tokenizeText(vsObjText, newVerseObjects, wordMap, nonWordVerseObjectCount, verseText, parentIndex);
     }
   }
   return nonWordVerseObjectCount;