proskomma-core 0.9.10 → 0.9.13

package/src/model/document_helpers/blocks.cjs

@@ -0,0 +1,135 @@
+const utils = require('../../util/index.cjs');
+const ByteArray = utils.ByteArray;
+const {
+  pushSuccinctGraftBytes,
+  pushSuccinctScopeBytes,
+  pushSuccinctTokenBytes,
+} = utils.succinct;
+const { itemEnum } = utils.itemDefs;
+const {
+  scopeEnum, scopeEnumLabels, nComponentsForScope,
+} = utils.scopeDefs;
+const { tokenCategory, tokenEnum } = utils.tokenDefs;
+const { headerBytes } = utils.succinct;
+
+const { updateBlockGrafts } = require('../doc_set_helpers/update.cjs');
+
+const deleteBlock = (document, seqId, blockN, buildCV) => {
+  if (buildCV !== false) {
+    buildCV = true;
+  }
+
+  if (!(seqId in document.sequences)) {
+    return false;
+  }
+
+  const sequence = document.sequences[seqId];
+
+  if (blockN < 0 || blockN >= sequence.blocks.length) {
+    return false;
+  }
+  sequence.blocks.splice(blockN, 1);
+
+  if (buildCV) {
+    document.buildChapterVerseIndex(this);
+  }
+  return true;
+};
+
+const newBlock = (document, seqId, blockN, blockScope, blockGrafts, buildCV) => {
+  if (buildCV !== false) {
+    buildCV = true;
+  }
+
+  if (!(seqId in document.sequences)) {
+    return false;
+  }
+
+  const sequence = document.sequences[seqId];
+
+  if (blockN < 0 || blockN > sequence.blocks.length) {
+    return false;
+  }
+
+  const docSet = document.processor.docSets[document.docSetId];
+  docSet.maybeBuildPreEnums();
+
+  const newBlock = {
+    bs: new ByteArray(0),
+    bg: new ByteArray(0),
+    c: new ByteArray(0),
+    os: new ByteArray(0),
+    is: new ByteArray(0),
+    nt: new ByteArray(0),
+  };
+  const scopeBits = blockScope.split('/');
+  const scopeTypeByte = scopeEnum[scopeBits[0]];
+  const expectedNScopeBits = nComponentsForScope(scopeBits[0]);
+
+  if (scopeBits.length !== expectedNScopeBits) {
+    throw new Error(`Scope ${blockScope} has ${scopeBits.length} component(s) (expected ${expectedNScopeBits}`);
+  }
+
+  const scopeBitBytes = scopeBits.slice(1).map(b => docSet.enumForCategoryValue('scopeBits', b, true));
+  pushSuccinctScopeBytes(newBlock.bs, itemEnum[`startScope`], scopeTypeByte, scopeBitBytes);
+  newBlock.bs.trim();
+
+  if (blockGrafts) {
+    updateBlockGrafts(
+      docSet,
+      document.id,
+      seqId,
+      blockN,
+      blockGrafts,
+    );
+  }
+  sequence.blocks.splice(blockN, 0, newBlock);
+
+  if (buildCV) {
+    document.buildChapterVerseIndex();
+  }
+  return true;
+};
+
+const rewriteBlock = (block, oldToNew) => {
+  for (const blockKey of ['bs', 'bg', 'c', 'is', 'os']) {
+    const oldBa = block[blockKey];
+    const newBa = new ByteArray(oldBa.length);
+    let pos = 0;
+
+    while (pos < oldBa.length) {
+      const [itemLength, itemType, itemSubtype] = headerBytes(oldBa, pos);
+
+      if (itemType === itemEnum['token']) {
+        if (itemSubtype === tokenEnum.wordLike) {
+          pushSuccinctTokenBytes(newBa, itemSubtype, oldToNew.wordLike[oldBa.nByte(pos + 2)]);
+        } else {
+          pushSuccinctTokenBytes(newBa, itemSubtype, oldToNew.notWordLike[oldBa.nByte(pos + 2)]);
+        }
+      } else if (itemType === itemEnum['graft']) {
+        pushSuccinctGraftBytes(newBa, oldToNew.graftTypes[itemSubtype], oldToNew.ids[oldBa.nByte(pos + 2)]);
+      } else {
+        let nScopeBitBytes = nComponentsForScope(scopeEnumLabels[itemSubtype]);
+        const scopeBitBytes = [];
+        let offset = 2;
+
+        while (nScopeBitBytes > 1) {
+          const scopeBitByte = oldToNew.scopeBits[oldBa.nByte(pos + offset)];
+          scopeBitBytes.push(scopeBitByte);
+          offset += oldBa.nByteLength(scopeBitByte);
+          nScopeBitBytes--;
+        }
+        pushSuccinctScopeBytes(newBa, itemType, itemSubtype, scopeBitBytes);
+      }
+      pos += itemLength;
+    }
+    newBa.trim();
+    block[blockKey] = newBa;
+  }
+};
+
+module.exports = {
+  newBlock,
+  deleteBlock,
+  rewriteBlock,
+};

package/src/model/document_helpers/chapter_verse.cjs

@@ -0,0 +1,291 @@
+const BitSet = require('bitset');
+
+const utils = require('../../util/index.cjs');
+const ByteArray = utils.ByteArray;
+const { itemEnum } = utils.itemDefs;
+const { tokenEnum } = utils.tokenDefs;
+
+const emptyCVIndexType = 0;
+const shortCVIndexType = 2;
+const longCVIndexType = 3;
+
+const buildChapterVerseIndex = document => {
+  const mainSequence = document.sequences[document.mainId];
+  const docSet = document.processor.docSets[document.docSetId];
+  docSet.buildPreEnums();
+  docSet.buildEnumIndexes();
+  const chapterVerseIndexes = {};
+  const chapterIndexes = {};
+  let chapterN = '0';
+  let verseN = '0';
+  let verses = '1';
+  let nextTokenN = 0;
+
+  mainSequence.chapterVerses = {};
+
+  if (docSet.enums.wordLike.length === 0) {
+    throw new Error('No wordLike content in docSet - probably a USFM issue, maybe missing \\mt?');
+  }
+  mainSequence.tokensPresent = new BitSet(
+    new Array(docSet.enums.wordLike.length)
+      .fill(0)
+      .map(b => b.toString())
+      .join(''),
+  );
+
+  for (const [blockN, block] of mainSequence.blocks.entries()) {
+    let pos = 0;
+    let succinct = block.c;
+    let itemN = -1;
+
+    while (pos < succinct.length) {
+      itemN++;
+      const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(succinct, pos);
+
+      if (itemType === itemEnum['startScope']) {
+        let scopeLabel = docSet.succinctScopeLabel(succinct, itemSubtype, pos);
+
+        if (scopeLabel.startsWith('chapter/')) {
+          chapterN = scopeLabel.split('/')[1];
+          chapterVerseIndexes[chapterN] = {};
+          chapterIndexes[chapterN] = {
+            startBlock: blockN,
+            startItem: itemN,
+            nextToken: nextTokenN,
+          };
+        } else if (scopeLabel.startsWith('verse/')) {
+          verseN = scopeLabel.split('/')[1];
+
+          if (verseN === '1' && !('0' in chapterVerseIndexes[chapterN])) {
+            if (chapterIndexes[chapterN].nextToken < nextTokenN) {
+              chapterVerseIndexes[chapterN]['0'] = [{
+                startBlock: chapterIndexes[chapterN].startBlock,
+                startItem: chapterIndexes[chapterN].startItem,
+                endBlock: blockN,
+                endItem: Math.max(itemN - 1, 0),
+                nextToken: chapterIndexes[chapterN].nextToken,
+                verses: '0',
+              }];
+            }
+          }
+
+          if (!(verseN in chapterVerseIndexes[chapterN])) {
+            chapterVerseIndexes[chapterN][verseN] = [];
+          }
+          chapterVerseIndexes[chapterN][verseN].push({
+            startBlock: blockN,
+            startItem: itemN,
+            nextToken: nextTokenN,
+          });
+        } else if (scopeLabel.startsWith('verses/')) {
+          verses = scopeLabel.split('/')[1];
+        }
+      } else if (itemType === itemEnum['endScope']) {
+        let scopeLabel = docSet.succinctScopeLabel(succinct, itemSubtype, pos);
+
+        if (scopeLabel.startsWith('chapter/')) {
+          chapterN = scopeLabel.split('/')[1];
+          let chapterRecord = chapterIndexes[chapterN];
+
+          if (chapterRecord) { // Check start chapter has not been deleted
+            chapterRecord.endBlock = blockN;
+            chapterRecord.endItem = itemN;
+          }
+        } else if (scopeLabel.startsWith('verse/')) {
+          verseN = scopeLabel.split('/')[1];
+          let versesRecord = chapterVerseIndexes[chapterN][verseN];
+
+          if (versesRecord) { // Check start verse has not been deleted
+            const verseRecord = chapterVerseIndexes[chapterN][verseN][chapterVerseIndexes[chapterN][verseN].length - 1];
+            verseRecord.endBlock = blockN;
+            verseRecord.endItem = itemN;
+            verseRecord.verses = verses;
+          }
+        }
+      } else if (itemType === itemEnum['token'] && itemSubtype === tokenEnum['wordLike']) {
+        mainSequence.tokensPresent
+          .set(
+            succinct.nByte(pos + 2),
+            1,
+          );
+        nextTokenN++;
+      }
+      pos += itemLength;
+    }
+  }
+
+  for (const [chapterN, chapterVerses] of Object.entries(chapterVerseIndexes)) {
+    const ba = new ByteArray();
+    mainSequence.chapterVerses[chapterN] = ba;
+    const sortedVerses = Object.keys(chapterVerses)
+      .map(n => parseInt(n))
+      .sort((a, b) => a - b);
+
+    if (sortedVerses.length === 0) {
+      continue;
+    }
+
+    const maxVerse = sortedVerses[sortedVerses.length - 1];
+    const verseSlots = Array.from(Array(maxVerse + 1).keys());
+    let pos = 0;
+
+    for (const verseSlot of verseSlots) {
+      const verseKey = `${verseSlot}`;
+
+      if (verseKey in chapterVerses) {
+        const verseElements = chapterVerses[verseKey];
+        const nVerseElements = verseElements.length;
+
+        for (const [verseElementN, verseElement] of verseElements.entries()) {
+          if (!verseElement.verses) {
+            console.log(`** VerseElement without verses for ${verseSlot} in buildChapterVerseIndex`);
+            continue;
+          }
+
+          const versesEnumIndex = docSet.enumForCategoryValue('scopeBits', verseElement.verses);
+          const recordType = verseElement.startBlock === verseElement.endBlock ? shortCVIndexType : longCVIndexType;
+          ba.pushByte(0);
+
+          if (recordType === shortCVIndexType) {
+            ba.pushNBytes([
+              verseElement.startBlock,
+              verseElement.startItem,
+              verseElement.endItem,
+              verseElement.nextToken,
+              versesEnumIndex,
+            ]);
+          } else {
+            ba.pushNBytes([
+              verseElement.startBlock,
+              verseElement.endBlock,
+              verseElement.startItem,
+              verseElement.endItem,
+              verseElement.nextToken,
+              versesEnumIndex,
+            ]);
+          }
+          ba.setByte(pos, makeVerseLengthByte(recordType, verseElementN === (nVerseElements - 1), ba.length - pos));
+          pos = ba.length;
+        }
+      } else {
+        ba.pushByte(makeVerseLengthByte(emptyCVIndexType, true, 1));
+        pos++;
+      }
+    }
+    ba.trim();
+  }
+  mainSequence.chapters = {};
+
+  for (const [chapterN, chapterElement] of Object.entries(chapterIndexes)) {
+    if (!('startBlock' in chapterElement) || !('endBlock' in chapterElement)) {
+      continue;
+    }
+
+    const ba = new ByteArray();
+    mainSequence.chapters[chapterN] = ba;
+    const recordType = chapterElement.startBlock === chapterElement.endBlock ? shortCVIndexType : longCVIndexType;
+    ba.pushByte(0);
+
+    if (recordType === shortCVIndexType) {
+      ba.pushNBytes([chapterElement.startBlock, chapterElement.startItem, chapterElement.endItem, chapterElement.nextToken]);
+    } else {
+      ba.pushNBytes([chapterElement.startBlock, chapterElement.endBlock, chapterElement.startItem, chapterElement.endItem, chapterElement.nextToken]);
+    }
+    ba.setByte(0, makeVerseLengthByte(recordType, true, ba.length));
+    ba.trim();
+  }
+};
+
+const chapterVerseIndex = (document, chapN) => {
+  const docSet = document.processor.docSets[document.docSetId];
+  docSet.buildEnumIndexes();
+  const ret = [];
+  const succinct = document.sequences[document.mainId].chapterVerses[chapN];
+
+  if (succinct) {
+    let pos = 0;
+    let currentVerseRecord = [];
+
+    while (pos < succinct.length) {
+      const [recordType, isLast, recordLength] = verseLengthByte(succinct, pos);
+
+      if (recordType === shortCVIndexType) {
+        const nBytes = succinct.nBytes(pos + 1, 5);
+
+        currentVerseRecord.push({
+          startBlock: nBytes[0],
+          endBlock: nBytes[0],
+          startItem: nBytes[1],
+          endItem: nBytes[2],
+          nextToken: nBytes[3],
+          verses: docSet.enums.scopeBits.countedString(docSet.enumIndexes.scopeBits[nBytes[4]]),
+        });
+      } else if (recordType === longCVIndexType) {
+        const nBytes = succinct.nBytes(pos + 1, 6);
+
+        currentVerseRecord.push({
+          startBlock: nBytes[0],
+          endBlock: nBytes[1],
+          startItem: nBytes[2],
+          endItem: nBytes[3],
+          nextToken: nBytes[4],
+          verses: docSet.enums.scopeBits.countedString(docSet.enumIndexes.scopeBits[nBytes[5]]),
+        });
+      }
+
+      if (isLast) {
+        ret.push(currentVerseRecord);
+        currentVerseRecord = [];
+      }
+      pos += recordLength;
+    }
+  }
+  return ret;
+};
+
+const chapterIndex = (document, chapN) => {
+  const succinct = document.sequences[document.mainId].chapters[chapN];
+
+  if (succinct) {
+    const recordType = verseLengthByte(succinct, 0)[0];
+
+    if (recordType === shortCVIndexType) {
+      const nBytes = succinct.nBytes(1, 4);
+
+      return {
+        startBlock: nBytes[0],
+        endBlock: nBytes[0],
+        startItem: nBytes[1],
+        endItem: nBytes[2],
+        nextToken: nBytes[3],
+      };
+    } else if (recordType === longCVIndexType) {
+      const nBytes = succinct.nBytes(1, 5);
+
+      return {
+        startBlock: nBytes[0],
+        endBlock: nBytes[1],
+        startItem: nBytes[2],
+        endItem: nBytes[3],
+        nextToken: nBytes[4],
+      };
+    }
+  }
+};
+
+const makeVerseLengthByte = (recordType, isLast, length) => length + (isLast ? 32 : 0) + (recordType * 64);
+
+const verseLengthByte= (succinct, pos) => {
+  const sByte = succinct.byte(pos);
+  return [
+    sByte >> 6,
+    (sByte >> 5) % 2 === 1,
+    sByte % 32,
+  ];
+};
+
+module.exports = {
+  buildChapterVerseIndex,
+  chapterVerseIndex,
+  chapterIndex,
+};

package/src/model/document_helpers/pre_enums.cjs

@@ -0,0 +1,67 @@
+const utils = require('../../util/index.cjs');
+
+const recordPreEnums = (docSet, seq) => {
+  docSet.recordPreEnum('scopeBits', '0');
+
+  for (const [blockN, block] of seq.blocks.entries()) {
+    for (const [itemN, item] of [...block.items, block.bs, ...block.bg].entries()) {
+      if (item.subType === 'wordLike') {
+        docSet.recordPreEnum('wordLike', item.payload);
+      } else if (['lineSpace', 'eol', 'punctuation', 'softLineBreak', 'bareSlash', 'unknown'].includes(item.subType)) {
+        docSet.recordPreEnum('notWordLike', item.payload);
+      } else if (item.type === 'graft') {
+        docSet.recordPreEnum('graftTypes', item.subType);
+      } else if (item.subType === 'start') {
+        const labelBits = item.payload.split('/');
+
+        if (labelBits.length !== utils.scopeDefs.nComponentsForScope(labelBits[0])) {
+          throw new Error(`Scope ${item.payload} has unexpected number of components`);
+        }
+
+        for (const labelBit of labelBits.slice(1)) {
+          docSet.recordPreEnum('scopeBits', labelBit);
+        }
+      }
+    }
+  }
+};
+
+const rerecordPreEnums = (docSet, seq) => {
+  docSet.recordPreEnum('scopeBits', '0');
+  docSet.recordPreEnum('ids', seq.id);
+
+  for (const block of seq.blocks) {
+    for (const blockKey of ['bs', 'bg', 'c', 'is', 'os']) {
+      rerecordBlockPreEnums(docSet, block[blockKey]);
+    }
+  }
+};
+
+const rerecordBlockPreEnums= (docSet, ba) => {
+  for (const item of docSet.unsuccinctifyItems(ba, {}, 0)) {
+    if (item[0] === 'token') {
+      if (item[1] === 'wordLike') {
+        docSet.recordPreEnum('wordLike', item[2]);
+      } else {
+        docSet.recordPreEnum('notWordLike', item[2]);
+      }
+    } else if (item[0] === 'graft') {
+      docSet.recordPreEnum('graftTypes', item[1]);
+    } else if (item[0] === 'scope' && item[1] === 'start') {
+      const labelBits = item[2].split('/');
+
+      if (labelBits.length !== utils.scopeDefs.nComponentsForScope(labelBits[0])) {
+        throw new Error(`Scope ${item[2]} has unexpected number of components`);
+      }
+
+      for (const labelBit of labelBits.slice(1)) {
+        docSet.recordPreEnum('scopeBits', labelBit);
+      }
+    }
+  }
+};
+
+module.exports = {
+  recordPreEnums,
+  rerecordPreEnums,
+};

package/src/model/document_helpers/sequences.cjs

@@ -0,0 +1,205 @@
+const deepCopy = require('deep-copy-all');
+
+const utils = require('../../util/index.cjs');
+
+const gcSequences = document => {
+  const usedSequences = new Set();
+  const docSet = document.processor.docSets[document.docSetId];
+  docSet.maybeBuildEnumIndexes();
+
+  const followGrafts = (document, sequence, used) => {
+    used.add(sequence.id);
+
+    for (const block of sequence.blocks) {
+      for (const blockGraft of docSet.unsuccinctifyGrafts(block.bg)) {
+        if (!used.has(blockGraft[2])) {
+          followGrafts(document, document.sequences[blockGraft[2]], used);
+        }
+      }
+
+      for (const inlineGraft of docSet.unsuccinctifyItems(block.c, { grafts: true }, 0)) {
+        if (!used.has(inlineGraft[2])) {
+          followGrafts(document, document.sequences[inlineGraft[2]], used);
+        }
+      }
+    }
+  };
+
+  followGrafts(document, document.sequences[document.mainId], usedSequences);
+  let changed = false;
+
+  for (const sequenceId of Object.keys(document.sequences)) {
+    if (!usedSequences.has(sequenceId)) {
+      delete document.sequences[sequenceId];
+      changed = true;
+    }
+  }
+
+  return changed;
+};
+
+const newSequence = (document, seqType, tags) => {
+  const seqId = utils.generateId();
+
+  document.sequences[seqId] = {
+    id: seqId,
+    type: seqType,
+    tags: new Set(tags || []),
+    isBaseType: (seqType in document.baseSequenceTypes),
+    blocks: [],
+  };
+
+  return seqId;
+};
+
+const deleteSequence = (document, seqId) => {
+  if (!(seqId in document.sequences)) {
+    return false;
+  }
+
+  if (document.sequences[seqId].type === 'main') {
+    throw new Error('Cannot delete main sequence');
+  }
+
+  if (document.sequences[seqId].type in document.baseSequenceTypes) {
+    gcSequenceReferences(document,'block', seqId);
+  } else {
+    gcSequenceReferences(document,'inline', seqId);
+  }
+  delete document.sequences[seqId];
+  document.buildChapterVerseIndex(this);
+  document.gcSequences();
+  return true;
+};
+
+const gcSequenceReferences = (document, seqContext, seqId) => {
+  const docSet = document.processor.docSets[document.docSetId];
+
+  for (const sequence of Object.values(document.sequences)) {
+    for (const block of sequence.blocks) {
+      const succinct = seqContext === 'block' ? block.bg : block.c;
+      let pos = 0;
+
+      while (pos < succinct.length) {
+        const [itemLength, itemType] = utils.succinct.headerBytes(succinct, pos);
+
+        if (itemType !== utils.itemDefs.itemEnum.graft) {
+          pos += itemLength;
+        } else {
+          const graftSeqId = utils.succinct.succinctGraftSeqId(docSet.enums, docSet.enumIndexes, succinct, pos);
+
+          if (graftSeqId === seqId) {
+            succinct.deleteItem(pos);
+          } else {
+            pos += itemLength;
+          }
+        }
+      }
+    }
+  }
+};
+
+const modifySequence = (
+  document,
+  seqId,
+  sequenceRewriteFunc,
+  blockFilterFunc,
+  itemFilterFunc,
+  blockRewriteFunc,
+  itemRewriteFunc,
+) => {
+  const docSet = document.processor.docSets[document.docSetId];
+  docSet.maybeBuildEnumIndexes();
+  sequenceRewriteFunc = sequenceRewriteFunc || (s => s);
+  const oldSequence = document.sequences[seqId];
+  const newSequence = sequenceRewriteFunc({
+    id: seqId,
+    type: oldSequence.type,
+    tags: oldSequence.tags,
+    isBaseType: oldSequence.isBaseType,
+    verseMapping: oldSequence.verseMapping,
+  });
+
+  pushModifiedBlocks(
+    oldSequence,
+    newSequence,
+    blockFilterFunc,
+    itemFilterFunc,
+    blockRewriteFunc,
+    itemRewriteFunc,
+  );
+  document.sequences[seqId] = newSequence;
+
+  if (newSequence.type === 'main') {
+    document.buildChapterVerseIndex();
+  }
+  return newSequence;
+};
+
+const pushModifiedBlocks = (
+  oldSequence,
+  newSequence,
+  blockFilterFunc,
+  itemFilterFunc,
+  blockRewriteFunc,
+  itemRewriteFunc,
+) => {
+  blockFilterFunc = blockFilterFunc || ((oldSequence, blockN, block) => !!block);
+  itemFilterFunc = itemFilterFunc ||
+    ((oldSequence, oldBlockN, block, itemN, itemType, itemSubType, pos) => !!block || pos);
+  blockRewriteFunc = blockRewriteFunc || ((oldSequence, blockN, block) => block);
+  itemRewriteFunc = itemRewriteFunc ||
+    (
+      (oldSequence, oldBlockN, oldBlock, newBlock, itemN, itemLength, itemType, itemSubType, pos) => {
+        for (let n = 0; n < itemLength; n++) {
+          newBlock.c.pushByte(oldBlock.c.byte(pos + n));
+        }
+      }
+    );
+  newSequence.blocks = [];
+
+  for (const [blockN, block] of oldSequence.blocks.entries()) {
+    if (blockFilterFunc(oldSequence, blockN, block)) {
+      const newBlock = blockRewriteFunc(oldSequence, blockN, deepCopy(block));
+      newBlock.c.clear();
+      modifyBlockItems(
+        oldSequence,
+        blockN,
+        block,
+        newBlock,
+        itemFilterFunc,
+        itemRewriteFunc,
+      );
+      newSequence.blocks.push(newBlock);
+    }
+  }
+};
+
+const modifyBlockItems = (
+  oldSequence,
+  oldBlockN,
+  oldBlock,
+  newBlock,
+  itemFilterFunc,
+  itemRewriteFunc,
+) => {
+  let pos = 0;
+  let itemN = -1;
+
+  while (pos < oldBlock.c.length) {
+    itemN++;
+    const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(oldBlock.c, pos);
+
+    if (itemFilterFunc(oldSequence, oldBlockN, oldBlock, itemN, itemType, itemSubtype, pos)) {
+      itemRewriteFunc(oldSequence, oldBlockN, oldBlock, newBlock, itemN, itemLength, itemType, itemSubtype, pos);
+    }
+    pos += itemLength;
+  }
+};
+
+module.exports = {
+  newSequence,
+  gcSequences,
+  deleteSequence,
+  modifySequence,
+};