proskomma-core 0.9.10 → 0.9.13

package/src/model/doc_set.cjs

@@ -0,0 +1,389 @@
+/* eslint-disable no-unused-vars */
+const checksum = require('checksum');
+const utils = require('../util/index.cjs');
+const ByteArray = utils.ByteArray;
+const {
+  addTag, removeTag, validateTags,
+} = utils.tags;
+const {
+  succinctGraftName,
+  succinctGraftSeqId,
+  succinctScopeLabel,
+  succinctTokenChars,
+  headerBytes,
+  enumIndex,
+  enumIndexes,
+} = utils.succinct;
+const { itemEnum } = utils.itemDefs;
+const { tokenEnumLabels } = utils.tokenDefs;
+const { validateSelectors } = require('./doc_set_helpers/selectors.cjs');
+const {
+  blocksWithScriptureCV,
+  allBlockScopes,
+  anyScopeInBlock,
+  allScopesInBlock,
+  blockHasBlockScope,
+  blockHasChars,
+  blockHasMatchingItem,
+} = require('./doc_set_helpers/block.cjs');
+const {
+  unsuccinctifyBlock,
+  unsuccinctifyItems,
+  unsuccinctifyItem,
+  unsuccinctifyPrunedItems,
+  unsuccinctifyItemsWithScriptureCV,
+} = require('./doc_set_helpers/unsuccinctify.cjs');
+const {
+  buildPreEnum,
+  recordPreEnum,
+  buildEnum,
+  enumForCategoryValue,
+} = require('./doc_set_helpers/enum.cjs');
+const {
+  countItems,
+  itemsByIndex,
+  sequenceItemsByScopes,
+  sequenceItemsByMilestones,
+} = require('./doc_set_helpers/item.cjs');
+const {
+  rehash,
+  makeRehashEnumMap,
+} = require('./doc_set_helpers/rehash.cjs');
+const {
+  updateItems,
+  updateBlockGrafts,
+  updateBlockScope,
+  updateOpenScopes,
+  updateIncludedScopes,
+  updateBlockIndexesAfterEdit,
+  updateBlockIndexesAfterFilter,
+} = require('./doc_set_helpers/update.cjs');
+const { serializeSuccinct } = require('./doc_set_helpers/serialize.cjs');
+class DocSet {
+  constructor(processor, selectors, tags, succinctJson) {
+    this.processor = processor;
+    this.preEnums = {};
+    this.enumIndexes = {};
+    this.docIds = [];
+
+    if (succinctJson) {
+      this.fromSuccinct(processor, succinctJson);
+    } else {
+      this.fromScratch(processor, selectors, tags);
+    }
+    validateTags(this.tags);
+  }
+
+  fromScratch(processor, selectors, tags) {
+    const defaultedSelectors = selectors || processor.selectors;
+    this.selectors = validateSelectors(this, defaultedSelectors);
+    this.id = this.selectorString();
+    this.tags = new Set(tags || []);
+    this.enums = {
+      ids: new ByteArray(512),
+      wordLike: new ByteArray(8192),
+      notWordLike: new ByteArray(256),
+      scopeBits: new ByteArray(256),
+      graftTypes: new ByteArray(10),
+    };
+  }
+
+  fromSuccinct(processor, succinctJson) {
+    const populatedByteArray = (succinct) => {
+      const ret = new ByteArray(256);
+      ret.fromBase64(succinct);
+      ret.trim();
+      return ret;
+    };
+
+    this.id = succinctJson.id;
+    this.selectors = validateSelectors(this, succinctJson.metadata.selectors);
+    this.tags = new Set(succinctJson.tags);
+    validateTags(this.tags);
+    this.preEnums = {};
+    this.enums = {
+      ids: populatedByteArray(succinctJson.enums.ids),
+      wordLike: populatedByteArray(succinctJson.enums.wordLike),
+      notWordLike: populatedByteArray(succinctJson.enums.notWordLike),
+      scopeBits: populatedByteArray(succinctJson.enums.scopeBits),
+      graftTypes: populatedByteArray(succinctJson.enums.graftTypes),
+    };
+    this.enumIndexes = {};
+    this.docIds = [];
+  }
+
+  addTag(tag) {
+    addTag(this.tags, tag);
+  }
+
+  removeTag(tag) {
+    removeTag(this.tags, tag);
+  }
+
+  selectorString() {
+    return this.processor.selectorString(this.selectors);
+  }
+
+  documents() {
+    return this.docIds.map(did => this.processor.documents[did]);
+  }
+
+  documentWithBook(bookCode) {
+    const docsWithBook = Object.values(this.documents()).filter(doc => 'bookCode' in doc.headers && doc.headers['bookCode'] === bookCode);
+    return docsWithBook.length === 1 ? docsWithBook[0] : null;
+  }
+
+  maybeBuildPreEnums() {
+    if (Object.keys(this.preEnums).length === 0) {
+      this.buildPreEnums();
+    }
+  }
+
+  buildPreEnums() {
+    for (const [category, succinct] of Object.entries(this.enums)) {
+      this.preEnums[category] = buildPreEnum(this, succinct);
+    }
+  }
+
+  recordPreEnum(category, value) {
+    recordPreEnum(this, category, value);
+  }
+
+  sortPreEnums() {
+    for (const catKey of Object.keys(this.preEnums)) {
+      this.preEnums[catKey] = new Map([...this.preEnums[catKey].entries()].sort((a, b) => b[1].frequency - a[1].frequency));
+
+      let count = 0;
+
+      for (const [k, v] of this.preEnums[catKey]) {
+        v.enum = count++;
+      }
+    }
+  }
+
+  enumForCategoryValue(category, value, addUnknown) {
+    return enumForCategoryValue(this, category, value, addUnknown);
+  }
+
+  buildEnums() {
+    for (const [category, catOb] of Object.entries(this.preEnums)) {
+      this.enums[category].clear();
+      this.buildEnum(category, catOb);
+    }
+  }
+
+  buildEnum(category, preEnumOb) {
+    buildEnum(this, category, preEnumOb);
+  }
+
+  maybeBuildEnumIndexes() {
+    if (Object.keys(this.enumIndexes).length === 0) {
+      this.buildEnumIndexes();
+    }
+  }
+
+  buildEnumIndexes() {
+    this.enumIndexes = enumIndexes(this.enums);
+  }
+
+  buildEnumIndex(category) {
+    this.enumIndexes[category] = enumIndex(category, this.enums[category]);
+  }
+
+  unsuccinctifyBlock(block, options) {
+    return unsuccinctifyBlock(this, block, options);
+  }
+
+  unsuccinctifyItems(succinct, options, nextToken, openScopes) {
+    return unsuccinctifyItems(this, succinct, options, nextToken, openScopes);
+  }
+
+  unsuccinctifyItem(succinct, pos, options) {
+    return unsuccinctifyItem(this, succinct, pos, options);
+  }
+
+  unsuccinctifyPrunedItems(block, options) {
+    return unsuccinctifyPrunedItems(this, block, options);
+  }
+
+  unsuccinctifyScopes(succinct) {
+    const ret = [];
+    let pos = 0;
+
+    while (pos < succinct.length) {
+      const [itemLength, itemType, itemSubtype] = headerBytes(succinct, pos);
+      ret.push(this.unsuccinctifyScope(succinct, itemType, itemSubtype, pos));
+      pos += itemLength;
+    }
+    return ret;
+  }
+
+  unsuccinctifyGrafts(succinct) {
+    const ret = [];
+    let pos = 0;
+
+    while (pos < succinct.length) {
+      const [itemLength, itemType, itemSubtype] = headerBytes(succinct, pos);
+      ret.push(this.unsuccinctifyGraft(succinct, itemSubtype, pos));
+      pos += itemLength;
+    }
+    return ret;
+  }
+
+  unsuccinctifyToken(succinct, itemSubtype, pos) {
+    try {
+      return [
+        'token',
+        tokenEnumLabels[itemSubtype],
+        this.succinctTokenChars(succinct, itemSubtype, pos),
+      ];
+    } catch (err) {
+      throw new Error(`Error from unsuccinctifyToken: ${err}`);
+    }
+  }
+
+  unsuccinctifyScope(succinct, itemType, itemSubtype, pos) {
+    try {
+      return [
+        'scope',
+        (itemType === itemEnum.startScope) ? 'start' : 'end',
+        this.succinctScopeLabel(succinct, itemSubtype, pos),
+      ];
+    } catch (err) {
+      throw new Error(`Error from unsuccinctifyScope: ${err}`);
+    }
+  }
+
+  unsuccinctifyGraft(succinct, itemSubtype, pos) {
+    try {
+      return [
+        'graft',
+        this.succinctGraftName(itemSubtype),
+        this.succinctGraftSeqId(succinct, pos),
+      ];
+    } catch (err) {
+      throw new Error(`Error from unsuccinctifyGraft: ${err}`);
+    }
+  }
+
+  unsuccinctifyBlockScopeLabelsSet(block) {
+    const [itemLength, itemType, itemSubtype] = headerBytes(block.bs, 0);
+    const blockScope = this.unsuccinctifyScope(block.bs, itemType, itemSubtype, 0);
+    return new Set(
+      this.unsuccinctifyScopes(block.os).concat(
+        this.unsuccinctifyScopes(block.is),
+      ).concat([blockScope])
+        .map(ri => ri[2]));
+  }
+
+  unsuccinctifyItemsWithScriptureCV(block, cv, options) {
+    return unsuccinctifyItemsWithScriptureCV(this, block, cv, options);
+  }
+
+  succinctTokenChars(succinct, itemSubtype, pos) {
+    return succinctTokenChars(this.enums, this.enumIndexes, succinct, itemSubtype, pos);
+  }
+
+  succinctScopeLabel(succinct, itemSubtype, pos) {
+    return succinctScopeLabel(this.enums, this.enumIndexes, succinct, itemSubtype, pos);
+  }
+
+  succinctGraftName(itemSubtype) {
+    return succinctGraftName(this.enums, this.enumIndexes, itemSubtype);
+  }
+
+  succinctGraftSeqId(succinct, pos) {
+    return succinctGraftSeqId(this.enums, this.enumIndexes, succinct, pos);
+  }
+
+  countItems(succinct) {
+    return countItems(this, succinct);
+  }
+
+  itemsByIndex(mainSequence, index, includeContext) {
+    return itemsByIndex(this, mainSequence, index, includeContext);
+  }
+
+  blocksWithScriptureCV(blocks, cv) {
+    return blocksWithScriptureCV(this, blocks, cv);
+  }
+
+  allBlockScopes(block) {
+    return allBlockScopes(this, block);
+  }
+
+  allScopesInBlock(block, scopes) {
+    return allScopesInBlock(this, block, scopes);
+  }
+
+  anyScopeInBlock(block, scopes) {
+    return anyScopeInBlock(this, block, scopes);
+  }
+
+  blockHasBlockScope(block, scope) {
+    return blockHasBlockScope(this, block, scope);
+  }
+
+  blockHasChars(block, charsIndexes) {
+    return blockHasChars(this, block, charsIndexes);
+  }
+
+  blockHasMatchingItem(block, testFunction, options) {
+    return blockHasMatchingItem(this, block, testFunction, options);
+  }
+
+  sequenceItemsByScopes(blocks, byScopes) {
+    return sequenceItemsByScopes(this, blocks, byScopes);
+  }
+
+  sequenceItemsByMilestones(blocks, byMilestones) {
+    return sequenceItemsByMilestones(this, blocks, byMilestones);
+  }
+
+  rehash() {
+    return rehash(this);
+  }
+
+  makeRehashEnumMap() {
+    return makeRehashEnumMap(this);
+  }
+
+  updateItems(documentId, sequenceId, blockPosition, itemObjects) {
+    return updateItems(this, documentId, sequenceId, blockPosition, itemObjects);
+  }
+
+  updateBlockGrafts(documentId, sequenceId, blockPosition, itemObjects) {
+    return updateBlockGrafts(this, documentId, sequenceId, blockPosition, itemObjects);
+  }
+
+  updateBlockScope(documentId, sequenceId, blockPosition, bsObject) {
+    return updateBlockScope(this, documentId, sequenceId, blockPosition, bsObject);
+  }
+
+  updateOpenScopes(documentId, sequenceId, blockPosition, osObjects) {
+    return updateOpenScopes(this, documentId, sequenceId, blockPosition, osObjects);
+  }
+
+  updateIncludedScopes(documentId, sequenceId, blockPosition, isObjects) {
+    return updateIncludedScopes(this, documentId, sequenceId, blockPosition, isObjects);
+  }
+
+  updateBlockIndexesAfterEdit(sequence, blockPosition) {
+    updateBlockIndexesAfterEdit(this, sequence, blockPosition);
+  }
+
+  updateBlockIndexesAfterFilter(sequence) {
+    updateBlockIndexesAfterFilter(this, sequence);
+  }
+
+  serializeSuccinct() {
+    return serializeSuccinct(this);
+  }
+
+  checksum() {
+    const docIdsString = [...this.docIds].sort().join(' ');
+    return checksum(docIdsString);
+  }
+}
+
+module.exports = { DocSet };

package/src/model/doc_set_helpers/block.cjs

@@ -0,0 +1,232 @@
+const xre = require('xregexp');
+const utils = require('../../util/index.cjs');
+
+const blocksWithScriptureCV = (docSet, blocks, cv) => {
+  const hasMiddleChapter = (b, fromC, toC) => {
+    const blockChapterScopes = [
+      ...docSet.unsuccinctifyScopes(b.os).map(s => s[2]),
+      ...docSet.unsuccinctifyScopes(b.is).map(s => s[2]),
+    ].filter(s => s.startsWith('chapter/'));
+    return blockChapterScopes.map(s => parseInt(s.split('/')[1])).filter(n => n > fromC && n < toC).length > 0;
+  };
+
+  const hasFirstChapter = (b, fromC, fromV) => {
+    const hasFirstChapterScope = [
+      ...docSet.unsuccinctifyScopes(b.os).map(s => s[2]),
+      ...docSet.unsuccinctifyScopes(b.is).map(s => s[2]),
+    ].includes(`chapter/${fromC}`);
+    return hasFirstChapterScope &&
+      docSet.blockHasMatchingItem(
+        b,
+        (item, openScopes) => {
+          if (!openScopes.has(`chapter/${fromC}`)) {
+            return false;
+          }
+          return (
+            Array.from(openScopes)
+              .filter(s => s.startsWith('verse/'))
+              .filter(s => parseInt(s.split('/')[1]) >= fromV).length
+            > 0
+            ||
+            (
+              fromV === 0 &&
+              item[0] === 'token' &&
+              item[2] &&
+              Array.from(openScopes)
+                .filter(s => s.startsWith('verse'))
+                .length === 0
+            )
+          );
+        },
+        {},
+      );
+  };
+
+  const hasLastChapter = (b, toC, toV) => {
+    const hasLastChapterScope = [
+      ...docSet.unsuccinctifyScopes(b.os).map(s => s[2]),
+      ...docSet.unsuccinctifyScopes(b.is).map(s => s[2]),
+    ].includes(`chapter/${toC}`);
+    return hasLastChapterScope &&
+      docSet.blockHasMatchingItem(
+        b,
+        (item, openScopes) => {
+          if (!openScopes.has(`chapter/${toC}`)) {
+            return false;
+          }
+          return (
+            Array.from(openScopes)
+              .filter(s => s.startsWith('verse/'))
+              .filter(s => parseInt(s.split('/')[1]) <= toV).length
+            > 0
+            ||
+            (
+              toV === 0 &&
+              item[0] === 'token' &&
+              item[2] &&
+              Array.from(openScopes)
+                .filter(s => s.startsWith('verse'))
+                .length === 0
+            )
+
+          );
+        },
+        {},
+      );
+  };
+
+  if (xre.exec(cv, xre('^[1-9][0-9]*$'))) {
+    const scopes = [`chapter/${cv}`];
+    return blocks.filter(b => docSet.allScopesInBlock(b, scopes));
+  } else if (xre.exec(cv, xre('^[1-9][0-9]*-[1-9][0-9]*$'))) {
+    const [fromC, toC] = cv.split('-').map(v => parseInt(v));
+
+    if (fromC > toC) {
+      throw new Error(`Chapter range must be from min to max, not '${cv}'`);
+    }
+
+    const scopes = [...Array((toC - fromC) + 1).keys()].map(n => `chapter/${n + fromC}`);
+    return blocks.filter(b => docSet.anyScopeInBlock(b, scopes));
+  } else if (xre.exec(cv, xre('^[1-9][0-9]*:[0-9]+$'))) {
+    const [fromC, fromV] = cv.split(':').map(v => parseInt(v));
+
+    if (fromV === 0) {
+      const scopes = [`chapter/${fromC}`];
+      return blocks
+        .filter(b => docSet.allScopesInBlock(b, scopes))
+        .filter(
+          b =>
+            [...docSet.allBlockScopes(b)]
+              .filter(s => s.startsWith('verse')).length === 0,
+        );
+    } else {
+      const scopes = [`chapter/${fromC}`, `verse/${fromV}`];
+      return blocks.filter(b => docSet.allScopesInBlock(b, scopes));
+    }
+  } else if (xre.exec(cv, xre('^[1-9][0-9]*:[0-9]+-[1-9][0-9]*$'))) {
+    const [fromC, vs] = cv.split(':');
+    const [fromV, toV] = vs.split('-').map(v => parseInt(v));
+
+    if (fromV > toV) {
+      throw new Error(`Verse range must be from min to max, not '${vs}'`);
+    }
+
+    const chapterScopes = [`chapter/${fromC}`];
+    const verseScopes = [...Array((toV - fromV) + 1).keys()].map(n => `verse/${n + fromV}`);
+    return blocks
+      .filter(b => docSet.allScopesInBlock(b, chapterScopes))
+      .filter(
+        b =>
+          docSet.anyScopeInBlock(b, verseScopes) ||
+          (
+            fromV === 0 &&
+            [...docSet.allBlockScopes(b)]
+              .filter(s => s.startsWith('verse')).length === 0
+          ),
+      );
+  } else if (xre.exec(cv, xre('^[1-9][0-9]*:[0-9]+-[1-9][0-9]*:[0-9]+$'))) {
+    const [fromCV, toCV] = cv.split('-');
+    const [fromC, fromV] = fromCV.split(':').map(c => parseInt(c));
+    const [toC, toV] = toCV.split(':').map(v => parseInt(v));
+
+    if (fromC > toC) {
+      throw new Error(`Chapter range must be from min to max, not '${fromC}-${toV}'`);
+    }
+
+    const chapterScopes = [...Array((toC - fromC) + 1).keys()].map(n => `chapter/${n + fromC}`);
+    const chapterBlocks = blocks.filter(b => docSet.anyScopeInBlock(b, chapterScopes));
+    return chapterBlocks.filter(b => hasMiddleChapter(b, fromC, toC) || hasFirstChapter(b, fromC, fromV) || hasLastChapter(b, toC, toV));
+  } else {
+    throw new Error(`Bad cv reference '${cv}'`);
+  }
+};
+
+const allBlockScopes = (docSet, block) => {
+  const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(block.bs, 0);
+  const blockScope = docSet.unsuccinctifyScope(block.bs, itemType, itemSubtype, 0);
+  return new Set([
+    ...docSet.unsuccinctifyScopes(block.os).map(s => s[2]),
+    ...docSet.unsuccinctifyScopes(block.is).map(s => s[2]),
+    blockScope[2],
+  ],
+  );
+};
+
+const allScopesInBlock = (docSet, block, scopes) => {
+  const allBlockScopes = docSet.allBlockScopes(block);
+
+  for (const scope of scopes) {
+    if (!allBlockScopes.has(scope)) {
+      return false;
+    }
+  }
+  return true;
+};
+
+const anyScopeInBlock = (docSet, block, scopes) => {
+  const allBlockScopes = docSet.allBlockScopes(block);
+
+  for (const scope of scopes) {
+    if (allBlockScopes.has(scope)) {
+      return true;
+    }
+  }
+  return false;
+};
+
+const blockHasBlockScope = (docSet, block, scope) => {
+  const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(block.bs, 0);
+  const blockScope = docSet.unsuccinctifyScope(block.bs, itemType, itemSubtype, 0);
+  return (blockScope[2] === scope);
+};
+
+const blockHasChars = (docSet, block, charsIndexes) => {
+  let ret = false;
+  let pos = 0;
+  const succinct = block.c;
+
+  if (charsIndexes.includes(-1)) {
+    return false;
+  }
+
+  while (!ret && (pos < succinct.length)) {
+    const [itemLength, itemType] = utils.succinct.headerBytes(succinct, pos);
+
+    if (itemType === utils.itemDefs.itemEnum['token']) {
+      if (charsIndexes.includes(succinct.nByte(pos + 2))) {
+        ret = true;
+      }
+    }
+    pos += itemLength;
+  }
+  return ret;
+};
+
+const blockHasMatchingItem = (docSet, block, testFunction, options) => {
+  const openScopes = new Set(docSet.unsuccinctifyScopes(block.os).map(ri => ri[2]));
+
+  for (const item of docSet.unsuccinctifyItems(block.c, options, 0)) {
+    if (item[0] === 'scope' && item[1] === 'start') {
+      openScopes.add(item[2]);
+    }
+
+    if (testFunction(item, openScopes)) {
+      return true;
+    }
+
+    if (item[0] === 'scope' && item[1] === 'end') {
+      openScopes.delete(item[2]);
+    }
+  }
+  return false;
+};
+
+module.exports = {
+  blocksWithScriptureCV,
+  allBlockScopes,
+  anyScopeInBlock,
+  allScopesInBlock,
+  blockHasBlockScope,
+  blockHasChars,
+  blockHasMatchingItem,
+};

package/src/model/doc_set_helpers/enum.cjs

@@ -0,0 +1,87 @@
+const buildPreEnum = (docSet, succinct) => {
+  const ret = new Map();
+  let pos = 0;
+  let enumCount = 0;
+
+  while (pos < succinct.length) {
+    ret.set(
+      succinct.countedString(pos),
+      {
+        'enum': enumCount++,
+        'frequency': 0,
+      },
+    );
+    pos += succinct.byte(pos) + 1;
+  }
+
+  return ret;
+};
+
+const recordPreEnum = (docSet, category, value) => {
+  if (!(category in docSet.preEnums)) {
+    throw new Error(`Unknown category ${category} in recordPreEnum. Maybe call buildPreEnums()?`);
+  }
+
+  if (value.length > 255) {
+    console.log('Value length of', value.length, 'in recordPreEnum');
+  }
+
+  if (!docSet.preEnums[category].has(value)) {
+    docSet.preEnums[category].set(
+      value,
+      {
+        'enum': docSet.preEnums[category].size,
+        'frequency': 1,
+      },
+    );
+  } else {
+    docSet.preEnums[category].get(value).frequency++;
+  }
+};
+
+const buildEnum = (docSet, category, preEnumOb) => {
+  const sortedPreEnums = new Map([...preEnumOb.entries()]);
+
+  for (const enumText of sortedPreEnums.keys()) {
+    if (enumText.length > 255) {
+      console.log('enum text for', category, 'has length', enumText.length, 'in buildEnum - truncating');
+    }
+    docSet.enums[category].pushCountedString(enumText.substring(0, 255));
+  }
+  docSet.enums[category].trim();
+};
+
+const enumForCategoryValue = (docSet, category, value, addUnknown) => {
+  if (!addUnknown) {
+    addUnknown = false;
+  }
+
+  if (!(category in docSet.preEnums)) {
+    throw new Error(`Unknown category ${category} in preEnums. Maybe call buildPreEnums()?`);
+  }
+
+  if (docSet.preEnums[category].has(value)) {
+    return docSet.preEnums[category].get(value).enum;
+  } else if (addUnknown) {
+    docSet.preEnums[category].set(
+      value,
+      {
+        'enum': docSet.preEnums[category].size,
+        'frequency': 1,
+      },
+    );
+    docSet.enums[category].pushCountedString(value);
+    docSet.buildEnumIndex(category);
+    return docSet.preEnums[category].get(value).enum;
+  } else {
+    throw new Error(`Unknown value '${value}' for category ${category} in enumForCategoryValue. Maybe call buildPreEnums()?`);
+  }
+};
+
+
+module.exports = {
+  buildPreEnum,
+  recordPreEnum,
+  buildEnum,
+  enumForCategoryValue,
+};