proskomma-core 0.9.10 → 0.9.13

package/src/model/doc_set_helpers/item.cjs

@@ -0,0 +1,198 @@
+const utils = require('../../util/index.cjs');
+
+const countItems = (docSet, succinct) => {
+  let count = 0;
+  let pos = 0;
+
+  while (pos < succinct.length) {
+    count++;
+    const headerByte = succinct.byte(pos);
+    const itemLength = headerByte & 0x0000003F;
+    pos += itemLength;
+  }
+  return count;
+};
+
+const itemsByIndex = (docSet, mainSequence, index, includeContext) => {
+  let ret = [];
+
+  if (!index) {
+    return ret;
+  }
+
+  let currentBlock = index.startBlock;
+  let nextToken = index.nextToken;
+
+  while (currentBlock <= index.endBlock) {
+    let blockItems = docSet.unsuccinctifyItems(mainSequence.blocks[currentBlock].c, {}, nextToken);
+    const blockScope = docSet.unsuccinctifyScopes(mainSequence.blocks[currentBlock].bs)[0];
+    const blockGrafts = docSet.unsuccinctifyGrafts(mainSequence.blocks[currentBlock].bg);
+
+    if (currentBlock === index.startBlock && currentBlock === index.endBlock) {
+      blockItems = blockItems.slice(index.startItem, index.endItem + 1);
+    } else if (currentBlock === index.startBlock) {
+      blockItems = blockItems.slice(index.startItem);
+    } else if (currentBlock === index.endBlock) {
+      blockItems = blockItems.slice(0, index.endItem + 1);
+    }
+
+    if (includeContext) {
+      let extendedBlockItems = [];
+
+      for (const bi of blockItems) {
+        extendedBlockItems.push(bi.concat([bi[0] === 'token' && bi[1] === 'wordLike' ? nextToken++ : null]));
+      }
+      blockItems = extendedBlockItems;
+    }
+    ret.push([...blockGrafts, ['scope', 'start', blockScope[2]], ...blockItems, ['scope', 'end', blockScope[2]]]);
+    currentBlock++;
+  }
+  return ret;
+};
+
+const sequenceItemsByScopes = (docSet, blocks, byScopes) => {
+  // Return array of [scopes, items]
+  // Scan block items, track scopes
+  // If all scopes found:
+  //   - turn found scopes into string
+  //   - if that scope string doesn't exist, add to lookup table and push array
+  //   - add item to array matching scope string
+  let allBlockScopes = [];
+
+  const allScopesPresent = () => {
+    for (const requiredScope of byScopes) {
+      if (!matchingScope(requiredScope)) {
+        return false;
+      }
+    }
+    return true;
+  };
+
+  const matchingScope = (scopeToMatch) => {
+    for (const blockScope of allBlockScopes) {
+      if (blockScope.startsWith(scopeToMatch)) {
+        return blockScope;
+      }
+    }
+    return null;
+  };
+
+  docSet.maybeBuildEnumIndexes();
+  const ret = [];
+  let waitingScopes = new Set([]);
+  let scopeMatchEnded = true; // Always start new itemGroup the first time
+
+  for (const [blockN, block] of blocks.entries()) {
+    const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(block.bs, 0);
+    const blockScope = docSet.unsuccinctifyScope(block.bs, itemType, itemSubtype, 0)[2];
+    const startBlockScope = ['scope', 'start', blockScope];
+    const endBlockScope = ['scope', 'end', blockScope];
+    const blockGrafts = docSet.unsuccinctifyGrafts(block.bg);
+
+    allBlockScopes = new Set(docSet.unsuccinctifyScopes(block.os)
+      .map(s => s[2])
+      .concat([blockScope]),
+    );
+
+    for (
+      const item of blockGrafts.concat(
+        [
+          startBlockScope,
+          ...docSet.unsuccinctifyItems(block.c, {}, block.nt.nByte(0), allBlockScopes),
+          endBlockScope,
+        ],
+      ).concat(
+        (blockN !== blocks.length - 1) ? [['token', 'lineSpace', ' ']] : [],
+      )
+    ) {
+      if (item[0] === 'scope' && item[1] === 'start') {
+        waitingScopes.add(item[2]);
+      }
+
+      if (item[0] === 'token' && waitingScopes.size > 0) {
+        for (const waiting of Array.from(waitingScopes)) {
+          allBlockScopes.add(waiting);
+        }
+        waitingScopes.clear();
+      }
+
+      if (allScopesPresent()) {
+        if (scopeMatchEnded) {
+          ret.push([[...allBlockScopes], []]);
+        }
+        ret[ret.length - 1][1].push(item);
+        scopeMatchEnded = false;
+      } else {
+        scopeMatchEnded = true;
+      }
+
+      if (item[0] === 'scope' && item[1] === 'end') {
+        allBlockScopes.delete(item[2]);
+        waitingScopes.delete(item[2]);
+      }
+    }
+  }
+  return ret;
+};
+
+const sequenceItemsByMilestones = (docSet, blocks, byMilestones) => {
+  // Return array of [scopes, items]
+  // Scan block items
+  // If milestone found:
+  //   - add array
+  // push item to last array
+  let allBlockScopes = new Set([]);
+  const milestoneFound = (item) => item[0] === 'scope' && item[1] === 'start' && byMilestones.includes(item[2]);
+
+  docSet.maybeBuildEnumIndexes();
+  const ret = [[[], []]];
+
+  for (const block of blocks) {
+    const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(block.bs, 0);
+    const blockScope = docSet.unsuccinctifyScope(block.bs, itemType, itemSubtype, 0)[2];
+    const blockGrafts = docSet.unsuccinctifyGrafts(block.bg);
+    allBlockScopes.add(blockScope);
+    docSet.unsuccinctifyScopes(block.os).forEach(s => allBlockScopes.add(s[2]));
+    const items = blockGrafts.concat(
+      [blockScope].concat(
+        docSet.unsuccinctifyItems(block.c, {}, block.nt.nByte(0)),
+      ),
+    );
+
+    for (const item of items) {
+      if (item[0] === 'scope' && item[1] === 'start') {
+        allBlockScopes.add(item[2]);
+      }
+
+      if (milestoneFound(item)) {
+        ret[ret.length - 1][0] = [...allBlockScopes].sort();
+        ret.push([[], []]);
+
+        for (
+          const bs of [...allBlockScopes]
+            .filter(
+              s => {
+                const excludes = ['blockTag', 'verse', 'verses', 'chapter'];
+                return excludes.includes(s.split('/')[0]) || byMilestones.includes(s);
+              },
+            )
+        ) {
+          allBlockScopes.delete(bs);
+        }
+        allBlockScopes.add(blockScope);
+      }
+      ret[ret.length - 1][1].push(item);
+    }
+    ret[ret.length - 1][1].push(['scope', 'end', blockScope]);
+    ret[ret.length - 1][1].push(['token', 'punctuation', '\n']);
+  }
+  ret[ret.length - 1][0] = [...allBlockScopes].sort();
+  return ret;
+};
+
+module.exports = {
+  countItems,
+  itemsByIndex,
+  sequenceItemsByMilestones,
+  sequenceItemsByScopes,
+};

package/src/model/doc_set_helpers/rehash.cjs

@@ -0,0 +1,53 @@
+const rehash = docSet => {
+  docSet.preEnums = {};
+
+  for (const category of Object.keys(docSet.enums)) {
+    docSet.preEnums[category] = new Map();
+  }
+  docSet.maybeBuildEnumIndexes();
+
+  for (const document of docSet.documents()) {
+    for (const sequence of Object.values(document.sequences)) {
+      document.rerecordPreEnums(docSet, sequence);
+    }
+  }
+  docSet.sortPreEnums();
+  const oldToNew = docSet.makeRehashEnumMap();
+
+  for (const document of docSet.documents()) {
+    for (const sequence of Object.values(document.sequences)) {
+      document.rewriteSequenceBlocks(sequence.id, oldToNew);
+    }
+  }
+  docSet.buildEnums();
+  docSet.buildEnumIndexes();
+  return true;
+};
+
+const makeRehashEnumMap = docSet => {
+  const ret = {};
+
+  for (const [category, enumSuccinct] of Object.entries(docSet.enums)) {
+    ret[category] = [];
+    let pos = 0;
+
+    while (pos < enumSuccinct.length) {
+      const stringLength = enumSuccinct.byte(pos);
+      const enumString = enumSuccinct.countedString(pos);
+
+      if (docSet.preEnums[category].has(enumString)) {
+        ret[category].push(docSet.preEnums[category].get(enumString).enum);
+      } else {
+        ret[category].push(null);
+      }
+
+      pos += stringLength + 1;
+    }
+  }
+  return ret;
+};
+
+module.exports = {
+  rehash,
+  makeRehashEnumMap,
+};

package/src/model/doc_set_helpers/selectors.cjs

@@ -0,0 +1,58 @@
+const xre = require('xregexp');
+
+const validateSelectors = (docSet, selectors) => {
+  if (typeof selectors !== 'object') {
+    throw new Error(`DocSet constructor expects selectors to be object, found ${typeof docSet.selectors}`);
+  }
+
+  const expectedSelectors = {};
+
+  for (const selector of docSet.processor.selectors) {
+    expectedSelectors[selector.name] = selector;
+  }
+
+  for (const [name, value] of Object.entries(selectors)) {
+    if (!(name in expectedSelectors)) {
+      throw new Error(`Unexpected selector '${name}' (expected one of [${Object.keys(expectedSelectors).join(', ')}])`);
+    }
+
+    if (
+      (typeof value === 'string' && expectedSelectors[name].type !== 'string') ||
+      (typeof value === 'number' && expectedSelectors[name].type !== 'integer')
+    ) {
+      throw new Error(`Selector '${name}' is of type ${typeof value} (expected ${expectedSelectors[name].type})`);
+    }
+
+    if (typeof value === 'number') {
+      if (!Number.isInteger(value)) {
+        throw new Error(`Value '${value}' of integer selector '${name}' is not an integer`);
+      }
+
+      if ('min' in expectedSelectors[name] && value < expectedSelectors[name].min) {
+        throw new Error(`Value '${value}' is less than ${expectedSelectors[name].min}`);
+      }
+
+      if ('max' in expectedSelectors[name] && value > expectedSelectors[name].max) {
+        throw new Error(`Value '${value}' is greater than ${expectedSelectors[name].max}`);
+      }
+    } else {
+      if ('regex' in expectedSelectors[name] && !xre.exec(value, xre(expectedSelectors[name].regex), 0)) {
+        throw new Error(`Value '${value}' does not match regex '${expectedSelectors[name].regex}'`);
+      }
+    }
+
+    if ('enum' in expectedSelectors[name] && !expectedSelectors[name].enum.includes(value)) {
+      throw new Error(`Value '${value}' of selector '${name}' is not in enum`);
+    }
+  }
+
+  for (const name of Object.keys(expectedSelectors)) {
+    if (!(name in selectors)) {
+      throw new Error(`Expected selector '${name}' not found`);
+    }
+  }
+  return selectors;
+};
+
+module.exports = { validateSelectors };
+

package/src/model/doc_set_helpers/serialize.cjs

@@ -0,0 +1,22 @@
+const serializeSuccinct = docSet => {
+  const ret = {
+    id: docSet.id,
+    metadata: { selectors: docSet.selectors },
+    enums: {},
+    docs: {},
+    tags: Array.from(docSet.tags),
+  };
+
+  for (const [eK, eV] of Object.entries(docSet.enums)) {
+    eV.trim();
+    ret.enums[eK] = eV.base64();
+  }
+  ret.docs = {};
+
+  for (const docId of docSet.docIds) {
+    ret.docs[docId] = docSet.processor.documents[docId].serializeSuccinct();
+  }
+  return ret;
+};
+
+module.exports = { serializeSuccinct };

package/src/model/doc_set_helpers/unsuccinctify.cjs

@@ -0,0 +1,288 @@
+const xre = require('xregexp');
+const utils = require('../../util/index.cjs');
+
+const unsuccinctifyBlock = (docSet, block, options) => {
+  docSet.maybeBuildEnumIndexes();
+  const succinctBlockScope = block.bs;
+  const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(succinctBlockScope, 0);
+  const blockScope = docSet.unsuccinctifyScope(succinctBlockScope, itemType, itemSubtype, 0);
+  const blockGrafts = docSet.unsuccinctifyGrafts(block.bg);
+  const openScopes = docSet.unsuccinctifyScopes(block.os);
+  const includedScopes = docSet.unsuccinctifyScopes(block.is);
+  const nextToken = block.nt.nByte(0);
+  const blockItems = docSet.unsuccinctifyItems(block.c, options || {}, nextToken);
+  return {
+    bs: blockScope,
+    bg: blockGrafts,
+    c: blockItems,
+    os: openScopes,
+    is: includedScopes,
+    nt: nextToken,
+  };
+};
+
+const unsuccinctifyItems = (docSet, succinct, options, nextToken, openScopes) => {
+  if (nextToken === undefined) {
+    throw new Error('nextToken (previously includeContext) must now be provided to unsuccinctifyItems');
+  }
+
+  if (nextToken !== null && typeof nextToken !== 'number') {
+    throw new Error(`nextToken (previously includeContext) must be null or an integer, not ${typeof nextToken} '${JSON.stringify(nextToken)}' in unsuccinctifyItems`);
+  }
+
+  const ret = [];
+  let pos = 0;
+  let tokenCount = nextToken || 0;
+  const scopes = new Set(openScopes || []);
+
+  while (pos < succinct.length) {
+    const [item, itemLength] = docSet.unsuccinctifyItem(succinct, pos, {});
+
+    if (item[0] === 'token') {
+      if ((Object.keys(options).length === 0) || options.tokens) {
+        if (nextToken !== null) {
+          item.push(item[0] === 'token' && item[1] === 'wordLike' ? tokenCount++ : null);
+          item.push([...scopes]);
+        }
+        ret.push(item);
+      }
+    } else if (item[0] === 'scope' && item[1] === 'start') {
+      scopes.add(item[2]);
+
+      if ((Object.keys(options).length === 0) || options.scopes) {
+        ret.push(item);
+      }
+    } else if (item[0] === 'scope' && item[1] === 'end') {
+      scopes.delete(item[2]);
+
+      if ((Object.keys(options).length === 0) || options.scopes) {
+        ret.push(item);
+      }
+    } else if (item[0] === 'graft') {
+      if ((Object.keys(options).length === 0) || options.grafts) {
+        ret.push(item);
+      }
+    }
+    pos += itemLength;
+  }
+  return ret;
+};
+
+const unsuccinctifyItem = (docSet, succinct, pos, options) => {
+  let item = null;
+  const [itemLength, itemType, itemSubtype] = utils.succinct.headerBytes(succinct, pos);
+
+  switch (itemType) {
+  case utils.itemDefs.itemEnum.token:
+    if (Object.keys(options).length === 0 || options.tokens) {
+      item = docSet.unsuccinctifyToken(succinct, itemSubtype, pos);
+    }
+    break;
+  case utils.itemDefs.itemEnum.startScope:
+  case utils.itemDefs.itemEnum.endScope:
+    if (Object.keys(options).length === 0 || options.scopes) {
+      item = docSet.unsuccinctifyScope(succinct, itemType, itemSubtype, pos);
+    }
+    break;
+  case utils.itemDefs.itemEnum.graft:
+    if (Object.keys(options).length === 0 || options.grafts) {
+      item = docSet.unsuccinctifyGraft(succinct, itemSubtype, pos);
+    }
+    break;
+  }
+  return [item, itemLength];
+};
+
+const unsuccinctifyPrunedItems = (docSet, block, options) => {
+  const openScopes = new Set(docSet.unsuccinctifyScopes(block.os).map(ri => ri[2]));
+  const requiredScopes = options.requiredScopes || [];
+  const anyScope = options.anyScope || false;
+
+  const allScopesInItem = () => {
+    for (const scope of requiredScopes) {
+      if (!openScopes.has(scope)) {
+        return false;
+      }
+    }
+    return true;
+  };
+
+  const anyScopeInItem = () => {
+    for (const scope of requiredScopes) {
+      if (openScopes.has(scope)) {
+        return true;
+      }
+    }
+    return (requiredScopes.length === 0);
+  };
+
+  const scopeTest = anyScope ? anyScopeInItem : allScopesInItem;
+  const charsTest = (item) =>
+    !options.withChars ||
+    options.withChars.length === 0 ||
+    (item[0] === 'token' && options.withChars.includes(item[2]));
+  const ret = [];
+
+  for (const item of docSet.unsuccinctifyItems(block.c, options, block.nt.nByte(0), openScopes)) {
+    if (item[0] === 'scope' && item[1] === 'start') {
+      openScopes.add(item[2]);
+    }
+
+    if (scopeTest() && charsTest(item)) {
+      ret.push(item);
+    }
+
+    if (item[0] === 'scope' && item[1] === 'end') {
+      openScopes.delete(item[2]);
+    }
+  }
+  return ret;
+};
+
+const unsuccinctifyItemsWithScriptureCV = (docSet, block, cv, options) => {
+  options = options || {};
+  const openScopes = new Set(docSet.unsuccinctifyScopes(block.os).map(ri => ri[2]));
+
+  const cvMatchFunction = () => {
+    if (xre.exec(cv, xre('^[1-9][0-9]*$'))) {
+      return () => openScopes.has(`chapter/${cv}`);
+    } else if (xre.exec(cv, xre('^[1-9][0-9]*-[1-9][0-9]*$'))) {
+      return () => {
+        const [fromC, toC] = cv.split('-').map(v => parseInt(v));
+
+        if (fromC > toC) {
+          throw new Error(`Chapter range must be from min to max, not '${cv}'`);
+        }
+
+        for (const scope of [...Array((toC - fromC) + 1).keys()].map(n => `chapter/${n + fromC}`)) {
+          if (openScopes.has(scope)) {
+            return true;
+          }
+        }
+        return false;
+      };
+    } else if (xre.exec(cv, xre('^[1-9][0-9]*:[0-9]+$'))) {
+      return () => {
+        const [fromC, fromV] = cv.split(':').map(v => parseInt(v));
+
+        if (fromV === 0) {
+          return (
+            openScopes.has(`chapter/${fromC}`) &&
+            [...openScopes].filter(s => s.startsWith('verse')).length === 0
+          );
+        } else {
+          for (const scope of [`chapter/${fromC}`, `verse/${fromV}`]) {
+            if (!openScopes.has(scope)) {
+              return false;
+            }
+          }
+          return true;
+        }
+      };
+    } else if (xre.exec(cv, xre('^[1-9][0-9]*:[0-9]+-[1-9][0-9]*$'))) {
+      return () => {
+        const [fromC, vs] = cv.split(':');
+        const [fromV, toV] = vs.split('-').map(v => parseInt(v));
+
+        if (fromV > toV) {
+          throw new Error(`Verse range must be from min to max, not '${vs}'`);
+        }
+
+        const chapterScope = `chapter/${fromC}`;
+        const verseScopes = [...Array((toV - fromV) + 1).keys()].map(n => `verse/${n + fromV}`);
+
+        if (!openScopes.has(chapterScope)) {
+          return false;
+        }
+
+        for (const scope of verseScopes) {
+          if (openScopes.has(scope)) {
+            return true;
+          }
+        }
+        return fromV === 0 && [...openScopes].filter(s => s.startsWith('verse')).length === 0;
+      };
+    } else if (xre.exec(cv, xre('^[1-9][0-9]*:[0-9]+-[1-9][0-9]*:[0-9]+$'))) {
+      return () => {
+        const [fromCV, toCV] = cv.split('-');
+        const [fromC, fromV] = fromCV.split(':').map(c => parseInt(c));
+        const [toC, toV] = toCV.split(':').map(v => parseInt(v));
+
+        if (fromC > toC) {
+          throw new Error(`Chapter range must be from min to max, not '${fromC}-${toV}'`);
+        }
+
+        const scopeArray = [...openScopes];
+        const chapterScopes = scopeArray.filter(s => s.startsWith('chapter/'));
+
+        if (chapterScopes.length > 1) {
+          throw new Error(`Expected zero or one chapter for item, found ${chapterScopes.length}`);
+        }
+
+        const chapterNo = parseInt(chapterScopes[0].split('/')[1]);
+
+        if ((chapterNo < fromC) || (chapterNo > toC)) {
+          return false;
+        } else if (chapterNo === fromC) {
+          return scopeArray.filter(
+            s =>
+              s.startsWith('verse/') &&
+              parseInt(s.split('/')[1]) >= fromV,
+          ).length > 0 ||
+            (fromV === 0 && scopeArray.filter(s => s.startsWith('verse')).length === 0);
+        } else if (chapterNo === toC) {
+          return scopeArray.filter(
+            s =>
+              s.startsWith('verse/') &&
+              parseInt(s.split('/')[1]) <= toV,
+          ).length > 0 ||
+            (toV === 0 && scopeArray.filter(s => s.startsWith('verse')).length === 0);
+        } else {
+          return true;
+        }
+      };
+    } else {
+      throw new Error(`Bad cv reference '${cv}'`);
+    }
+  };
+
+  const itemMatchesCV = cvMatchFunction();
+
+  const itemInOptions = (item) => {
+    if (!options || Object.keys(options).length === 0) {
+      return true;
+    } else {
+      const itemType = item[0];
+      return (
+        (itemType === 'token' && 'tokens' in options) ||
+        (itemType === 'graft' && 'grafts' in options) ||
+        (itemType === 'scope' && 'scopes' in options)
+      );
+    }
+  };
+
+  const ret = [];
+
+  for (const item of docSet.unsuccinctifyItems(block.c, {}, block.nt.nByte(0))) {
+    if (item[0] === 'scope' && item[1] === 'start') {
+      openScopes.add(item[2]);
+    }
+
+    if (itemMatchesCV() && itemInOptions(item)) {
+      ret.push(item);
+    }
+
+    if (item[0] === 'scope' && item[1] === 'end') {
+      openScopes.delete(item[2]);
+    }
+  }
+  return ret;
+};
+
+module.exports = {
+  unsuccinctifyBlock,
+  unsuccinctifyItem,
+  unsuccinctifyItems,
+  unsuccinctifyItemsWithScriptureCV,
+  unsuccinctifyPrunedItems,
+};