proskomma-core 0.9.10 → 0.9.13

package/src/util/scriptlike/open_scriptures_hebrew_lexicon_to_entries.cjs

@@ -0,0 +1,85 @@
+const fse = require('fs-extra');
+const { tokenizeString } = require('./blocksSpec.cjs');
+
+const itemTokens = str => tokenizeString(str).map(tr => ({
+  type: 'token',
+  subType: tr[1],
+  payload: tr[0],
+}));
+
+const rowRecord = rowArray => ({
+  'bs': {
+    'type': 'scope',
+    'subType': 'start',
+    'payload': `kvPrimary/${rowArray[0]}`,
+  },
+  'bg': [],
+  'os': [],
+  'is': [
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': `kvSecondary/strong/${rowArray[1]}`,
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/briefDef',
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/fullDef',
+    },
+  ],
+  'items': [
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': `kvSecondary/strong/${rowArray[1]}`,
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/briefDef',
+    },
+    ...itemTokens(rowArray[4].length > 0 ? rowArray[4].replace(/; $/, '') : rowArray[5]),
+    {
+      'type': 'scope',
+      'subType': 'end',
+      'payload': 'kvField/briefDef',
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/fullDef',
+    },
+    ...itemTokens(rowArray[3]),
+    {
+      'type': 'scope',
+      'subType': 'end',
+      'payload': 'kvField/fullDef',
+    },
+    {
+      'type': 'scope',
+      'subType': 'end',
+      'payload': `kvSecondary/strong/${rowArray[1]}`,
+    },
+  ],
+});
+
+const lexicon_tsv = fse.readFileSync(process.argv[2])
+  .toString()
+  .split('\n')
+  .map(row => row.split('\t'))
+  .filter(row => row.length > 1);
+
+console.log(
+  JSON.stringify(
+    lexicon_tsv.map(
+      row => rowRecord(row),
+    ),
+    null,
+    2,
+  ),
+);

package/src/util/succinct.cjs

@@ -0,0 +1,159 @@
+const tokenDefs = require('./tokenDefs.cjs');
+const scopeDefs = require('./scopeDefs.cjs');
+const itemDefs = require('./itemDefs.cjs');
+
+const headerBytes = (succinct, pos) =>{
+  const headerByte = succinct.byte(pos);
+  const itemType = headerByte >> 6;
+  const itemLength = headerByte & 0x0000003F;
+  const itemSubtype = succinct.byte(pos + 1);
+  return [itemLength, itemType, itemSubtype];
+};
+
+const succinctTokenChars = (enums, enumIndexes, succinct, itemSubtype, pos) => {
+  const itemCategory = tokenDefs.tokenCategory[tokenDefs.tokenEnumLabels[itemSubtype]];
+  const itemIndex = enumIndexes[itemCategory][succinct.nByte(pos + 2)];
+  return enums[itemCategory].countedString(itemIndex);
+};
+
+const succinctScopeLabel = (enums, enumIndexes, succinct, itemSubtype, pos) => {
+  const scopeType = scopeDefs.scopeEnumLabels[itemSubtype];
+  let nScopeBits = scopeDefs.nComponentsForScope(scopeType);
+  let offset = 2;
+  let scopeBits = '';
+
+  while (nScopeBits > 1) {
+    const itemIndexIndex = succinct.nByte(pos + offset);
+    const itemIndex = enumIndexes.scopeBits[itemIndexIndex];
+    const scopeBitString = enums.scopeBits.countedString(itemIndex);
+    scopeBits += `/${scopeBitString}`;
+    offset += succinct.nByteLength(itemIndexIndex);
+    nScopeBits--;
+  }
+  return `${scopeType}${scopeBits}`;
+};
+
+const succinctGraftName = (enums, enumIndexes, itemSubtype) => {
+  const graftIndex = enumIndexes.graftTypes[itemSubtype];
+  return enums.graftTypes.countedString(graftIndex);
+};
+
+const succinctGraftSeqId = (enums, enumIndexes, succinct, pos) => {
+  const seqIndex = enumIndexes.ids[succinct.nByte(pos + 2)];
+  return enums.ids.countedString(seqIndex);
+};
+
+const enumIndexes = (enums) => {
+  const ret = {};
+
+  for (const [category, succinct] of Object.entries(enums)) {
+    ret[category] = enumIndex(category, succinct);
+  }
+  return ret;
+};
+
+const enumIndex = (category, enumSuccinct) => {
+  const indexSuccinct = new Uint32Array(enumSuccinct.length);
+  let pos = 0;
+  let count = 0;
+
+  while (pos < enumSuccinct.length) {
+    indexSuccinct[count] = pos;
+    const stringLength = enumSuccinct.byte(pos);
+    pos += (stringLength + 1);
+    count += 1;
+  }
+  return indexSuccinct;
+};
+
+const unpackEnum = (succinct, includeIndex) => {
+  if (!includeIndex) {
+    includeIndex = false;
+  }
+
+  let pos = 0;
+  let count = 0;
+  const ret = [];
+
+  while (pos < succinct.length) {
+    const stringLength = succinct.byte(pos);
+    const unpacked = succinct.countedString(pos);
+    ret.push(includeIndex ? [count, unpacked] : unpacked);
+    pos += stringLength + 1;
+    count++;
+  }
+  return ret;
+};
+
+const undefinedArgError = (func, field) => {
+  throw new Error(`Undefined or null argument '${field}' in '${func}'`);
+};
+
+const pushSuccinctTokenBytes = (bA, tokenEnumIndex, charsEnumIndex) => {
+  if (tokenEnumIndex === undefined || tokenEnumIndex === null) {
+    undefinedArgError('pushSuccinctTokenBytes', 'tokenEnumIndex');
+  }
+
+  if (charsEnumIndex === undefined || charsEnumIndex === null) {
+    undefinedArgError('pushSuccinctTokenBytes', 'charsEnumIndex');
+  }
+
+  const lengthPos = bA.length;
+  bA.pushByte(0);
+  bA.pushByte(tokenEnumIndex);
+  bA.pushNByte(charsEnumIndex);
+  bA.setByte(lengthPos, (bA.length - lengthPos) | itemDefs.itemEnum.token << 6);
+};
+
+const pushSuccinctGraftBytes = (bA, graftTypeEnumIndex, seqEnumIndex) => {
+  if (graftTypeEnumIndex === undefined || graftTypeEnumIndex === null) {
+    undefinedArgError('pushSuccinctGraftBytes', 'graftTypeEnumIndex');
+  }
+
+  if (seqEnumIndex === undefined || seqEnumIndex === null) {
+    undefinedArgError('pushSuccinctGraftBytes', 'seqEnumIndex');
+  }
+
+  const lengthPos = bA.length;
+  bA.pushByte(0);
+  bA.pushByte(graftTypeEnumIndex);
+  bA.pushNByte(seqEnumIndex);
+  bA.setByte(lengthPos, (bA.length - lengthPos) | itemDefs.itemEnum.graft << 6);
+};
+
+const pushSuccinctScopeBytes = (bA, itemTypeByte, scopeTypeByte, scopeBitBytes) => {
+  if (itemTypeByte === undefined || itemTypeByte === null) {
+    undefinedArgError('pushSuccinctScopeBytes', 'itemTypeByte');
+  }
+
+  if (scopeTypeByte === undefined || scopeTypeByte === null) {
+    undefinedArgError('pushSuccinctScopeBytes', 'scopeTypeByte');
+  }
+
+  if (scopeBitBytes === undefined || scopeBitBytes === null) {
+    undefinedArgError('pushSuccinctScopeBytes', 'scopeBitBytes');
+  }
+
+  const lengthPos = bA.length;
+  bA.pushByte(0);
+  bA.pushByte(scopeTypeByte);
+
+  for (const sbb of scopeBitBytes) {
+    bA.pushNByte(sbb);
+  }
+  bA.setByte(lengthPos, (bA.length - lengthPos) | itemTypeByte << 6);
+};
+
+module.exports = {
+  enumIndex,
+  enumIndexes,
+  headerBytes,
+  pushSuccinctTokenBytes,
+  pushSuccinctGraftBytes,
+  pushSuccinctScopeBytes,
+  succinctTokenChars,
+  succinctScopeLabel,
+  succinctGraftName,
+  succinctGraftSeqId,
+  unpackEnum,
+};

package/src/util/tags.cjs

@@ -0,0 +1,28 @@
+const xre = require('xregexp');
+
+const validateTags = tags => {
+  for (const tag of tags) {
+    validateTag(tag);
+  }
+};
+
+const validateTag = tag => {
+  if (!xre.exec(tag, /^[a-z][A-za-z0-9]*(:.+)?$/)) {
+    throw new Error(`Tag '${tag}' is not valid (should be [a-z][A-za-z0-9]*(:.+)?)`);
+  }
+};
+
+const addTag = (tags, tag) => {
+  validateTag(tag);
+  tags.add(tag);
+};
+
+const removeTag = (tags, tag) => {
+  validateTag(tag);
+  tags.delete(tag);
+};
+
+module.exports = {
+  validateTags, validateTag, addTag, removeTag,
+};
+

package/src/util/tokenDefs.cjs

@@ -0,0 +1,27 @@
+const tokenEnum = {
+  wordLike: 0,
+  punctuation: 1,
+  lineSpace: 2,
+  eol: 3,
+  softLineBreak: 4,
+  noBreakSpace: 5,
+  bareSlash: 6,
+  unknown: 7,
+};
+
+const tokenEnumLabels = Object.entries(tokenEnum).sort((a, b) => a[1] - b[1]).map(kv => kv[0]);
+
+const tokenCategory = {
+  wordLike: 'wordLike',
+  punctuation: 'notWordLike',
+  lineSpace: 'notWordLike',
+  eol: 'notWordLike',
+  softLineBreak: 'notWordLike',
+  noBreakSpace: 'notWordLike',
+  bareSlash: 'notWordLike',
+  unknown: 'notWordLike',
+};
+
+module.exports = {
+  tokenEnum, tokenEnumLabels, tokenCategory,
+};

package/src/util/versification.cjs

@@ -0,0 +1,364 @@
+const xre = require('xregexp');
+
+const ByteArray = require('./byteArray.cjs');
+
+const cvMappingType = 2;
+const bcvMappingType = 3;
+
+const bookCodes = [ // From Paratext via Scripture Burrito
+  'GEN',
+  'EXO',
+  'LEV',
+  'NUM',
+  'DEU',
+  'JOS',
+  'JDG',
+  'RUT',
+  '1SA',
+  '2SA',
+  '1KI',
+  '2KI',
+  '1CH',
+  '2CH',
+  'EZR',
+  'NEH',
+  'EST',
+  'JOB',
+  'PSA',
+  'PRO',
+  'ECC',
+  'SNG',
+  'ISA',
+  'JER',
+  'LAM',
+  'EZK',
+  'DAN',
+  'HOS',
+  'JOL',
+  'AMO',
+  'OBA',
+  'JON',
+  'MIC',
+  'NAM',
+  'HAB',
+  'ZEP',
+  'HAG',
+  'ZEC',
+  'MAL',
+  'MAT',
+  'MRK',
+  'LUK',
+  'JHN',
+  'ACT',
+  'ROM',
+  '1CO',
+  '2CO',
+  'GAL',
+  'EPH',
+  'PHP',
+  'COL',
+  '1TH',
+  '2TH',
+  '1TI',
+  '2TI',
+  'TIT',
+  'PHM',
+  'HEB',
+  'JAS',
+  '1PE',
+  '2PE',
+  '1JN',
+  '2JN',
+  '3JN',
+  'JUD',
+  'REV',
+  'TOB',
+  'JDT',
+  'ESG',
+  'WIS',
+  'SIR',
+  'BAR',
+  'LJE',
+  'S3Y',
+  'SUS',
+  'BEL',
+  '1MA',
+  '2MA',
+  '3MA',
+  '4MA',
+  '1ES',
+  '2ES',
+  'MAN',
+  'PS2',
+  'ODA',
+  'PSS',
+  'JSA',
+  'JDB',
+  'TBS',
+  'SST',
+  'DNT',
+  'BLT',
+  'EZA',
+  '5EZ',
+  '6EZ',
+  'DAG',
+  'PS3',
+  '2BA',
+  'LBA',
+  'JUB',
+  'ENO',
+  '1MQ',
+  '2MQ',
+  '3MQ',
+  'REP',
+  '4BA',
+  'LAO',
+];
+
+const bookCodeIndex = () => {
+  const ret = {};
+
+  for (const [bookN, book] of Object.entries(bookCodes)) {
+    ret[book] = parseInt(bookN);
+  }
+  return ret;
+};
+
+const vrs2json = vrsString => {
+  const ret = {};
+
+  for (
+    const vrsLineBits of
+    vrsString
+      .split(/[\n\r]+/)
+      .map(l => l.trim())
+      .map(l => xre.exec(l, xre('^([A-Z1-6]{3} [0-9]+:[0-9]+(-[0-9]+)?) = ([A-Z1-6]{3} [0-9]+:[0-9]+[a-z]?(-[0-9]+)?)$')))
+  ) {
+    if (!vrsLineBits) {
+      continue;
+    }
+
+    if (!(vrsLineBits[1] in ret)) {
+      ret[vrsLineBits[1]] = [];
+    }
+    ret[vrsLineBits[1]].push(vrsLineBits[3]);
+  }
+  return { mappedVerses: ret };
+};
+
+const reverseVersification = vrsJson => {
+  // Assumes each verse is only mapped from once
+  const ret = {};
+
+  for (const [fromSpec, toSpecs] of Object.entries(vrsJson.mappedVerses)) {
+    for (const toSpec of toSpecs) {
+      if (toSpec in ret) {
+        ret[toSpec].push(fromSpec);
+      } else {
+        ret[toSpec] = [fromSpec];
+      }
+    }
+  }
+  return { reverseMappedVerses: ret };
+};
+
+const preSuccinctVerseMapping = mappingJson => {
+  const ret = {};
+
+  for (let [fromSpec, toSpecs] of Object.entries(mappingJson)) {
+    if (typeof toSpecs === 'string') {
+      toSpecs = [toSpecs];
+    }
+
+    const [fromBook, fromCVV] = fromSpec.split(' ');
+    const toBook = toSpecs[0].split(' ')[0];
+    const record = toBook === fromBook ? ['cv'] : ['bcv'];
+    let [fromCh, fromV] = fromCVV.split(':');
+    let toV = fromV;
+
+    if (fromV.includes('-')) {
+      const vBits = fromV.split('-');
+      fromV = vBits[0];
+      toV = vBits[1];
+    }
+    record.push([parseInt(fromV), parseInt(toV)]);
+    record.push([]);
+
+    for (const toCVV of toSpecs.map(ts => ts.split(' ')[1])) {
+      let [toCh, fromV] = toCVV.split(':');
+      let toV = fromV;
+
+      if (fromV.includes('-')) {
+        const vBits = fromV.split('-');
+        fromV = vBits[0];
+        toV = vBits[1];
+      }
+
+      if (record[0] === 'cv') {
+        record[2].push([parseInt(toCh), parseInt(fromV), parseInt(toV)]);
+      } else {
+        record[2].push([parseInt(toCh), parseInt(fromV), parseInt(toV), toBook]);
+      }
+    }
+
+    if (!(fromBook in ret)) {
+      ret[fromBook] = {};
+    }
+
+    if (!(fromCh in ret[fromBook])) {
+      ret[fromBook][fromCh] = [];
+    }
+    ret[fromBook][fromCh].push(record);
+  }
+  return ret;
+};
+
+const succinctifyVerseMappings = preSuccinct => {
+  const ret = {};
+  const bci = bookCodeIndex();
+
+  for (const [book, chapters] of Object.entries(preSuccinctVerseMapping(preSuccinct))) {
+    ret[book] = {};
+
+    for (const [chapter, mappings] of Object.entries(chapters)) {
+      ret[book][chapter] = succinctifyVerseMapping(mappings, bci);
+    }
+  }
+  return ret;
+};
+
+const succinctifyVerseMapping = (preSuccinctBC, bci) => {
+  const makeMappingLengthByte = (recordType, length) =>
+    length + (recordType * 64);
+
+  const ret = new ByteArray(64);
+
+  for (const [recordTypeStr, [fromVerseStart, fromVerseEnd], mappings] of preSuccinctBC) {
+    const pos = ret.length;
+    const recordType = recordTypeStr === 'bcv' ? bcvMappingType : cvMappingType;
+    ret.pushNBytes([0, fromVerseStart, fromVerseEnd]);
+
+    if (recordType === bcvMappingType) {
+      const bookIndex = bci[mappings[0][3]];
+      ret.pushNByte(bookIndex);
+    }
+    ret.pushNByte(mappings.length);
+
+    for (const [ch, fromV] of mappings) {
+      ret.pushNBytes([ch, fromV]);
+    }
+
+    const recordLength = ret.length - pos;
+
+    if (recordLength > 63) {
+      throw new Error(`Mapping in succinctifyVerseMapping ${JSON.stringify(mappings)} is too long (${recordLength} bytes)`);
+    }
+    ret.setByte(pos, makeMappingLengthByte(recordType, recordLength));
+  }
+  ret.trim();
+  return ret;
+};
+
+const mappingLengthByte = (succinct, pos) => {
+  const sByte = succinct.byte(pos);
+  return [
+    sByte >> 6,
+    sByte % 64,
+  ];
+};
+
+// eslint-disable-next-line no-unused-vars
+const unsuccinctifyVerseMapping = (succinctBC, fromBookCode, bci) => {
+  const ret = [];
+  let pos = 0;
+
+  while (pos < succinctBC.length) {
+    let recordPos = pos;
+    const unsuccinctRecord = {};
+    const [recordType, recordLength] = mappingLengthByte(succinctBC, pos);
+    recordPos++;
+    unsuccinctRecord.fromVerseStart = succinctBC.nByte(recordPos);
+    recordPos += succinctBC.nByteLength(unsuccinctRecord.fromVerseStart);
+    unsuccinctRecord.fromVerseEnd = succinctBC.nByte(recordPos);
+    recordPos += succinctBC.nByteLength(unsuccinctRecord.fromVerseEnd);
+    unsuccinctRecord.bookCode = fromBookCode;
+
+    if (recordType === bcvMappingType) {
+      const bookIndex = succinctBC.nByte(recordPos);
+      unsuccinctRecord.bookCode = bookCodes[bookIndex];
+      recordPos += succinctBC.nByteLength(bookIndex);
+    }
+
+    const nMappings = succinctBC.nByte(recordPos);
+    recordPos += succinctBC.nByteLength(nMappings);
+    const mappings = [];
+
+    while (mappings.length < nMappings) {
+      const mapping = {};
+      mapping.ch = succinctBC.nByte(recordPos);
+      recordPos += succinctBC.nByteLength(mapping.ch);
+      mapping.verseStart = succinctBC.nByte(recordPos);
+      recordPos += succinctBC.nByteLength(mapping.verseStart);
+      mappings.push(mapping);
+    }
+    unsuccinctRecord.mapping = mappings;
+    ret.push(unsuccinctRecord);
+    pos += recordLength;
+  }
+  return ret;
+};
+
+const mapVerse = (succinct, b, c, v) => {
+  // Succinct for one chapter.
+  // Pass book and chapter to provide complete response in each case.
+  let ret = null;
+  let pos = 0;
+
+  while (pos < succinct.length) {
+    let recordPos = pos;
+    const [recordType, recordLength] = mappingLengthByte(succinct, pos);
+    recordPos++;
+    const fromVerseStart = succinct.nByte(recordPos);
+    recordPos += succinct.nByteLength(fromVerseStart);
+    const fromVerseEnd = succinct.nByte(recordPos);
+    recordPos += succinct.nByteLength(fromVerseEnd);
+
+    if (v < fromVerseStart || v > fromVerseEnd) {
+      pos += recordLength;
+      continue;
+    }
+
+    let bookCode = b;
+
+    if (recordType === bcvMappingType) {
+      const bookIndex = succinct.nByte(recordPos);
+      bookCode = bookCodes[bookIndex];
+      recordPos += succinct.nByteLength(bookIndex);
+    }
+    ret = [bookCode, []];
+    const nMappings = succinct.nByte(recordPos);
+    recordPos += succinct.nByteLength(nMappings);
+
+    while (ret[1].length < nMappings) {
+      const ch = succinct.nByte(recordPos);
+      recordPos += succinct.nByteLength(ch);
+      const verseStart = succinct.nByte(recordPos);
+      recordPos += succinct.nByteLength(verseStart);
+      ret[1].push([ch, (v - fromVerseStart) + verseStart]);
+    }
+    break;
+  }
+  return ret || [b, [[c, v]]];
+};
+
+module.exports = {
+  vrs2json,
+  reverseVersification,
+  preSuccinctVerseMapping,
+  bookCodes,
+  succinctifyVerseMapping,
+  succinctifyVerseMappings,
+  unsuccinctifyVerseMapping,
+  bookCodeIndex,
+  mapVerse,
+};