proskomma-core 0.9.18 → 0.9.20

package/src/util/scriptlike/open_scriptures_hebrew_lexicon_to_entries.cjs

@@ -1,85 +0,0 @@
-const fse = require('fs-extra');
-const { tokenizeString } = require('../../parser/lib/tokenize.cjs');
-
-const itemTokens = str => tokenizeString(str).map(tr => ({
-  type: 'token',
-  subType: tr[1],
-  payload: tr[0],
-}));
-
-const rowRecord = rowArray => ({
-  'bs': {
-    'type': 'scope',
-    'subType': 'start',
-    'payload': `kvPrimary/${rowArray[0]}`,
-  },
-  'bg': [],
-  'os': [],
-  'is': [
-    {
-      'type': 'scope',
-      'subType': 'start',
-      'payload': `kvSecondary/strong/${rowArray[1]}`,
-    },
-    {
-      'type': 'scope',
-      'subType': 'start',
-      'payload': 'kvField/briefDef',
-    },
-    {
-      'type': 'scope',
-      'subType': 'start',
-      'payload': 'kvField/fullDef',
-    },
-  ],
-  'items': [
-    {
-      'type': 'scope',
-      'subType': 'start',
-      'payload': `kvSecondary/strong/${rowArray[1]}`,
-    },
-    {
-      'type': 'scope',
-      'subType': 'start',
-      'payload': 'kvField/briefDef',
-    },
-    ...itemTokens(rowArray[4].length > 0 ? rowArray[4].replace(/; $/, '') : rowArray[5]),
-    {
-      'type': 'scope',
-      'subType': 'end',
-      'payload': 'kvField/briefDef',
-    },
-    {
-      'type': 'scope',
-      'subType': 'start',
-      'payload': 'kvField/fullDef',
-    },
-    ...itemTokens(rowArray[3]),
-    {
-      'type': 'scope',
-      'subType': 'end',
-      'payload': 'kvField/fullDef',
-    },
-    {
-      'type': 'scope',
-      'subType': 'end',
-      'payload': `kvSecondary/strong/${rowArray[1]}`,
-    },
-  ],
-});
-
-const lexicon_tsv = fse.readFileSync(process ?? process.argv[2])
-  .toString()
-  .split('\n')
-  .map(row => row.split('\t'))
-  .filter(row => row.length > 1);
-
-console.log(
-  JSON.stringify(
-    lexicon_tsv.map(
-      row => rowRecord(row),
-    ),
-    null,
-    2,
-  ),
-);

package/src/util/succinct.cjs

@@ -1,159 +0,0 @@
-const tokenDefs = require('./tokenDefs.cjs');
-const scopeDefs = require('./scopeDefs.cjs');
-const itemDefs = require('./itemDefs.cjs');
-
-const headerBytes = (succinct, pos) =>{
-  const headerByte = succinct.byte(pos);
-  const itemType = headerByte >> 6;
-  const itemLength = headerByte & 0x0000003F;
-  const itemSubtype = succinct.byte(pos + 1);
-  return [itemLength, itemType, itemSubtype];
-};
-
-const succinctTokenChars = (enums, enumIndexes, succinct, itemSubtype, pos) => {
-  const itemCategory = tokenDefs.tokenCategory[tokenDefs.tokenEnumLabels[itemSubtype]];
-  const itemIndex = enumIndexes[itemCategory][succinct.nByte(pos + 2)];
-  return enums[itemCategory].countedString(itemIndex);
-};
-
-const succinctScopeLabel = (enums, enumIndexes, succinct, itemSubtype, pos) => {
-  const scopeType = scopeDefs.scopeEnumLabels[itemSubtype];
-  let nScopeBits = scopeDefs.nComponentsForScope(scopeType);
-  let offset = 2;
-  let scopeBits = '';
-
-  while (nScopeBits > 1) {
-    const itemIndexIndex = succinct.nByte(pos + offset);
-    const itemIndex = enumIndexes.scopeBits[itemIndexIndex];
-    const scopeBitString = enums.scopeBits.countedString(itemIndex);
-    scopeBits += `/${scopeBitString}`;
-    offset += succinct.nByteLength(itemIndexIndex);
-    nScopeBits--;
-  }
-  return `${scopeType}${scopeBits}`;
-};
-
-const succinctGraftName = (enums, enumIndexes, itemSubtype) => {
-  const graftIndex = enumIndexes.graftTypes[itemSubtype];
-  return enums.graftTypes.countedString(graftIndex);
-};
-
-const succinctGraftSeqId = (enums, enumIndexes, succinct, pos) => {
-  const seqIndex = enumIndexes.ids[succinct.nByte(pos + 2)];
-  return enums.ids.countedString(seqIndex);
-};
-
-const enumIndexes = (enums) => {
-  const ret = {};
-
-  for (const [category, succinct] of Object.entries(enums)) {
-    ret[category] = enumIndex(category, succinct);
-  }
-  return ret;
-};
-
-const enumIndex = (category, enumSuccinct) => {
-  const indexSuccinct = new Uint32Array(enumSuccinct.length);
-  let pos = 0;
-  let count = 0;
-
-  while (pos < enumSuccinct.length) {
-    indexSuccinct[count] = pos;
-    const stringLength = enumSuccinct.byte(pos);
-    pos += (stringLength + 1);
-    count += 1;
-  }
-  return indexSuccinct;
-};
-
-const unpackEnum = (succinct, includeIndex) => {
-  if (!includeIndex) {
-    includeIndex = false;
-  }
-
-  let pos = 0;
-  let count = 0;
-  const ret = [];
-
-  while (pos < succinct.length) {
-    const stringLength = succinct.byte(pos);
-    const unpacked = succinct.countedString(pos);
-    ret.push(includeIndex ? [count, unpacked] : unpacked);
-    pos += stringLength + 1;
-    count++;
-  }
-  return ret;
-};
-
-const undefinedArgError = (func, field) => {
-  throw new Error(`Undefined or null argument '${field}' in '${func}'`);
-};
-
-const pushSuccinctTokenBytes = (bA, tokenEnumIndex, charsEnumIndex) => {
-  if (tokenEnumIndex === undefined || tokenEnumIndex === null) {
-    undefinedArgError('pushSuccinctTokenBytes', 'tokenEnumIndex');
-  }
-
-  if (charsEnumIndex === undefined || charsEnumIndex === null) {
-    undefinedArgError('pushSuccinctTokenBytes', 'charsEnumIndex');
-  }
-
-  const lengthPos = bA.length;
-  bA.pushByte(0);
-  bA.pushByte(tokenEnumIndex);
-  bA.pushNByte(charsEnumIndex);
-  bA.setByte(lengthPos, (bA.length - lengthPos) | itemDefs.itemEnum.token << 6);
-};
-
-const pushSuccinctGraftBytes = (bA, graftTypeEnumIndex, seqEnumIndex) => {
-  if (graftTypeEnumIndex === undefined || graftTypeEnumIndex === null) {
-    undefinedArgError('pushSuccinctGraftBytes', 'graftTypeEnumIndex');
-  }
-
-  if (seqEnumIndex === undefined || seqEnumIndex === null) {
-    undefinedArgError('pushSuccinctGraftBytes', 'seqEnumIndex');
-  }
-
-  const lengthPos = bA.length;
-  bA.pushByte(0);
-  bA.pushByte(graftTypeEnumIndex);
-  bA.pushNByte(seqEnumIndex);
-  bA.setByte(lengthPos, (bA.length - lengthPos) | itemDefs.itemEnum.graft << 6);
-};
-
-const pushSuccinctScopeBytes = (bA, itemTypeByte, scopeTypeByte, scopeBitBytes) => {
-  if (itemTypeByte === undefined || itemTypeByte === null) {
-    undefinedArgError('pushSuccinctScopeBytes', 'itemTypeByte');
-  }
-
-  if (scopeTypeByte === undefined || scopeTypeByte === null) {
-    undefinedArgError('pushSuccinctScopeBytes', 'scopeTypeByte');
-  }
-
-  if (scopeBitBytes === undefined || scopeBitBytes === null) {
-    undefinedArgError('pushSuccinctScopeBytes', 'scopeBitBytes');
-  }
-
-  const lengthPos = bA.length;
-  bA.pushByte(0);
-  bA.pushByte(scopeTypeByte);
-
-  for (const sbb of scopeBitBytes) {
-    bA.pushNByte(sbb);
-  }
-  bA.setByte(lengthPos, (bA.length - lengthPos) | itemTypeByte << 6);
-};
-
-module.exports = {
-  enumIndex,
-  enumIndexes,
-  headerBytes,
-  pushSuccinctTokenBytes,
-  pushSuccinctGraftBytes,
-  pushSuccinctScopeBytes,
-  succinctTokenChars,
-  succinctScopeLabel,
-  succinctGraftName,
-  succinctGraftSeqId,
-  unpackEnum,
-};

package/src/util/tags.cjs

@@ -1,28 +0,0 @@
-const xre = require('xregexp');
-
-const validateTags = tags => {
-  for (const tag of tags) {
-    validateTag(tag);
-  }
-};
-
-const validateTag = tag => {
-  if (!xre.exec(tag, /^[a-z][A-za-z0-9]*(:.+)?$/)) {
-    throw new Error(`Tag '${tag}' is not valid (should be [a-z][A-za-z0-9]*(:.+)?)`);
-  }
-};
-
-const addTag = (tags, tag) => {
-  validateTag(tag);
-  tags.add(tag);
-};
-
-const removeTag = (tags, tag) => {
-  validateTag(tag);
-  tags.delete(tag);
-};
-
-module.exports = {
-  validateTags, validateTag, addTag, removeTag,
-};
-

package/src/util/tokenDefs.cjs

@@ -1,27 +0,0 @@
-const tokenEnum = {
-  wordLike: 0,
-  punctuation: 1,
-  lineSpace: 2,
-  eol: 3,
-  softLineBreak: 4,
-  noBreakSpace: 5,
-  bareSlash: 6,
-  unknown: 7,
-};
-
-const tokenEnumLabels = Object.entries(tokenEnum).sort((a, b) => a[1] - b[1]).map(kv => kv[0]);
-
-const tokenCategory = {
-  wordLike: 'wordLike',
-  punctuation: 'notWordLike',
-  lineSpace: 'notWordLike',
-  eol: 'notWordLike',
-  softLineBreak: 'notWordLike',
-  noBreakSpace: 'notWordLike',
-  bareSlash: 'notWordLike',
-  unknown: 'notWordLike',
-};
-
-module.exports = {
-  tokenEnum, tokenEnumLabels, tokenCategory,
-};

package/src/util/versification.cjs

@@ -1,364 +0,0 @@
-const xre = require('xregexp');
-
-const ByteArray = require('./byteArray.cjs');
-
-const cvMappingType = 2;
-const bcvMappingType = 3;
-
-const bookCodes = [ // From Paratext via Scripture Burrito
-  'GEN',
-  'EXO',
-  'LEV',
-  'NUM',
-  'DEU',
-  'JOS',
-  'JDG',
-  'RUT',
-  '1SA',
-  '2SA',
-  '1KI',
-  '2KI',
-  '1CH',
-  '2CH',
-  'EZR',
-  'NEH',
-  'EST',
-  'JOB',
-  'PSA',
-  'PRO',
-  'ECC',
-  'SNG',
-  'ISA',
-  'JER',
-  'LAM',
-  'EZK',
-  'DAN',
-  'HOS',
-  'JOL',
-  'AMO',
-  'OBA',
-  'JON',
-  'MIC',
-  'NAM',
-  'HAB',
-  'ZEP',
-  'HAG',
-  'ZEC',
-  'MAL',
-  'MAT',
-  'MRK',
-  'LUK',
-  'JHN',
-  'ACT',
-  'ROM',
-  '1CO',
-  '2CO',
-  'GAL',
-  'EPH',
-  'PHP',
-  'COL',
-  '1TH',
-  '2TH',
-  '1TI',
-  '2TI',
-  'TIT',
-  'PHM',
-  'HEB',
-  'JAS',
-  '1PE',
-  '2PE',
-  '1JN',
-  '2JN',
-  '3JN',
-  'JUD',
-  'REV',
-  'TOB',
-  'JDT',
-  'ESG',
-  'WIS',
-  'SIR',
-  'BAR',
-  'LJE',
-  'S3Y',
-  'SUS',
-  'BEL',
-  '1MA',
-  '2MA',
-  '3MA',
-  '4MA',
-  '1ES',
-  '2ES',
-  'MAN',
-  'PS2',
-  'ODA',
-  'PSS',
-  'JSA',
-  'JDB',
-  'TBS',
-  'SST',
-  'DNT',
-  'BLT',
-  'EZA',
-  '5EZ',
-  '6EZ',
-  'DAG',
-  'PS3',
-  '2BA',
-  'LBA',
-  'JUB',
-  'ENO',
-  '1MQ',
-  '2MQ',
-  '3MQ',
-  'REP',
-  '4BA',
-  'LAO',
-];
-
-const bookCodeIndex = () => {
-  const ret = {};
-
-  for (const [bookN, book] of Object.entries(bookCodes)) {
-    ret[book] = parseInt(bookN);
-  }
-  return ret;
-};
-
-const vrs2json = vrsString => {
-  const ret = {};
-
-  for (
-    const vrsLineBits of
-    vrsString
-      .split(/[\n\r]+/)
-      .map(l => l.trim())
-      .map(l => xre.exec(l, xre('^([A-Z1-6]{3} [0-9]+:[0-9]+(-[0-9]+)?) = ([A-Z1-6]{3} [0-9]+:[0-9]+[a-z]?(-[0-9]+)?)$')))
-  ) {
-    if (!vrsLineBits) {
-      continue;
-    }
-
-    if (!(vrsLineBits[1] in ret)) {
-      ret[vrsLineBits[1]] = [];
-    }
-    ret[vrsLineBits[1]].push(vrsLineBits[3]);
-  }
-  return { mappedVerses: ret };
-};
-
-const reverseVersification = vrsJson => {
-  // Assumes each verse is only mapped from once
-  const ret = {};
-
-  for (const [fromSpec, toSpecs] of Object.entries(vrsJson.mappedVerses)) {
-    for (const toSpec of toSpecs) {
-      if (toSpec in ret) {
-        ret[toSpec].push(fromSpec);
-      } else {
-        ret[toSpec] = [fromSpec];
-      }
-    }
-  }
-  return { reverseMappedVerses: ret };
-};
-
-const preSuccinctVerseMapping = mappingJson => {
-  const ret = {};
-
-  for (let [fromSpec, toSpecs] of Object.entries(mappingJson)) {
-    if (typeof toSpecs === 'string') {
-      toSpecs = [toSpecs];
-    }
-
-    const [fromBook, fromCVV] = fromSpec.split(' ');
-    const toBook = toSpecs[0].split(' ')[0];
-    const record = toBook === fromBook ? ['cv'] : ['bcv'];
-    let [fromCh, fromV] = fromCVV.split(':');
-    let toV = fromV;
-
-    if (fromV.includes('-')) {
-      const vBits = fromV.split('-');
-      fromV = vBits[0];
-      toV = vBits[1];
-    }
-    record.push([parseInt(fromV), parseInt(toV)]);
-    record.push([]);
-
-    for (const toCVV of toSpecs.map(ts => ts.split(' ')[1])) {
-      let [toCh, fromV] = toCVV.split(':');
-      let toV = fromV;
-
-      if (fromV.includes('-')) {
-        const vBits = fromV.split('-');
-        fromV = vBits[0];
-        toV = vBits[1];
-      }
-
-      if (record[0] === 'cv') {
-        record[2].push([parseInt(toCh), parseInt(fromV), parseInt(toV)]);
-      } else {
-        record[2].push([parseInt(toCh), parseInt(fromV), parseInt(toV), toBook]);
-      }
-    }
-
-    if (!(fromBook in ret)) {
-      ret[fromBook] = {};
-    }
-
-    if (!(fromCh in ret[fromBook])) {
-      ret[fromBook][fromCh] = [];
-    }
-    ret[fromBook][fromCh].push(record);
-  }
-  return ret;
-};
-
-const succinctifyVerseMappings = preSuccinct => {
-  const ret = {};
-  const bci = bookCodeIndex();
-
-  for (const [book, chapters] of Object.entries(preSuccinctVerseMapping(preSuccinct))) {
-    ret[book] = {};
-
-    for (const [chapter, mappings] of Object.entries(chapters)) {
-      ret[book][chapter] = succinctifyVerseMapping(mappings, bci);
-    }
-  }
-  return ret;
-};
-
-const succinctifyVerseMapping = (preSuccinctBC, bci) => {
-  const makeMappingLengthByte = (recordType, length) =>
-    length + (recordType * 64);
-
-  const ret = new ByteArray(64);
-
-  for (const [recordTypeStr, [fromVerseStart, fromVerseEnd], mappings] of preSuccinctBC) {
-    const pos = ret.length;
-    const recordType = recordTypeStr === 'bcv' ? bcvMappingType : cvMappingType;
-    ret.pushNBytes([0, fromVerseStart, fromVerseEnd]);
-
-    if (recordType === bcvMappingType) {
-      const bookIndex = bci[mappings[0][3]];
-      ret.pushNByte(bookIndex);
-    }
-    ret.pushNByte(mappings.length);
-
-    for (const [ch, fromV] of mappings) {
-      ret.pushNBytes([ch, fromV]);
-    }
-
-    const recordLength = ret.length - pos;
-
-    if (recordLength > 63) {
-      throw new Error(`Mapping in succinctifyVerseMapping ${JSON.stringify(mappings)} is too long (${recordLength} bytes)`);
-    }
-    ret.setByte(pos, makeMappingLengthByte(recordType, recordLength));
-  }
-  ret.trim();
-  return ret;
-};
-
-const mappingLengthByte = (succinct, pos) => {
-  const sByte = succinct.byte(pos);
-  return [
-    sByte >> 6,
-    sByte % 64,
-  ];
-};
-
-// eslint-disable-next-line no-unused-vars
-const unsuccinctifyVerseMapping = (succinctBC, fromBookCode, bci) => {
-  const ret = [];
-  let pos = 0;
-
-  while (pos < succinctBC.length) {
-    let recordPos = pos;
-    const unsuccinctRecord = {};
-    const [recordType, recordLength] = mappingLengthByte(succinctBC, pos);
-    recordPos++;
-    unsuccinctRecord.fromVerseStart = succinctBC.nByte(recordPos);
-    recordPos += succinctBC.nByteLength(unsuccinctRecord.fromVerseStart);
-    unsuccinctRecord.fromVerseEnd = succinctBC.nByte(recordPos);
-    recordPos += succinctBC.nByteLength(unsuccinctRecord.fromVerseEnd);
-    unsuccinctRecord.bookCode = fromBookCode;
-
-    if (recordType === bcvMappingType) {
-      const bookIndex = succinctBC.nByte(recordPos);
-      unsuccinctRecord.bookCode = bookCodes[bookIndex];
-      recordPos += succinctBC.nByteLength(bookIndex);
-    }
-
-    const nMappings = succinctBC.nByte(recordPos);
-    recordPos += succinctBC.nByteLength(nMappings);
-    const mappings = [];
-
-    while (mappings.length < nMappings) {
-      const mapping = {};
-      mapping.ch = succinctBC.nByte(recordPos);
-      recordPos += succinctBC.nByteLength(mapping.ch);
-      mapping.verseStart = succinctBC.nByte(recordPos);
-      recordPos += succinctBC.nByteLength(mapping.verseStart);
-      mappings.push(mapping);
-    }
-    unsuccinctRecord.mapping = mappings;
-    ret.push(unsuccinctRecord);
-    pos += recordLength;
-  }
-  return ret;
-};
-
-const mapVerse = (succinct, b, c, v) => {
-  // Succinct for one chapter.
-  // Pass book and chapter to provide complete response in each case.
-  let ret = null;
-  let pos = 0;
-
-  while (pos < succinct.length) {
-    let recordPos = pos;
-    const [recordType, recordLength] = mappingLengthByte(succinct, pos);
-    recordPos++;
-    const fromVerseStart = succinct.nByte(recordPos);
-    recordPos += succinct.nByteLength(fromVerseStart);
-    const fromVerseEnd = succinct.nByte(recordPos);
-    recordPos += succinct.nByteLength(fromVerseEnd);
-
-    if (v < fromVerseStart || v > fromVerseEnd) {
-      pos += recordLength;
-      continue;
-    }
-
-    let bookCode = b;
-
-    if (recordType === bcvMappingType) {
-      const bookIndex = succinct.nByte(recordPos);
-      bookCode = bookCodes[bookIndex];
-      recordPos += succinct.nByteLength(bookIndex);
-    }
-    ret = [bookCode, []];
-    const nMappings = succinct.nByte(recordPos);
-    recordPos += succinct.nByteLength(nMappings);
-
-    while (ret[1].length < nMappings) {
-      const ch = succinct.nByte(recordPos);
-      recordPos += succinct.nByteLength(ch);
-      const verseStart = succinct.nByte(recordPos);
-      recordPos += succinct.nByteLength(verseStart);
-      ret[1].push([ch, (v - fromVerseStart) + verseStart]);
-    }
-    break;
-  }
-  return ret || [b, [[c, v]]];
-};
-
-module.exports = {
-  vrs2json,
-  reverseVersification,
-  preSuccinctVerseMapping,
-  bookCodes,
-  succinctifyVerseMapping,
-  succinctifyVerseMappings,
-  unsuccinctifyVerseMapping,
-  bookCodeIndex,
-  mapVerse,
-};