proskomma-core 0.9.10 → 0.9.13

package/src/util/generateId.cjs

@@ -0,0 +1,6 @@
+const UUID = require('pure-uuid');
+const btoa = require('btoa');
+
+const generateId = () => btoa(new UUID(4)).substring(0, 12);
+
+module.exports = { generateId };

package/src/util/graftDefs.cjs

@@ -0,0 +1,15 @@
+const graftLocation = {
+  heading: 'block',
+  title: 'block',
+  endTitle: 'block',
+  remark: 'block',
+  footnote: 'inline',
+  xref: 'inline',
+  noteCaller: 'inline',
+  esbCat: 'inline',
+  table: 'block',
+  tree: 'block',
+  kv: 'block',
+};
+
+module.exports = { graftLocation };

package/src/util/index.cjs

@@ -0,0 +1,29 @@
+const ByteArray = require('./byteArray.cjs');
+const canons = require('./canons.cjs');
+const enums = require('./enums.cjs');
+const { generateId } = require('./generateId.cjs');
+const graftDefs = require('./graftDefs.cjs');
+const inspect = require('./inspectSuccinct.cjs');
+const itemDefs = require('./itemDefs.cjs');
+const parserConstants = require('./parserConstantDefs.cjs');
+const scopeDefs = require('./scopeDefs.cjs');
+const succinct = require('./succinct.cjs');
+const tags = require('./tags.cjs');
+const tokenDefs = require('./tokenDefs.cjs');
+const versification = require('./versification.cjs');
+
+module.exports = {
+  ByteArray,
+  canons,
+  enums,
+  generateId,
+  graftDefs,
+  inspect,
+  itemDefs,
+  parserConstants,
+  scopeDefs,
+  succinct,
+  tags,
+  tokenDefs,
+  versification,
+};

package/src/util/inspectSuccinct.cjs

@@ -0,0 +1,59 @@
+const ByteArray = require('./byteArray.cjs');
+const succinct = require('./succinct.cjs');
+const itemDefs = require('./itemDefs.cjs');
+const tokenDefs = require('./tokenDefs.cjs');
+const scopeDefs = require('./scopeDefs.cjs');
+
+const inspectEnum = enumString => {
+  const ba = new ByteArray();
+  ba.fromBase64(enumString);
+  const ret = [];
+  ret.push(`* Char length ${ba.length} *`);
+
+  for (const [count, text] of succinct.unpackEnum(ba, true)) {
+    ret.push(`${count}\t"${text}"`);
+  }
+  return ret.join('\n');
+};
+
+const inspectSuccinct = (succinctdoc, enumStrings) => {
+  const ba = new ByteArray();
+  ba.fromBase64(succinctdoc);
+  const enums = {};
+
+  for (const [category, enumString] of Object.entries(enumStrings)) {
+    enums[category] = new ByteArray();
+    enums[category].fromBase64(enumString);
+  }
+
+  const indexes = succinct.enumIndexes(enums);
+  const ret = [];
+  ret.push(`* Char length ${ba.length} *`);
+  let pos = 0;
+
+  while (pos < ba.length) {
+    const [itemLength, itemType, itemSubtype] = succinct.headerBytes(ba, pos);
+    let subtypeLabel = itemSubtype;
+    let extra = '';
+
+    switch (itemDefs.itemEnumLabels[itemType]) {
+    case 'token':
+      subtypeLabel = tokenDefs.tokenEnumLabels[itemSubtype];
+      extra = `"${succinct.succinctTokenChars(enums, indexes, ba, itemSubtype, pos)}"`;
+      break;
+    case 'startScope':
+    case 'endScope':
+      subtypeLabel = scopeDefs.scopeEnumLabels[itemSubtype];
+      extra = succinct.succinctScopeLabel(enums, indexes, ba, itemSubtype, pos);
+      break;
+    case 'graft':
+      subtypeLabel = succinct.succinctGraftName(enums, indexes, itemSubtype);
+      extra = succinct.succinctGraftSeqId(enums, indexes, ba, pos);
+    }
+    ret.push(`${itemDefs.itemEnumLabels[itemType]}\t${subtypeLabel}\t(${itemLength})\t${extra}`);
+    pos += itemLength;
+  }
+  return ret.join('\n');
+};
+
+module.exports = { inspectEnum, inspectSuccinct };

package/src/util/itemDefs.cjs

@@ -0,0 +1,33 @@
+const itemEnum = {
+  token: 0,
+  graft: 1,
+  startScope: 2,
+  endScope: 3,
+};
+
+const itemEnumLabels = Object.entries(itemEnum).sort((a, b) => a[1] - b[1]).map(kv => kv[0]);
+
+const itemArray2Object = a => ({
+  type: a[0],
+  subType: a[1],
+  payload: a[2],
+});
+
+const itemObject2Array = ob => [
+  ob.type,
+  ob.subType,
+  ob.payload,
+];
+
+const itemArrays2Objects = aa => aa.map(a => itemArray2Object(a));
+
+const itemObjects2Arrays = obs => obs.map(ob => itemObject2Array(ob));
+
+module.exports = {
+  itemEnum,
+  itemEnumLabels,
+  itemArray2Object,
+  itemObject2Array,
+  itemArrays2Objects,
+  itemObjects2Arrays,
+};

package/src/util/parserConstantDefs.cjs

@@ -0,0 +1,30 @@
+module.exports = {
+  usfm: {
+    baseSequenceTypes: {
+      main: '1',
+      introduction: '*',
+      introTitle: '?',
+      introEndTitle: '?',
+      title: '?',
+      endTitle: '?',
+      heading: '*',
+      header: '*',
+      remark: '*',
+      sidebar: '*',
+      table: '*',
+      tree: '*',
+      kv: '*',
+    },
+    inlineSequenceTypes: {
+      footnote: '*',
+      noteCaller: '*',
+      xref: '*',
+      pubNumber: '*',
+      altNumber: '*',
+      esbCat: '*',
+      fig: '*',
+      temp: '?',
+    },
+  },
+};
+

package/src/util/scopeDefs.cjs

@@ -0,0 +1,173 @@
+const xre = require('xregexp');
+
+const scopeEnum = {
+  'blockTag': 0,
+  'inline': 1,
+  'chapter': 2,
+  'pubChapter': 3,
+  'altChapter': 4,
+  'verses': 5,
+  'verse': 6,
+  'pubVerse': 7,
+  'altVerse': 8,
+  'esbCat': 9,
+  'span': 10,
+  'table': 11,
+  'cell': 12,
+  'milestone': 13,
+  'spanWithAtts': 14,
+  'attribute': 15,
+  'hangingGraft': 16,
+  'orphanTokens': 17,
+  'tTableRow': 18,
+  'tTableCol': 19,
+  'tTreeNode': 20,
+  'tTreeParent': 21,
+  'tTreeChild': 22,
+  'tTreeContent': 23,
+  'kvPrimary': 24,
+  'kvSecondary': 25,
+  'kvField': 26,
+};
+
+const scopeEnumLabels = Object.entries(scopeEnum).sort((a, b) => a[1] - b[1]).map(kv => kv[0]);
+
+const splitTagNumber = fullTagName => {
+  const tagBits = xre.exec(fullTagName, xre('([^1-9]+)(.*)'));
+  const tagName = tagBits[1];
+  const tagNo = tagBits[2].length > 0 ? tagBits[2] : '1';
+  return [tagName, tagNo];
+};
+
+const cellScope = fullTagName => {
+  const tagProps = {
+    'th': {
+      type: 'colHeading',
+      align: 'left',
+    },
+    'thr': {
+      type: 'colHeading',
+      align: 'right',
+    },
+    'tc': {
+      type: 'body',
+      align: 'left',
+    },
+    'tcr': {
+      type: 'body',
+      align: 'right',
+    },
+  };
+  const [tagName, tagNo] = splitTagNumber(fullTagName);
+  let tagField = '1';
+
+  if (tagNo.includes('-')) {
+    const [fromN, toN] = tagNo.split('-');
+    tagField = `${(parseInt(toN) - parseInt(fromN)) + 1}`;
+  }
+  return `cell/${tagProps[tagName].type}/${tagProps[tagName].align}/${tagField}`;
+};
+
+const labelForScope = (scopeType, scopeFields) => {
+  switch (scopeType) {
+  case 'blockTag':
+    return `blockTag/${scopeFields[0]}`;
+  case 'inline':
+    return `inline/${scopeFields[0]}`;
+  case 'chapter':
+    return `chapter/${scopeFields[0]}`;
+  case 'verses':
+    return `verses/${scopeFields[0]}`;
+  case 'verse':
+    return `verse/${scopeFields[0]}`;
+  case 'span':
+    return `span/${scopeFields[0]}`;
+  case 'table':
+    return 'table';
+  case 'cell':
+    return cellScope(scopeFields[0]);
+  case 'milestone':
+    return `milestone/${scopeFields[0]}`;
+  case 'spanWithAtts':
+    return `spanWithAtts/${scopeFields[0]}`;
+  case 'attribute':
+    return `attribute/${scopeFields[0]}/${scopeFields[1]}/${scopeFields[2]}/${scopeFields[3]}`;
+  case 'orphanTokens':
+    return `orphanTokens`;
+  case 'hangingGraft':
+    return `hangingGraft`;
+  case 'pubChapter':
+    return `pubChapter/${scopeFields[0]}`;
+  case 'pubVerse':
+    return `pubVerse/${scopeFields[0]}`;
+  case 'altChapter':
+    return `altChapter/${scopeFields[0]}`;
+  case 'altVerse':
+    return `altVerse/${scopeFields[0]}`;
+  case 'esbCat':
+    return `esbCat/${scopeFields[0]}`;
+  case 'tTableRow':
+    return `tTableRow/${scopeFields[0]}`;
+  case 'tTableCol':
+    return `tTableCol/${scopeFields[0]}`;
+  case 'tTreeNode':
+    return `tTreeNode/${scopeFields[0]}`;
+  case 'tTreeParent':
+    return `tTreeParent/${scopeFields[0]}`;
+  case 'tTreeChild':
+    return `tTreeChild/${scopeFields[0]}/${scopeFields[1]}`;
+  case 'tTreeContent':
+    return `tTreeContent/${scopeFields[0]}`;
+  case 'kvPrimary':
+    return `kvPrimary/${scopeFields[0]}`;
+  case 'kvSecondary':
+    return `kvSecondary/${scopeFields[0]}/${scopeFields[1]}`;
+  case 'kvField':
+    return `kvField/${scopeFields[0]}`;
+  default:
+    throw new Error(`Unknown scope type '${scopeType}' in labelForScope`);
+  }
+};
+
+const nComponentsForScope = (scopeType) => {
+  switch (scopeType) {
+  case 'orphanTokens':
+  case 'hangingGraft':
+  case 'table':
+    return 1;
+  case 'blockTag':
+  case 'inline':
+  case 'chapter':
+  case 'verses':
+  case 'verse':
+  case 'span':
+  case 'milestone':
+  case 'spanWithAtts':
+  case 'pubChapter':
+  case 'altChapter':
+  case 'pubVerse':
+  case 'altVerse':
+  case 'esbCat':
+  case 'tTableRow':
+  case 'tTableCol':
+  case 'tTreeNode':
+  case 'tTreeParent':
+  case 'tTreeContent':
+  case 'kvPrimary':
+  case 'kvField':
+    return 2;
+  case 'tTreeChild':
+  case 'kvSecondary':
+    return 3;
+  case 'cell':
+    return 4;
+  case 'attribute':
+    return 6;
+  default:
+    throw new Error(`Unknown scope type '${scopeType}' in nComponentsForScope`);
+  }
+};
+
+module.exports = {
+  scopeEnum, scopeEnumLabels, labelForScope, nComponentsForScope,
+};

package/src/util/scriptlike/blocksSpec.cjs

@@ -0,0 +1,192 @@
+const utils = require('../index.cjs');
+const { labelForScope } = utils.scopeDefs;
+const { tokenizeString } = require('../../parser/lib/tokenize.cjs');
+const {
+  flattenNodes,
+  numberNodes,
+} = require('../../parser/lexers/nodes.cjs');
+
+const tsvToInputBlock = (tsv, hasHeadings) => {
+  const ret = [];
+  const rows = tsv.split(/[\n\r]+/);
+
+  for (const [rowN, rowTSV] of rows.entries()) {
+    if (hasHeadings && rowN === 0) {
+      continue;
+    }
+
+    const row = rowTSV.split('\t');
+
+    for (const [cellN, cellString] of row.entries()) {
+      const cellRecord = {
+        os: [],
+        bg: [],
+        bs: {
+          type: 'scope',
+          subType: 'start',
+          payload: labelForScope('tTableRow', [`${rowN}`]),
+        },
+        is: [],
+        items: [],
+      };
+      const colScope = `tTableCol/${cellN}`;
+
+      cellRecord.is.push({
+        type: 'scope',
+        subType: 'start',
+        payload: colScope,
+      });
+      cellRecord.items.push({
+        type: 'scope',
+        subType: 'start',
+        payload: colScope,
+      });
+
+      for (const [token, tokenType] of tokenizeString(cellString)) {
+        cellRecord.items.push({
+          type: 'token',
+          subType: tokenType,
+          payload: token,
+        });
+      }
+      cellRecord.items.push({
+        type: 'scope',
+        subType: 'end',
+        payload: colScope,
+      });
+      ret.push(cellRecord);
+    }
+  }
+  return ret;
+};
+
+const tsvHeadingTags = tsv => {
+  const firstRow = tsv.split(/[\n\r]+/)[0];
+  return firstRow.split('\t').map((c, n) => `col${n}:${c.trim()}`);
+};
+
+const treeToInputBlock = treeJson => {
+  const ret = [];
+
+  for (const node of flattenNodes(numberNodes(treeJson))) {
+    const nodeRecord = {
+      os: [],
+      bg: [],
+      bs: {
+        type: 'scope',
+        subType: 'start',
+        payload: labelForScope('tTreeNode', [`${node.id}`]),
+      },
+      is: [],
+      items: [],
+    };
+    const scopePayload = labelForScope('tTreeParent', [`${node.parentId}`]);
+
+    nodeRecord.items.push({
+      type: 'scope',
+      subType: 'start',
+      payload: scopePayload,
+    });
+
+    nodeRecord.is.push({
+      type: 'scope',
+      subType: 'start',
+      payload: scopePayload,
+    });
+
+    if (node.content) {
+      for (const [name, content] of Object.entries(node.content)) {
+        const treeContentStart = nodeRecord.items.length;
+        const tokenized = tokenizeString(content);
+        const scopePayload = labelForScope('tTreeContent', [name, node.id, `${treeContentStart}`, `${tokenized.length}`]);
+
+        nodeRecord.items.push({
+          type: 'scope',
+          subType: 'start',
+          payload: scopePayload,
+        });
+
+        nodeRecord.is.push({
+          type: 'scope',
+          subType: 'start',
+          payload: scopePayload,
+        });
+
+        for (const [payload, subType] of tokenized) {
+          nodeRecord.items.push({
+            type: 'token',
+            subType,
+            payload,
+          });
+        }
+        nodeRecord.items.push({
+          type: 'scope',
+          subType: 'end',
+          payload: scopePayload,
+        });
+      }
+    }
+
+    if (node.children) {
+      for (const [childN, childNodeN] of node.children.entries()) {
+        const scopePayload = labelForScope('tTreeChild', [childN, childNodeN]);
+
+        nodeRecord.items.push({
+          type: 'scope',
+          subType: 'start',
+          payload: scopePayload,
+        });
+        nodeRecord.is.push({
+          type: 'scope',
+          subType: 'start',
+          payload: scopePayload,
+        });
+        nodeRecord.items.push({
+          type: 'scope',
+          subType: 'end',
+          payload: scopePayload,
+        });
+      }
+    }
+    nodeRecord.items.push({
+      type: 'scope',
+      subType: 'end',
+      payload: scopePayload,
+    });
+    ret.push(nodeRecord);
+  }
+  return ret;
+};
+
+const escapePayload =
+    str =>
+      str.replace(/\\/g, '\\\\')
+        .replace(/"/g, '\\"')
+        .replace(/\n/g, '\\n')
+        .replace(/\t/g, '\\t')
+        .replace(/\r/g, '\\r');
+
+const object2Query = obs =>
+  '[' +
+  obs.map(
+    ob => `\n    {\n      type: "${ob.type}" \n      subType: "${ob.subType}" \n      payload: "${escapePayload(ob.payload)}"\n    }`,
+  ).join(',') +
+  ']';
+const oneObject2Query = ob => `{\n      type: "${ob.type}" \n      subType: "${ob.subType}" \n      payload: "${escapePayload(ob.payload)}"}`;
+const blocksSpec2Query =
+    bSpec =>
+      '[\n' +
+      bSpec.map(
+        b =>
+          `  {\n    bs: ${oneObject2Query(b.bs)}, \n    bg: ${object2Query(b.bg)}, \n    os: ${object2Query(b.os)}, \n    is: ${object2Query(b.is)}, \n    items: ${object2Query(b.items)}}\n`) +
+      ']';
+
+module.exports = {
+  tokenizeString,
+  tsvToInputBlock,
+  tsvHeadingTags,
+  treeToInputBlock,
+  blocksSpec2Query,
+  object2Query,
+  oneObject2Query,
+};

package/src/util/scriptlike/dodson_lexicon.xsl

@@ -0,0 +1,19 @@
+<?xml version="1.0" ?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+
+    <xsl:output method="text"/>
+
+    <xsl:template match="/">
+        <xsl:for-each select="/*/*">
+            <xsl:value-of select="normalize-space(substring-before(@n, '|'))"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="normalize-space(substring-after(@n, '|'))"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="normalize-space(*[2])"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="normalize-space(*[3])"/>
+            <xsl:text>&#x0A;</xsl:text>
+        </xsl:for-each>
+     </xsl:template>
+
+</xsl:stylesheet>

package/src/util/scriptlike/dodson_lexicon_to_entries.cjs

@@ -0,0 +1,85 @@
+const fse = require('fs-extra');
+const { tokenizeString } = require('./blocksSpec.cjs');
+
+const itemTokens = str => tokenizeString(str).map(tr => ({
+  type: 'token',
+  subType: tr[1],
+  payload: tr[0],
+}));
+
+const rowRecord = rowArray => ({
+  'bs': {
+    'type': 'scope',
+    'subType': 'start',
+    'payload': `kvPrimary/${rowArray[0]}`,
+  },
+  'bg': [],
+  'os': [],
+  'is': [
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': `kvSecondary/strong/${rowArray[1]}`,
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/briefDef',
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/fullDef',
+    },
+  ],
+  'items': [
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': `kvSecondary/strong/${rowArray[1]}`,
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/briefDef',
+    },
+    ...itemTokens(rowArray[2]),
+    {
+      'type': 'scope',
+      'subType': 'end',
+      'payload': 'kvField/briefDef',
+    },
+    {
+      'type': 'scope',
+      'subType': 'start',
+      'payload': 'kvField/fullDef',
+    },
+    ...itemTokens(rowArray[3]),
+    {
+      'type': 'scope',
+      'subType': 'end',
+      'payload': 'kvField/fullDef',
+    },
+    {
+      'type': 'scope',
+      'subType': 'end',
+      'payload': `kvSecondary/strong/${rowArray[1]}`,
+    },
+  ],
+});
+
+const lexicon_tsv = fse.readFileSync(process.argv[2])
+  .toString()
+  .split('\n')
+  .map(row => row.split('\t'))
+  .filter(row => row.length > 1);
+
+console.log(
+  JSON.stringify(
+    lexicon_tsv.map(
+      row => rowRecord(row),
+    ),
+    null,
+    2,
+  ),
+);

package/src/util/scriptlike/open_scriptures_hebrew_lexicon.xsl

@@ -0,0 +1,28 @@
+<?xml version="1.0" ?>
+<xsl:stylesheet
+        version="1.0"
+        xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+        xmlns:os="http://openscriptures.github.com/morphhb/namespace">
+
+    <xsl:output method="text"/>
+
+    <xsl:template match="/os:lexicon">
+        <xsl:for-each select="os:entry">
+            <xsl:value-of select="os:w/text()"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="@id"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="os:w/@pos"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="os:meaning"/>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:for-each select="os:meaning/os:def">
+                <xsl:value-of select="concat(./text(), '; .cjs')"/>
+            </xsl:for-each>
+            <xsl:text>&#x09;</xsl:text>
+            <xsl:value-of select="os:usage/text()"/>
+            <xsl:text>&#x0A;</xsl:text>
+        </xsl:for-each>
+     </xsl:template>
+
+</xsl:stylesheet>