proskomma-core 0.9.10 → 0.9.13

package/src/parser/lexers/object_for_fragment.cjs

@@ -0,0 +1,123 @@
+const xre = require('xregexp');
+
+const makePrintable = (subclass, matchedBits) => ({
+  subclass,
+  printValue: matchedBits[0],
+});
+
+const makeChapter = (subclass, matchedBits) => ({
+  subclass,
+  numberString: matchedBits[2],
+  number: parseInt(matchedBits[2]),
+  printValue: `\\c ${matchedBits[2]}\n`,
+});
+
+const makeVerses = (subclass, matchedBits) => {
+  const ret = {
+    subclass,
+    numberString: matchedBits[2],
+    printValue: `\\v ${matchedBits[2]}\n`,
+  };
+
+  if (ret.numberString.includes('-')) {
+    const [fromV, toV] = ret.numberString.split('-').map(v => parseInt(v));
+    ret.numbers = Array.from(Array((toV - fromV) + 1).keys()).map(v => v + fromV);
+  } else {
+    ret.numbers = [parseInt(ret.numberString)];
+  }
+  return ret;
+};
+
+const makeAttribute = (subclass, matchedBits) => {
+  let ret;
+
+  if (subclass === 'defaultAttribute') {
+    ret = {
+      subclass,
+      key: 'default',
+      valueString: matchedBits[2].trim().replace(/\//g, '÷'),
+    };
+  } else {
+    ret = {
+      subclass,
+      key: matchedBits[2],
+      valueString: matchedBits[3].trim().replace(/\//g, '÷'),
+    };
+  }
+  ret.values = ret.valueString.split(',').map(vb => vb.trim());
+  ret.printValue = `| ${ret.key}="${ret.valueString}"`;
+  return ret;
+};
+
+const makePubChapter = (subclass, matchedBits) => ({
+  subclass,
+  numberString: matchedBits[2],
+  printValue: `\\cp ${matchedBits[2]}\n`,
+});
+
+const makeMilestone = (subclass, matchedBits) => {
+  const ret = {
+    subclass,
+    sOrE: null,
+  };
+
+  if (subclass === 'endMilestoneMarker') {
+    ret.printValue = '\\*';
+  } else {
+    ret.tagName = matchedBits[2];
+
+    if (subclass === 'emptyMilestone') {
+      ret.printValue = `\\${ret.tagName}\\*`;
+    } else {
+      ret.printValue = `\\${ret.tagName}`;
+      ret.sOrE = matchedBits[3];
+    }
+  }
+  return ret;
+};
+
+const makeTag = (subclass, matchedBits) => {
+  const ret = {
+    subclass,
+    tagName: matchedBits[2],
+    isNested: false,
+  };
+
+  if (ret.tagName.startsWith('+')) {
+    ret.isNested = true;
+    ret.tagName = ret.tagName.substring(1);
+  }
+  ret.tagLevel = matchedBits[3] !== '' ? parseInt(matchedBits[3]) : 1;
+  ret.fullTagName = `${ret.tagName}${matchedBits[3] === '1' ? '' : matchedBits[3]}`;
+  ret.printValue = subclass === 'startTag' ? `\\${ret.fullTagName} ` : `\\${ret.fullTagName}*`;
+  return ret;
+};
+
+const constructorForFragment = {
+  printable: makePrintable,
+  chapter: makeChapter,
+  pubchapter: makePubChapter,
+  verses: makeVerses,
+  tag: makeTag,
+  break: makePrintable,
+  milestone: makeMilestone,
+  attribute: makeAttribute,
+  bad: makePrintable,
+};
+
+const preTokenObjectForFragment = (fragment, lexingRegexes) => {
+  for (let n = 0; n < lexingRegexes.length; n++) {
+    let [tClass, tSubclass, tRE] = lexingRegexes[n];
+    let matchedBits = xre.exec(fragment, tRE, 0, 'sticky');
+
+    if (matchedBits) {
+      return constructorForFragment[tClass](tSubclass, matchedBits);
+    }
+  }
+  throw new Error(`Could not match preToken fragment '${fragment}'`);
+};
+
+module.exports = {
+  constructorForFragment,
+  preTokenObjectForFragment,
+};

package/src/parser/lexers/tsv.cjs

@@ -0,0 +1,45 @@
+const utils = require('../../util/index.cjs');
+const { labelForScope } = utils.scopeDefs;
+const { Sequence } = require('../model/index.cjs');
+const { tokenizeString } = require('../lib/tokenize.cjs');
+
+const parseTableToDocument = (str, parser, bookCode) => {
+  const { rows } = JSON.parse(str);
+  parser.headers.id = bookCode;
+  parser.headers.bookCode = bookCode;
+  const tableSequence = new Sequence('table');
+
+  for (const [rowN, row] of rows.entries()) {
+    for (const [cellN, cell] of row.entries()) {
+      tableSequence.newBlock(labelForScope('tTableRow', [`${rowN}`]));
+      const lastBlock = tableSequence.lastBlock();
+
+      lastBlock.addItem({
+        type: 'scope',
+        subType: 'start',
+        payload: `tTableCol/${cellN}`,
+      });
+
+      for (const [token, tokenType] of tokenizeString(cell)) {
+        lastBlock.addItem({
+          type: 'token',
+          subType: tokenType,
+          payload: token,
+        });
+      }
+      lastBlock.addItem({
+        type: 'scope',
+        subType: 'end',
+        payload: `tTableCol/${cellN}`,
+      });
+    }
+  }
+  parser.sequences.table.push(tableSequence);
+  parser.sequences.main.addBlockGraft({
+    type: 'graft',
+    subType: 'table',
+    payload: tableSequence.id,
+  });
+};
+
+module.exports = { parseTableToDocument };

package/src/parser/lexers/usfm.cjs

@@ -0,0 +1,17 @@
+const xre = require('xregexp');
+
+const {
+  lexingRegexes,
+  mainRegex,
+} = require('./lexingRegexes.cjs');
+const { preTokenObjectForFragment } = require('./object_for_fragment.cjs');
+
+const parseUsfm = (str, parser) => {
+  const matches = xre.match(str, mainRegex, 'all');
+
+  for (let n = 0; n < matches.length; n++) {
+    parser.parseItem(preTokenObjectForFragment(matches[n], lexingRegexes));
+  }
+};
+
+module.exports = { parseUsfm };

package/src/parser/lexers/usx/index.cjs

@@ -0,0 +1,7 @@
+const { UsxLexer } = require('./usx_lexer.cjs');
+
+const parseUsx = (str, parser) => {
+  new UsxLexer().lexAndParse(str, parser);
+};
+
+module.exports = { parseUsx };

package/src/parser/lexers/usx/usx_lexer.cjs

@@ -0,0 +1,321 @@
+const xre = require('xregexp');
+
+const sax = require('sax');
+
+const {
+  lexingRegexes,
+  mainRegex,
+} = require('../lexingRegexes.cjs');
+const {
+  preTokenObjectForFragment,
+  constructorForFragment,
+} = require('../object_for_fragment.cjs');
+
+class UsxLexer {
+  constructor() {
+    this.sax = sax.parser(true);
+    this.sax.ontext = text => this.handleSaxText(text);
+    this.sax.onopentag = ot => this.handleSaxOpenTag(ot);
+    this.sax.onclosetag = ct => this.handleSaxCloseTag(ct);
+    this.lexed = [];
+    this.elementStack = [];
+    this.currentText = '';
+    this.openTagHandlers = {
+      usx: this.ignoreHandler,
+      book: this.handleBookOpen,
+      chapter: this.handleChapter,
+      verse: this.handleVerses,
+      para: this.handleParaOpen,
+      table: this.ignoreHandler,
+      row: this.handleRowOpen,
+      cell: this.handleCellOpen,
+      char: this.handleCharOpen,
+      ms: this.handleMSOpen,
+      note: this.handleNoteOpen,
+      sidebar: this.handleSidebarOpen,
+      periph: this.notHandledHandler,
+      figure: this.handleFigureOpen,
+      optbreak: this.handleOptBreakOpen,
+      ref: this.ignoreHandler, // this.handleRefOpen,
+    };
+    this.closeTagHandlers = {
+      usx: this.ignoreHandler,
+      book: this.handleBookClose,
+      chapter: this.ignoreHandler,
+      verse: this.ignoreHandler,
+      para: this.handleParaClose,
+      table: this.ignoreHandler,
+      row: this.handleRowClose,
+      cell: this.handleCellClose,
+      char: this.handleCharClose,
+      ms: this.handleMSClose,
+      note: this.handleNoteClose,
+      sidebar: this.handleSidebarClose,
+      periph: this.notHandledHandler,
+      figure: this.handleFigureClose,
+      optbreak: this.handleOptBreakClose,
+      ref: this.ignoreHandler, // this.handleRefClose,
+    };
+  }
+
+  lexAndParse(str, parser) {
+    this.parser = parser;
+    this.lexed = [];
+    this.elementStack = [];
+    this.sax.write(str).close();
+  }
+
+  handleSaxText(text) {
+    this.currentText = this.replaceEntities(text);
+    xre.match(this.currentText, mainRegex, 'all')
+      .map(f => preTokenObjectForFragment(f, lexingRegexes))
+      .forEach(t => this.parser.parseItem(t));
+  }
+
+  replaceEntities(text) {
+    return text.replace('&lt;', '<')
+      .replace('&gt;', '>')
+      .replace('&apos;', '\'')
+      .replace('&quot;', '"')
+      .replace('&amp;', '&');
+  }
+
+  handleSaxOpenTag(tagOb) {
+    const name = tagOb.name;
+    const atts = tagOb.attributes;
+
+    if (name in this.openTagHandlers) {
+      this.openTagHandlers[name](this, 'open', name, atts);
+    } else {
+      throw new Error(`Unexpected open element tag '${name}' in UsxParser`);
+    }
+  }
+
+  handleSaxCloseTag(name) {
+    this.closeTagHandlers[name](this, 'close', name);
+  }
+
+  notHandledHandler(lexer, oOrC, tag) {
+    console.error(`WARNING: ${oOrC} element tag '${tag}' is not handled by UsxParser`);
+  }
+
+  stackPush(name, atts) {
+    this.elementStack.push([name, atts]);
+  }
+
+  stackPop() {
+    return this.elementStack.pop();
+  }
+
+  splitTagNumber(fullTagName) {
+    const tagBits = xre.exec(fullTagName, xre('([^1-9]+)(.*)'));
+    const tagName = tagBits[1];
+    const tagNo = tagBits[2].length > 0 ? tagBits[2] : '1';
+    return [tagName, tagNo];
+  }
+
+  ignoreHandler(lexer, oOrC, tag) {
+  }
+
+  handleParaOpen(lexer, oOrC, name, atts) {
+    lexer.currentText = '';
+    const [tagName, tagNo] = lexer.splitTagNumber(atts.style);
+
+    if (!['cp'].includes(tagName)) {
+      lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, tagName, tagNo]));
+    }
+    lexer.stackPush(name, atts);
+  }
+
+  handleParaClose(lexer) {
+    const sAtts = lexer.stackPop()[1];
+    const [tagName, tagNo] = lexer.splitTagNumber(sAtts.style);
+
+    if (['cp'].includes(tagName)) {
+      lexer.parser.parseItem(constructorForFragment.pubchapter('pubchapter', [null, null, lexer.currentText]));
+    } else {
+      lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, tagName, tagNo]));
+    }
+    lexer.currentText = '';
+  }
+
+  handleCharOpen(lexer, oOrC, name, atts) {
+    const [tagName, tagNo] = lexer.splitTagNumber(atts.style);
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, `+${tagName}`, tagNo]));
+    const ignoredAtts = ['sid', 'eid', 'style', 'srcloc', 'link-href', 'link-title', 'link-id', 'closed'];
+
+    for (const [attName, attValue] of Object.entries(atts)) {
+      if (!ignoredAtts.includes(attName)) {
+        lexer.parser.parseItem(constructorForFragment.attribute('attribute', [null, null, attName, attValue]));
+      }
+    }
+    lexer.stackPush(name, atts);
+  }
+
+  handleCharClose(lexer) {
+    const sAtts = lexer.stackPop()[1];
+    const [tagName, tagNo] = lexer.splitTagNumber(sAtts.style);
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, `+${tagName}`, tagNo]));
+  }
+
+  handleRefOpen(lexer) {
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, 'xt', '']));
+  }
+
+  handleRefClose(lexer) {
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, 'xt', '']));
+  }
+
+  handleRowOpen(lexer, oOrC, name, atts) {
+    const [tagName, tagNo] = lexer.splitTagNumber(atts.style);
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, tagName, tagNo]));
+    lexer.stackPush(name, atts);
+  }
+
+  handleRowClose(lexer) {
+    const sAtts = lexer.stackPop()[1];
+    const [tagName, tagNo] = lexer.splitTagNumber(sAtts.style);
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, tagName, tagNo]));
+  }
+
+  handleCellOpen(lexer, oOrC, name, atts) {
+    const [tagName, tagNo] = lexer.splitTagNumber(atts.style);
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, tagName, tagNo]));
+    lexer.stackPush(name, atts);
+  }
+
+  handleCellClose(lexer) {
+    const sAtts = lexer.stackPop()[1];
+    const [tagName, tagNo] = lexer.splitTagNumber(sAtts.style);
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, tagName, tagNo]));
+  }
+
+  handleBookOpen(lexer, oOrC, name, atts) {
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, 'id', '']));
+    lexer.parser.parseItem(constructorForFragment.printable('wordLike', [atts.code]));
+    lexer.parser.parseItem(constructorForFragment.printable('lineSpace', [' ']));
+    lexer.stackPush(name, atts);
+  }
+
+  handleBookClose(lexer) {
+    lexer.stackPop();
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, 'id', '']));
+  }
+
+  handleChapter(lexer, oOrC, name, atts) {
+    if (atts.number) {
+      lexer.parser.parseItem(constructorForFragment.chapter('chapter', [null, null, atts.number]));
+
+      if (atts.pubnumber) {
+        lexer.parser.parseItem(constructorForFragment.pubchapter('pubchapter', [null, null, atts.pubnumber]));
+      }
+
+      if (atts.altnumber) {
+        lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, '+ca', '']));
+        lexer.parser.parseItem(constructorForFragment.printable('wordLike', [atts.altnumber]));
+        lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, '+ca', '']));
+      }
+    }
+  }
+
+  handleVerses(lexer, oOrC, name, atts) {
+    if (atts.number) {
+      lexer.parser.parseItem(constructorForFragment.verses('verses', [null, null, atts.number]));
+
+      if (atts.pubnumber) {
+        lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, '+vp', '']));
+        lexer.parser.parseItem(constructorForFragment.printable('wordLike', [atts.pubnumber]));
+        lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, '+vp', '']));
+      }
+
+      if (atts.altnumber) {
+        lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, '+va', '']));
+        lexer.parser.parseItem(constructorForFragment.printable('wordLike', [atts.altnumber]));
+        lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, '+va', '']));
+      }
+    }
+  }
+
+  handleNoteOpen(lexer, oOrC, name, atts) {
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, atts.style, '']));
+    lexer.parser.parseItem(constructorForFragment.printable('punctuation', [atts.caller]));
+    lexer.stackPush(name, atts);
+  }
+
+  handleNoteClose(lexer) {
+    const sAtts = lexer.stackPop()[1];
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, sAtts.style, '']));
+  }
+
+  handleSidebarOpen(lexer, oOrC, name, atts) {
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, 'esb', '']));
+
+    if ('category' in atts) {
+      lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, 'cat', '']));
+      lexer.parser.parseItem(constructorForFragment.printable('wordLike', [atts.category]));
+      lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, 'cat', '']));
+    }
+    lexer.stackPush(name, atts);
+  }
+
+  handleSidebarClose(lexer) {
+    lexer.stackPop();
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, 'esbe', '']));
+  }
+
+  handleMSOpen(lexer, oOrC, name, atts) {
+    let matchBits = xre.exec(atts.style, xre('(([a-z1-9]+)-([se]))'));
+
+    if (matchBits) {
+      const startMS = constructorForFragment.milestone('startMilestoneTag', [null, null, matchBits[2], matchBits[3]]);
+      lexer.parser.parseItem(startMS);
+      const ignoredAtts = ['sid', 'eid', 'style', 'srcloc', 'link-href', 'link-title', 'link-id'];
+
+      for (const [attName, attValue] of Object.entries(atts)) {
+        if (!ignoredAtts.includes(attName)) {
+          lexer.parser.parseItem(constructorForFragment.attribute('attribute', [null, null, attName, attValue]));
+        }
+      }
+      lexer.parser.parseItem(constructorForFragment.milestone('endMilestoneMarker'));
+    } else {
+      const emptyMS = constructorForFragment.milestone('emptyMilestone', [null, null, atts.style, '']);
+      lexer.parser.parseItem(emptyMS);
+    }
+    lexer.stackPush(name, atts);
+  }
+
+  handleMSClose(lexer) {
+    lexer.stackPop();
+  }
+
+  handleFigureOpen(lexer, oOrC, name, atts) {
+    lexer.parser.parseItem(constructorForFragment.tag('startTag', [null, null, '+fig', '']));
+
+    for (const [attName, attValue] of Object.entries(atts)) {
+      if (attName === 'style') {
+        continue;
+      }
+
+      const scopeAttName = (attName === 'file') ? 'src' : attName;
+      lexer.parser.parseItem(constructorForFragment.attribute('attribute', [null, null, scopeAttName, attValue]));
+    }
+    lexer.stackPush(name, atts);
+  }
+
+  handleFigureClose(lexer) {
+    const sAtts = lexer.stackPop()[1];
+    lexer.parser.parseItem(constructorForFragment.tag('endTag', [null, null, `+fig`, '']));
+  }
+
+
+  handleOptBreakOpen(lexer, oOrC, name, atts) {
+    lexer.parser.parseItem(constructorForFragment.printable('softLineBreak', ['//']));
+    lexer.stackPush(name, atts);
+  }
+
+  handleOptBreakClose(lexer) {
+    lexer.stackPop();
+  }
+}
+
+module.exports = { UsxLexer };

package/src/parser/lib/tokenize.cjs

@@ -0,0 +1,37 @@
+const xre = require('xregexp');
+const { lexingRegexes } = require('../lexers/lexingRegexes.cjs');
+
+const tokenTypes = {};
+const unionComponents = [];
+
+for (const lr of lexingRegexes) {
+  if (['wordLike', 'eol', 'lineSpace', 'punctuation', 'unknown'].includes(lr[1])) {
+    tokenTypes[lr[1]] = xre(`^${lr[2].xregexp.source}$`);
+    unionComponents.push(lr[2]);
+  }
+}
+
+const tokenizeString = str => {
+  const unionRegex = xre.union(unionComponents);
+  const ret = [];
+
+  for (const token of xre.match(str, unionRegex, 'all')) {
+    let tokenType;
+
+    if (xre.test(token, tokenTypes['wordLike'])) {
+      tokenType = 'wordLike';
+    } else if (xre.test(token, tokenTypes['punctuation'])) {
+      tokenType = 'punctuation';
+    } else if (xre.test(token, tokenTypes['lineSpace'])) {
+      tokenType = 'lineSpace';
+    } else if (xre.test(token, tokenTypes['eol'])) {
+      tokenType = 'eol';
+    } else {
+      tokenType = 'unknown';
+    }
+    ret.push([token, tokenType]);
+  }
+  return ret;
+};
+
+module.exports = { tokenizeString };