proskomma-core 0.9.10 → 0.9.13

package/src/model/doc_set_helpers/update.cjs

@@ -0,0 +1,290 @@
+const utils = require('../../util/index.cjs');
+const ByteArray = utils.ByteArray;
+const {
+  pushSuccinctGraftBytes,
+  pushSuccinctScopeBytes,
+  pushSuccinctTokenBytes,
+} = utils.succinct;
+const { itemEnum } = utils.itemDefs;
+const { scopeEnum } = utils.scopeDefs;
+const { tokenCategory, tokenEnum } = utils.tokenDefs;
+
+const updateItems1 = (
+  docSet,
+  documentId,
+  sequenceId,
+  blockPosition,
+  typedArrayName,
+  itemObjects) => {
+  const document = docSet.processor.documents[documentId];
+
+  if (!document) {
+    throw new Error(`Document '${documentId}' not found`);
+  }
+
+  let sequence;
+
+  if (sequenceId) {
+    sequence = document.sequences[sequenceId];
+
+    if (!sequence) {
+      throw new Error(`Sequence '${sequenceId}' not found`);
+    }
+  } else {
+    sequence = document.sequences[document.mainId];
+  }
+
+  if (sequence.blocks.length <= blockPosition) {
+    throw new Error(`Could not find block ${blockPosition} (length=${sequence.blocks.length})`);
+  }
+
+  const block = sequence.blocks[blockPosition];
+  const newItemsBA = new ByteArray(itemObjects.length);
+  docSet.maybeBuildPreEnums();
+
+  let nextToken = 0;
+
+  if (blockPosition < 0) {
+    nextToken = sequence.blocks[blockPosition - 1].nt.nByte(0);
+  }
+
+  let charsEnumIndex, graftTypeEnumIndex, seqEnumIndex, scopeBits, scopeTypeByte, scopeBitBytes = null;
+
+  for (const item of itemObjects) {
+    switch (item.type) {
+    case 'token':
+      charsEnumIndex = docSet.enumForCategoryValue(tokenCategory[item.subType], item.payload, true);
+      pushSuccinctTokenBytes(newItemsBA, tokenEnum[item.subType], charsEnumIndex);
+      nextToken++;
+      break;
+    case 'graft':
+      graftTypeEnumIndex = docSet.enumForCategoryValue('graftTypes', item.subType, true);
+      seqEnumIndex = docSet.enumForCategoryValue('ids', item.payload, true);
+      pushSuccinctGraftBytes(newItemsBA, graftTypeEnumIndex, seqEnumIndex);
+      break;
+    case 'scope':
+      scopeBits = item.payload.split('/');
+      scopeTypeByte = scopeEnum[scopeBits[0]];
+
+      if (!scopeTypeByte && scopeTypeByte !== 0) {
+        throw new Error(`"${scopeBits[0]}" is not a scope type`);
+      }
+
+      scopeBitBytes = scopeBits.slice(1).map(b => docSet.enumForCategoryValue('scopeBits', b, true));
+      pushSuccinctScopeBytes(newItemsBA, itemEnum[`${item.subType}Scope`], scopeTypeByte, scopeBitBytes);
+      break;
+    }
+  }
+  newItemsBA.trim();
+  block[typedArrayName] = newItemsBA;
+
+  if (typedArrayName === 'c') {
+    block.nt.clear();
+    block.nt.pushNByte(nextToken);
+    // document.buildChapterVerseIndex();
+  }
+  return true;
+};
+
+const updateItems = (
+  docSet,
+  documentId,
+  sequenceId,
+  blockPosition,
+  itemObjects) =>
+  updateItems1(
+    docSet,
+    documentId,
+    sequenceId,
+    blockPosition,
+    'c',
+    itemObjects,
+  );
+
+const updateBlockGrafts = (
+  docSet,
+  documentId,
+  sequenceId,
+  blockPosition,
+  itemObjects) =>
+  updateItems1(
+    docSet,
+    documentId,
+    sequenceId,
+    blockPosition,
+    'bg',
+    itemObjects,
+  );
+
+const updateBlockScope = (
+  docSet,
+  documentId,
+  sequenceId,
+  blockPosition,
+  bsObject) =>
+  updateItems1(
+    docSet,
+    documentId,
+    sequenceId,
+    blockPosition,
+    'bs',
+    [bsObject],
+  );
+
+const updateOpenScopes = (
+  docSet,
+  documentId,
+  sequenceId,
+  blockPosition,
+  osObjects) =>
+  updateItems1(
+    docSet,
+    documentId,
+    sequenceId,
+    blockPosition,
+    'os',
+    osObjects,
+  );
+
+const updateIncludedScopes = (
+  docSet,
+  documentId,
+  sequenceId,
+  blockPosition,
+  isObjects) =>
+  updateItems1(
+    docSet,
+    documentId,
+    sequenceId,
+    blockPosition,
+    'is',
+    isObjects,
+  );
+
+const updateBlockIndexesAfterEdit = (docSet, sequence, blockPosition) => {
+  const labelsMatch = (firstA, secondA) => {
+    for (const first of Array.from(firstA)) {
+      if (!secondA.has(first)) {
+        return false;
+      }
+    }
+
+    for (const second of Array.from(secondA)) {
+      if (!firstA.has(second)) {
+        return false;
+      }
+    }
+    return true;
+  };
+
+  const addSuccinctScope = (docSet, succinct, scopeLabel) => {
+    const scopeBits = scopeLabel.split('/');
+    const scopeTypeByte = scopeEnum[scopeBits[0]];
+
+    if (!scopeTypeByte && scopeTypeByte !== 0) {
+      throw new Error(`"${scopeBits[0]}" is not a scope type`);
+    }
+
+    const scopeBitBytes = scopeBits.slice(1).map(b => docSet.enumForCategoryValue('scopeBits', b, true));
+    pushSuccinctScopeBytes(succinct, itemEnum[`startScope`], scopeTypeByte, scopeBitBytes);
+  };
+
+  const block = sequence.blocks[blockPosition];
+  const includedScopeLabels = new Set();
+  const openScopeLabels = new Set();
+
+  for (const openScope of docSet.unsuccinctifyScopes(block.os)) {
+    openScopeLabels.add(openScope[2]);
+  }
+
+  for (const scope of docSet.unsuccinctifyItems(block.c, { scopes: true }, null)) {
+    if (scope[1] === 'start') {
+      includedScopeLabels.add(scope[2]);
+      openScopeLabels.add(scope[2]);
+    } else {
+      openScopeLabels.delete(scope[2]);
+    }
+  }
+
+  const isArray = Array.from(includedScopeLabels);
+  const isBA = new ByteArray(isArray.length);
+
+  for (const scopeLabel of isArray) {
+    addSuccinctScope(docSet, isBA, scopeLabel);
+  }
+  isBA.trim();
+  block.is = isBA;
+
+  if (blockPosition < (sequence.blocks.length - 1)) {
+    const nextOsBlock = sequence.blocks[blockPosition + 1];
+    const nextOsBA = nextOsBlock.os;
+    const nextOSLabels = new Set(docSet.unsuccinctifyScopes(nextOsBA).map(s => s[2]));
+
+    if (!labelsMatch(openScopeLabels, nextOSLabels)) {
+      const osBA = new ByteArray(nextOSLabels.length);
+
+      for (const scopeLabel of Array.from(openScopeLabels)) {
+        addSuccinctScope(docSet, osBA, scopeLabel);
+      }
+      osBA.trim();
+      nextOsBlock.os = osBA;
+      docSet.updateBlockIndexesAfterEdit(sequence, blockPosition + 1);
+    }
+  }
+};
+
+const updateBlockIndexesAfterFilter = (docSet, sequence) => {
+  const addSuccinctScope = (docSet, succinct, scopeLabel) => {
+    const scopeBits = scopeLabel.split('/');
+    const scopeTypeByte = scopeEnum[scopeBits[0]];
+
+    if (!scopeTypeByte && scopeTypeByte !== 0) {
+      throw new Error(`"${scopeBits[0]}" is not a scope type`);
+    }
+
+    const scopeBitBytes = scopeBits.slice(1).map(b => docSet.enumForCategoryValue('scopeBits', b, true));
+    pushSuccinctScopeBytes(succinct, itemEnum[`startScope`], scopeTypeByte, scopeBitBytes);
+  };
+
+  const openScopeLabels = new Set();
+
+  for (const block of sequence.blocks) {
+    const osArray = Array.from(openScopeLabels);
+    const osBA = new ByteArray(osArray.length);
+
+    for (const scopeLabel of osArray) {
+      addSuccinctScope(docSet, osBA, scopeLabel);
+    }
+    osBA.trim();
+    block.os = osBA;
+    const includedScopeLabels = new Set();
+
+    for (const scope of docSet.unsuccinctifyItems(block.c, { scopes: true }, null)) {
+      if (scope[1] === 'start') {
+        includedScopeLabels.add(scope[2]);
+        openScopeLabels.add(scope[2]);
+      } else {
+        openScopeLabels.delete(scope[2]);
+      }
+    }
+
+    const isArray = Array.from(includedScopeLabels);
+    const isBA = new ByteArray(isArray.length);
+
+    for (const scopeLabel of isArray) {
+      addSuccinctScope(docSet, isBA, scopeLabel);
+    }
+    isBA.trim();
+    block.is = isBA;
+  }
+};
+
+module.exports = {
+  updateItems,
+  updateBlockGrafts,
+  updateBlockScope,
+  updateOpenScopes,
+  updateIncludedScopes,
+  updateBlockIndexesAfterEdit,
+  updateBlockIndexesAfterFilter,
+};

package/src/model/document.cjs

@@ -0,0 +1,369 @@
+const {
+  PerfRenderFromProskomma,
+  SofriaRenderFromProskomma,
+  render,
+  PipelineHandler,
+} = require('proskomma-json-tools');
+const utils = require('../util/index.cjs');
+const {
+  parseUsfm,
+  parseUsx,
+  parseTableToDocument,
+  parseNodes,
+} = require('../parser/lexers/index.cjs');
+const { Parser } = require('../parser/index.cjs');
+const pipelines = require('../pipelines/perf2x/index.cjs');
+const customTransforms = require('../transforms/index.cjs');
+const {
+  buildChapterVerseIndex,
+  chapterVerseIndex,
+  chapterIndex,
+} = require('./document_helpers/chapter_verse.cjs');
+const {
+  modifySequence,
+  deleteSequence,
+  gcSequences,
+  newSequence,
+} = require('./document_helpers/sequences.cjs');
+const {
+  deleteBlock,
+  newBlock,
+  rewriteBlock,
+} = require('./document_helpers/blocks.cjs');
+const { succinctFilter } = require('./document_helpers/succinct_filter.cjs');
+const { serializeSuccinct } = require('./document_helpers/serialize_succinct.cjs');
+const {
+  recordPreEnums,
+  rerecordPreEnums,
+} = require('./document_helpers/pre_enums.cjs');
+
+const {
+  addTag,
+  removeTag,
+  validateTags,
+} = utils.tags;
+const parserConstantDef = utils.parserConstants;
+// const maybePrint = str => console.log(str);
+const maybePrint = str => str;
+
+class Document {
+  constructor(processor, docSetId, contentType, contentString, filterOptions, customTags, emptyBlocks, tags) {
+    this.processor = processor;
+    this.docSetId = docSetId;
+    this.baseSequenceTypes = parserConstantDef.usfm.baseSequenceTypes;
+
+    if (contentType) {
+      this.id = utils.generateId();
+      this.filterOptions = filterOptions;
+      this.customTags = customTags;
+      this.emptyBlocks = emptyBlocks;
+      this.tags = new Set(tags || []);
+      validateTags(this.tags);
+      this.headers = {};
+      this.mainId = null;
+      this.sequences = {};
+
+      switch (contentType.toLowerCase()) {
+      case 'usfm':
+      case 'sfm':
+        this.processUsfm(contentString);
+        break;
+      case 'usx':
+        this.processUsx(contentString);
+        break;
+      case 'tsv':
+        this.processTSV(contentString);
+        break;
+      case 'nodes':
+        this.processNodes(contentString);
+        break;
+      default:
+        throw new Error(`Unknown document contentType '${contentType}'`);
+      }
+    }
+  }
+
+  addTag(tag) {
+    addTag(this.tags, tag);
+  }
+
+  removeTag(tag) {
+    removeTag(this.tags, tag);
+  }
+
+  makeParser() {
+    return new Parser(
+      this.filterOptions,
+      this.customTags,
+      this.emptyBlocks,
+    );
+  }
+
+  processUsfm(usfmString) {
+    const parser = this.makeParser();
+    const t = Date.now();
+    parseUsfm(usfmString, parser);
+    const t2 = Date.now();
+    maybePrint(`\nParse USFM in ${t2 - t} msec`);
+    this.postParseScripture(parser);
+    maybePrint(`Total USFM import time = ${Date.now() - t} msec (parse = ${((t2 - t) * 100) / (Date.now() - t)}%)`);
+  }
+
+  processUsx(usxString) {
+    const parser = this.makeParser();
+    const t = Date.now();
+    parseUsx(usxString, parser);
+    const t2 = Date.now();
+    maybePrint(`\nParse USX in ${t2 - t} msec`);
+    this.postParseScripture(parser);
+    maybePrint(`Total USX import time = ${Date.now() - t} msec (parse = ${((t2 - t) * 100) / (Date.now() - t)}%)`);
+  }
+
+  processTSV(tsvString) {
+    const parser = this.makeParser();
+    const bookCode = `T${this.processor.nextTable > 9 ? this.processor.nextTable : '0' + this.processor.nextTable}`;
+    this.processor.nextTable++;
+    parseTableToDocument(tsvString, parser, bookCode);
+    this.headers = parser.headers;
+    this.succinctPass1(parser);
+    this.succinctPass2(parser);
+    buildChapterVerseIndex(this);
+
+    const tableSequence = Object.values(this.sequences).filter(s => s.type === 'table')[0];
+
+    for (const [colN, colHead] of JSON.parse(tsvString).headings.entries()) {
+      tableSequence.tags.add(`col${colN}:${colHead}`);
+    }
+  }
+
+  processNodes(nodesString) {
+    const parser = this.makeParser();
+    const bookCode = `N${this.processor.nextNodes > 9 ? this.processor.nextNodes : '0' + this.processor.nextNodes}`;
+    this.processor.nextNodes++;
+    parseNodes(nodesString, parser, bookCode);
+    this.headers = parser.headers;
+    this.succinctPass1(parser);
+    this.succinctPass2(parser);
+    buildChapterVerseIndex(this);
+  }
+
+  postParseScripture(parser) {
+    let t = Date.now();
+    parser.tidy();
+    maybePrint(`Tidy in ${Date.now() - t} msec`);
+    t = Date.now();
+    const fo = parser.filterOptions; // CHANGE THIS WHEN REFACTORING PARSER
+    this.headers = parser.headers;
+    this.succinctPass1(parser);
+    maybePrint(`Succinct pass 1 in ${Date.now() - t} msec`);
+    t = Date.now();
+    this.succinctPass2(parser);
+    maybePrint(`Succinct pass 2 in ${Date.now() - t} msec`);
+    t = Date.now();
+    this.succinctFilter(fo);
+    maybePrint(`Filter in ${Date.now() - t} msec`);
+    t = Date.now();
+    buildChapterVerseIndex(this);
+    maybePrint(`CV indexes in ${Date.now() - t} msec`);
+  }
+
+  succinctFilter(filterOptions) {
+    succinctFilter(this, filterOptions);
+  }
+
+  succinctPass1(parser) {
+    const docSet = this.processor.docSets[this.docSetId];
+
+    let t = Date.now();
+
+    for (const seq of parser.allSequences()) {
+      docSet.recordPreEnum('ids', seq.id);
+      this.recordPreEnums(docSet, seq);
+    }
+    maybePrint(`   recordPreEnums in ${Date.now() - t} msec`);
+    t = Date.now();
+
+    if (docSet.enums.wordLike.length === 0) {
+      docSet.sortPreEnums();
+      maybePrint(`   sortPreEnums in ${Date.now() - t} msec`);
+      t = Date.now();
+    }
+    docSet.buildEnums();
+    maybePrint(`   buildEnums in ${Date.now() - t} msec`);
+  }
+
+  recordPreEnums(docSet, seq) {
+    recordPreEnums(docSet, seq);
+  }
+
+  rerecordPreEnums(docSet, seq) {
+    rerecordPreEnums(docSet, seq);
+  }
+
+  succinctPass2(parser) {
+    const docSet = this.processor.docSets[this.docSetId];
+    this.mainId = parser.sequences.main.id;
+
+    for (const seq of parser.allSequences()) {
+      this.sequences[seq.id] = {
+        id: seq.id,
+        type: seq.type,
+        tags: new Set(seq.tags),
+        isBaseType: (seq.type in parser.baseSequenceTypes),
+        blocks: seq.succinctifyBlocks(docSet),
+      };
+    }
+    this.sequences[this.mainId].verseMapping = {};
+  }
+
+  modifySequence(
+    seqId,
+    sequenceRewriteFunc,
+    blockFilterFunc,
+    itemFilterFunc,
+    blockRewriteFunc,
+    itemRewriteFunc,
+  ) {
+    modifySequence(
+      this,
+      seqId,
+      sequenceRewriteFunc,
+      blockFilterFunc,
+      itemFilterFunc,
+      blockRewriteFunc,
+      itemRewriteFunc,
+    );
+  }
+
+  buildChapterVerseIndex() {
+    buildChapterVerseIndex(this);
+  }
+
+  chapterVerseIndexes() {
+    const ret = {};
+
+    for (const chapN of Object.keys(this.sequences[this.mainId].chapterVerses)) {
+      ret[chapN] = chapterVerseIndex(this, chapN);
+    }
+    return ret;
+  }
+
+  chapterVerseIndex(chapN) {
+    return chapterVerseIndex(this, chapN);
+  }
+
+  chapterIndexes() {
+    const ret = {};
+
+    for (const chapN of Object.keys(this.sequences[this.mainId].chapters)) {
+      ret[chapN] = chapterIndex(this, chapN);
+    }
+    return ret;
+  }
+
+  chapterIndex(chapN) {
+    return chapterIndex(this, chapN);
+  }
+
+  rewriteSequenceBlocks(sequenceId, oldToNew) {
+    const sequence = this.sequences[sequenceId];
+
+    for (const block of sequence.blocks) {
+      this.rewriteSequenceBlock(block, oldToNew);
+    }
+  }
+
+  rewriteSequenceBlock(block, oldToNew) {
+    rewriteBlock(block, oldToNew);
+  }
+
+  serializeSuccinct() {
+    return serializeSuccinct(this);
+  }
+
+  gcSequences() {
+    return gcSequences(this);
+  }
+
+  newSequence(seqType, tags) {
+    return newSequence(this, seqType, tags);
+  }
+
+  deleteSequence(seqId) {
+    return deleteSequence(this, seqId);
+  }
+
+  deleteBlock(seqId, blockN, buildCV) {
+    return deleteBlock(this, seqId, blockN, buildCV);
+  }
+
+  newBlock(seqId, blockN, blockScope, blockGrafts, buildCV) {
+    return newBlock(this, seqId, blockN, blockScope, blockGrafts, buildCV);
+  }
+
+  perf(indent) {
+    const cl = new PerfRenderFromProskomma(
+      {
+        proskomma: this.processor,
+        actions: render.perfToPerf.renderActions.identityActions,
+      },
+    );
+    const output = {};
+
+    cl.renderDocument(
+      {
+        docId: this.id,
+        config: {},
+        output,
+      },
+    );
+    return indent ? JSON.stringify(output.perf, null, indent) : JSON.stringify(output.perf);
+  }
+
+  async usfm() {
+    const perf = JSON.parse(this.perf());
+
+    try {
+      const pipelineHandler = new PipelineHandler({
+        pipelines:pipelines,
+        transforms:customTransforms,
+        proskomma:this.processor,
+      });
+      const output = await pipelineHandler.runPipeline('perf2usfmPipeline', { perf });
+      return output.usfm;
+    } catch (err) {
+      console.error('pipelineHandler Error :\n', err);
+    }
+  }
+
+  sofria(indent, chapter) {
+    const cl = new SofriaRenderFromProskomma(
+      {
+        proskomma: this.processor,
+        actions: render.sofriaToSofria.renderActions.identityActions,
+      },
+    );
+    const output = {};
+    const config = {};
+
+    if (chapter) {
+      config.chapters = [`${chapter}`];
+    }
+
+    try {
+      cl.renderDocument(
+        {
+          docId: this.id,
+          config,
+          output,
+        },
+      );
+    } catch (err) {
+      console.log(err);
+      throw err;
+    }
+    return indent ? JSON.stringify(output.sofria, null, indent) : JSON.stringify(output.sofria);
+  }
+}
+
+module.exports = { Document };