proskomma-core 0.9.7 → 0.9.12

package/test/code/main/milestone_attributes.cjs

@@ -0,0 +1,163 @@
+const test = require('tape');
+
+const { pkWithDoc } = require('../../lib/load.js');
+
+const testGroup = 'Milestones & Attributes';
+
+const pk = pkWithDoc('../test_data/usfm/milestone_attributes.usfm', {
+  lang: 'fra',
+  abbr: 'hello',
+})[0];
+const pk2 = pkWithDoc('../test_data/usx/milestone_attributes.usx', {
+  lang: 'fra',
+  abbr: 'hello',
+})[0];
+const pk3 = pkWithDoc('../test_data/usfm/milestone_attributes_default.usfm', {
+  lang: 'fra',
+  abbr: 'hello',
+})[0];
+
+const topItems = items => {
+  if (items.length === 0) {
+    return [];
+  } else if (items[0].subType === 'start' && items[0].payload === 'milestone/zaln') {
+    return items;
+  } else {
+    return topItems(items.slice(1));
+  }
+};
+
+const tailItems = items => {
+  if (items.length === 0) {
+    return [];
+  } else if (items[items.length - 1].subType === 'end' && items[items.length - 1].payload === 'milestone/zaln') {
+    return items;
+  } else {
+    return tailItems(items.slice(0, items.length - 2));
+  }
+};
+
+const topNTailItems = items => tailItems(topItems(items));
+
+const zalnScopes = ['x-strong/0/H5662', 'x-lemma/0/עֹבַדְיָה', 'x-morph/0/He', 'x-morph/1/Np', 'x-occurrence/0/1', 'x-occurrences/0/1'];
+const wScopes = ['x-occurrence/0/1', 'x-occurrences/0/1'];
+const query =
+  '{ documents { mainSequence { id } sequences { id blocks { items { type subType payload } } } } }';
+
+const checkResult = (t, result) => {
+  const mainSequenceItems = result.data.documents[0].sequences.filter(s => s.id === result.data.documents[0].mainSequence.id)[0].blocks[0].items;
+  const content = topNTailItems(mainSequenceItems);
+  const zalnAtt = suffix => `attribute/milestone/zaln/${suffix}`;
+  const wAtt = suffix => `attribute/spanWithAtts/w/${suffix}`;
+  t.equal(mainSequenceItems[0].subType, 'start');
+  t.equal(mainSequenceItems[0].payload, 'milestone/ts');
+  t.equal(mainSequenceItems[1].subType, 'end');
+  t.equal(mainSequenceItems[1].payload, 'milestone/ts');
+  t.equal(content.length, 23);
+  t.equal(content[0].subType, 'start');
+  t.equal(content[0].payload, 'milestone/zaln');
+
+  for (const [n, s] of zalnScopes.entries()) {
+    t.equal(content[n + 1].subType, 'start');
+    t.equal(content[n + 1].payload, zalnAtt(s));
+  }
+  t.equal(content[8].subType, 'start');
+  t.equal(content[8].payload, 'spanWithAtts/w');
+
+  for (const [n, s] of wScopes.entries()) {
+    t.equal(content[n + 9].subType, 'start');
+    t.equal(content[n + 9].payload, wAtt(s));
+  }
+  t.equal(content[11].subType, 'wordLike');
+  t.equal(content[11].payload, 'Obadiah');
+
+  for (const [n, s] of wScopes.entries()) {
+    t.equal(content[13 - n].subType, 'end');
+    t.equal(content[13 - n].payload, wAtt(s));
+  }
+  t.equal(content[14].subType, 'end');
+  t.equal(content[14].payload, 'spanWithAtts/w');
+  t.equal(content[22].subType, 'end');
+
+  for (const [n, s] of zalnScopes.entries()) {
+    t.equal(content[21 - n].subType, 'end');
+    t.equal(content[21 - n].payload, zalnAtt(s));
+  }
+  t.equal(content[22].payload, 'milestone/zaln');
+};
+
+test(
+  `USFM (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(16 + (4 * zalnScopes.length) + (4 * wScopes.length));
+      const result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      checkResult(t, result);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `USFM with default attribute(${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(2);
+      const result = await pk3.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      t.equal(
+        result.data.documents[0].sequences
+          .filter(s => s.id === result.data.documents[0].mainSequence.id)[0]
+          .blocks[0].items.filter(
+            i => i.payload === 'attribute/spanWithAtts/w/lemma/0/O-bi-di-ah' && i.subType === 'start',
+          ).length,
+        1);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `USX (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(16 + (4 * zalnScopes.length) + (4 * wScopes.length));
+      const result = await pk2.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      checkResult(t, result);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `Literal slash in attribute value (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(5);
+      let pk;
+
+      t.doesNotThrow(() => {
+        pk = pkWithDoc('../test_data/usfm/slash_in_att.usfm', {
+          lang: 'fra',
+          abbr: 'hello',
+        })[0];
+      });
+
+      t.equal(pk.nDocuments(), 1);
+      const query = '{documents { headers { key value } sequences { type text } } }';
+      const result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const firstDoc = result.data.documents[0];
+      const headerKeys = firstDoc.headers.map(h => h.key);
+      t.ok(headerKeys.includes('id'));
+      t.ok(headerKeys.includes('bookCode'));
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);

package/test/code/main/mixed_sequence_documents.cjs

@@ -0,0 +1,137 @@
+const path = require('path');
+const fse = require('fs-extra');
+const test = require('tape');
+
+const { pkWithDoc } = require('../../lib/load.js');
+
+const pk = pkWithDoc('../test_data/usfm/66-JUD-ust.usfm', {
+  lang: 'eng',
+  abbr: 'ust',
+})[0];
+
+const tsvToTable = (tsv, hasHeadings) => {
+  const ret = {
+    headings: [],
+    rows: [],
+  };
+  let rows = tsv.split(/[\n\r]+/);
+
+  if (hasHeadings) {
+    ret.headings = rows[0].split('\t');
+    rows = rows.slice(1);
+  }
+
+  for (const row of rows) {
+    ret.rows.push(row.split('\t'));
+  }
+  return ret;
+};
+
+const importTSV = (pk, tsvPath, hasHeadings) => {
+  pk.importDocument({
+    lang: 'eng',
+    abbr: 'ust',
+  }, 'tsv',
+  JSON.stringify(
+    tsvToTable(
+      fse.readFileSync(
+        path.resolve(__dirname, tsvPath),
+      )
+        .toString(),
+      hasHeadings,
+    ),
+    {},
+  ),
+  null,
+  2,
+  );
+};
+
+const importNodes = pk => {
+  pk.importDocument({
+    lang: 'eng',
+    abbr: 'ust',
+  }, 'nodes',
+  fse.readFileSync(path.resolve(__dirname, '../../test_data/tree/jude.json')),
+  {},
+  );
+};
+
+importNodes(pk);
+importTSV(pk, '../../test_data/tsv/en_tn_66-JUD.tsv', true);
+
+const testGroup = 'Mixed Sequence Documents';
+
+test(
+  `As vanilla sequences (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(1);
+      let query = '{documents { mainSequence { id } sequences { id type tags blocks { bs { payload} bg { payload } is { payload } tokens { type subType payload } } } } }';
+      let result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `As cast sequences (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(2);
+      let query = `{
+      docSets {
+        tableDoc: document(bookCode:"T00") { sequences(types:"table") { id } }
+        treeDoc: document(bookCode:"N00") { sequences(types:"tree") { id } }
+        }
+      }`;
+      let result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const tableSequenceId = result.data.docSets[0].tableDoc.sequences[0].id;
+      const treeSequenceId = result.data.docSets[0].treeDoc.sequences[0].id;
+      const bookN = 65;
+      const chapterN = 1;
+      const verseN = 23;
+
+      const padInt = n => {
+        const padded = `00${n}`;
+        return padded.substring(padded.length -3);
+      };
+
+      query = `{
+        docSets {
+          scripture: document(bookCode:"JUD") {
+            cv (chapter:"${chapterN}" verses:["${verseN}"]) { text(normalizeSpace:true) }
+          }
+          table: document(bookCode:"T00") {
+            tableSequence(id:"${tableSequenceId}") { rows(equals:[{colN:1 values:"${chapterN}"}, {colN:2 values:"${verseN}"}] columns:[3, 4, 5, 6, 7, 8]) { text } }
+          }
+          tree: document(bookCode:"N00") {
+            treeSequence(id:"${treeSequenceId}") {
+              tribos(
+                query:"nodes[and(==(content('elementType'), 'Node'), startsWith(content('morphId'), '${bookN}${padInt(chapterN)}${padInt(verseN)}'))]/node{@Cat, @text, @English}") }
+          }
+      }}`;
+      result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      /*
+      const docSet = result.data.docSets[0];
+      const scripture = docSet.scripture.cv[0].text;
+      const table = docSet.table.tableSequence.rows.map(r => r.map(c => c.text));
+      const tree = JSON.parse(docSet.tree.treeSequence.tribos).data.map(n => n.content);
+
+      console.log(
+        JSON.stringify({
+          scripture,
+          table,
+          tree,
+        },
+        null,
+        2));*/
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);

package/test/code/main/mixed_sequences.cjs

@@ -0,0 +1,217 @@
+const path = require('path');
+const fse = require('fs-extra');
+const test = require('tape');
+
+const { pkWithDoc } = require('../../lib/load.js');
+const {
+  blocksSpec2Query,
+  tsvToInputBlock,
+  treeToInputBlock,
+} = require('../../../src/util/scriptlike/blocksSpec.cjs');
+
+const [pk, pkDoc] = pkWithDoc('../test_data/usfm/66-JUD-ust.usfm', {
+  lang: 'eng',
+  abbr: 'ust',
+});
+
+const testGroup = 'Mixed Sequences';
+
+test(
+  `TN Table mutation (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(2);
+      const tableQuery = blocksSpec2Query(
+        tsvToInputBlock(
+          fse.readFileSync(path.resolve(__dirname, '../../test_data/tsv/en_tn_66-JUD.tsv')).toString(),
+        ),
+      );
+
+      let query = `mutation { newSequence(` +
+        ` documentId: "${pkDoc.id}"` +
+        ` type: "table"` +
+        ` blocksSpec: ${tableQuery}` +
+        ` graftToMain: true) }`;
+
+      let result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const newSeqId = result.data.newSequence;
+      query = `mutation { addSequenceTags(docSetId: "eng_ust", documentId: "${pkDoc.id}", sequenceId: "${newSeqId}", tags: ["tnotes"]) }`;
+      result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `TWL Table mutation (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(2);
+      const tableQuery = blocksSpec2Query(
+        tsvToInputBlock(
+          fse.readFileSync(path.resolve(__dirname, '../../test_data/tsv/twl_JUD.tsv')).toString(),
+        ),
+      );
+
+      let query = `mutation { newSequence(` +
+        ` documentId: "${pkDoc.id}"` +
+        ` type: "table"` +
+        ` blocksSpec: ${tableQuery}` +
+        ` graftToMain: true) }`;
+
+      let result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const newSeqId = result.data.newSequence;
+      query = `mutation { addSequenceTags(docSetId: "eng_ust", documentId: "${pkDoc.id}", sequenceId: "${newSeqId}", tags: ["twords"]) }`;
+      result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `Tree mutation (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(2);
+      const treeQuery = blocksSpec2Query(
+        treeToInputBlock(
+          fse.readJSONSync(path.resolve(__dirname, '../../test_data/tree/jude.json')),
+        ),
+      );
+
+      let query = `mutation { newSequence(` +
+        ` documentId: "${pkDoc.id}"` +
+        ` type: "tree"` +
+        ` blocksSpec: ${treeQuery}` +
+        ` graftToMain: true) }`;
+
+      let result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const newSeqId = result.data.newSequence;
+      query = `mutation { addSequenceTags(docSetId: "eng_ust", documentId: "${pkDoc.id}", sequenceId: "${newSeqId}", tags: ["stree"]) }`;
+      result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `kv mutation (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(2);
+      const kvQuery = blocksSpec2Query(
+        fse.readJSONSync(path.resolve(__dirname, '../../test_data/inputBlockSpec/kv.json')),
+      );
+
+      let query = `mutation { newSequence(` +
+        ` documentId: "${pkDoc.id}"` +
+        ` type: "kv"` +
+        ` blocksSpec: ${kvQuery}` +
+        ` graftToMain: true) }`;
+
+      let result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const newSeqId = result.data.newSequence;
+      query = `mutation { addSequenceTags(docSetId: "eng_ust", documentId: "${pkDoc.id}", sequenceId: "${newSeqId}", tags: ["kv"]) }`;
+      result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `As vanilla sequences (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(6);
+      let query = '{documents { mainSequence { id } sequences { id type tags blocks { bs { payload} bg { payload } is { payload } tokens { type subType payload } } } } }';
+      let result = await pk.gqlQuery(query);
+
+      for (const sType of ['main', 'title', 'table', 'tree', 'kv']) {
+        t.ok(result.data.documents[0].sequences.map(s => s.type).includes(sType));
+      }
+      t.equal(result.errors, undefined);
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+
+test(
+  `As cast sequences (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(1);
+      const bookN = 65;
+      const chapterN = 1;
+      const verseN = 23;
+
+      const padInt = n => {
+        const padded = `00${n}`;
+        return padded.substring(padded.length -3);
+      };
+
+      const query = `{
+        docSets {
+          document(bookCode:"JUD") {
+            cv (chapter:"${chapterN}" verses:["${verseN}"]) { tokens {subType payload scopes} }
+            tnTable: tableSequences(withTags:"tnotes") { tags rows(equals:[{colN:1 values:"${chapterN}"}, {colN:2 values:"${verseN}"}] columns:[3, 4, 5, 6, 7, 8]) { text } }
+            twlTable: tableSequences(withTags:"twords") { tags rows(equals:[{colN:0 values:"${chapterN}:${verseN}"}] columns:[1, 2, 3, 4, 5]) { text } }
+            kv: kvSequences(withTags:"kv") { nEntries, entries(secondaryEquals:[{key:"service", values:"dessert"}, {key:"color", values:"green"}]) { key secondaryKeys { key value } itemGroups { scopeLabels(startsWith:"kvField") text } } }
+            sTree: treeSequences(withTags:"stree") {
+              tribos(
+                query:"nodes[and(==(content('elementType'), 'Node'), startsWith(content('morphId'), '${bookN}${padInt(chapterN)}${padInt(verseN)}'))]/node{@Cat, @text, @English}"
+              )
+            }
+          }
+      }}`;
+      const result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+
+      // const document = result.data.docSets[0].document;
+      // const scripture = document.cv[0].tokens;
+      // const tNotes = document.tnTable[0].rows.map(r => r.map(c => c.text));
+      // const tWords = document.twlTable[0].rows.map(r => r.map(c => c.text));
+      // const sTree = JSON.parse(document.sTree[0].tribos).data.map(n => n.content);
+      /*
+      const cleanKV = entry => {
+        const fields = {};
+        entry.itemGroups.forEach(ig => fields[ig.scopeLabels[0].split('/')[1]] = ig.text);
+        const secondaryKeys = {};
+        entry.secondaryKeys.forEach(k => secondaryKeys[k.key] = k.value);
+        return {
+          key: entry.key,
+          secondaryKeys,
+          fields,
+        };
+      };
+      const kv = document.kv[0].entries.map(kv => cleanKV(kv));
+
+      console.log(
+        JSON.stringify({
+          scripture,
+          sTree,
+          tNotes,
+          tWords,
+          kv,
+        },
+        null,
+        2));
+       */
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);

package/test/code/main/modify_sequence.cjs

@@ -0,0 +1,29 @@
+const test = require('tape');
+const deepCopy = require('deep-copy-all');
+const deepEqual = require('deep-equal');
+
+const { pkWithDoc } = require('../../lib/load.js');
+
+const testGroup = 'Modify Sequence Method';
+
+const pk = pkWithDoc('../test_data/usfm/hello.usfm', {
+  lang: 'eng',
+  abbr: 'web',
+})[0];
+
+test(
+  `Identity Transform (${testGroup})`,
+  function (t) {
+    try {
+      t.plan(1);
+      const myPk = deepCopy(pk);
+      const document = Object.values(myPk.documents)[0];
+      const oldSequence = deepCopy(document.sequences[document.mainId]);
+      document.modifySequence(oldSequence.id);
+      const newSequence = document.sequences[document.mainId];
+      t.ok(deepEqual(oldSequence, newSequence));
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);

package/test/code/main/multiple_chapters.cjs

@@ -0,0 +1,58 @@
+const test = require('tape');
+
+const { pkWithDoc } = require('../../lib/load.js');
+
+const testGroup = 'Multiple Chapters';
+
+const pk = pkWithDoc(
+  '../test_data/usfm/1pe_webbe.usfm',
+  {
+    lang: 'fra',
+    abbr: 'hello',
+  },
+  {
+    includeScopes: ['chapter', 'verse/'],
+    includeGrafts: [],
+  })[0];
+
+test(
+  `Chapter Closing (${testGroup})`,
+  async function (t) {
+    try {
+      const scopeRecords = [
+        {
+          os: [],
+          is: ['chapter/1', 'verse/1'],
+        },
+        {
+          os: [],
+          is: ['chapter/2', 'verse/1', 'verse/2'],
+        },
+        {
+          os: ['chapter/2', 'verse/2'],
+          is: ['verse/3'],
+        },
+      ];
+      t.plan(2 + (2 * scopeRecords.length) + 8);
+      const query =
+        '{ documents { mainSequence { blocks { os { payload } is { payload } } } } }';
+      const result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const blocks = result.data.documents[0].mainSequence.blocks;
+      t.equal(scopeRecords.length, blocks.length);
+
+      for (const [n, scopeRecord] of scopeRecords.entries()) {
+        for (const blockField of ['os', 'is']) {
+          t.equal(blocks[n][blockField].length, scopeRecord[blockField].length);
+          const blockScopes = blocks[n][blockField].map(f => f.payload);
+
+          for (const scopeField of scopeRecord[blockField]) {
+            t.ok(blockScopes.includes(scopeField));
+          }
+        }
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);

package/test/code/main/multiple_content.cjs

@@ -0,0 +1,105 @@
+const test = require('tape');
+
+const {
+  pkWithDocs,
+  pkWithDocSetDocs,
+} = require('../../lib/load.js');
+
+const testGroup = 'Multiple Content';
+
+test(
+  `DocSets (${testGroup})`,
+  async function (t) {
+    try {
+      const expected = [
+        {
+          'lang': 'fra',
+          'abbr': 'hello',
+          'nDocs': 1,
+        },
+        {
+          'lang': 'fra',
+          'abbr': 'cp_vp',
+          'nDocs': 1,
+        },
+      ];
+      t.plan(2 + (3 * expected.length));
+      const pk = pkWithDocs([
+        ['../test_data/usfm/hello.usfm', {
+          lang: 'fra',
+          abbr: 'hello',
+        }],
+        ['../test_data/usfm/cp_vp.usfm', {
+          lang: 'fra',
+          abbr: 'cp_vp',
+        }],
+      ]);
+      const query = '{ docSets { lang: selector(id:"lang") abbr: selector(id:"abbr") documents { id } } }';
+      const result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      t.equal(result.data.docSets.length, 2);
+
+      for (const [n, docSet] of result.data.docSets.entries()) {
+        t.equal(docSet.lang, expected[n].lang);
+        t.equal(docSet.abbr, expected[n].abbr);
+        t.equal(docSet.documents.length, expected[n].nDocs);
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `Documents (${testGroup})`,
+  async function (t) {
+    try {
+      t.plan(5);
+      const pk = pkWithDocSetDocs(
+        [
+          '../test_data/usfm/hello.usfm',
+          '../test_data/usfm/cp_vp.usfm',
+        ],
+        {
+          lang: 'eng',
+          abbr: 'ust',
+        },
+        {},
+      )[0];
+      const query = '{ docSets { lang: selector(id:"lang") abbr: selector(id:"abbr") documents { header(id:"id") mainSequence { blocks { text } } } } }';
+      const result = await pk.gqlQuery(query);
+      t.equal(result.errors, undefined);
+      const documents = result.data.docSets[0].documents;
+      t.ok(documents[0].header.startsWith('MRK'));
+      t.ok(documents[0].mainSequence.blocks[0].text.startsWith('This is how'));
+      t.ok(documents[1].header.startsWith('ESG'));
+      t.ok(documents[1].mainSequence.blocks[0].text.startsWith('Here is the text'));
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);
+
+test(
+  `Duplicate Documents (${testGroup})`,
+  function (t) {
+    try {
+      t.plan(1);
+      t.throws(() => pkWithDocSetDocs(
+        [
+          '../test_data/usfm/hello.usfm',
+          '../test_data/usfm/hello.usfm',
+        ],
+        {
+          lang: 'eng',
+          abbr: 'ust',
+        },
+        {},
+      ),
+      /MRK.+already exists/,
+      );
+    } catch (err) {
+      console.log(err);
+    }
+  },
+);