theprogrammablemind 7.6.0 → 7.7.0-beta.1

package/client.js

@@ -173,22 +173,44 @@ const setupArgs = (args, config, logs, hierarchy) => {
   args.theDebugger = {
     breakOnSemantics: (value) => args.breakOnSemantics = value
   }
-  args.s = (c) => config.getSemantics(logs).apply(args, c)
-  args.g = (c) => config.getGenerators(logs).apply(args, c)
-  args.gp = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: true, isResponse: false, response: false})
-  args.gr = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: false, isResponse: true })
   if (!logs) {
     debugger
   }
-  args.e = (c) => config.getEvaluator(args.s, args.calls, logs, c)
   args.log = (message) => logs.push(message)
-  args.gs = gs(args.g)
-  args.gsp = gs(args.gp)
-  args.gsr = gs(args.gr)
+
+  args.addAssumedScoped = (args, assumed) => {
+    const addAssumed = (args, ...moreAssumed) => {
+      return { ...args, assumed: Object.assign({}, (args.assumed || {}), ...moreAssumed) }
+    }
+
+    args.s = (c) => config.getSemantics(logs).apply(args, c)
+    args.g = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
+    }
+    args.gp = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, {paraphrase: true, isResponse: false, response: false}), c, {paraphrase: true, isResponse: false, response: false})
+    }
+    args.gr = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, {paraphrase: false, isResponse: true}), { ...c, paraphrase: false, isResponse: true })
+    }
+    args.e = (c) => {
+      return config.getEvaluator(args.s, args.calls, logs, c)
+    }
+    args.gs = gs(args.g)
+    args.gsp = gs(args.gp)
+    args.gsr = gs(args.gr)
+  }
+  // for semantics
+  args.addAssumedScoped(args, {})
   config.getAddedArgs(args)
 }
 
 const gs = (g) => (contexts, separator, lastSeparator) => {
+  if (!Array.isArray(contexts)) {
+    debugger
+    throw new Error("Expected a list")
+  }
+
   let s = ''
   if (!separator) {
     separator = ' '
@@ -276,8 +298,9 @@ const processContext = (context, { objects = {}, config, logs = [] }) => {
   setupArgs(args, config, logs, hierarchy)
 
   context = semantics.apply(args, context)
-  const generated = generators.apply(args, context)[0]
-  const paraphrases = generators.apply(args, context, { paraphrase: true, response: false, isResponse: false })[0]
+  const generated = generators.apply(args, context)
+  const assumed = { paraphrase: true, response: false, isResponse: false }
+  const paraphrases = generators.apply({...args, assumed}, context, { paraphrase: true, response: false, isResponse: false })
   let responses = []
   if (context.isResponse) {
     responses = generated
@@ -447,37 +470,25 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
         continue
       }
       let assumed = { isResponse: true };
-      const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
+      const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed) : ''
       let generatedParenthesized = []
       if (generateParenthesized) {
         config.parenthesized = true
-        generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
+        generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed) : ''
         config.parenthesized = false
       }
       // assumed = { paraphrase: true, response: false };
-      assumed = { paraphrase: true };
-      args.g = (c) => config.getGenerators(json.logs).apply(args, c, assumed)
-      args.gp = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: true, isResponse: false, response: false }, assumed)
-      args.gr = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: false }, assumed)
-      args.gs = gs(args.g)
-      args.gsp = gs(args.gsp)
-      args.gsr = gs(args.gr)
+      assumed = { paraphrase: true, isResponse: false, response: false };
       if (generateParenthesized) {
         config.parenthesized = false
       }
-      const paraphrases = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
+      const paraphrases = config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed)
       let paraphrasesParenthesized = []
       if (generateParenthesized) {
         config.parenthesized = true
-        paraphrasesParenthesized = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
+        paraphrasesParenthesized = config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed)
         config.parenthesized = false
       }
-      args.g = (c) => config.getGenerators(json.logs).apply(args, c)
-      args.gp = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: true, isResponse: false, response: false })
-      args.gr = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: false })
-      args.gs = gs(args.g)
-      args.gsp = gs(args.gp)
-      args.gsr = gs(args.gr)
       contextsPrime.push(contextPrime)
       generatedPrime.push(generated)
       paraphrasesPrime.push(paraphrases)
@@ -614,7 +625,12 @@ const loadInstance = (config, instance) => {
       if (results.skipSemantics) {
         config.config.skipSemantics = results.skipSemantics
       }
-      processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}`, instance: instance.queries[i] })
+      const args = { config, hierarchy, json: results, commandLineArgs: {} }
+      if (instance.queries) {
+        args.isInstance = `instance${i}`
+        args.instance = instance.queries[i]
+      }
+      processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null
       }
@@ -660,6 +676,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         opChoices: []
       },
       times: 0.0,
+      clientSideTimes: 0.0,
       trace: '',
       contexts: [],
       generated: [],
@@ -701,8 +718,16 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       if (json.status !== 200) {
         throw json
       } else {
+        let clientSideTime
+        if (isTest) {
+          start = runtime.performance.performance.now()
+        }
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
                                   processContextsB({ isTest, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        if (isTest) {
+          end = runtime.performance.performance.now()
+          clientSideTime = end - start
+        }
         response.associations = json.associations
         response.learned_contextual_priorities = json.learned_contextual_priorities
         response.hierarchy = json.hierarchy
@@ -713,6 +738,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         // appendNoDups(response.metadata.priorities, json.metadata.priorities)
         appendNoDups(response.metadata.opChoices, json.metadata.opChoices)
         response.times += json.times
+        response.clientSideTimes += clientSideTime
         response.trace = response.trace.concat(json.trace)
         response.version = json.version
         response.explain_priorities = json.explain_priorities
@@ -778,10 +804,6 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug }) =
   const test = expected.query
   // initialize in between test so state is not preserved since the test was adding without state
   config.rebuild()
-  if (!args.dontAddAssociations) {
-    config.addAssociationsFromTests(config.tests)
-  }
-  // config.addAssocationsFromTests(
   const errorHandler = (error) => {
     if (error.metadata) {
       const priorities = analyzeMetaData(expected.metadata, error.metadata)
@@ -806,10 +828,10 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug }) =
         defaultInnerProcess(config, errorHandler, result)
       }
       if (verbose) {
-        const widths = [100, 20]
+        const widths = [100, 60]
         const lines = new Lines(widths)
         lines.setElement(0, 0, test)
-        lines.setElement(0, 1, `time on server ${result.times.toFixed(2)}`)
+        lines.setElement(0, 1, `time on server: ${result.times.toFixed(2)} client: ${(result.clientSideTimes/1000).toFixed(2)}`)
         lines.log()
       }
       const expected_objects = sortJson(convertToStable(expected.objects), { depth: 25 })
@@ -1173,7 +1195,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
 
   if (responses.explain_priorities) {
-    console.log("Explain Priorities")
+    console.log("Explain Priorities (listed from lower priority to higher priority)")
     for ([inputss, outpus, reason] of responses.explain_priorities) {
       console.log(`    ${JSON.stringify(inputss)} reason: ${reason}`)
     }
@@ -1232,7 +1254,7 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
 }
 
 // builtTemplate saveInstance
-const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
+const rebuildTemplate = async ({ config, target, previousResultss, startOfChanges, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
@@ -1245,7 +1267,7 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
       finish()
       return
     }
-    const { property, hierarchy, query: queryOrExtraConfig, initializer, skipSemantics } = queries.shift()
+    const { property, hierarchy, query: queryOrExtraConfig, previousResults, initializer, skipSemantics } = queries.shift()
     // queries are strings or { query: "blah", development: true/false }
     if (typeof queryOrExtraConfig === 'string' || queryOrExtraConfig.query) {
       let query = queryOrExtraConfig;
@@ -1267,17 +1289,26 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
         }
       }
       try {
-        const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
+        let results
+        let prMessage = ''
+        if (previousResults && previousResults.query == query.query) {
+          results = previousResults
+          prMessage = ' Using previous results. use -rtf for a hard rebuild of everything on the server side.'
+          loadInstance(config, { resultss: [results] })
+        } else {
+          results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
+        }
         if (config.config.debug) {
           // TODO pass in the error handler like the other ones
           defaultInnerProcess(config, defaultErrorHandler, results)
         }
         if (results.contexts.length > 1) {
           console.log(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
+          throw new Error(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
         } else if (results.paraphrases[0] != query.query) {
-          console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".`)
+          console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".${prMessage}`)
         } else {
-          console.log(`query ${query.query}`)
+          console.log(`query "${query.query}".${prMessage}`)
         }
         global.transitoryMode = transitoryMode
         config.config.skipSemantics = null
@@ -1299,14 +1330,18 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
       // it will just get added to the config
       const extraConfig = queryOrExtraConfig
       console.log('config', extraConfig)
-      try {
-        config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
-      } catch ( e ) {
-        const where = extraConfig.where ? ` ${extraConfig.where}` : ''
-        throw new Error(`Error processing extra config${where}: ${e.stack}}`)
+      if (extraConfig.stop) {
+        await looper([])
+      } else {
+        try {
+          config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
+        } catch ( e ) {
+          const where = extraConfig.where ? ` ${extraConfig.where}` : ''
+          throw new Error(`Error processing extra config${where}: ${e.stack}}`)
+        }
+        accumulators[property].push({ extraConfig: true, ...extraConfig })
+        await looper(queries)
       }
-      accumulators[property].push({ extraConfig: true, ...extraConfig })
-      await looper(queries)
     }
   }
 
@@ -1327,6 +1362,7 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
           } else {
             delete result.load_cache_time
             delete result.times
+            delete result.clientSideTimes
             delete result.memory_free_percent
             delete result.logs
             delete result.version
@@ -1360,13 +1396,41 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
   }
   let todo = []
   todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
-  todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false || query.skipSemantics} }))
+  todo = todo.concat((template.queries || []).map((query, index) => { 
+    let pr
+    if (index < startOfChanges) {
+      pr = previousResultss[index] 
+    }
+    return { property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics} 
+  }))
   todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
   todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
   await looper(Object.assign([], todo))
 }
 
-const knowledgeModule = async ({
+const checkTemplate = (template) => {
+  return
+  if (!template) {
+    return
+  }
+  if (template.checks) {
+    throw new Error("The 'checks' property should be in the 'test' property not the 'template' property")
+  }
+}
+
+const checkTest = (testConfig) => {
+  if (!testConfig) {
+    return
+  }
+  if (!testConfig.name) {
+    throw new Error("The 'test' property is missing the 'name' property that contains the name of the '<km>.test.json' file")
+  }
+  if (!testConfig.contents) {
+    throw new Error("The 'test' property is missing the 'contents' property that contains contents of the '<km>.test.json' file")
+  }
+}
+
+const knowledgeModuleImpl = async ({
   module: moduleFromJSFile,
   description,
   section,
@@ -1399,9 +1463,10 @@ const knowledgeModule = async ({
   if (!description) {
     throw new Error("'description' is a required parameter. The value should the description of the knowledge module.")
   }
-  if (!test) {
+  if (!testConfig) {
     throw new Error("'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }.")
   }
+  checkTest(testConfig)
 
   const isProcess = require.main === moduleFromJSFile
 
@@ -1452,13 +1517,14 @@ const knowledgeModule = async ({
     parser.add_argument('-tnp', '--testNoParenthesized', { action: 'store_true', help: 'Don\' check parenthesized differences for the tests' })
     parser.add_argument('-n', '--count', { help: 'Number of times to run the tests. Default is one. Use this to check for flakey test. If possible the system will print out a message with the word "hint" suggesting how to fix the problem' })
     // parser.add_argument('-b', '--build', { help: 'Specify the template file name of the form <kmName>. There should be a file called <baseKmName>.<kmName>.template.json with the queries to run. For example { queries: [...] }. The template file will be run and generate an instantiation called <baseKmName>.<kmName>.instance.json and a file called <kmName>.js that will load the template file (this is file generated only if not already existing) and a test file called <KmName>.tests.json. This can then be loaded into an instance of the current knowledge module to setup initial conditions.' })
-    parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild' })
+    parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild. Using optimization where if the query/config has not changed it will use the previous value. One there is a change all subsequence query/configs will be run.' })
+    parser.add_argument('-rtf', '--rebuildTemplateFull', { action: 'store_true', help: 'Force a template rebuild. Skip the optimization' })
     parser.add_argument('-l', '--loop', { action: 'store_true', help: 'Run a loop so that multiply queries may be run' })
     parser.add_argument('-i', '--info', { action: 'store_true', help: 'Print meta-data for the module' })
     parser.add_argument('-v', '--vimdiff', { action: 'store_true', help: 'For failures run vimdiff' })
     parser.add_argument('-g', '--greg', { action: 'store_true', help: 'Set the server to be localhost so I can debug stuff' })
     parser.add_argument('-cl', '--checkForLoop', { nargs: "?", help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
-    parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
+    parser.add_argument('-r', '--reset', { action: 'store_true', help: 'Get the server to bypass the cache and rebuild everything' })
     parser.add_argument('-q', '--query', { help: 'Run the specified query' })
     parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
     parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
@@ -1466,7 +1532,6 @@ const knowledgeModule = async ({
     parser.add_argument('--parenthesized', { action: 'store_true', help: 'Show the generated phrases with parenthesis.' })
     parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
     parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
-    parser.add_argument('-daa', '--dontAddAssociations', { action: 'store_true', help: 'Do not add associations from the tests.' })
     parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a == associations j == JSON sent to server. for example --print wb' })
     parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
     parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
@@ -1484,6 +1549,10 @@ const knowledgeModule = async ({
     const args = parser.parse_args()
     args.count = args.count || 1
 
+    if (args.rebuildTemplateFull) {
+      args.rebuildTemplate = true
+    }
+
     if (args.parenthesized) {
       config.parenthesized = true
     }
@@ -1671,24 +1740,40 @@ const knowledgeModule = async ({
       }
     }
 
+    checkTemplate(template)
+
     if (template) {
-      const needsRebuild = config.needsRebuild(template.template, template.instance, options)
-      if (needsRebuild) {
+      let needsRebuild
+      if (args.rebuildTemplate && !args.rebuildTemplateFull) {
+        // get the startOfChanges for the partial rebuild
+        needsRebuild = config.needsRebuild(template.template, template.instance, { ...options, rebuild: false })
+      } else {
+        // do a check or full rebuild
+        needsRebuild = config.needsRebuild(template.template, template.instance, options)
+      }
+
+      if (needsRebuild.needsRebuild) {
+        if (needsRebuild.previousResultss) {
+          console.log("Rebuild using the optimization to use previous results until a change is hit. For a full rebuild use -rtf")
+        }
         console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
         options.rebuild = true
         config.config.rebuild = true
       }
-      config.load(template.template, template.instance, { rebuild: needsRebuild })
-      printConfig()
-      if (needsRebuild) {
+      try {
+        config.load(template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
+      } catch( e ) {
+        console.error(`Error loading template for ${config.name}. ${e.error ? e.error : e}`)
+        runtime.process.exit(-1)
+      }
+      if (!args.query) {
+        printConfig()
+      }
+      if (needsRebuild.needsRebuild) {
         return
       }
     }
 
-    if (!args.save && !args.rebuildTemplate && !args.dontAddAssociations) {
-      config.addAssociationsFromTests(config.tests);
-    }
-
     if (args.retrain) {
       config.config.retrain = true
     }
@@ -1696,7 +1781,6 @@ const knowledgeModule = async ({
     if (args.test || args.testVerbose || args.testAllVerbose || args.save) {
       global.transitoryMode = true
     }
-
     if (!args.query && !args.test && !args.info && (args.save || args.saveDeveloper)) {
       global.transitoryMode = true
       saveTests(config, test, testConfig, args.saveDeveloper)
@@ -1718,7 +1802,6 @@ const knowledgeModule = async ({
             useTestConfig = config.getConfigs()[args.testModuleName].getTestConfig()
             useTestConfig.testModuleName = args.testModuleName
             test = useTestConfig.name
-
           }
           runTests(config, test, { args, debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
             let newError = false
@@ -1762,17 +1845,21 @@ const knowledgeModule = async ({
                       console.log(`    actual ${label}  `, actual)
                       newError = true
                       headerShown = true
+                      if (args.vimdiff) {
+                        vimdiff(result.actual.paraphrasesParenthesized, result.expected.paraphrasesParenthesized)
+                      }
                     }
                   }
                   show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
                   if (!args.testNoParenthesized) {
                     show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
                   }
+                  /*
+                  }
                   show('responses', result.expected.responses, result.actual.responses)
                   if (!args.testNoParenthesized) {
                     show('responses parenthesized', result.expected.generatedParenthesized, result.actual.generatedParenthesized)
                   }
-                  /*
                   if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
                     if (!headerShown) {
                       console.log('  Failure')
@@ -1798,10 +1885,10 @@ const knowledgeModule = async ({
                     }
                     const widths = [4, 18, 72]
                     const lines = new Lines(widths)
-                    lines.setElement(1, 1, 'expected checked')
+                    lines.setElement(1, 1, 'expected checked objects')
                     lines.setElement(2, 2, JSON.stringify(result.expected.checked, null, 2))
                     lines.log()
-                    lines.setElement(1, 1, 'actual checked')
+                    lines.setElement(1, 1, 'actual checked objects')
                     lines.setElement(2, 2, JSON.stringify(result.actual.checked, null, 2))
                     lines.log()
                     if (args.vimdiff) {
@@ -1816,10 +1903,10 @@ const knowledgeModule = async ({
                     }
                     const widths = [4, 18, 72]
                     const lines = new Lines(widths)
-                    lines.setElement(1, 1, 'expected checkedContexts', true)
+                    lines.setElement(1, 1, 'expected checked contexts', true)
                     lines.setElement(2, 2, JSON.stringify(result.expected.checkedContexts, null, 2))
                     lines.log()
-                    lines.setElement(1, 1, 'actual checkedContexts', true)
+                    lines.setElement(1, 1, 'actual checked contexts', true)
                     lines.setElement(2, 2, JSON.stringify(result.actual.checkedContexts, null, 2))
                     lines.log()
                     if (args.vimdiff) {
@@ -1836,16 +1923,18 @@ const knowledgeModule = async ({
                   }
                 }
               }
-              if (!headerShown) {
+              if (hasError) {
+                if (!headerShown) {
+                  if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
+                    console.log('There are failures due to things other than paraphrases, responses and checked properties being different. They are not shown because you ran -tv or -tva which only shows difference in paraphrase and results. Usually what I do is -s and do a diff to make sure there are no other problems. If the paraphrases or results were different they would have shown here.')
+                  }
+                }
                 if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
-                  console.log('There are failures due to things other than paraphrases, responses and checked properties being different. They are not shown because you ran -tv or -tva which only shows difference in paraphrase and results. Usually what I do is -s and do a diff to make sure there are no other problems. If the paraphrases or results were different they would have shown here.')
+                  console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
+                  // console.log(JSON.stringify(contexts))
+                  console.log('**************************** THERE WERE ERRORS ************************')
                 }
               }
-              if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
-                console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
-                // console.log(JSON.stringify(contexts))
-                console.log('**************************** ERRORS ************************')
-              }
             }
             // const contexts = { failures: results }
             l(n - 1, hasError || newError)
@@ -1929,7 +2018,7 @@ const knowledgeModule = async ({
       }
 
       if (template) {
-        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess })) {
+        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess }).needsRebuild) {
           const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
           throw new Error(error)
         }
@@ -1939,8 +2028,6 @@ const knowledgeModule = async ({
           errorHandler(e)
         }
       }
-
-      config.addAssociationsFromTests(config.tests);
     }
 
     createConfigExport = () => {
@@ -1951,7 +2038,6 @@ const knowledgeModule = async ({
     }
     moduleFromJSFile.exports = createConfigExport
   }
-
 }
 
 /*
@@ -1999,6 +2085,10 @@ function w(func) {
   return func
 }
 
+const knowledgeModule = async (...args) => {
+  await knowledgeModuleImpl(...args).catch((e) => console.error(e))
+}
+
 module.exports = {
   process: _process,
   where,

package/package.json

@@ -12,6 +12,7 @@
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
     "test:debug": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
+    "td": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
     "tod": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
     "lint:fix": "eslint \"**/*.js\" --fix",
     "lint": "eslint \"**/*.js\"",
@@ -64,6 +65,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.6.0",
+  "version": "7.7.0-beta.1",
   "license": "ISC"
 }

package/runtime.js

@@ -15,4 +15,5 @@ module.exports = {
   jsonDiff: require('json-diff'),
   sortJson: require('sort-json'),
   util: require('util'),
+  performance: require('perf_hooks'),
 }

package/src/config.js

@@ -27,7 +27,7 @@ const config_toServer = (config) => {
 
 const debugPriority = (priority) => {
   if (global.entodictonDebugPriority) {
-    if (helpers.safeEquals(entodictonDebugPriority, priority)) {
+    if (helpers.subPriority(entodictonDebugPriority, priority)) {
       debugger; // debug hierarchy hit
     }
   }
@@ -965,7 +965,7 @@ class Config {
   }
 
   // { rebuild: false, isModule: false }
-  needsRebuild(template, instance, options) {
+  needsRebuild (template, instance, options) {
     if (options.rebuild) {
       return true
     }
@@ -1058,7 +1058,21 @@ class Config {
       return elements.map( toCanonicalQuery )
     }
 
-    const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
+    const templateQueries = toCanonicalQueries(template.queries || []).map(helpers.updateQueries)
+    const instanceQueries = toCanonicalQueries(instance.queries || [])
+    let sameQueries = true
+    let startOfChanges;
+    for (let iq = 0; iq < templateQueries.length; ++iq) {
+      if (!helpers.safeEquals(templateQueries[iq], instanceQueries[iq])) {
+        sameQueries = false
+        startOfChanges = iq
+      }
+    }
+    // things were deleted case
+    if (templateQueries.length < instanceQueries.length) {
+      startOfChanges = instanceQueries.length
+    }
+    // const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
 
     const debug = false
     if (debug) {
@@ -1067,16 +1081,20 @@ class Config {
           debugger
           debugger
         }
-        console.log("instance", instance)
+        // console.log("instance", instance)
         console.log("sameQueries", sameQueries)
         console.log("sameFragments", sameFragments)
-        console.log("templateFragments", templateFragments)
-        console.log("instanceFragments", instanceFragments)
-        console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
-        console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
+        // console.log("templateFragments", templateFragments)
+        // console.log("instanceFragments", instanceFragments)
+        // console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
+        // console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
       }
     }
-    return !(instance && sameQueries && sameFragments)
+    if (startOfChanges || instance.resultss) {
+      return { needsRebuild: !(instance && sameQueries && sameFragments), startOfChanges, previousResultss: instance.resultss }
+    } else {
+      return { needsRebuild: !(instance && sameQueries && sameFragments) }
+    }
   }
 
   validifyTemplate (template) {
@@ -1097,7 +1115,7 @@ class Config {
   }
 
   // loadTemplate
-  load (template, instance, options = { rebuild: false } ) {
+  load (template, instance, options = { rebuild: false, previousResultss: undefined, startOfChanges: undefined } ) {
     this.validifyTemplate(template)
     instance.template = template
     this.logs.push(`loading template for ${this.name}`)
@@ -1105,7 +1123,7 @@ class Config {
       // TODO fix beforeQuery
       template = { fragments: [], queries: [], ...template }
       template.fragments = template.fragments.concat(this.dynamicFragments)
-      client.rebuildTemplate({ config: this, target: this.name, beforeQuery: () => {}, template, ...options })
+      client.rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
     } else {
       // no change
       // this.initInstances.push({ ...instance, name: config.name })
@@ -1137,12 +1155,6 @@ class Config {
     return this.config.objects.namespaced[this._uuid]
   }
 
-  addAssociationsFromTests(tests = []) {
-    for (let test of tests) {
-      this.addAssociations(test.associations || []);
-    }
-  }
-
   addAssociations (associations) {
     for (let association of associations) {
       this.addAssociation(association)

package/src/flatten.js

@@ -81,6 +81,9 @@ const flatten = (markers, value) => {
     return flattenListHelper(markers, value)
   }
 
+  if (value.flatten === false) {
+    return [[value], false]
+  }
   const marker = value.marker
   let properties = value
 

package/src/generators.js

@@ -73,7 +73,8 @@ class Generator {
     return matches
   }
 
-  apply (baseArgs, objects, g, gs, context, hierarchy, config, response, log, options = {}) {
+  // apply (baseArgs, objects, g, gs, context, hierarchy, config, response, log, options = {}) {
+  apply (baseArgs, objects, context, hierarchy, config, response, log, options = {}) {
     if (!log) {
       throw new Error('generators.apply argument log is required')
     }
@@ -103,12 +104,12 @@ class Generator {
             log, 
             global: 
             objects, 
-            g, 
+            // g, 
             n, 
             hierarchy, 
             context, 
             uuid: this.uuid, 
-            gs, 
+            // gs, 
             config, 
             response, 
             api: this.getAPI(config), 
@@ -165,136 +166,124 @@ class Generators {
     this.logs = logs
   };
 
-  // assumed - properties added to contexts before the generators are called. For setting paraphrase property
-  apply (args, contexts, assumed = {}, options = {}) {
+  // assumed - properties added to context before the generators are called. For setting paraphrase property
+  apply (args, context, assumed = {}, options = {}) {
+    if (Array.isArray(context)) {
+      throw new Error("Expected a context not an array")
+    }
+    if (typeof context !== 'object') {
+      return String(context)
+    }
+
     const config = args.config
     const objects = args.objects
     const hierarchy = args.hierarchy
     const response = args.response
-    if (Array.isArray(contexts)) {
-      // no-op
-    } else if (typeof contexts === 'object') {
-      contexts = [contexts]
-    } else {
-      return String(contexts)
-    }
-
-    const contextsPrime = []
-    for (const icontext in contexts) {
-      let context = contexts[icontext]
-      context = Object.assign({}, context, assumed)
-      // let context_prime = JSON.stringify(context);
-      let generated = context
-      let applied = false
-      const stack = args.calls.push()
-      for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
-        const generator = this.generators[igenerator]
-        if (generator.matches(args, objects, context, hierarchy, config, options)) {
-          const g = (context, options) => {
-            const r = this.apply(args, context, Object.assign({}, ((options||{}).assumed || {}), assumed), options)
-            if (Array.isArray(r)) {
-              return r.join(' ')
-            } else {
-              return r
-            }
-          }
-          const log = (message) => { this.logs.push(message) }
-          // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
-          let errorMessage = 'The apply function did not return a value'
-          try {
-            generated= generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
-          } catch( e ) {
-            // the next if handle this
-            generated = null
-            e.retryCall = () => generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
-            const help = 'The error has a retryCall property that will recall the function that failed.'
-            if (e.stack && e.message) {
-              const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
-              errorMessage = `Error applying generator '${info}. Error is ${e.toString()} stack is ${e.stack}. Generator is ${generator.toString()}. ${help}`
-            } else if (e.error) {
-              const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
-              errorMessage = `Error applying generator '${info}. Error is ${e.error.join()}. Generator is ${generator.toString()}. ${help}`
-            } else {
-							errorMessage = e.toString()
-						}
+    
+    // args = { ...args, ...args.getAssumedScoped(assumed) }
+    args.addAssumedScoped(args, assumed) 
+    context = Object.assign({}, context, args.assumed)
+    let generated = ''
+    let applied = false
+    const stack = args.calls.push()
+    for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
+      const generator = this.generators[igenerator]
+      if (generator.matches(args, objects, context, hierarchy, config, options)) {
+        const log = (message) => { this.logs.push(message) }
+        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
+        let errorMessage = 'The apply function did not return a value'
+        try {
+          generated= generator.apply(args, objects, context, hierarchy, config, response, log)
+        } catch( e ) {
+          // the next if handle this
+          generated = null
+          e.retryCall = () => generator.apply(args, objects, context, hierarchy, config, response, log)
+          const help = 'The error has a retryCall property that will recall the function that failed.'
+          if (e.stack && e.message) {
+            const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
+            errorMessage = `Error applying generator '${info}. Error is ${e.toString()} stack is ${e.stack}. Generator is ${generator.toString()}. ${help}`
+          } else if (e.error) {
+            const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
+            errorMessage = `Error applying generator '${info}. Error is ${e.error.join()}. Generator is ${generator.toString()}. ${help}`
+          } else {
+            errorMessage = e.toString()
           }
-          if (!generated && generated !== '') {
-            const widths = [10, 10, 90]
-            const lines = new Lines(widths)
-            lines.setElement(0, 0, 'Generator')
-            const source = `${generator.km}/#${generator.index}`
-            lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
-            lines.newRow()
-            lines.setElement(0, 2, generator.toString())
-            lines.newRow()
-            lines.setElement(0, 1, 'TO')
-            lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
-            lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
-            lines.newRow()
-            lines.setElement(0, 1, 'STACK')
-            lines.setElement(0, 2, stack)
-						lines.newRow()
-						lines.setElement(0, 1, 'DEBUG')
-						lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
-            lines.newRow()
-            lines.setElement(0, 1, 'ERROR')
-            lines.setElement(0, 2, errorMessage)
-            this.logs.push(lines.toString())
-            const message = `ERROR while applying (${source}) ${generator.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
-            // this.logs.push(message)
-            // return [message]
-            args.calls.pop()
-            throw { error: [message], logs: this.logs }
-          }
-          if (((config || {}).config || {}).debug) {
-            const widths = [10, 10, 90]
-            const lines = new Lines(widths)
-            lines.setElement(0, 0, 'Generator')
-            if (generator.index > -1 && generator.km) {
-              // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
-              lines.setElement(0, 2, generator.toLabel())
-            }
-            lines.newRow()
-            lines.setElement(0, 1, 'APPLIED')
+        }
+        if (!generated && generated !== '') {
+          const widths = [10, 10, 90]
+          const lines = new Lines(widths)
+          lines.setElement(0, 0, 'Generator')
+          const source = `${generator.km}/#${generator.index}`
+          lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
+          lines.newRow()
+          lines.setElement(0, 2, generator.toString())
+          lines.newRow()
+          lines.setElement(0, 1, 'TO')
+          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
+          lines.newRow()
+          lines.setElement(0, 1, 'ERROR')
+          lines.setElement(0, 2, errorMessage)
+          this.logs.push(lines.toString())
+          const message = `ERROR while applying (${source}) ${generator.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
+          // this.logs.push(message)
+          // return [message]
+          args.calls.pop()
+          throw { error: [message], logs: this.logs }
+        }
+        if (((config || {}).config || {}).debug) {
+          const widths = [10, 10, 90]
+          const lines = new Lines(widths)
+          lines.setElement(0, 0, 'Generator')
+          if (generator.index > -1 && generator.km) {
+            // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
             lines.setElement(0, 2, generator.toLabel())
-            lines.newRow()
-            lines.setElement(0, 2, generator.toString())
-            lines.newRow()
-            lines.setElement(0, 1, 'RESULT')
-            lines.setElement(0, 2, generated)
-            lines.newRow()
-            lines.setElement(0, 1, 'STACK')
-            lines.setElement(0, 2, stack)
-						lines.newRow()
-						lines.setElement(0, 1, 'DEBUG')
-						lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
-						lines.newRow()
-            lines.setElement(0, 1, 'TO')
-            lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
-            lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
-            this.logs.push(lines.toString())
           }
-          applied = true
-          break
+          lines.newRow()
+          lines.setElement(0, 1, 'APPLIED')
+          lines.setElement(0, 2, generator.toLabel())
+          lines.newRow()
+          lines.setElement(0, 2, generator.toString())
+          lines.newRow()
+          lines.setElement(0, 1, 'RESULT')
+          lines.setElement(0, 2, generated)
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
+          lines.newRow()
+          lines.setElement(0, 1, 'TO')
+          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          this.logs.push(lines.toString())
         }
+        applied = true
+        break
       }
-      args.calls.pop()
-      if (!applied && ((config || {}).config || {}).debug) {
-        const widths = [10, 10, 90]
-        const lines = new Lines(widths)
-        lines.setElement(0, 0, 'Generator')
-        lines.setElement(0, 2, 'No generator applied')
-        lines.newRow()
-        lines.setElement(0, 1, 'STACK')
-        lines.setElement(0, 2, stack)
-        lines.newRow()
-        lines.setElement(0, 1, 'TO')
-        lines.setElement(0, 2, JSON.stringify(context, null, 2))
-        this.logs.push(lines.toString())
-      }
-      contextsPrime.push((config || {}).parenthesized ? "(" + generated + ")" : generated)
     }
-    return contextsPrime
+    args.calls.pop()
+    if (!applied && ((config || {}).config || {}).debug) {
+      const widths = [10, 10, 90]
+      const lines = new Lines(widths)
+      lines.setElement(0, 0, 'Generator')
+      lines.setElement(0, 2, 'No generator applied')
+      lines.newRow()
+      lines.setElement(0, 1, 'STACK')
+      lines.setElement(0, 2, stack)
+      lines.newRow()
+      lines.setElement(0, 1, 'TO')
+      lines.setElement(0, 2, JSON.stringify(context, null, 2))
+      this.logs.push(lines.toString())
+    }
+    return ((config || {}).parenthesized ? "(" + generated + ")" : generated)
   }
 }
 

package/src/helpers.js

@@ -334,6 +334,45 @@ const ecatch = (where, call) => {
   }
 }
 
+const equalKey = (key1, key2) => {
+  return key1[0] == key2[0] && key1[1] == key2[1]
+}
+
+// matches for { context: ..., [ordered], choose: ... } exactely OR
+//             [ <id1>, <id2> ] - where id1 is chosen
+const subPriority = (sub, sup) => {
+  if (Array.isArray(sub)) {
+    const subChoosen = sub[0]
+    const subOther = sub[1]
+    const hasChoosen = sup.choose.find( (index) => equalKey(sup.context[index], subChoosen)) != undefined
+    const hasOtherChosen = sup.choose.find( (index) => equalKey(sup.context[index], subOther)) != undefined
+    const hasOther = sup.context.find( (other) => equalKey(other, subOther) ) !== undefined
+    return !!(hasChoosen && hasOther) && !hasOtherChosen
+  }
+
+  if (!safeEquals([...sub.choose].sort(), [...sup.choose].sort())) {
+    return false
+  }
+
+  const choose = (priority) => {
+    const chosen = []
+    for (const i of priority.choose) {
+      chosen.push(priority.context[i])
+    }
+    return chosen
+  }
+  const chosen1 = choose(sub)
+  const chosen2 = choose(sup)
+  const sameId = (id1, id2) => id1[0] == id2[0] && id1[1] == id2[1]
+  // same length so only need one way
+  const missing1 = chosen1.find( (id1) => !chosen2.find( (id2) => sameId(id1, id2)) )
+  if (missing1) {
+    return false
+  }
+
+  return true
+}
+
 module.exports = { 
   ecatch,
   functionsToStrings,
@@ -354,5 +393,6 @@ module.exports = {
   isCompound, 
   InitCalls, 
   hashCode, 
-  sortJson 
+  sortJson,
+  subPriority,
 }

package/src/project.js

@@ -16,6 +16,9 @@ const project = (object, filter) => {
       return object.map( element => project(element, filter) )
     } else {
       for (let properties of filter) {
+        if (typeof properties == 'function') {
+          properties = properties(object)
+        }
         if (typeof properties == 'object') {
           if (properties.propertyLists) {
             for (const propertyList in properties.propertyLists) {
@@ -45,7 +48,6 @@ const project = (object, filter) => {
           if (properties.property) {
             const property = properties.property
             if (properties.isPropertyList) {
-              debugger
               if (!Array.isArray(object[property])) {
                 return projection
               }