theprogrammablemind_4wp 7.7.0 → 7.8.0-beta.0

package/client.js

@@ -30,18 +30,29 @@ const pickObjects = (testConfig, objects) => {
   return project(objects, getConfig_getObjectsCheck(testConfig))
 }
 
+// move ask to the KM's since verbatim is called
 const getAsk = (config) => (uuid) => (asks) => {
   for (let ask of asks) {
+    let oneShot = true // default
+    if (ask.oneShot === false) {
+      oneShot = false
+    }
     config.addSemantic({
       uuid,
-      oneShot: true,
+      oneShot,
       match: (args) => ask.matchr(args),
       apply: (args) => ask.applyr(args)
     })
   }
+  let oneShot = true
+  for (let ask of asks) {
+    if (ask.oneShot === false) {
+      oneShot = false
+    }
+  }
   config.addSemantic({
       uuid,
-      oneShot: true,
+      oneShot,
       match: ({context}) => context.marker == 'controlEnd' || context.marker == 'controlBetween',
       apply: (args) => {
         for (let ask of asks) {
@@ -57,14 +68,18 @@ const getAsk = (config) => (uuid) => (asks) => {
           }
           if (matchq(args)) {
             // args.context.motivationKeep = true
+            args.verbatim(applyq(args))
+            /*
             args.context.verbatim = applyq(args)
             args.context.isResponse = true;
             delete args.context.controlRemove;
+            */
             args.context.controlKeepMotivation = true;
             break
           }
         }
-     }
+        args.context.cascade = true
+      }
   })
 }
 
@@ -169,6 +184,7 @@ const setupArgs = (args, config, logs, hierarchy) => {
       apis: getAPIs(uuid)
     }
   }
+  Object.assign(args, args.getUUIDScoped(this.uuid))
   args.breakOnSemantics = false
   args.theDebugger = {
     breakOnSemantics: (value) => args.breakOnSemantics = value
@@ -179,11 +195,23 @@ const setupArgs = (args, config, logs, hierarchy) => {
   args.log = (message) => logs.push(message)
 
   args.addAssumedScoped = (args, assumed) => {
+    const addAssumed = (args, ...moreAssumed) => {
+      return { ...args, assumed: Object.assign({}, (args.assumed || {}), ...moreAssumed) }
+    }
+
     args.s = (c) => config.getSemantics(logs).apply(args, c)
-    args.g = (c) => config.getGenerators(logs).apply(args, c)
-    args.gp = (c) => config.getGenerators(logs).apply({...args, assumed: {paraphrase: true, isResponse: false, response: false}}, c, {paraphrase: true, isResponse: false, response: false})
-    args.gr = (c) => config.getGenerators(logs).apply({...args, assumed: {paraphrase: false, isResponse: true}}, { ...c, paraphrase: false, isResponse: true })
-    args.e = (c) => config.getEvaluator(args.s, args.calls, logs, c)
+    args.g = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
+    }
+    args.gp = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, {paraphrase: true, isResponse: false, response: false}), c, {paraphrase: true, isResponse: false, response: false})
+    }
+    args.gr = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, {paraphrase: false, isResponse: true}), { ...c, paraphrase: false, isResponse: true })
+    }
+    args.e = (c) => {
+      return config.getEvaluator(args.s, args.calls, logs, c)
+    }
     args.gs = gs(args.g)
     args.gsp = gs(args.gp)
     args.gsr = gs(args.gr)
@@ -467,8 +495,6 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
       }
       // assumed = { paraphrase: true, response: false };
       assumed = { paraphrase: true, isResponse: false, response: false };
-      // args.g = (c) => config.getGenerators(json.logs).apply({...args, assumed}, c, assumed)
-      // args.gs = gs(args.g)
       if (generateParenthesized) {
         config.parenthesized = false
       }
@@ -479,8 +505,6 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
         paraphrasesParenthesized = config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed)
         config.parenthesized = false
       }
-      // args.g = (c) => config.getGenerators(json.logs).apply(args, c)
-      // args.gs = gs(args.g)
       contextsPrime.push(contextPrime)
       generatedPrime.push(generated)
       paraphrasesPrime.push(paraphrases)
@@ -613,11 +637,25 @@ const loadInstance = (config, instance) => {
       // config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
       // config.addInternal(config.template.queries[i], { handleCalculatedProps: true } )
       config.addInternal(instance.template.queries[i], { addFirst: true, handleCalculatedProps: true } )
+    } else if (results.apply) {
+      const objects = config.get('objects')
+      const args = { objects, getObjects: getObjects(objects) }
+      if (instance.queries) {
+        args.isInstance = `instance${i}`
+        args.instance = instance.queries[i]
+      }
+      setupArgs(args, config, config.logs, hierarchy)
+      results.apply(args)
     } else {
       if (results.skipSemantics) {
         config.config.skipSemantics = results.skipSemantics
       }
-      processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}`, instance: instance.queries[i] })
+      const args = { config, hierarchy, json: results, commandLineArgs: {} }
+      if (instance.queries) {
+        args.isInstance = `instance${i}`
+        args.instance = instance.queries[i]
+      }
+      processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null
       }
@@ -1183,8 +1221,8 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
 
   if (responses.explain_priorities) {
     console.log("Explain Priorities")
-    for ([inputss, outpus, reason] of responses.explain_priorities) {
-      console.log(`    ${JSON.stringify(inputss)} reason: ${reason}`)
+    for ([inputs, output, reason] of responses.explain_priorities) {
+      console.log(`    inputs: ${JSON.stringify(inputs)} output: ${JSON.stringify(output)}  reason: ${reason}`)
     }
   }
   const objects = config.get('objects').namespaced[config.uuid]
@@ -1241,7 +1279,7 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
 }
 
 // builtTemplate saveInstance
-const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
+const rebuildTemplate = async ({ config, target, previousResultss, startOfChanges, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
@@ -1254,7 +1292,7 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
       finish()
       return
     }
-    const { property, hierarchy, query: queryOrExtraConfig, initializer, skipSemantics } = queries.shift()
+    const { property, hierarchy, query: queryOrExtraConfig, previousResults, initializer, skipSemantics } = queries.shift()
     // queries are strings or { query: "blah", development: true/false }
     if (typeof queryOrExtraConfig === 'string' || queryOrExtraConfig.query) {
       let query = queryOrExtraConfig;
@@ -1276,17 +1314,26 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
         }
       }
       try {
-        const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
+        let results
+        let prMessage = ''
+        if (previousResults && previousResults.query == query.query) {
+          results = previousResults
+          prMessage = ' Using previous results. use -rtf for a hard rebuild of everything on the server side.'
+          loadInstance(config, { resultss: [results] })
+        } else {
+          results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
+        }
         if (config.config.debug) {
           // TODO pass in the error handler like the other ones
           defaultInnerProcess(config, defaultErrorHandler, results)
         }
         if (results.contexts.length > 1) {
           console.log(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
+          throw new Error(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
         } else if (results.paraphrases[0] != query.query) {
-          console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".`)
+          console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".${prMessage}`)
         } else {
-          console.log(`query ${query.query}`)
+          console.log(`query "${query.query}".${prMessage}`)
         }
         global.transitoryMode = transitoryMode
         config.config.skipSemantics = null
@@ -1303,19 +1350,32 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
         config.config.skipSemantics = null
         errorHandler(error)
       }
+    } else if (typeof queryOrExtraConfig == 'function') {
+      console.log('calling initialize function')
+      const initFunction = queryOrExtraConfig
+      const objects = config.get('objects')
+      const args = { objects, getObjects: getObjects(objects) }
+      setupArgs(args, config, config.logs, hierarchy)
+      initFunction(args)
+      accumulators[property].push({ apply: queryOrExtraConfig })
+      await looper(queries)
     } else {
       // extra config is def from a time like operators or bridges or words etc
       // it will just get added to the config
       const extraConfig = queryOrExtraConfig
       console.log('config', extraConfig)
-      try {
-        config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
-      } catch ( e ) {
-        const where = extraConfig.where ? ` ${extraConfig.where}` : ''
-        throw new Error(`Error processing extra config${where}: ${e.stack}}`)
+      if (extraConfig.stop) {
+        await looper([])
+      } else {
+        try {
+          config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
+        } catch ( e ) {
+          const where = extraConfig.where ? ` ${extraConfig.where}` : ''
+          throw new Error(`Error processing extra config${where}: ${e.stack}}`)
+        }
+        accumulators[property].push({ extraConfig: true, ...extraConfig })
+        await looper(queries)
       }
-      accumulators[property].push({ extraConfig: true, ...extraConfig })
-      await looper(queries)
     }
   }
 
@@ -1331,8 +1391,11 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
     const stabilizeOutput = (template) => {
       stabilizeAssociations(template.associations)
       const stabilize = (results) => {
-        for (let result of results) {
-          if (result.extraConfig) {
+        for (let i = 0; i < results.length; ++i) {
+          const result = results[i]
+          if (result.apply) {
+            result.apply = result.apply.toString()
+          } else if (result.extraConfig) {
           } else {
             delete result.load_cache_time
             delete result.times
@@ -1370,7 +1433,13 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
   }
   let todo = []
   todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
-  todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false || query.skipSemantics} }))
+  todo = todo.concat((template.queries || []).map((query, index) => { 
+    let pr
+    if (index < startOfChanges) {
+      pr = previousResultss[index] 
+    }
+    return { property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics} 
+  }))
   todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
   todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
   await looper(Object.assign([], todo))
@@ -1398,7 +1467,7 @@ const checkTest = (testConfig) => {
   }
 }
 
-const knowledgeModule = async ({
+const knowledgeModuleImpl = async ({
   module: moduleFromJSFile,
   description,
   section,
@@ -1485,13 +1554,14 @@ const knowledgeModule = async ({
     parser.add_argument('-tnp', '--testNoParenthesized', { action: 'store_true', help: 'Don\' check parenthesized differences for the tests' })
     parser.add_argument('-n', '--count', { help: 'Number of times to run the tests. Default is one. Use this to check for flakey test. If possible the system will print out a message with the word "hint" suggesting how to fix the problem' })
     // parser.add_argument('-b', '--build', { help: 'Specify the template file name of the form <kmName>. There should be a file called <baseKmName>.<kmName>.template.json with the queries to run. For example { queries: [...] }. The template file will be run and generate an instantiation called <baseKmName>.<kmName>.instance.json and a file called <kmName>.js that will load the template file (this is file generated only if not already existing) and a test file called <KmName>.tests.json. This can then be loaded into an instance of the current knowledge module to setup initial conditions.' })
-    parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild' })
+    parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild. Using optimization where if the query/config has not changed it will use the previous value. One there is a change all subsequence query/configs will be run.' })
+    parser.add_argument('-rtf', '--rebuildTemplateFull', { action: 'store_true', help: 'Force a template rebuild. Skip the optimization' })
     parser.add_argument('-l', '--loop', { action: 'store_true', help: 'Run a loop so that multiply queries may be run' })
     parser.add_argument('-i', '--info', { action: 'store_true', help: 'Print meta-data for the module' })
     parser.add_argument('-v', '--vimdiff', { action: 'store_true', help: 'For failures run vimdiff' })
     parser.add_argument('-g', '--greg', { action: 'store_true', help: 'Set the server to be localhost so I can debug stuff' })
     parser.add_argument('-cl', '--checkForLoop', { nargs: "?", help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
-    parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
+    parser.add_argument('-r', '--reset', { action: 'store_true', help: 'Get the server to bypass the cache and rebuild everything' })
     parser.add_argument('-q', '--query', { help: 'Run the specified query' })
     parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
     parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
@@ -1516,6 +1586,10 @@ const knowledgeModule = async ({
     const args = parser.parse_args()
     args.count = args.count || 1
 
+    if (args.rebuildTemplateFull) {
+      args.rebuildTemplate = true
+    }
+
     if (args.parenthesized) {
       config.parenthesized = true
     }
@@ -1606,6 +1680,10 @@ const knowledgeModule = async ({
       config.config.debug = true
     }
 
+    if (args.reset) {
+      config.config.skip_cache = true
+    }
+
     if (args.explainPriorities) {
       config.config.explain_priorities = true
     }
@@ -1706,17 +1784,33 @@ const knowledgeModule = async ({
     checkTemplate(template)
 
     if (template) {
-      const needsRebuild = config.needsRebuild(template.template, template.instance, options)
-      if (needsRebuild) {
+      let needsRebuild
+      if (args.rebuildTemplate && !args.rebuildTemplateFull) {
+        // get the startOfChanges for the partial rebuild
+        needsRebuild = config.needsRebuild(template.template, template.instance, { ...options, rebuild: false })
+      } else {
+        // do a check or full rebuild
+        needsRebuild = config.needsRebuild(template.template, template.instance, options)
+      }
+
+      if (needsRebuild.needsRebuild) {
+        if (needsRebuild.previousResultss) {
+          console.log("Rebuild using the optimization to use previous results until a change is hit. For a full rebuild use -rtf")
+        }
         console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
         options.rebuild = true
         config.config.rebuild = true
       }
-      config.load(template.template, template.instance, { rebuild: needsRebuild })
+      try {
+        config.load(template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
+      } catch( e ) {
+        console.error(`Error loading template for ${config.name}. ${e.error ? e.error : e}${e.stack ? e.stack : ''}`)
+        runtime.process.exit(-1)
+      }
       if (!args.query) {
         printConfig()
       }
-      if (needsRebuild) {
+      if (needsRebuild.needsRebuild) {
         return
       }
     }
@@ -1792,14 +1886,14 @@ const knowledgeModule = async ({
                       console.log(`    actual ${label}  `, actual)
                       newError = true
                       headerShown = true
+                      if (args.vimdiff) {
+                        vimdiff(result.actual.paraphrasesParenthesized, result.expected.paraphrasesParenthesized)
+                      }
                     }
                   }
                   show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
                   if (!args.testNoParenthesized) {
                     show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
-                    if (args.vimdiff) {
-                      vimdiff(result.actual.paraphrasesParenthesized, result.expected.paraphrasesParenthesized)
-                    }
                   }
                   /*
                   }
@@ -1832,10 +1926,10 @@ const knowledgeModule = async ({
                     }
                     const widths = [4, 18, 72]
                     const lines = new Lines(widths)
-                    lines.setElement(1, 1, 'expected checked')
+                    lines.setElement(1, 1, 'expected checked objects')
                     lines.setElement(2, 2, JSON.stringify(result.expected.checked, null, 2))
                     lines.log()
-                    lines.setElement(1, 1, 'actual checked')
+                    lines.setElement(1, 1, 'actual checked objects')
                     lines.setElement(2, 2, JSON.stringify(result.actual.checked, null, 2))
                     lines.log()
                     if (args.vimdiff) {
@@ -1850,10 +1944,10 @@ const knowledgeModule = async ({
                     }
                     const widths = [4, 18, 72]
                     const lines = new Lines(widths)
-                    lines.setElement(1, 1, 'expected checkedContexts', true)
+                    lines.setElement(1, 1, 'expected checked contexts', true)
                     lines.setElement(2, 2, JSON.stringify(result.expected.checkedContexts, null, 2))
                     lines.log()
-                    lines.setElement(1, 1, 'actual checkedContexts', true)
+                    lines.setElement(1, 1, 'actual checked contexts', true)
                     lines.setElement(2, 2, JSON.stringify(result.actual.checkedContexts, null, 2))
                     lines.log()
                     if (args.vimdiff) {
@@ -1879,7 +1973,7 @@ const knowledgeModule = async ({
                 if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
                   console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
                   // console.log(JSON.stringify(contexts))
-                  console.log('**************************** ERRORS ************************')
+                  console.log('**************************** THERE WERE ERRORS ************************')
                 }
               }
             }
@@ -1965,7 +2059,7 @@ const knowledgeModule = async ({
       }
 
       if (template) {
-        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess })) {
+        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess }).needsRebuild) {
           const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
           throw new Error(error)
         }
@@ -1985,7 +2079,6 @@ const knowledgeModule = async ({
     }
     moduleFromJSFile.exports = createConfigExport
   }
-
 }
 
 /*
@@ -2033,6 +2126,10 @@ function w(func) {
   return func
 }
 
+const knowledgeModule = async (...args) => {
+  await knowledgeModuleImpl(...args).catch((e) => console.error(e))
+}
+
 module.exports = {
   process: _process,
   where,

package/package.json

@@ -1,7 +1,7 @@
 {
   "devDependencies": {
     "eslint-plugin-import": "^2.23.4",
-    "jest": "^26.6.3",
+    "jest": "^29.7.0",
     "@typescript-eslint/parser": "^4.28.4",
     "@typescript-eslint/eslint-plugin": "^4.28.4",
     "eslint-plugin-node": "^11.1.0",
@@ -12,6 +12,7 @@
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
     "test:debug": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
+    "td": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
     "tod": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
     "lint:fix": "eslint \"**/*.js\" --fix",
     "lint": "eslint \"**/*.js\"",
@@ -64,6 +65,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.7.0",
+  "version": "7.8.0-beta.0",
   "license": "ISC"
 }

package/src/config.js

@@ -27,7 +27,7 @@ const config_toServer = (config) => {
 
 const debugPriority = (priority) => {
   if (global.entodictonDebugPriority) {
-    if (helpers.safeEquals(entodictonDebugPriority, priority)) {
+    if (helpers.subPriority(entodictonDebugPriority, priority)) {
       debugger; // debug hierarchy hit
     }
   }
@@ -965,7 +965,7 @@ class Config {
   }
 
   // { rebuild: false, isModule: false }
-  needsRebuild(template, instance, options) {
+  needsRebuild (template, instance, options) {
     if (options.rebuild) {
       return true
     }
@@ -983,6 +983,18 @@ class Config {
       if (typeof queryOrConfig == 'string') {
         const query = queryOrConfig
         return query
+      } else if (typeof queryOrConfig == 'function') {
+        if (options.isModule) {
+          return { apply: 'function in the browser has webpack rewrites so can not be compared' }
+        } else {
+          return { apply: queryOrConfig.toString() }
+        }
+      } else if (queryOrConfig.apply) {
+        if (options.isModule) {
+          return { apply: 'function in the browser has webpack rewrites so can not be compared' }
+        } else {
+          return { apply: queryOrConfig.apply }
+        }
       } else {
         const config = { ...queryOrConfig }
         delete config.where
@@ -1058,7 +1070,27 @@ class Config {
       return elements.map( toCanonicalQuery )
     }
 
-    const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
+    const templateQueries = toCanonicalQueries(template.queries || []).map(helpers.updateQueries)
+    const instanceQueries = toCanonicalQueries(instance.queries || [])
+    let sameQueries = true
+    let startOfChanges;
+    for (let iq = 0; iq < templateQueries.length; ++iq) {
+      if (!helpers.safeEquals(templateQueries[iq], instanceQueries[iq])) {
+        sameQueries = false
+        startOfChanges = iq
+      }
+    }
+
+    if (startOfChanges) {
+      console.log('templateQueries[startOfChanges]', templateQueries[startOfChanges]);
+      console.log('instanceQueries[startOfChanges]', instanceQueries[startOfChanges]);
+    }
+
+    // things were deleted case
+    if (templateQueries.length < instanceQueries.length) {
+      startOfChanges = instanceQueries.length
+    }
+    // const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
 
     const debug = false
     if (debug) {
@@ -1067,16 +1099,20 @@ class Config {
           debugger
           debugger
         }
-        console.log("instance", instance)
+        // console.log("instance", instance)
         console.log("sameQueries", sameQueries)
         console.log("sameFragments", sameFragments)
-        console.log("templateFragments", templateFragments)
-        console.log("instanceFragments", instanceFragments)
-        console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
-        console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
+        // console.log("templateFragments", templateFragments)
+        // console.log("instanceFragments", instanceFragments)
+        // console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
+        // console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
       }
     }
-    return !(instance && sameQueries && sameFragments)
+    if (startOfChanges || instance.resultss) {
+      return { needsRebuild: !(instance && sameQueries && sameFragments), startOfChanges, previousResultss: instance.resultss }
+    } else {
+      return { needsRebuild: !(instance && sameQueries && sameFragments) }
+    }
   }
 
   validifyTemplate (template) {
@@ -1097,7 +1133,7 @@ class Config {
   }
 
   // loadTemplate
-  load (template, instance, options = { rebuild: false } ) {
+  load (template, instance, options = { rebuild: false, previousResultss: undefined, startOfChanges: undefined } ) {
     this.validifyTemplate(template)
     instance.template = template
     this.logs.push(`loading template for ${this.name}`)
@@ -1105,7 +1141,7 @@ class Config {
       // TODO fix beforeQuery
       template = { fragments: [], queries: [], ...template }
       template.fragments = template.fragments.concat(this.dynamicFragments)
-      client.rebuildTemplate({ config: this, target: this.name, beforeQuery: () => {}, template, ...options })
+      client.rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
     } else {
       // no change
       // this.initInstances.push({ ...instance, name: config.name })
@@ -1120,6 +1156,13 @@ class Config {
         return !properties.find( (property) => instance[property] && instance[property].length > 0 )
       }
       if (!isEmpty(instance)) {
+        // fix up apply functions
+        for (let i = 0; i < instance.resultss.length; ++i) {
+          const result = instance.resultss[i]
+          if (result.apply) {
+            result.apply = template.queries[i]
+          }
+        }
         instance.name = this.name
         this.initInstances.push(instance)
         this.instances.push(instance)
@@ -2219,8 +2262,6 @@ class Config {
     const addInternals = []
     const inits = []
     const initAfterApis = []
-    const reverseIt = true
-    const interleaved = true
     this.configs.forEach((km) => {
       const namespace = km.namespace
       this.config.objects.namespaced[km._uuid] = {}
@@ -2286,58 +2327,30 @@ class Config {
           inits.unshift( () => config._api.initialize({ config: this, km: kmFn, ...args, api: config._api }) )
           // config._api.initialize({ config, api: config._api })
         } else {
-          if (interleaved) {
-            inits.unshift(null)
-          }
+          inits.unshift(null)
         }
         // config._api.objects = namespacedObjects
         // config._api.config = () => this
         // config._api.uuid = config._uuid
       } else {
-        if (interleaved) {
-          inits.unshift(null)
-        }
+        inits.unshift(null)
       }
       config.setUUIDs()
       config.applyNamespace(config.config, namespace, config.uuid)
       if (!isSelf) {
-        if (!reverseIt) {
-          addInternals.push(config)
-        } else {
-          addInternals.unshift(config)
-        }
+        addInternals.unshift(config)
       } else {
-        if (interleaved) {
-          addInternals.unshift(null)
-        }
+        addInternals.unshift(null)
       }
       km.valid()
     })
 
     const generators = this.config.generators
     const semantics = this.config.semantics
-    if (reverseIt) {
-      this.config.generators = []
-      this.config.semantics = []
-    }
+    this.config.generators = []
+    this.config.semantics = []
 
-    if (!interleaved) {
-      for (const config of addInternals) {
-        if (!reverseIt) {
-          this.addInternal(config, { includeNamespace: false, allowNameToBeNull: true })
-        } else {
-          this.addInternalR(config, true, false, false, true)
-        }
-      }
-      // console.log('inits from config', inits)
-      for (const init of inits) {
-        init()
-      }
-      for (let init of initAfterApis) {
-        init.config.initializerFn({ ...init.args, kms: this.getConfigs(), isAfterApi: true })
-      }
-      this.instances.forEach((instance) => client.loadInstance(this, instance))
-    } else {
+    {
       const base = {
         operators: this.config.operators,
         bridges: this.config.bridges,
@@ -2353,13 +2366,16 @@ class Config {
       this.config.priorities = []
       this.config.associations = { positive: [], negative: [] }
       this.config.words = {}
-
+      
       for (let i = 0; i < addInternals.length; ++i) {
         let name;
         if (addInternals[i]) {
           this.addInternalR(addInternals[i], true, false, false, true)
           name = addInternals[i].name
         } else{
+          // the ones defined in config must come after the ones in the templates
+          this.config.generators = generators.concat(this.config.generators)
+          this.config.semantics = semantics.concat(this.config.semantics)
           this.addInternalR(base, true, false, false, true)
           name = this.name
         }
@@ -2380,10 +2396,6 @@ class Config {
       }
     }
 
-    if (reverseIt) {
-      this.config.generators = generators.concat(this.config.generators)
-      this.config.semantics = semantics.concat(this.config.semantics)
-    }
     this.hierarchy.edges = this.config.hierarchy
     this.valid()
     this.checkBridges()

package/src/generators.js

@@ -169,7 +169,7 @@ class Generators {
   // assumed - properties added to context before the generators are called. For setting paraphrase property
   apply (args, context, assumed = {}, options = {}) {
     if (Array.isArray(context)) {
-      return new Error("Expected a context not an array")
+      throw new Error("Expected a context not an array")
     }
     if (typeof context !== 'object') {
       return String(context)
@@ -179,7 +179,6 @@ class Generators {
     const objects = args.objects
     const hierarchy = args.hierarchy
     const response = args.response
-
     
     // args = { ...args, ...args.getAssumedScoped(assumed) }
     args.addAssumedScoped(args, assumed) 

package/src/helpers.js

@@ -272,6 +272,8 @@ const mapInPlace = (list, fn) => {
 const updateQueries = (queryOrConfig) => {
   if (typeof queryOrConfig == 'string' || queryOrConfig.query) {
     return queryOrConfig
+  } else if (typeof queryOrConfig == 'function') {
+    return { apply: queryOrConfig.toString() }
   } else {
     const config = queryOrConfig
     return functionsToStrings(config)
@@ -334,6 +336,45 @@ const ecatch = (where, call) => {
   }
 }
 
+const equalKey = (key1, key2) => {
+  return key1[0] == key2[0] && key1[1] == key2[1]
+}
+
+// matches for { context: ..., [ordered], choose: ... } exactely OR
+//             [ <id1>, <id2> ] - where id1 is chosen
+const subPriority = (sub, sup) => {
+  if (Array.isArray(sub)) {
+    const subChoosen = sub[0]
+    const subOther = sub[1]
+    const hasChoosen = sup.choose.find( (index) => equalKey(sup.context[index], subChoosen)) != undefined
+    const hasOtherChosen = sup.choose.find( (index) => equalKey(sup.context[index], subOther)) != undefined
+    const hasOther = sup.context.find( (other) => equalKey(other, subOther) ) !== undefined
+    return !!(hasChoosen && hasOther) && !hasOtherChosen
+  }
+
+  if (!safeEquals([...sub.choose].sort(), [...sup.choose].sort())) {
+    return false
+  }
+
+  const choose = (priority) => {
+    const chosen = []
+    for (const i of priority.choose) {
+      chosen.push(priority.context[i])
+    }
+    return chosen
+  }
+  const chosen1 = choose(sub)
+  const chosen2 = choose(sup)
+  const sameId = (id1, id2) => id1[0] == id2[0] && id1[1] == id2[1]
+  // same length so only need one way
+  const missing1 = chosen1.find( (id1) => !chosen2.find( (id2) => sameId(id1, id2)) )
+  if (missing1) {
+    return false
+  }
+
+  return true
+}
+
 module.exports = { 
   ecatch,
   functionsToStrings,
@@ -354,5 +395,6 @@ module.exports = {
   isCompound, 
   InitCalls, 
   hashCode, 
-  sortJson 
+  sortJson,
+  subPriority,
 }

package/src/project.js

@@ -16,6 +16,9 @@ const project = (object, filter) => {
       return object.map( element => project(element, filter) )
     } else {
       for (let properties of filter) {
+        if (typeof properties == 'function') {
+          properties = properties(object)
+        }
         if (typeof properties == 'object') {
           if (properties.propertyLists) {
             for (const propertyList in properties.propertyLists) {