theprogrammablemind 7.7.0-beta.0 → 7.7.0-beta.2

package/client.js

@@ -179,11 +179,23 @@ const setupArgs = (args, config, logs, hierarchy) => {
   args.log = (message) => logs.push(message)
 
   args.addAssumedScoped = (args, assumed) => {
+    const addAssumed = (args, ...moreAssumed) => {
+      return { ...args, assumed: Object.assign({}, (args.assumed || {}), ...moreAssumed) }
+    }
+
     args.s = (c) => config.getSemantics(logs).apply(args, c)
-    args.g = (c) => config.getGenerators(logs).apply(args, c)
-    args.gp = (c) => config.getGenerators(logs).apply({...args, assumed: {paraphrase: true, isResponse: false, response: false}}, c, {paraphrase: true, isResponse: false, response: false})
-    args.gr = (c) => config.getGenerators(logs).apply({...args, assumed: {paraphrase: false, isResponse: true}}, { ...c, paraphrase: false, isResponse: true })
-    args.e = (c) => config.getEvaluator(args.s, args.calls, logs, c)
+    args.g = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
+    }
+    args.gp = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, {paraphrase: true, isResponse: false, response: false}), c, {paraphrase: true, isResponse: false, response: false})
+    }
+    args.gr = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, {paraphrase: false, isResponse: true}), { ...c, paraphrase: false, isResponse: true })
+    }
+    args.e = (c) => {
+      return config.getEvaluator(args.s, args.calls, logs, c)
+    }
     args.gs = gs(args.g)
     args.gsp = gs(args.gp)
     args.gsr = gs(args.gr)
@@ -467,8 +479,6 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
       }
       // assumed = { paraphrase: true, response: false };
       assumed = { paraphrase: true, isResponse: false, response: false };
-      // args.g = (c) => config.getGenerators(json.logs).apply({...args, assumed}, c, assumed)
-      // args.gs = gs(args.g)
       if (generateParenthesized) {
         config.parenthesized = false
       }
@@ -479,8 +489,6 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
         paraphrasesParenthesized = config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed)
         config.parenthesized = false
       }
-      // args.g = (c) => config.getGenerators(json.logs).apply(args, c)
-      // args.gs = gs(args.g)
       contextsPrime.push(contextPrime)
       generatedPrime.push(generated)
       paraphrasesPrime.push(paraphrases)
@@ -617,7 +625,12 @@ const loadInstance = (config, instance) => {
       if (results.skipSemantics) {
         config.config.skipSemantics = results.skipSemantics
       }
-      processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}`, instance: instance.queries[i] })
+      const args = { config, hierarchy, json: results, commandLineArgs: {} }
+      if (instance.queries) {
+        args.isInstance = `instance${i}`
+        args.instance = instance.queries[i]
+      }
+      processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null
       }
@@ -1241,7 +1254,7 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
 }
 
 // builtTemplate saveInstance
-const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
+const rebuildTemplate = async ({ config, target, previousResultss, startOfChanges, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
@@ -1254,7 +1267,7 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
       finish()
       return
     }
-    const { property, hierarchy, query: queryOrExtraConfig, initializer, skipSemantics } = queries.shift()
+    const { property, hierarchy, query: queryOrExtraConfig, previousResults, initializer, skipSemantics } = queries.shift()
     // queries are strings or { query: "blah", development: true/false }
     if (typeof queryOrExtraConfig === 'string' || queryOrExtraConfig.query) {
       let query = queryOrExtraConfig;
@@ -1276,7 +1289,15 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
         }
       }
       try {
-        const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
+        let results
+        let prMessage = ''
+        if (previousResults && previousResults.query == query.query) {
+          results = previousResults
+          prMessage = ' Using previous results. use -rtf for a hard rebuild of everything on the server side.'
+          loadInstance(config, { resultss: [results] })
+        } else {
+          results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
+        }
         if (config.config.debug) {
           // TODO pass in the error handler like the other ones
           defaultInnerProcess(config, defaultErrorHandler, results)
@@ -1285,9 +1306,9 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
           console.log(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
           throw new Error(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
         } else if (results.paraphrases[0] != query.query) {
-          console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".`)
+          console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".${prMessage}`)
         } else {
-          console.log(`query ${query.query}`)
+          console.log(`query "${query.query}".${prMessage}`)
         }
         global.transitoryMode = transitoryMode
         config.config.skipSemantics = null
@@ -1375,7 +1396,13 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
   }
   let todo = []
   todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
-  todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false || query.skipSemantics} }))
+  todo = todo.concat((template.queries || []).map((query, index) => { 
+    let pr
+    if (index < startOfChanges) {
+      pr = previousResultss[index] 
+    }
+    return { property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics} 
+  }))
   todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
   todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
   await looper(Object.assign([], todo))
@@ -1490,13 +1517,14 @@ const knowledgeModuleImpl = async ({
     parser.add_argument('-tnp', '--testNoParenthesized', { action: 'store_true', help: 'Don\' check parenthesized differences for the tests' })
     parser.add_argument('-n', '--count', { help: 'Number of times to run the tests. Default is one. Use this to check for flakey test. If possible the system will print out a message with the word "hint" suggesting how to fix the problem' })
     // parser.add_argument('-b', '--build', { help: 'Specify the template file name of the form <kmName>. There should be a file called <baseKmName>.<kmName>.template.json with the queries to run. For example { queries: [...] }. The template file will be run and generate an instantiation called <baseKmName>.<kmName>.instance.json and a file called <kmName>.js that will load the template file (this is file generated only if not already existing) and a test file called <KmName>.tests.json. This can then be loaded into an instance of the current knowledge module to setup initial conditions.' })
-    parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild' })
+    parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild. Using optimization where if the query/config has not changed it will use the previous value. One there is a change all subsequence query/configs will be run.' })
+    parser.add_argument('-rtf', '--rebuildTemplateFull', { action: 'store_true', help: 'Force a template rebuild. Skip the optimization' })
     parser.add_argument('-l', '--loop', { action: 'store_true', help: 'Run a loop so that multiply queries may be run' })
     parser.add_argument('-i', '--info', { action: 'store_true', help: 'Print meta-data for the module' })
     parser.add_argument('-v', '--vimdiff', { action: 'store_true', help: 'For failures run vimdiff' })
     parser.add_argument('-g', '--greg', { action: 'store_true', help: 'Set the server to be localhost so I can debug stuff' })
     parser.add_argument('-cl', '--checkForLoop', { nargs: "?", help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
-    parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
+    parser.add_argument('-r', '--reset', { action: 'store_true', help: 'Get the server to bypass the cache and rebuild everything' })
     parser.add_argument('-q', '--query', { help: 'Run the specified query' })
     parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
     parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
@@ -1521,6 +1549,10 @@ const knowledgeModuleImpl = async ({
     const args = parser.parse_args()
     args.count = args.count || 1
 
+    if (args.rebuildTemplateFull) {
+      args.rebuildTemplate = true
+    }
+
     if (args.parenthesized) {
       config.parenthesized = true
     }
@@ -1711,23 +1743,33 @@ const knowledgeModuleImpl = async ({
     checkTemplate(template)
 
     if (template) {
-      const needsRebuild = config.needsRebuild(template.template, template.instance, options)
-      if (needsRebuild) {
+      let needsRebuild
+      if (args.rebuildTemplate && !args.rebuildTemplateFull) {
+        // get the startOfChanges for the partial rebuild
+        needsRebuild = config.needsRebuild(template.template, template.instance, { ...options, rebuild: false })
+      } else {
+        // do a check or full rebuild
+        needsRebuild = config.needsRebuild(template.template, template.instance, options)
+      }
+
+      if (needsRebuild.needsRebuild) {
+        if (needsRebuild.previousResultss) {
+          console.log("Rebuild using the optimization to use previous results until a change is hit. For a full rebuild use -rtf")
+        }
         console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
         options.rebuild = true
         config.config.rebuild = true
       }
       try {
-        config.load(template.template, template.instance, { rebuild: needsRebuild })
+        config.load(template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
       } catch( e ) {
-        debugger
         console.error(`Error loading template for ${config.name}. ${e.error ? e.error : e}`)
         runtime.process.exit(-1)
       }
       if (!args.query) {
         printConfig()
       }
-      if (needsRebuild) {
+      if (needsRebuild.needsRebuild) {
         return
       }
     }
@@ -1890,7 +1932,7 @@ const knowledgeModuleImpl = async ({
                 if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
                   console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
                   // console.log(JSON.stringify(contexts))
-                  console.log('**************************** ERRORS ************************')
+                  console.log('**************************** THERE WERE ERRORS ************************')
                 }
               }
             }
@@ -1976,7 +2018,7 @@ const knowledgeModuleImpl = async ({
       }
 
       if (template) {
-        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess })) {
+        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess }).needsRebuild) {
           const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
           throw new Error(error)
         }

package/package.json

@@ -12,6 +12,7 @@
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
     "test:debug": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
+    "td": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
     "tod": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
     "lint:fix": "eslint \"**/*.js\" --fix",
     "lint": "eslint \"**/*.js\"",
@@ -64,6 +65,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.7.0-beta.0",
+  "version": "7.7.0-beta.2",
   "license": "ISC"
 }

package/src/config.js

@@ -965,7 +965,7 @@ class Config {
   }
 
   // { rebuild: false, isModule: false }
-  needsRebuild(template, instance, options) {
+  needsRebuild (template, instance, options) {
     if (options.rebuild) {
       return true
     }
@@ -1058,7 +1058,21 @@ class Config {
       return elements.map( toCanonicalQuery )
     }
 
-    const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
+    const templateQueries = toCanonicalQueries(template.queries || []).map(helpers.updateQueries)
+    const instanceQueries = toCanonicalQueries(instance.queries || [])
+    let sameQueries = true
+    let startOfChanges;
+    for (let iq = 0; iq < templateQueries.length; ++iq) {
+      if (!helpers.safeEquals(templateQueries[iq], instanceQueries[iq])) {
+        sameQueries = false
+        startOfChanges = iq
+      }
+    }
+    // things were deleted case
+    if (templateQueries.length < instanceQueries.length) {
+      startOfChanges = instanceQueries.length
+    }
+    // const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
 
     const debug = false
     if (debug) {
@@ -1067,16 +1081,20 @@ class Config {
           debugger
           debugger
         }
-        console.log("instance", instance)
+        // console.log("instance", instance)
         console.log("sameQueries", sameQueries)
         console.log("sameFragments", sameFragments)
-        console.log("templateFragments", templateFragments)
-        console.log("instanceFragments", instanceFragments)
-        console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
-        console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
+        // console.log("templateFragments", templateFragments)
+        // console.log("instanceFragments", instanceFragments)
+        // console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
+        // console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
       }
     }
-    return !(instance && sameQueries && sameFragments)
+    if (startOfChanges || instance.resultss) {
+      return { needsRebuild: !(instance && sameQueries && sameFragments), startOfChanges, previousResultss: instance.resultss }
+    } else {
+      return { needsRebuild: !(instance && sameQueries && sameFragments) }
+    }
   }
 
   validifyTemplate (template) {
@@ -1097,7 +1115,7 @@ class Config {
   }
 
   // loadTemplate
-  load (template, instance, options = { rebuild: false } ) {
+  load (template, instance, options = { rebuild: false, previousResultss: undefined, startOfChanges: undefined } ) {
     this.validifyTemplate(template)
     instance.template = template
     this.logs.push(`loading template for ${this.name}`)
@@ -1105,7 +1123,7 @@ class Config {
       // TODO fix beforeQuery
       template = { fragments: [], queries: [], ...template }
       template.fragments = template.fragments.concat(this.dynamicFragments)
-      client.rebuildTemplate({ config: this, target: this.name, beforeQuery: () => {}, template, ...options })
+      client.rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
     } else {
       // no change
       // this.initInstances.push({ ...instance, name: config.name })

package/src/generators.js

@@ -179,7 +179,6 @@ class Generators {
     const objects = args.objects
     const hierarchy = args.hierarchy
     const response = args.response
-
     
     // args = { ...args, ...args.getAssumedScoped(assumed) }
     args.addAssumedScoped(args, assumed)