theprogrammablemind 7.6.0-beta.5 → 7.6.0-beta.7

package/client.js

@@ -175,8 +175,8 @@ const setupArgs = (args, config, logs, hierarchy) => {
   }
   args.s = (c) => config.getSemantics(logs).apply(args, c)
   args.g = (c) => config.getGenerators(logs).apply(args, c)
-  args.gp = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: true, isResponse: false, response: false})
-  args.gr = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: false, isResponse: true })
+  args.gp = (c) => config.getGenerators(logs).apply({...args, assumed: {paraphrase: true, isResponse: false, response: false}}, c, {paraphrase: true, isResponse: false, response: false})
+  args.gr = (c) => config.getGenerators(logs).apply({...args, assumed: {paraphrase: false, isResponse: true}}, { ...c, paraphrase: false, isResponse: true })
   if (!logs) {
     debugger
   }
@@ -189,6 +189,11 @@ const setupArgs = (args, config, logs, hierarchy) => {
 }
 
 const gs = (g) => (contexts, separator, lastSeparator) => {
+  if (!Array.isArray(contexts)) {
+    debugger
+    throw new Error("Expected a list")
+  }
+
   let s = ''
   if (!separator) {
     separator = ' '
@@ -276,8 +281,9 @@ const processContext = (context, { objects = {}, config, logs = [] }) => {
   setupArgs(args, config, logs, hierarchy)
 
   context = semantics.apply(args, context)
-  const generated = generators.apply(args, context)[0]
-  const paraphrases = generators.apply(args, context, { paraphrase: true, response: false, isResponse: false })[0]
+  const generated = generators.apply(args, context)
+  const assumed = { paraphrase: true, response: false, isResponse: false }
+  const paraphrases = generators.apply({...args, assumed}, context, { paraphrase: true, response: false, isResponse: false })
   let responses = []
   if (context.isResponse) {
     responses = generated
@@ -447,37 +453,29 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
         continue
       }
       let assumed = { isResponse: true };
-      const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
+      const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed) : ''
       let generatedParenthesized = []
       if (generateParenthesized) {
         config.parenthesized = true
-        generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
+        generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed) : ''
         config.parenthesized = false
       }
       // assumed = { paraphrase: true, response: false };
-      assumed = { paraphrase: true };
-      args.g = (c) => config.getGenerators(json.logs).apply(args, c, assumed)
-      args.gp = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: true, isResponse: false, response: false }, assumed)
-      args.gr = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: false }, assumed)
+      assumed = { paraphrase: true, isResponse: false, response: false };
+      args.g = (c) => config.getGenerators(json.logs).apply({...args, assumed}, c, assumed)
       args.gs = gs(args.g)
-      args.gsp = gs(args.gsp)
-      args.gsr = gs(args.gr)
       if (generateParenthesized) {
         config.parenthesized = false
       }
-      const paraphrases = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
+      const paraphrases = config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed)
       let paraphrasesParenthesized = []
       if (generateParenthesized) {
         config.parenthesized = true
-        paraphrasesParenthesized = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
+        paraphrasesParenthesized = config.getGenerators(json.logs).apply({...args, assumed}, contextPrime, assumed)
         config.parenthesized = false
       }
       args.g = (c) => config.getGenerators(json.logs).apply(args, c)
-      args.gp = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: true, isResponse: false, response: false })
-      args.gr = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: false })
       args.gs = gs(args.g)
-      args.gsp = gs(args.gp)
-      args.gsr = gs(args.gr)
       contextsPrime.push(contextPrime)
       generatedPrime.push(generated)
       paraphrasesPrime.push(paraphrases)
@@ -660,6 +658,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         opChoices: []
       },
       times: 0.0,
+      clientSideTimes: 0.0,
       trace: '',
       contexts: [],
       generated: [],
@@ -701,8 +700,16 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       if (json.status !== 200) {
         throw json
       } else {
+        let clientSideTime
+        if (isTest) {
+          start = runtime.performance.performance.now()
+        }
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
                                   processContextsB({ isTest, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        if (isTest) {
+          end = runtime.performance.performance.now()
+          clientSideTime = end - start
+        }
         response.associations = json.associations
         response.learned_contextual_priorities = json.learned_contextual_priorities
         response.hierarchy = json.hierarchy
@@ -713,6 +720,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         // appendNoDups(response.metadata.priorities, json.metadata.priorities)
         appendNoDups(response.metadata.opChoices, json.metadata.opChoices)
         response.times += json.times
+        response.clientSideTimes += clientSideTime
         response.trace = response.trace.concat(json.trace)
         response.version = json.version
         response.explain_priorities = json.explain_priorities
@@ -802,10 +810,10 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug }) =
         defaultInnerProcess(config, errorHandler, result)
       }
       if (verbose) {
-        const widths = [100, 20]
+        const widths = [100, 60]
         const lines = new Lines(widths)
         lines.setElement(0, 0, test)
-        lines.setElement(0, 1, `time on server ${result.times.toFixed(2)}`)
+        lines.setElement(0, 1, `time on server: ${result.times.toFixed(2)} client: ${(result.clientSideTimes/1000).toFixed(2)}`)
         lines.log()
       }
       const expected_objects = sortJson(convertToStable(expected.objects), { depth: 25 })
@@ -1323,6 +1331,7 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
           } else {
             delete result.load_cache_time
             delete result.times
+            delete result.clientSideTimes
             delete result.memory_free_percent
             delete result.logs
             delete result.version
@@ -1362,6 +1371,28 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
   await looper(Object.assign([], todo))
 }
 
+const checkTemplate = (template) => {
+  return
+  if (!template) {
+    return
+  }
+  if (template.checks) {
+    throw new Error("The 'checks' property should be in the 'test' property not the 'template' property")
+  }
+}
+
+const checkTest = (testConfig) => {
+  if (!testConfig) {
+    return
+  }
+  if (!testConfig.name) {
+    throw new Error("The 'test' property is missing the 'name' property that contains the name of the '<km>.test.json' file")
+  }
+  if (!testConfig.contents) {
+    throw new Error("The 'test' property is missing the 'contents' property that contains contents of the '<km>.test.json' file")
+  }
+}
+
 const knowledgeModule = async ({
   module: moduleFromJSFile,
   description,
@@ -1395,9 +1426,10 @@ const knowledgeModule = async ({
   if (!description) {
     throw new Error("'description' is a required parameter. The value should the description of the knowledge module.")
   }
-  if (!test) {
+  if (!testConfig) {
     throw new Error("'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }.")
   }
+  checkTest(testConfig)
 
   const isProcess = require.main === moduleFromJSFile
 
@@ -1666,6 +1698,8 @@ const knowledgeModule = async ({
       }
     }
 
+    checkTemplate(template)
+
     if (template) {
       const needsRebuild = config.needsRebuild(template.template, template.instance, options)
       if (needsRebuild) {
@@ -1689,7 +1723,6 @@ const knowledgeModule = async ({
     if (args.test || args.testVerbose || args.testAllVerbose || args.save) {
       global.transitoryMode = true
     }
-
     if (!args.query && !args.test && !args.info && (args.save || args.saveDeveloper)) {
       global.transitoryMode = true
       saveTests(config, test, testConfig, args.saveDeveloper)
@@ -1711,7 +1744,6 @@ const knowledgeModule = async ({
             useTestConfig = config.getConfigs()[args.testModuleName].getTestConfig()
             useTestConfig.testModuleName = args.testModuleName
             test = useTestConfig.name
-
           }
           runTests(config, test, { args, debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
             let newError = false
@@ -1760,12 +1792,16 @@ const knowledgeModule = async ({
                   show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
                   if (!args.testNoParenthesized) {
                     show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
+                    if (args.vimdiff) {
+                      vimdiff(result.actual.paraphrasesParenthesized, result.expected.paraphrasesParenthesized)
+                    }
+                  }
+                  /*
                   }
                   show('responses', result.expected.responses, result.actual.responses)
                   if (!args.testNoParenthesized) {
                     show('responses parenthesized', result.expected.generatedParenthesized, result.actual.generatedParenthesized)
                   }
-                  /*
                   if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
                     if (!headerShown) {
                       console.log('  Failure')
@@ -1829,16 +1865,18 @@ const knowledgeModule = async ({
                   }
                 }
               }
-              if (!headerShown) {
+              if (hasError) {
+                if (!headerShown) {
+                  if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
+                    console.log('There are failures due to things other than paraphrases, responses and checked properties being different. They are not shown because you ran -tv or -tva which only shows difference in paraphrase and results. Usually what I do is -s and do a diff to make sure there are no other problems. If the paraphrases or results were different they would have shown here.')
+                  }
+                }
                 if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
-                  console.log('There are failures due to things other than paraphrases, responses and checked properties being different. They are not shown because you ran -tv or -tva which only shows difference in paraphrase and results. Usually what I do is -s and do a diff to make sure there are no other problems. If the paraphrases or results were different they would have shown here.')
+                  console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
+                  // console.log(JSON.stringify(contexts))
+                  console.log('**************************** ERRORS ************************')
                 }
               }
-              if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
-                console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
-                // console.log(JSON.stringify(contexts))
-                console.log('**************************** ERRORS ************************')
-              }
             }
             // const contexts = { failures: results }
             l(n - 1, hasError || newError)

package/package.json

@@ -64,6 +64,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.6.0-beta.5",
+  "version": "7.6.0-beta.7",
   "license": "ISC"
 }

package/runtime.js

@@ -15,4 +15,5 @@ module.exports = {
   jsonDiff: require('json-diff'),
   sortJson: require('sort-json'),
   util: require('util'),
+  performance: require('perf_hooks'),
 }

package/src/generators.js

@@ -165,136 +165,130 @@ class Generators {
     this.logs = logs
   };
 
-  // assumed - properties added to contexts before the generators are called. For setting paraphrase property
-  apply (args, contexts, assumed = {}, options = {}) {
+  // assumed - properties added to context before the generators are called. For setting paraphrase property
+  apply (args, context, assumed = {}, options = {}) {
+    if (Array.isArray(context)) {
+      return new Error("Expected a context not an array")
+    }
+    if (typeof context !== 'object') {
+      return String(context)
+    }
+
     const config = args.config
     const objects = args.objects
     const hierarchy = args.hierarchy
     const response = args.response
-    if (Array.isArray(contexts)) {
-      // no-op
-    } else if (typeof contexts === 'object') {
-      contexts = [contexts]
-    } else {
-      return String(contexts)
-    }
 
-    const contextsPrime = []
-    for (const icontext in contexts) {
-      let context = contexts[icontext]
-      context = Object.assign({}, context, assumed)
-      // let context_prime = JSON.stringify(context);
-      let generated = context
-      let applied = false
-      const stack = args.calls.push()
-      for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
-        const generator = this.generators[igenerator]
-        if (generator.matches(args, objects, context, hierarchy, config, options)) {
-          const g = (context, options) => {
-            const r = this.apply(args, context, Object.assign({}, ((options||{}).assumed || {}), assumed), options)
-            if (Array.isArray(r)) {
-              return r.join(' ')
-            } else {
-              return r
-            }
-          }
-          const log = (message) => { this.logs.push(message) }
-          // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
-          let errorMessage = 'The apply function did not return a value'
-          try {
-            generated= generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
-          } catch( e ) {
-            // the next if handle this
-            generated = null
-            e.retryCall = () => generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
-            const help = 'The error has a retryCall property that will recall the function that failed.'
-            if (e.stack && e.message) {
-              const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
-              errorMessage = `Error applying generator '${info}. Error is ${e.toString()} stack is ${e.stack}. Generator is ${generator.toString()}. ${help}`
-            } else if (e.error) {
-              const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
-              errorMessage = `Error applying generator '${info}. Error is ${e.error.join()}. Generator is ${generator.toString()}. ${help}`
-            } else {
-							errorMessage = e.toString()
-						}
+    context = Object.assign({}, context, args.assumed)
+    let generated = ''
+    let applied = false
+    const stack = args.calls.push()
+    for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
+      const generator = this.generators[igenerator]
+      if (generator.matches(args, objects, context, hierarchy, config, options)) {
+        const g = (context, options) => {
+          const r = this.apply(args, context, Object.assign({}, ((options||{}).assumed || {}), assumed), options)
+          if (Array.isArray(r)) {
+            return r.join(' ')
+          } else {
+            return r
           }
-          if (!generated && generated !== '') {
-            const widths = [10, 10, 90]
-            const lines = new Lines(widths)
-            lines.setElement(0, 0, 'Generator')
-            const source = `${generator.km}/#${generator.index}`
-            lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
-            lines.newRow()
-            lines.setElement(0, 2, generator.toString())
-            lines.newRow()
-            lines.setElement(0, 1, 'TO')
-            lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
-            lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
-            lines.newRow()
-            lines.setElement(0, 1, 'STACK')
-            lines.setElement(0, 2, stack)
-						lines.newRow()
-						lines.setElement(0, 1, 'DEBUG')
-						lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
-            lines.newRow()
-            lines.setElement(0, 1, 'ERROR')
-            lines.setElement(0, 2, errorMessage)
-            this.logs.push(lines.toString())
-            const message = `ERROR while applying (${source}) ${generator.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
-            // this.logs.push(message)
-            // return [message]
-            args.calls.pop()
-            throw { error: [message], logs: this.logs }
+        }
+        const log = (message) => { this.logs.push(message) }
+        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
+        let errorMessage = 'The apply function did not return a value'
+        try {
+          generated= generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
+        } catch( e ) {
+          // the next if handle this
+          generated = null
+          e.retryCall = () => generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
+          const help = 'The error has a retryCall property that will recall the function that failed.'
+          if (e.stack && e.message) {
+            const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
+            errorMessage = `Error applying generator '${info}. Error is ${e.toString()} stack is ${e.stack}. Generator is ${generator.toString()}. ${help}`
+          } else if (e.error) {
+            const info = `${generator.notes ? generator.notes : ''}${generator.where ? generator.where : ''}`
+            errorMessage = `Error applying generator '${info}. Error is ${e.error.join()}. Generator is ${generator.toString()}. ${help}`
+          } else {
+            errorMessage = e.toString()
           }
-          if (((config || {}).config || {}).debug) {
-            const widths = [10, 10, 90]
-            const lines = new Lines(widths)
-            lines.setElement(0, 0, 'Generator')
-            if (generator.index > -1 && generator.km) {
-              // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
-              lines.setElement(0, 2, generator.toLabel())
-            }
-            lines.newRow()
-            lines.setElement(0, 1, 'APPLIED')
+        }
+        if (!generated && generated !== '') {
+          const widths = [10, 10, 90]
+          const lines = new Lines(widths)
+          lines.setElement(0, 0, 'Generator')
+          const source = `${generator.km}/#${generator.index}`
+          lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
+          lines.newRow()
+          lines.setElement(0, 2, generator.toString())
+          lines.newRow()
+          lines.setElement(0, 1, 'TO')
+          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
+          lines.newRow()
+          lines.setElement(0, 1, 'ERROR')
+          lines.setElement(0, 2, errorMessage)
+          this.logs.push(lines.toString())
+          const message = `ERROR while applying (${source}) ${generator.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
+          // this.logs.push(message)
+          // return [message]
+          args.calls.pop()
+          throw { error: [message], logs: this.logs }
+        }
+        if (((config || {}).config || {}).debug) {
+          const widths = [10, 10, 90]
+          const lines = new Lines(widths)
+          lines.setElement(0, 0, 'Generator')
+          if (generator.index > -1 && generator.km) {
+            // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
             lines.setElement(0, 2, generator.toLabel())
-            lines.newRow()
-            lines.setElement(0, 2, generator.toString())
-            lines.newRow()
-            lines.setElement(0, 1, 'RESULT')
-            lines.setElement(0, 2, generated)
-            lines.newRow()
-            lines.setElement(0, 1, 'STACK')
-            lines.setElement(0, 2, stack)
-						lines.newRow()
-						lines.setElement(0, 1, 'DEBUG')
-						lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
-						lines.newRow()
-            lines.setElement(0, 1, 'TO')
-            lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
-            lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
-            this.logs.push(lines.toString())
           }
-          applied = true
-          break
+          lines.newRow()
+          lines.setElement(0, 1, 'APPLIED')
+          lines.setElement(0, 2, generator.toLabel())
+          lines.newRow()
+          lines.setElement(0, 2, generator.toString())
+          lines.newRow()
+          lines.setElement(0, 1, 'RESULT')
+          lines.setElement(0, 2, generated)
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
+          lines.newRow()
+          lines.setElement(0, 1, 'TO')
+          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          this.logs.push(lines.toString())
         }
+        applied = true
+        break
       }
-      args.calls.pop()
-      if (!applied && ((config || {}).config || {}).debug) {
-        const widths = [10, 10, 90]
-        const lines = new Lines(widths)
-        lines.setElement(0, 0, 'Generator')
-        lines.setElement(0, 2, 'No generator applied')
-        lines.newRow()
-        lines.setElement(0, 1, 'STACK')
-        lines.setElement(0, 2, stack)
-        lines.newRow()
-        lines.setElement(0, 1, 'TO')
-        lines.setElement(0, 2, JSON.stringify(context, null, 2))
-        this.logs.push(lines.toString())
-      }
-      contextsPrime.push((config || {}).parenthesized ? "(" + generated + ")" : generated)
     }
-    return contextsPrime
+    args.calls.pop()
+    if (!applied && ((config || {}).config || {}).debug) {
+      const widths = [10, 10, 90]
+      const lines = new Lines(widths)
+      lines.setElement(0, 0, 'Generator')
+      lines.setElement(0, 2, 'No generator applied')
+      lines.newRow()
+      lines.setElement(0, 1, 'STACK')
+      lines.setElement(0, 2, stack)
+      lines.newRow()
+      lines.setElement(0, 1, 'TO')
+      lines.setElement(0, 2, JSON.stringify(context, null, 2))
+      this.logs.push(lines.toString())
+    }
+    return ((config || {}).parenthesized ? "(" + generated + ")" : generated)
   }
 }
 

package/src/project.js

@@ -45,7 +45,6 @@ const project = (object, filter) => {
           if (properties.property) {
             const property = properties.property
             if (properties.isPropertyList) {
-              debugger
               if (!Array.isArray(object[property])) {
                 return projection
               }