theprogrammablemind 8.1.0 → 8.3.0-beta.0

package/client.js

@@ -441,7 +441,7 @@ const getConfigForTest = (config, testConfig) => {
   return configForTest
 }
 
-const runTest = async (config, expected, { args, verbose, testConfig, debug }) => {
+const runTest = async (config, expected, { args, verbose, testConfig, debug, timings={ server: 0, client: 0 } }) => {
   const test = expected.query
   if (args.query && args.query != test) {
     // no run this
@@ -472,6 +472,8 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug }) =
       const lines = new Lines(widths)
       lines.setElement(0, 0, test)
       lines.setElement(0, 1, `time on server: ${result.times.toFixed(2)} client: ${(result.clientSideTimes / 1000).toFixed(2)}`)
+      timings.server += result.times
+      timings.client += result.clientSideTimes / 1000
       lines.log()
     }
     const expected_objects = sortJson(convertToStable(expected.objects), { depth: 25 })
@@ -784,7 +786,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     responses.errors.forEach((error) => console.log(`    ${error}`))
   }
   console.log("KM's loaded (ordered)", config.configs.map((c) => c.name))
-  console.log('This is the global objects from running semantics:\n', config.objects)
+  // console.log('This is the global objects from running semantics:\n', config.objects)
   if (!_.isEmpty(responses.learned_contextual_priorities)) {
     console.log('\nThe learned contextual priorties are :\n')
     for (const lcp of responses.learned_contextual_priorities) {
@@ -801,10 +803,12 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
   console.log(responses.trace)
 
-  if (global.beforeObjects) {
-    console.log('objects', runtime.jsonDiff.diffString(global.beforeObjects, config.get('objects')))
-  } else {
-    console.log('objects', runtime.util.inspect(config.get('objects'), { depth: Infinity, sorted: true }))
+  if (false) {
+    if (global.beforeObjects) {
+      console.log('objects', runtime.jsonDiff.diffString(global.beforeObjects, config.get('objects')))
+    } else {
+      console.log('objects', runtime.util.inspect(config.get('objects'), { depth: Infinity, sorted: true }))
+    }
   }
 
   const pickEm = () => {
@@ -867,7 +871,9 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
   */
   console.log('')
-  const widths = [60, 10, 72]
+  const screen_width = process.stdout.columns
+  // || 0 for when running without a console
+  const widths = [60, 10, Math.max(80, screen_width-71 || 0)]
   const lines = new Lines(widths)
   lines.setElement(0, 0, '--- The paraphrases are ----------')
   lines.setElement(0, 2, '--- The response strings are ----------')
@@ -1229,6 +1235,7 @@ const knowledgeModuleImpl = async ({
       parser.add_argument('-do', '--debugOperator', { action: 'store_true', help: helpDebugOperator })
       parser.add_argument('-ep', '--explainPriorities', { action: 'store_true', help: 'The server will return all priorities including the generated one along with an explanation of there they came from' })
       parser.add_argument('-dic', '--debugIncludeConvolutions', { nargs: '?', help: 'When running with the --debugIncludeConvolutions flag the logs will include convolutions which are somewhat annoyingly verbose. Default is false' })
+      parser.add_argument('-bc', '--bypassCache', { action: 'store_true', help: 'Bypass the cache on the server side and rebuild' })
 
       const args = parser.parse_args()
       args.count = args.count || 1
@@ -1488,7 +1495,7 @@ const knowledgeModuleImpl = async ({
         }
       }
 
-      if (args.retrain) {
+      if (args.bypassCache) {
         config.config.retrain = true
       }
 
@@ -1520,7 +1527,8 @@ const knowledgeModuleImpl = async ({
               useTestConfig.testModuleName = args.testModuleName
               test = useTestConfig.name
             }
-            await runTests(config, test, { args, debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
+            const timings = { server: 0, client: 0 }
+            await runTests(config, test, { timings, args, debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
               let newError = false
               if (results.length > 0) {
                 let headerShown = false
@@ -1552,6 +1560,8 @@ const knowledgeModuleImpl = async ({
                   }
                 }
 
+                
+                console.log(`Total Server Time: ${timings.server.toFixed(2)}. Total Client Time: ${timings.client.toFixed(2)}`)
                 if (hasError) {
                   console.log('**************************** ERRORS ************************')
                   for (const result of results) {

package/package.json

@@ -65,6 +65,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "8.1.0",
+  "version": "8.3.0-beta.0",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -27,6 +27,10 @@ const config_toServer = (config) => {
   // cant change things because copy breaks something
 }
 
+const updateHierarchy = (config) => {
+  config.hierarchy = new DigraphInternal(config.config.hierarchy)
+}
+
 const initWords = (words) => {
   if (!words.literals) {
     words.literals = {}
@@ -266,7 +270,6 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       }
       if (bridge.children) {
         for (const child of bridge.children) {
-          // config.addHierarchy(child, bridge.id)
           if (child.child) {
             config.addHierarchy(child.child, bridge.id, child.instance)
           } else {
@@ -1366,8 +1369,25 @@ class Config {
     }
   }
 
-  addHierarchyProperties (edge) {
-    const { child, parent, instance } = edge
+  addHierarchyInternal (edge) {
+    debugHierarchy([edge[0], edge[1]])
+    // because this is called from the semantics and internally. in the semantics the config can be a pseudo config 
+    // where hierarchy and config.hierarchy do not match
+    if (this.hierarchy) {
+      this.hierarchy.addEdge(edge)
+      // no dups due to sharing
+      if (this.hierarchy._edges != this.config.hierarchy) {
+        this.config.hierarchy.push(edge)
+        this._delta.json.hierarchy.push(edge)
+      }
+    } else {
+      this.config.hierarchy.push(edge)
+      this._delta.json.hierarchy.push(edge)
+    }
+  }
+
+  addHierarchyProperties (properties) {
+    const { child, parent, instance } = properties
     if (typeof child !== 'string') {
       throw new Error(`addHierarchy expected child property to be a string. got ${JSON.stringify(child)}`)
     }
@@ -1377,10 +1397,8 @@ class Config {
     if (instance && typeof instance !== 'boolean') {
       throw new Error(`addHierarchy expected instance property to be a boolean or undefined. got ${JSON.stringify(instance)}`)
     }
-    debugHierarchy([child, parent])
-    this.config.hierarchy.push(edge)
-    // TODO greg11 this.hierarchy.addEdge(edge)
-    this._delta.json.hierarchy.push([child, parent, instance || false])
+    const edge = [child, parent, instance || false]
+    this.addHierarchyInternal(edge)
   }
 
   addHierarchyChildParent (child, parent, instance) {
@@ -1403,9 +1421,8 @@ class Config {
       return
     }
 
-    this.config.hierarchy.push([child, parent, instance || false])
-    // this.hierarchy.addEdge([child, parent])
-    this._delta.json.hierarchy.push([child, parent, instance || false])
+    const edge = [child, parent, instance || false]
+    this.addHierarchyInternal(edge)
   }
 
   getBridge (id, level) {
@@ -2116,6 +2133,7 @@ class Config {
     cp.configCounter = this.configCounter
     cp.testConfig = this.testConfig
     cp._server = this._server
+    cp.hierarchy = new DigraphInternal(this.config.hierarchy)
 
     cp.initConfig = _.cloneDeep(this.initConfig)
     cp.defaultConfig()
@@ -2506,6 +2524,7 @@ class Config {
       }
       if (!isSelf) {
         config.config = _.cloneDeep(config.initConfig)
+        config.hierarchy = new DigraphInternal(config.config.hierarchy)
       }
       config.wasInitialized = false
       // TODO change name of config: to baseConfig:
@@ -3132,10 +3151,15 @@ class Config {
 
         // console.log('key', key, 'XXX')
         // console.log('more', JSON.stringify(more, null, 2))
-        if (addFirst) {
-          this.config[key] = more[key].concat(this.config[key])
+        // hierarchy must update in place and does not care about the list order
+        if (key == 'hierarchy') {
+          this.config[key].push(...more[key])
         } else {
-          this.config[key] = this.config[key].concat(more[key])
+          if (addFirst) {
+            this.config[key] = more[key].concat(this.config[key])
+          } else {
+            this.config[key] = this.config[key].concat(more[key])
+          }
         }
       } else {
         if (!(key in this.config)) {

package/src/configHelpers.js

@@ -240,7 +240,8 @@ const setupProcessB = ({ config, initializer, allowDelta = false } = {}) => {
   delete data.generators
   // const semantics = new Semantics((data.semantics || []).map((g) => new Semantic(normalizeSemantic(g))))
   delete data.semantics
-  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+  // const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+  const hierarchy = config.hierarchy
 
   return {
     data,
@@ -406,7 +407,8 @@ const loadInstance = async (config, instance) => {
     }
   }
 
-  const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
+  // const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
+  const hierarchy = config.hierarchy
   for (const i in (instance.resultss || [])) {
     const results = instance.resultss[i]
     if (results.extraConfig) {

package/src/digraph.js

@@ -1,3 +1,5 @@
+const _ = require('lodash')
+
 const toA = (edge) => {
   if (Array.isArray(edge)) {
     return edge
@@ -45,6 +47,10 @@ class Digraph {
     }
   }
 
+  get length () {
+    return this._edges.length
+  }
+
   addEdges (edges) {
     for (const edge of edges) {
       this.addEdge(edge)
@@ -53,7 +59,11 @@ class Digraph {
 
   addEdge (edge) {
     edge = toA(edge)
+    if (this._edges.find((existing) => _.isEqual(edge, existing))) {
+      return false
+    }
     this._edges.push(edge)
+    return true
   }
 
   get edges () {

package/src/digraph_internal.js

@@ -1,3 +1,5 @@
+const _ = require('lodash') 
+
 const toA = (edge) => {
   if (Array.isArray(edge)) {
     return edge
@@ -21,7 +23,11 @@ class DigraphInternal {
 
   addEdge (edge) {
     edge = toA(edge)
+    if (this._edges.find((existing) => _.isEqual(edge, existing))) {
+      return false
+    }
     this._edges.push(edge)
+    return true
   }
 
   get edges () {
@@ -32,6 +38,10 @@ class DigraphInternal {
     this._edges = edges
   }
 
+  get length () {
+    return this._edges.length
+  }
+
   /*
   set edges(edges) {
     this._edges = edges.map( toA )