theprogrammablemind 7.4.1 → 7.4.3-beta.0

package/client.js

@@ -436,9 +436,12 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
   return { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime }
 }
 
-const doWithRetries = async (n, url, data) => {
+const doWithRetries = async (n, url, queryParams, data) => {
+  if (!queryParams) {
+    queryParams = ''
+  }
   for (let i = 0; i < n; ++i) {
-    const result = await fetch(`${url}/process`, {
+    const result = await fetch(`${url}/process${queryParams}`, {
       method: 'POST',
       body: JSON.stringify(data),
       timeout: 1000 * 60 * 5, // it does not respect this timeout so that's why I have the retries
@@ -467,13 +470,22 @@ const doWithRetries = async (n, url, data) => {
   }
 }
 
-const setupProcessB = ({ config }) => {
+const setupProcessB = ({ config, allowDelta=false } = {}) => {
   const key = config._key
 
-  const data = Object.assign({ key, version: '3' }, config.config)
-  for (const uuid of Object.keys(data.namespaces)) {
-    const km = config.configs.find((km) => km.uuid === uuid)
-    data.namespaces[uuid].name = km.name
+  const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
+  if (allowDelta && config.allowDelta && config.hasDelta()) {
+    // console.log('config', config)
+    data.delta = config.delta()
+  } else {
+    Object.assign(data, config.config)
+  }
+ 
+  if (data.namespaces) { 
+    for (const uuid of Object.keys(data.namespaces)) {
+      const km = config.configs.find((km) => km.uuid === uuid)
+      data.namespaces[uuid].name = km.name
+    }
   }
 
   // const generators = new Generators((data.generators || []).map((g) => new Generator(normalizeGenerator(g))))
@@ -511,6 +523,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
     config.server(credentials.server, credentials.key)
   }
   const url = config._server
+  const queryParams = config._queryParams || ''
   const retries = 2
   writeTests = !!writeTests
 
@@ -525,7 +538,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
     throw error
   }
 
-  const { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config })
+  let { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config, allowDelta: true })
   let queries = query.split('\\n')
 
   try {
@@ -549,8 +562,28 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
       if (queries.length === 0) {
         break;
       }
+
       data.utterance = queries[0]
-      const json = await doWithRetries(retries, url, data)
+      let json = await doWithRetries(retries, url, queryParams, data)
+      let resetData = false
+      if (json.code == 'NOT_IN_CACHE') {
+        resetData = true
+        const setupB = setupProcessB({ config, allowDelta: false })
+        data = setupB.data
+        hierarchy = setupB.hierarchy
+        data.utterance = queries[0]
+        json = await doWithRetries(retries, url, queryParams, data)
+      }
+      if (json.cacheKey) {
+        config.cacheKey = json.cacheKey
+        if (resetData) {
+          if (queries.length > 1) {
+            const setupB = setupProcessB({ config, allowDelta: true })
+            data = setupB.data
+            hierarchy = setupB.hierarchy
+          }
+        }
+      }
       json.contexts = json.results
       delete json.results
       if (json.status !== 200) {
@@ -1220,6 +1253,7 @@ const knowledgeModule = async ({
     parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
     parser.add_argument('-q', '--query', { help: 'Run the specified query' })
     parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
+    parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
     parser.add_argument('-qd', '--queryDelete', { help: 'Delete the specified query from the tests file' })
     parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
     parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
@@ -1278,6 +1312,10 @@ const knowledgeModule = async ({
       config.server(args.server)
     }
 
+    if (args.queryParams) {
+      config.setQueryParams(args.queryParams)
+    }
+
     if (args.debug) {
       config.config.debug = true
     }
@@ -1320,7 +1358,7 @@ const knowledgeModule = async ({
         }
       }
       if (args.print.includes('c')) {
-        const { data  } = setupProcessB({ config })
+        const { data } = setupProcessB({ config })
         console.log("Config as sent to server")
         console.log(JSON.stringify(data, null, 2));
       }
@@ -1479,7 +1517,19 @@ const knowledgeModule = async ({
         if (!('then' in promise)) {
           throw 'Return a promise from process in the definition of knowledgeModule'
         }
-        promise.then(() => f()).catch( (e) => f() )
+        promise
+          .then(() => {
+            f()
+          })
+          .catch( (e) => {
+            if (e.errno == 'ECONNREFUSED') {
+              console.log(e)
+              readline.close()
+            } else {
+              console.log(e)
+              f() 
+            }
+          })
       })
       f()
     } else if (args.query) {

package/package.json

@@ -61,6 +61,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.4.1",
+  "version": "7.4.3-beta.0",
   "license": "ISC"
 }

package/src/config.js

@@ -572,6 +572,7 @@ class Config {
       }
     }
     this.config.associations.positive.push(association)
+    this._delta.json.associations.push({ action: 'add', association })
   }
 
   // TODO add more error checking to these like addHierarchy has
@@ -580,6 +581,7 @@ class Config {
       this.config.priorities = []
     }
     this.config.priorities.push(priorities)
+    this._delta.json.priorities.push({ action: 'add', priorities })
   }
  
   addHierarchy (child, parent) {
@@ -603,6 +605,7 @@ class Config {
     }
     // this.config.hierarchy.push([properties.child, properties.parent])
     this.config.hierarchy.push(properties)
+    this._delta.json.hierarchy.push([child, parent])
   }
 
   addHierarchyChildParent (child, parent) {
@@ -630,6 +633,7 @@ class Config {
     }
 
     this.config.hierarchy.push([child, parent])
+    this._delta.json.hierarchy.push([child, parent])
   }
 
   getBridge (id, level) {
@@ -659,6 +663,7 @@ class Config {
     }
     bridges.push(def)
     this.checkBridges();
+    this._delta.json.bridges.push({ action: 'add', bridge: def })
   }
 
   addGenerator (match, apply) {
@@ -735,17 +740,19 @@ class Config {
 
     operators.unshift(operator)
     this.checkOperators()
+
+    this._delta.json.operators.push({ action: 'add', operator })
   }
 
   addWord (word, def) {
-    this.addWordInternal(this.config, word, def)
+    this.addWordInternal(word, def)
   }
 
-  addWordInternal (config, word, def) {
-    if (!config.words) {
-      config.words = {}
+  addWordInternal (word, def) {
+    if (!this.config.words) {
+      this.config.words = {}
     }
-    const words = config.words
+    const words = this.config.words
     def = Object.assign({}, def, { uuid: this._uuid })
     if (words[word]) {
       if (!words[word].some((e) => helpers.safeEquals(e, def))) {
@@ -754,6 +761,8 @@ class Config {
     } else {
       words[word] = [def]
     }
+
+    this._delta.json.words.push({ action: 'add', word, def })
   }
 
   getAPI (uuid) {
@@ -792,13 +801,22 @@ class Config {
     return this._key
   }
 
-  server (server, apiKey) {
+  getQueryParams() {
+    return this._queryParams
+  }
+
+  setQueryParams(queryParams) {
+    this._queryParams = queryParams
+  }
+
+  server (server, apiKey, queryParams) {
     if (server) {
       this._server = server
     }
     if (apiKey) {
       this._key = apiKey
     }
+    this._queryParams = queryParams
 
     if (this._api && this._api.multiApi) {
       for (const key of Object.keys(this._api.apis)) {
@@ -961,6 +979,9 @@ class Config {
       throw 'Excepted the config argument to be a hash not a Config object'
     }
 
+    this.allowDelta = false
+    this.resetDelta()
+
     this.addedArgss = []
     let isProcess = require.main === module
     if (global.theprogrammablemind && config) {
@@ -1042,6 +1063,15 @@ class Config {
         // this.config.words = {}
       }
       for (let bridge of (this.config.bridges || [])) {
+        const valid = [ 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generators', 'hierarchy', 'id', 'inverted', 'isA',
+                        'level', 'optional', 'selector', 'semantic',  'words' ]
+        for (let prop of Object.keys(bridge)) {
+          if (!valid.includes(prop)) {
+            if (!prop.endsWith("Bridge")) {
+              throw `Unknown property "${prop}" in the bridge. Valid properties are ${valid}. The bridge is ${JSON.stringify(bridge)}`
+            }
+          }
+        }
         /*
         if (bridge.generator) {
           this.config.generators.push({ 
@@ -1063,11 +1093,11 @@ class Config {
         if (bridge.words) {
           for (let def of bridge.words) {
             if (typeof def == 'string') {
-              this.addWordInternal(this.config, def, {"id": bridge.id, "initial": "{}" })
+              this.addWordInternal(def, {"id": bridge.id, "initial": `{ value: "${def}"}` })
             } else {
               const word = def.word
               def = { initial: JSON.stringify(def), id: bridge.id, word: undefined }
-              this.addWordInternal(this.config, word, def)
+              this.addWordInternal(word, def)
             }
           }
         }
@@ -1165,6 +1195,32 @@ class Config {
     }
   }
 
+  hasDelta () {
+    return this._delta.json
+  }
+
+  delta () {
+    return { cacheKey: this._delta.cacheKey, json: this._delta.json }
+  }
+
+  resetDelta (cacheKey) {
+    this._delta = {
+      cacheKey,
+      json: {
+        words: [],
+        operators: [],
+        bridges: [],
+        associations: [],
+        priorities: [],
+        hierarchy: [],
+      }
+    }
+  }
+
+  set cacheKey (cacheKey) {
+    this.resetDelta(cacheKey)
+  }
+
   get api () {
     if (this._api && this._api.multiApi) {
       return this._api.api(this._api)
@@ -1637,6 +1693,7 @@ class Config {
 
   // rebuild ({ isModule: mainIsModule = false } = {}) {
   rebuild ({ isModule: mainIsModule } = {}) {
+    this.resetDelta()
     const debug = this.config.debug;
     this.config = _.cloneDeep(this.initConfig)
     if (debug) {
@@ -2061,7 +2118,7 @@ class Config {
     more.valid()
     // copy so i don't have to copy later
     more = more.copy()
-    more.server(this._server, this._key)
+    more.server(this._server, this._key, this._queryParams)
 
     this.loadOrder.addList(more.configs.map((km) => km.name || km.uuid))