theprogrammablemind 7.4.0 → 7.4.1-beta.1

package/client.js

@@ -12,14 +12,16 @@ const { appendNoDups, InitCalls } = require('./src/helpers')
 const runtime = require('./runtime')
 const sortJson = runtime.sortJson
 
-const ask = (config) => (asks) => {
+const getAsk = (config) => (uuid) => (asks) => {
   for (let ask of asks) {
     config.addMotivation({
+      uuid,
       match: (args) => ask.matchr(args),
       apply: (args) => ask.applyr(args)
     })
   }
   config.addMotivation({
+      uuid,
       match: ({context}) => context.marker == 'controlEnd' || context.marker == 'controlBetween',
       apply: (args) => {
         for (let ask of asks) {
@@ -100,7 +102,25 @@ const setupArgs = (args, config, logs, hierarchy) => {
   args.listable = listable(hierarchy)
   args.asList = asList
   args.retry = () => { throw new RetryError() }
-  args.ask = ask(config)
+  const scopedAsk = getAsk(config)
+
+  const getAPI = (uuid) => {
+    if (config && config.getAPI) {
+      return config.getAPI(uuid)
+    }
+  }
+  const getAPIs = (uuid) => {
+    if (config && config.getAPIs) {
+      return config.getAPIs(uuid)
+    }
+  }
+  args.getUUIDScoped = (uuid) => {
+    return { 
+      ask: scopedAsk(uuid),
+      api: getAPI(uuid),
+      apis: getAPIs(uuid)
+    }
+  }
   args.motivation = (m) => config.addMotivation(m)
   args.s = (c) => config.getSemantics(logs).apply(args, c)
   args.g = (c) => config.getGenerators(logs).apply(args, c)
@@ -174,11 +194,13 @@ const processContexts = (contexts, params) => {
   return { contexts: contextsPrime, generated, logs }
 }
 
-const getObjects = (objects) => (uuid) => {
-  if (objects && objects.namespaced) {
-    objects = objects.namespaced[uuid]
+const getObjects = (objects) => {
+  return (uuid) => {
+      if (objects && objects.namespaced) {
+        return objects.namespaced[uuid]
+      }
+      return objects
   }
-  return objects
 }
 
 const processContext = (context, { objects = {}, config, logs = [] }) => {
@@ -414,9 +436,12 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
   return { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime }
 }
 
-const doWithRetries = async (n, url, data) => {
+const doWithRetries = async (n, url, queryParams, data) => {
+  if (!queryParams) {
+    queryParams = ''
+  }
   for (let i = 0; i < n; ++i) {
-    const result = await fetch(`${url}/process`, {
+    const result = await fetch(`${url}/process${queryParams}`, {
       method: 'POST',
       body: JSON.stringify(data),
       timeout: 1000 * 60 * 5, // it does not respect this timeout so that's why I have the retries
@@ -484,11 +509,12 @@ const processInstance = (config, instance) => {
   global.transitoryMode = transitoryMode
 }
 
-const _process = async (config, query, { credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler, beforeQuery } = {}) => {
+const _process = async (config, query, { credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
   const url = config._server
+  const queryParams = config._queryParams || ''
   const retries = 2
   writeTests = !!writeTests
 
@@ -497,7 +523,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
     if (writeTests) {
       config.rebuild()
       const objects = getObjects(config.config.objects)(config.uuid)
-      beforeQuery({ query, isModule: false, objects })
+      config.beforeQuery({ query, isModule: false, objects })
     }
   } catch(error) {
     throw error
@@ -528,7 +554,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
         break;
       }
       data.utterance = queries[0]
-      const json = await doWithRetries(retries, url, data)
+      const json = await doWithRetries(retries, url, queryParams, data)
       json.contexts = json.results
       delete json.results
       if (json.status !== 200) {
@@ -598,7 +624,7 @@ const getConfigForTest = (config, testConfig) => {
   return configForTest
 }
 
-const runTest = async (config, expected, { verbose, beforeQuery, afterTest, testConfig, debug }) => {
+const runTest = async (config, expected, { verbose, afterTest, testConfig, debug }) => {
   const test = expected.query
   // initialize in between test so state is not preserved since the test was adding without state
   config.rebuild()
@@ -616,7 +642,7 @@ const runTest = async (config, expected, { verbose, beforeQuery, afterTest, test
   }
 
   const objects = getObjects(config.config.objects)(config.uuid)
-  beforeQuery({ query: test, isModule: false, objects })
+  config.beforeQuery({ query: test, isModule: false, objects })
   config.resetMotivations()
     try {
       const result = await _process(config, test, { errorHandler, isTest: true })
@@ -642,8 +668,8 @@ const runTest = async (config, expected, { verbose, beforeQuery, afterTest, test
       const failed_config = !matching(actual_config, expected_config)
       let failed = failed_paraphrases || failed_responses || failed_contexts || failed_objects || failed_config
       if (!failed) {
-        if (afterTest) {
-          failed = afterTest({ query: test, expected, actual: result, config })
+        if (config.afterTest) {
+          failed = config.afterTest({ query: test, expected, actual: result, config })
           if (failed) {
             return {
               utterance: test,
@@ -703,21 +729,20 @@ const runTestsHelper = async (config, tests, failed, juicyBits) => {
 
 const runTests = async (config, testFile, juicyBits) => {
   const tests = JSON.parse(runtime.fs.readFileSync(testFile))
-  const { beforeTests, afterTests } = juicyBits
-  beforeTests()
+  config.beforeTests()
   if (juicyBits.verbose) {
     console.log('\n', testFile, '-----------------------------------------------', '\n')
   }
   const v = await runTestsHelper(config, [...tests], [], juicyBits)
-  afterTests()
+  config.afterTests()
   return v
 }
 
-const saveTest = async (testFile, config, test, expected, beforeQuery, testConfig, saveDeveloper) => {
+const saveTest = async (testFile, config, test, expected, testConfig, saveDeveloper) => {
   config.rebuild()
   const objects = getObjects(config.config.objects)(config.uuid)
   config.resetMotivations()
-  beforeQuery({ query: test, isModule: false, objects })
+  config.beforeQuery({ query: test, isModule: false, objects })
   console.log(test)
   const result = await _process(config, test, { isTest: true })
   // const actualObjects = config.config.objects
@@ -725,23 +750,23 @@ const saveTest = async (testFile, config, test, expected, beforeQuery, testConfi
   writeTest(testFile, test, config.config.objects, result.generated, result.paraphrases, result.responses, result.contexts, result.associations, result.metadata, actualConfig, saveDeveloper)
 }
 
-const saveTestsHelper = async (testFile, config, tests, todo, beforeQuery, testConfig, saveDeveloper) => {
+const saveTestsHelper = async (testFile, config, tests, todo, testConfig, saveDeveloper) => {
   if (todo.length === 0) {
     return
   }
   const test = todo.pop()
   config.rebuild()
-  const result = await saveTest(testFile, config, test, tests[test], beforeQuery, testConfig, saveDeveloper)
+  const result = await saveTest(testFile, config, test, tests[test], testConfig, saveDeveloper)
   // initialize in between test so state is not preserved since the test was adding without state
   // config.initialize({force: true})
   config.rebuild()
-  return saveTestsHelper(testFile, config, tests, todo, beforeQuery, testConfig, saveDeveloper)
+  return saveTestsHelper(testFile, config, tests, todo, testConfig, saveDeveloper)
 }
 
-const saveTests = (config, testFile, beforeQuery, testConfig) => {
+const saveTests = (config, testFile, testConfig) => {
   const tests = JSON.parse(runtime.fs.readFileSync(testFile))
   console.log(testFile)
-  return saveTestsHelper(testFile, config, tests, tests.map( (test) => test.query ), beforeQuery, testConfig)
+  return saveTestsHelper(testFile, config, tests, tests.map( (test) => test.query ), testConfig)
 }
 
 /*
@@ -962,7 +987,7 @@ entodicton.knowledgeModule( {
 `
 */
 
-const build = async ({ config, target, beforeQuery, template, errorHandler = defaultErrorHandler }) => {
+const build = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
@@ -1000,7 +1025,7 @@ const build = async ({ config, target, beforeQuery, template, errorHandler = def
       }
 
       try {
-        const results = await _process(config, query.query, { beforeQuery })
+        const results = await _process(config, query.query, {})
         if (config.config.debug) {
           // TODO pass in the error handler like the other ones
           defaultInnerProcess(config, defaultErrorHandler, results)
@@ -1097,11 +1122,40 @@ const knowledgeModule = async ({
   afterTest = () => {}
 } = {}) => {
 
-  const isProcess = require.main === moduleFromJSFile
+  config.beforeQuery = beforeQuery 
+  config.beforeTests = beforeTests 
+  config.afterTests = afterTests 
+  config.beforeTest = beforeTest 
+  config.afterTest = afterTest 
+
   const testConfig = test
 
+  if (!moduleFromJSFile) {
+    throw "'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module."
+  }
+  if (!config) {
+    throw "'config' is a required parameter. The value should the config that defines the knowledge module."
+  }
+  if (!config.name) {
+    throw "config must have 'name' set to the knowledge module name."
+  }
+  if (!description) {
+    throw "'description' is a required parameter. The value should the description of the knowledge module."
+  }
+  if (!test) {
+    throw "'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }."
+  }
+
+  const isProcess = require.main === moduleFromJSFile
+  let loadForTesting = false
+  if (global.theprogrammablemind) {
+    if (global.theprogrammablemind.loadForTesting[config.name]) {
+      loadForTesting = true
+    }
+  }
+
   // remove test only stuff
-  if (!isProcess) {
+  if (!isProcess && !loadForTesting) {
     config.config.operators = config.config.operators.filter( (operator) => {
       if (operator.development) {
         return false
@@ -1118,22 +1172,6 @@ const knowledgeModule = async ({
     })
   }
 
-  if (!moduleFromJSFile) {
-    throw "'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module."
-  }
-  if (!config) {
-    throw "'config' is a required parameter. The value should the config that defines the knowledge module."
-  }
-  if (!config.name) {
-    throw "config must have 'name' set to the knowledge module name."
-  }
-  if (!description) {
-    throw "'description' is a required parameter. The value should the description of the knowledge module."
-  }
-  if (!test) {
-    throw "'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }."
-  }
-
   let module
   if (_.isFunction(moduleFromJSFile)) {
     module = moduleFromJSFile
@@ -1172,6 +1210,7 @@ const knowledgeModule = async ({
       description: 'Entodicton knowledge module'
     })
 
+    parser.add_argument('-tfn', '--testFileName', { help: 'Override the test file for the module when the tests are being run' })
     parser.add_argument('-t', '--test', { action: 'store_true', help: 'Run the tests. Create tests by running with the --query + --save flag' })
     parser.add_argument('-tv', '--testVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. Create tests by running with the --query or --loop with the --save flag' })
     parser.add_argument('-tva', '--testAllVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. All the tests will be run instead of stopping at first failure. Create tests by running with the --query or --loop with the --save flag' })
@@ -1185,6 +1224,7 @@ const knowledgeModule = async ({
     parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
     parser.add_argument('-q', '--query', { help: 'Run the specified query' })
     parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
+    parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
     parser.add_argument('-qd', '--queryDelete', { help: 'Delete the specified query from the tests file' })
     parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
     parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
@@ -1243,6 +1283,10 @@ const knowledgeModule = async ({
       config.server(args.server)
     }
 
+    if (args.queryParams) {
+      config.setQueryParams(args.queryParams)
+    }
+
     if (args.debug) {
       config.config.debug = true
     }
@@ -1366,9 +1410,8 @@ const knowledgeModule = async ({
 
     if (!args.query && !args.test && !args.info && (args.save || args.saveDeveloper)) {
       global.transitoryMode = true
-      saveTests(config, test, beforeQuery, testConfig, args.saveDeveloper)
+      saveTests(config, test, testConfig, args.saveDeveloper)
     // } else if (args.build) {
-    // build({ config, target: args.build, beforeQuery, errorHandler })
     } else if (args.info) {
       showInfo(description, section, config)
     } else if (args.test || args.testVerbose || args.testAllVerbose) {
@@ -1381,7 +1424,7 @@ const knowledgeModule = async ({
             }
             return
           }
-          runTests(config, test, { debug: args.debug, testConfig: testConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose, beforeQuery, beforeTests, afterTests, beforeTest, afterTest }).then((results) => {
+          runTests(config, args.testFileName ? `${args.testFileName}.test.json` : test, { debug: args.debug, testConfig: testConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
             if (results.length > 0 && args.vimdiff) {
               for (const result of results) {
                 vimdiff(result.expected, result.actual)
@@ -1454,9 +1497,9 @@ const knowledgeModule = async ({
       if (args.objectDiff) {
         global.beforeObjects = _.cloneDeep(objects)
       }
-      beforeQuery({ query: args.query, isModule: false, objects })
+      config.beforeQuery({ query: args.query, isModule: false, objects })
       try {
-        processResults(_process(config, args.query, { dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test, beforeQuery }))
+        processResults(_process(config, args.query, { dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
       } catch( error ) {
         console.log('Error', error);
       }

package/package.json

@@ -61,6 +61,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.4.0",
+  "version": "7.4.1-beta.1",
   "license": "ISC"
 }

package/src/config.js

@@ -767,6 +767,23 @@ class Config {
     }
   }
 
+  getAPIs (uuid) {
+    let config
+    if (this._uuid === uuid) {
+      config = this
+    } else {
+      for (const km of this.configs) {
+        if (km._uuid === uuid) {
+          config = km.config
+          break
+        }
+      }
+    }
+    if (config && config._api && config._api.multiApi) {
+      return config._api.apis
+    }
+  }
+
   getServer() {
     return this._server
   }
@@ -775,13 +792,22 @@ class Config {
     return this._key
   }
 
-  server (server, apiKey) {
+  getQueryParams() {
+    return this._queryParams
+  }
+
+  setQueryParams(queryParams) {
+    this._queryParams = queryParams
+  }
+
+  server (server, apiKey, queryParams) {
     if (server) {
       this._server = server
     }
     if (apiKey) {
       this._key = apiKey
     }
+    this._queryParams = queryParams
 
     if (this._api && this._api.multiApi) {
       for (const key of Object.keys(this._api.apis)) {
@@ -888,6 +914,19 @@ class Config {
     return configs;
   }
 
+  getConfigByUUID (uuid) {
+    if (this.uuid === uuid) {
+      return this
+    }
+    for (const config of this.configs) {
+      if (config.config instanceof Config) {
+        if (config.uuid === uuid) {
+          return config.config
+        }
+      }
+    }
+  }
+
   getConfig (name) {
     if (this.name === name) {
       return this
@@ -932,7 +971,13 @@ class Config {
     }
 
     this.addedArgss = []
-    const isProcess = require.main === module
+    let isProcess = require.main === module
+    if (global.theprogrammablemind && config) {
+      if (global.theprogrammablemind.loadForTesting[config.name]) {
+        isProcess = true
+        this.loadedForTesting = true
+      }
+    }
     this.isModule = !isProcess
     if (this.isModule) {
       this.removeDevelopmentElements(config)
@@ -1198,6 +1243,9 @@ class Config {
 
   // motivation === { match, apply, uuid }
   addMotivation (motivation) {
+    if (!motivation.uuid) {
+      motivation.uuid = this.uuid
+    }
     this.motivations.push(motivation)
   }
 
@@ -1207,11 +1255,13 @@ class Config {
 
   doMotivations (args, context) {
     args = Object.assign({}, args, { context })
-    args.objects = args.getObjects(this.uuid)
+    // console.log('src/config doMotivations this.uuid', this.uuid)
+    // args.objects = args.getObjects(this.uuid)
     const motivations = this.motivations
     this.motivations = []
     let done = false
     for (const motivation of motivations) {
+      args.objects = args.getObjects(motivation.uuid)
       if (!done && motivation.match(args)) {
         motivation.apply(args)
         if (args.context.controlKeepMotivation || motivation.repeat) {
@@ -1255,6 +1305,7 @@ class Config {
     cp.tests = this.tests
     cp.motivations = this.motivations
     cp.isModule = this.isModule
+    cp.loadedForTesting = this.loadedForTesting
     cp.initInstances = this.initInstances.slice()
     cp.instances = this.instances.slice()
     cp.configCounter = this.configCounter
@@ -1603,6 +1654,12 @@ class Config {
     // already set
     // this.isModule = this.isModule || mainIsModule
     mainIsModule = (mainIsModule === undefined) ? this.isModule : mainIsModule
+    if (mainIsModule !== undefined) {
+      this.isModule = mainIsModule
+    }
+    if (this.loadedForTesting) {
+      this.isModule = false
+    }
     this.config.objects.namespaced = {}
     this.resetWasInitialized()
 
@@ -1640,7 +1697,9 @@ class Config {
       if (!(config instanceof Config)) {
         config = this
         isSelf = true
-        isModule = mainIsModule
+        isModule = config.isModule || mainIsModule
+      } else {
+        isModule = config.isModule
       }
       if (!isSelf) {
         config.config = _.cloneDeep(config.initConfig)
@@ -2011,7 +2070,7 @@ class Config {
     more.valid()
     // copy so i don't have to copy later
     more = more.copy()
-    more.server(this._server, this._key)
+    more.server(this._server, this._key, this._queryParams)
 
     this.loadOrder.addList(more.configs.map((km) => km.name || km.uuid))
 

package/src/generators.js

@@ -59,7 +59,7 @@ class Generator {
             api: this.getAPI(config),
             apis: this.getAPIs(config) 
           }
-    const args = Object.assign({}, baseArgs, moreArgs)
+    const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
     // return this.match(args)
     const matches = this.match(args)
     if ((matches && (options.debug || {}).match)
@@ -106,7 +106,7 @@ class Generator {
             api: this.getAPI(config), 
             apis: this.getAPIs(config) 
           }
-    const args = Object.assign({}, baseArgs, moreArgs)
+    const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
     // if (this.callId) {
     // greg
     /*

package/src/semantics.js

@@ -45,13 +45,16 @@ class Semantic {
   matches (baseArgs, context, options = {}) {
     const hierarchy = baseArgs.hierarchy
     const config = baseArgs.config
+
     const objects = baseArgs.getObjects(this.uuid)
+    // const ask = baseArgs.getAsk(this.uuid)
+
     // return this.matcher(Object.assign({}, argsBase, {args: contextArgs(context, hierarchy), objects: objects, global: objects, context: context, hierarchy: hierarchy, api: this.getAPI(config)})
     const callId = baseArgs.calls.current()
     const moreArgs = { 
             uuid: this.uuid, 
             args: contextArgs(context, hierarchy), 
-            objects: objects, 
+            objects,
             global: objects, 
             context: context, 
             // hierarchy: hierarchy, 
@@ -59,7 +62,7 @@ class Semantic {
             api: this.getAPI(config),
             apis: this.getAPIs(config)
           }
-    const args = Object.assign({}, baseArgs, moreArgs)
+    const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
 
     const matches = this.matcher(args)
     if (matches && (options.debug || {}).match
@@ -74,6 +77,7 @@ class Semantic {
   apply (baseArgs, context, s, log, options = {}) {
     const { hierarchy, config, response } = baseArgs
     const objects = baseArgs.getObjects(this.uuid)
+    // const ask = baseArgs.getAsk(this.uuid)
     if (!log) {
       console.trace()
       throw 'log is a required argument'
@@ -99,7 +103,7 @@ class Semantic {
             api: this.getAPI(config),
             apis: this.getAPIs(config)
     }
-    const args = Object.assign({}, baseArgs, moreArgs)
+    const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
     if ((options.debug || {}).apply
         ||
         callId == this.callId) {