theprogrammablemind 7.5.8-beta.3 → 7.5.8-beta.30

package/client.js

@@ -15,14 +15,16 @@ const sortJson = runtime.sortJson
 
 const getAsk = (config) => (uuid) => (asks) => {
   for (let ask of asks) {
-    config.addMotivation({
+    config.addSemantic({
       uuid,
+      oneShot: true,
       match: (args) => ask.matchr(args),
       apply: (args) => ask.applyr(args)
     })
   }
-  config.addMotivation({
+  config.addSemantic({
       uuid,
+      oneShot: true,
       match: ({context}) => context.marker == 'controlEnd' || context.marker == 'controlBetween',
       apply: (args) => {
         for (let ask of asks) {
@@ -37,7 +39,7 @@ const getAsk = (config) => (uuid) => (asks) => {
             }
           }
           if (matchq(args)) {
-            args.context.motivationKeep = true
+            // args.context.motivationKeep = true
             args.context.verbatim = applyq(args)
             args.context.isResponse = true;
             delete args.context.controlRemove;
@@ -109,11 +111,12 @@ class ErrorReason extends Error {
 
 const setupArgs = (args, config, logs, hierarchy) => {
   config.setArgs(args)
-  args.calls = new InitCalls(config.name)
+  args.calls = new InitCalls(args.isInstance ? `${args.isInstance}#${config.name}` : config.name)
   if (global.theprogrammablemind && global.theprogrammablemind.loadForTesting) {
     args.calls = new InitCalls(Object.keys(global.theprogrammablemind.loadForTesting)[0])
   }
   args.km = (name) => config.getConfig(name)
+  args.api = (name) => config.getConfig(name).api
   args.error = (context) => {
     throw new ErrorReason(context)
   }
@@ -143,7 +146,6 @@ const setupArgs = (args, config, logs, hierarchy) => {
       apis: getAPIs(uuid)
     }
   }
-  args.motivation = (m) => config.addMotivation(m)
   args.breakOnSemantics = false
   args.theDebugger = {
     breakOnSemantics: (value) => args.breakOnSemantics = value
@@ -157,10 +159,10 @@ const setupArgs = (args, config, logs, hierarchy) => {
   }
   args.e = (c) => config.getEvaluator(args.s, args.calls, logs, c)
   args.log = (message) => logs.push(message)
-  // config.getAddedArgs(args)
   args.gs = gs(args.g)
   args.gsp = gs(args.gp)
   args.gsr = gs(args.gr)
+  config.getAddedArgs(args)
 }
 
 const gs = (g) => (contexts, separator, lastSeparator) => {
@@ -298,7 +300,6 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   }
   associations.sort()
   // tests[query] = sortJson({ paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
-  debugger
   results = sortJson({ 
     query, 
     paraphrases, 
@@ -367,7 +368,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, query, data, retries, url, commandLineArgs }) => {
+const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, isInstance, query, data, retries, url, commandLineArgs }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -378,7 +379,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
   const contexts = setupContexts(json.contexts)
 
   const objects = config.get('objects')
-  const args = { objects, isResponse: true, response: json, isTest, getObjects: getObjects(objects) }
+  const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), isInstance }
   if (!json.logs) {
     json.logs = []
   }
@@ -418,19 +419,18 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
       if (json.has_errors) {
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
       }
+      const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
       if (!config.get('skipSemantics')) {
-        if (!config.doMotivations(args, context)) {
-          const semantics = config.getSemantics(json.logs)
-          try {
-            contextPrime = semantics.apply(args, context)
-          } catch( e ) {
-            if (e.message == 'Maximum call stack size exceeded') {
-              const mostCalled = semantics.getMostCalled()
-              e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
-            }
-            // contextPrime = semantics.apply(args, { marker: 'error', context, error: e })
-            contextPrime = semantics.apply(args, { marker: 'error', context, reason: e.reason })
+        const semantics = config.getSemantics(json.logs)
+        try {
+          contextPrime = semantics.apply(args, context)
+        } catch( e ) {
+          if (e.message == 'Maximum call stack size exceeded') {
+            const mostCalled = semantics.getMostCalled()
+            e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
           }
+          // contextPrime = semantics.apply(args, { marker: 'error', context, error: e })
+          contextPrime = semantics.apply(args, { marker: 'error', context, reason: e.reason })
         }
       }
       if (contextPrime.controlRemove) {
@@ -439,7 +439,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
       let assumed = { isResponse: true };
       const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
       let generatedParenthesized = []
-      if (isTest) {
+      if (generateParenthesized) {
         config.parenthesized = true
         generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
         config.parenthesized = false
@@ -452,12 +452,12 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
       args.gs = gs(args.g)
       args.gsp = gs(args.gsp)
       args.gsr = gs(args.gr)
-      if (isTest) {
+      if (generateParenthesized) {
         config.parenthesized = false
       }
       const paraphrases = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
       let paraphrasesParenthesized = []
-      if (isTest) {
+      if (generateParenthesized) {
         config.parenthesized = true
         paraphrasesParenthesized = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
         config.parenthesized = false
@@ -471,7 +471,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
       contextsPrime.push(contextPrime)
       generatedPrime.push(generated)
       paraphrasesPrime.push(paraphrases)
-      if (isTest) {
+      if (generateParenthesized) {
         paraphrasesParenthesizedPrime.push(paraphrasesParenthesized)
         generatedParenthesizedPrime.push(generatedParenthesized)
       }
@@ -549,8 +549,11 @@ const setupProcessB = ({ config, initializer, allowDelta=false } = {}) => {
     // console.log('config', config)
     data.delta = config.delta()
   } else {
-    Object.assign(data, config.config)
+    config.toData(data)
+    // Object.assign(data, config.config)
   }
+
+  // config.toServer(data)
  
   if (data.namespaces) { 
     for (const uuid of Object.keys(data.namespaces)) {
@@ -586,13 +589,13 @@ const processInstance = (config, instance) => {
       // config.addInternal(config.template.queries[i], { handleCalculatedProps: true } )
       config.addInternal(instance.template.queries[i], { handleCalculatedProps: true } )
     } else {
-      processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {} })
+      processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}` })
     }
   }
   global.transitoryMode = transitoryMode
 }
 
-const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
+const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -616,6 +619,9 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
   if (commandLineArgs && commandLineArgs.checkForLoop) {
     data.checkForLoop = true
   }
+  if (rebuildingTemplate) {
+    data.errors_ignore_contextual_priorities_non_existant_ops = true
+  }
   let queries = query.split('\\n')
 
   try {
@@ -694,7 +700,12 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
 
     if (writeTests) {
       const actual_config = getConfigForTest(config, testConfig)
-      writeTest(testsFN, query, config.config.objects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized)
+      const saveObjects = {...config.config.objects}
+      saveObjects.nameToUUID = {}
+      for (let km of config.configs) {
+        saveObjects.nameToUUID[km.name] = km.uuid
+      }
+      writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized)
     }
 
     return response
@@ -760,7 +771,6 @@ const runTest = async (config, expected, { args, verbose, afterTest, testConfig,
     testConfigName = testConfig.testModuleName
   }
   config.beforeQuery({ query: test, isModule: false, objects })
-  // config.resetMotivations()
     try {
       const result = await _process(config, test, { errorHandler, isTest: true })
       result.query = test
@@ -778,12 +788,17 @@ const runTest = async (config, expected, { args, verbose, afterTest, testConfig,
       delete expected_objects.nameToUUID
       const actual_objects = sortJson(convertToStable(config.config.objects), { depth: 25 })
       const failed_paraphrases = !matching(result.paraphrases, expected.paraphrases)
-      const failed_paraphrasesParenthesized = !matching(result.paraphrasesParenthesized, expected.paraphrasesParenthesized)
-      const failed_generatedParenthesized = !matching(result.generatedParenthesized, expected.generatedParenthesized)
+      let failed_paraphrasesParenthesized = !matching(result.paraphrasesParenthesized, expected.paraphrasesParenthesized)
+      let failed_generatedParenthesized = !matching(result.generatedParenthesized, expected.generatedParenthesized)
       const failed_responses = !matching(result.responses, expected.responses)
       const failed_contexts = !matching(result.contexts, expected.contexts)
       const failed_objects = !matching(actual_objects, expected_objects)
 
+      if (args.testNoParenthesized) {
+        failed_paraphrasesParenthesized = false
+        failed_generatedParenthesized = false
+      }
+
       const pickEm = (getObjects) => {
         const picked = {}
         for (let prop of (testConfig.check || [])) {
@@ -809,6 +824,11 @@ const runTest = async (config, expected, { args, verbose, afterTest, testConfig,
         }
         return expected.objects.namespaced[expected.objects.nameToUUID[name]]
       }
+      try {
+        sortJson(pickEm(expectedGetObjects), { depth: 25 })
+      } catch ( e ) {
+        debugger
+      }
       const expected_checked = sortJson(pickEm(expectedGetObjects), { depth: 25 })
       const actualGetObjects = (name) => {
         if (!name) {
@@ -935,7 +955,6 @@ const runTests = async (config, testFile, juicyBits) => {
 const saveTest = async (testFile, config, test, expected, testConfig, saveDeveloper) => {
   config.rebuild()
   const objects = getObjects(config.config.objects)(config.uuid)
-  //config.resetMotivations()
   config.beforeQuery({ query: test, isModule: false, objects })
   console.log(test)
   const result = await _process(config, test, { isTest: true })
@@ -1118,10 +1137,11 @@ const defaultErrorHandler = async (error) => {
     doErrorExit = true
   }
 
-  if (doErrorExit) {
+  if (typeof runtime.process.exit == 'function' && doErrorExit) {
     runtime.process.exit(-1)
   }
-  // throw error
+
+  throw error
 }
 
 const defaultInnerProcess = (config, errorHandler, responses) => {
@@ -1142,6 +1162,37 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   } else {
     console.log('objects', runtime.util.inspect(config.get('objects'), { depth: Infinity, sorted: true }))
   }
+
+  const pickEm = () => {
+    const picked = {}
+    const namespaced = config.get('objects')['namespaced']
+    for (let prop of (config.testConfig.check || [])) {
+      if (prop.km) {
+        /*
+        const objects = namespaced[prop.km]]
+        picked[prop.km] = {}
+        for (let p of c.testConfig.check) {
+          if (p.km) {
+            continue
+          }
+          picked[p] = objects[p]
+        }
+        */
+        console.log('TODO implement this if needed')
+      } else {
+        const objects = namespaced[config.uuid]
+        picked[prop] = objects[prop]
+      }
+    }
+    return picked
+  }
+
+  const picked = pickEm()
+  if (!_.isEmpty(picked)) {
+    console.log('--- checked object properties ---')
+    console.log(JSON.stringify(picked, null, 2))
+  }
+
   console.log('--- The contexts are ----------')
   console.log(JSON.stringify(sortJson(responses.contexts, { depth: 25 }), null, 2))
   console.log('')
@@ -1207,7 +1258,7 @@ entodicton.knowledgeModule( {
 `
 */
 
-const build = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
+const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
@@ -1231,8 +1282,6 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
       if (property == 'fragments') {
         global.transitoryMode = true
       }
-      // greg32 
-      // config.addInternal( query )
       if (hierarchy) {
         for (let edge of hierarchy) {
           if (Array.isArray(edge)) {
@@ -1244,7 +1293,7 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
       }
 
       try {
-        const results = await _process(config, query.query, {initializer})
+        const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
         if (config.config.debug) {
           // TODO pass in the error handler like the other ones
           defaultInnerProcess(config, defaultErrorHandler, results)
@@ -1274,6 +1323,12 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
       // it will just get added to the config
       const extraConfig = queryOrExtraConfig
       console.log('config', extraConfig)
+      try {
+        config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
+      } catch ( e ) {
+        const where = extraConfig.where ? ` ${extraConfig.where}` : ''
+        throw new Error(`Error processing extra config${where}: ${e.stack}}`)
+      }
       accumulators[property].push({ extraConfig: true, ...extraConfig })
       await looper(queries)
     }
@@ -1349,9 +1404,15 @@ const knowledgeModule = async ({
   beforeTests = () => {},
   afterTests = () => {},
   beforeTest = () => {},
-  afterTest = () => {}
+  afterTest = () => {},
+  ...rest
 } = {}) => {
 
+  const unknownArgs = Object.keys(rest)
+  if (unknownArgs.length > 0) {
+    throw new Error(`Unknown arguments to knowledgeModule: ${unknownArgs.join()}`)
+  }
+
   config.beforeQuery = beforeQuery 
   config.beforeTests = beforeTests 
   config.afterTests = afterTests 
@@ -1446,9 +1507,10 @@ const knowledgeModule = async ({
       description: 'Entodicton knowledge module'
     })
 
-    const helpDebugAssociation = 'In order to get a debug break when a specific association is created set the DEBUG_ASSOCIATION environment variable to the JSON of the association to break on. For example DEBUG_ASSOCIATION=\'["the#0", "mammel#0"]\' })'
-    const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'["cat#1", "mammel#1"]\' })'
-    const helpDebugPriority = 'In order to get a debug break when a specific set of priorities is created set set DEBUG_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_PRIORITY=\'["verb#0", "article#0"]\' })'
+    const helpDebugAssociation = 'In order to get a debug break when a specific association is created set the DEBUG_ASSOCIATION environment variable to the JSON of the association to break on. For example DEBUG_ASSOCIATION=\'[["the", 0], ["mammal", 1]]\' })'
+    const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'[["cat", 1], ["mammel", 1]]\' })'
+    const helpDebugPriority = 'In order to get a debug break when a specific set of priorities is created set set DEBUG_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_PRIORITY=\'[["verb", 0], ["article", 0]]\' })'
+    const helpDebugContextualPriority = 'In order to get a debug break when a specific set of contextual priorities is created set set DEBUG_CONTEXTUAL_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_CONTEXTUAL_PRIORITY=\'{ context: [["verb", 0], ["article", 0], select: 1}\' })'
     const helpDebugBridge = 'In order to get a debug break when a specific bridge is created set the DEBUG_BRIDGE environment variable to id/level to break on. For example DEBUG_BRIDGE=\'id#level\' })'
     const helpDebugOperator = 'In order to get a debug break when a specific hierarcy is created set the DEBUG_OPERATOR environment variable to debug any config loaded. For example DEBUG_OPERATOR=\'([operator] ([arg]))\' })'
 
@@ -1457,6 +1519,7 @@ const knowledgeModule = async ({
     parser.add_argument('-t', '--test', { action: 'store_true', help: 'Run the tests. Create tests by running with the --query + --save flag' })
     parser.add_argument('-tv', '--testVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. Create tests by running with the --query or --loop with the --save flag' })
     parser.add_argument('-tva', '--testAllVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. All the tests will be run instead of stopping at first failure. Create tests by running with the --query or --loop with the --save flag' })
+    parser.add_argument('-tnp', '--testNoParenthesized', { action: 'store_true', help: 'Don\' check parenthesized differences for the tests' })
     parser.add_argument('-n', '--count', { help: 'Number of times to run the tests. Default is one. Use this to check for flakey test. If possible the system will print out a message with the word "hint" suggesting how to fix the problem' })
     // parser.add_argument('-b', '--build', { help: 'Specify the template file name of the form <kmName>. There should be a file called <baseKmName>.<kmName>.template.json with the queries to run. For example { queries: [...] }. The template file will be run and generate an instantiation called <baseKmName>.<kmName>.instance.json and a file called <kmName>.js that will load the template file (this is file generated only if not already existing) and a test file called <KmName>.tests.json. This can then be loaded into an instance of the current knowledge module to setup initial conditions.' })
     parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild' })
@@ -1483,6 +1546,7 @@ const knowledgeModule = async ({
     parser.add_argument('-da', '--debugAssociation', { help: helpDebugAssociation })
     parser.add_argument('-dh', '--debugHierarchy', { help: helpDebugHierarchy })
     parser.add_argument('-dp', '--debugPriority', { help: helpDebugPriority })
+    parser.add_argument('-dcp', '--debugContextualPriority', { help: helpDebugContextualPriority })
     parser.add_argument('-db', '--debugBridge', { help: helpDebugBridge })
     parser.add_argument('-do', '--debugOperator', { help: helpDebugOperator })
 
@@ -1712,11 +1776,13 @@ const knowledgeModule = async ({
                 if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
                   hasError = true
                 }
-                if (JSON.stringify(result.expected.paraphrasesParenthesized) !== JSON.stringify(result.actual.paraphrasesParenthesized)) {
-                  hasError = true
-                }
-                if (JSON.stringify(result.expected.generatedParenthesized) !== JSON.stringify(result.actual.generatedParenthesized)) {
-                  hasError = true
+                if (!args.testNoParenthesized) {
+                    if (JSON.stringify(result.expected.paraphrasesParenthesized) !== JSON.stringify(result.actual.paraphrasesParenthesized)) {
+                      hasError = true
+                    }
+                    if (JSON.stringify(result.expected.generatedParenthesized) !== JSON.stringify(result.actual.generatedParenthesized)) {
+                      hasError = true
+                    }
                 }
                 if (JSON.stringify(result.expected.responses) !== JSON.stringify(result.actual.responses)) {
                   hasError = true
@@ -1742,9 +1808,13 @@ const knowledgeModule = async ({
                     }
                   }
                   show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
-                  show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
+                  if (!args.testNoParenthesized) {
+                    show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
+                  }
                   show('responses', result.expected.responses, result.actual.responses)
-                  show('responses parenthesized', result.expected.generatedParenthesized, result.actual.generatedParenthesized)
+                  if (!args.testNoParenthesized) {
+                    show('responses parenthesized', result.expected.generatedParenthesized, result.actual.generatedParenthesized)
+                  }
                   /*
                   if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
                     if (!headerShown) {
@@ -1918,7 +1988,7 @@ module.exports = {
   w,
   // submitBug,
   ensureTestFile,
-  build,
+  rebuildTemplate,
   processContext,
   processContexts,
   runTests,
@@ -1935,3 +2005,4 @@ module.exports = {
   flattens,
   writeTest
 }
+

package/index.js

@@ -23,5 +23,6 @@ module.exports = {
   Generator,
   Digraph,
   flattens: flattens.flattens,
+  flatten: flattens.flatten,
   unflatten: unflatten.unflatten
 }

package/package.json

@@ -63,6 +63,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.5.8-beta.3",
+  "version": "7.5.8-beta.30",
   "license": "ISC"
 }