theprogrammablemind 8.9.1-beta.27 → 8.9.1-beta.29

package/client.js

@@ -12,7 +12,7 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where } = require('./src/helpers')
+const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries } = require('./src/helpers')
 const runtime = require('./runtime')
 const sortJson = runtime.sortJson
 
@@ -35,6 +35,16 @@ const getConfig_getObjectsCheck = (config, testConfig) => {
   }
 }
 
+const getSuggestion = (diff) => {
+  return diff.map((element) => {
+    return element.marker
+  })
+}
+
+const getSuggestionMessage = (suggestion) => {
+  return `Try adding this to the associations: { context: ${JSON.stringify(getSuggestion(suggestion))}, choose: <indexOfMainElement> },\n If that does not work look at the logs and check when the operators become wrong during an interation. Deduce the change based on the previous iteration and what operator was applied.`
+}
+
 const getConfig_getContextCheck = (testConfig) => {
   return (testConfig.checks && testConfig.checks.context) || []
 }
@@ -277,7 +287,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     data.errors_ignore_contextual_priorities_non_existant_ops = true
   }
   let queries = query.split('\\n')
-
+  const summaries = [] // for error
   try {
     const response = {
       hierarchy: [],
@@ -295,11 +305,11 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       paraphrasesParenthesized: [],
       generatedParenthesized: [],
       responses: [],
-      associations: []
+      associations: [],
+      summaries: []
     }
 
     let startCounter = 0
-    const summaries = []
     while (true) {
       if (queries.length === 0) {
         break
@@ -328,9 +338,6 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         }
       }
       json.contexts = json.results
-      if (json.summaries) {
-        summaries.push(json.summaries)
-      }
       startCounter = json.end_counter + 1
       delete json.results
       if (json.status !== 200) {
@@ -340,6 +347,8 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         if (isTest) {
           start = runtime.performance.performance.now()
         }
+        const summary = { summaries: json.summaries, length: json.contexts.length }
+        summaries.push(summary)
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
                                   await processContextsB({ isTest, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         if (isTest) {
@@ -368,7 +377,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         response.paraphrasesParenthesized = response.paraphrasesParenthesized.concat(paraphrasesParenthesizedPrime)
         response.generatedParenthesized = response.generatedParenthesized.concat(generatedParenthesizedPrime)
         response.responses = response.responses.concat(responsesPrime)
-        response.summaries = summaries
+        response.summaries.push(summary)
         queries = queries.slice(1)
       }
     }
@@ -380,11 +389,12 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       for (const km of config.configs) {
         saveObjects.nameToUUID[km.name] = km.uuid
       }
-      writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized, summaries)
+      writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized, response.summaries)
     }
 
     return response
   } catch (error) {
+    error.summaries = summaries
     error.query = query
     errorHandler(error)
   }
@@ -544,7 +554,8 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
           checked: expected_checked,
           checkedContexts: pickedExpectedContexts,
           objects: expected_objects,
-          config: expected.config
+          config: expected.config,
+          summaries: expected.summaries
         },
         actual: {
           responses: result.responses,
@@ -555,7 +566,8 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
           checked: actual_checked,
           checkedContexts: pickedResultContexts,
           objects: actual_objects,
-          config: actual_config
+          config: actual_config,
+          summaries: result.summaries
         }
       }
     }
@@ -909,12 +921,13 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
 }
 
 // builtTemplate saveInstance
-const rebuildTemplate = async ({ config, target, previousResultss, startOfChanges, template, errorHandler = defaultErrorHandler }) => {
+const rebuildTemplate = async ({ config, instance, target, previousResultss, rebuild, startOfChanges, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
     semantics: [],
     associations: [],
+    summaries: [],
     learned_contextual_priorities: []
   }
   config.fragmentsBeingBuilt = []
@@ -948,12 +961,28 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
       try {
         let results
         let prMessage = ''
-        if (previousResults && previousResults.query == query.query) {
+        const suggestFix = (newSummaries) => {
+          if (!previousResults) {
+            return
+          }
+          const suggestion = suggestAssociationsFixFromSummaries(previousResults.summaries, newSummaries)
+          if (suggestion.length > 0) {
+            console.log(getSuggestionMessage(suggestion))
+          }
+        }
+        if (!rebuild && previousResults && previousResults.query == query.query) {
           results = previousResults
           prMessage = ' Using previous results. use -rtf for a hard rebuild of everything on the server side.'
           await loadInstance(config, { resultss: [results] })
         } else {
-          results = await _process(config, query.query, { initializer, rebuildingTemplate: true })
+          try {
+            results = await _process(config, query.query, { initializer, rebuildingTemplate: true })
+          } catch (e) {
+            if (e.summaries && e.summaries.length > 0) {
+              suggestFix(e.summaries)
+            }
+            throw e
+          }
         }
         if (config.config.debug) {
           // TODO pass in the error handler like the other ones
@@ -961,9 +990,11 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
         }
         if (results.contexts.length > 1) {
           console.log(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
+          suggestFix(results.summaries)
           throw new Error(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
         } else if (results.paraphrases[0].toLowerCase() !== query.query.toLowerCase()) {
           console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".${prMessage}`)
+          // suggestFix(results.summaries)
         } else {
           console.log(`query ${isFragment ? 'fragment' : ''}"${query.query}".${prMessage}`)
         }
@@ -978,6 +1009,7 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
         if (isFragment) {
           config.fragmentsBeingBuilt.push({ query: query.query, contexts: results.contexts })
         }
+        accumulators.summaries = accumulators.summaries.concat(results.summaries)
         accumulators.associations = accumulators.associations.concat(results.associations)
         accumulators.learned_contextual_priorities = accumulators.learned_contextual_priorities.concat(results.learned_contextual_priorities)
         await looper(configs)
@@ -1070,16 +1102,31 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
   }
   let todo = []
   todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
+  /*
   todo = todo.concat((template.configs || []).map((query, index) => {
     let pr
-    if (index < startOfChanges) {
+    if (index < startOfChanges || (!startOfChanges && index < previousResultss.length)) {
       pr = previousResultss[index]
     }
     return { property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics }
   }))
-  todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
-  todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
-  await looper(Object.assign([], todo))
+  */
+  if (template.configs) {
+    for (let configsIndex = 0; configsIndex < template.configs.length; ++configsIndex) {
+      const query = template.configs[configsIndex]
+      // account for things for example associations being added to the config while debugginer
+      const pr = previousResultss.find((pr) => pr.query == query)
+      todo.push({ property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics })
+    }
+  }
+  todo = todo.concat((template.fragments || []).map((query, index) => {
+    const pr = instance.fragments[index]
+    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: false })
+  }))
+  todo = todo.concat((template.semantics || []).map((definition) => {
+    return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true }
+  }))
+  await looper([...todo])
 }
 
 const checkTemplate = (template) => {
@@ -1577,13 +1624,18 @@ const knowledgeModuleImpl = async ({
                   console.log('**************************** ERRORS ************************')
                   for (const result of results) {
                     console.log('Utterance: ', result.utterance)
-                    const show = (label, expected, actual) => {
+                    if (!result.hasError) {
+                      continue
+                    }
+                    const show = (label, expected, actual, { console_log = true } = {}) => {
                       if (JSON.stringify(expected) !== JSON.stringify(actual)) {
-                        if (!headerShown) {
-                          console.log('  Failure')
+                        if (console_log) {
+                          if (!headerShown) {
+                            console.log('  Failure')
+                          }
+                          console.log(`    expected ${label}`, expected)
+                          console.log(`    actual ${label}  `, actual)
                         }
-                        console.log(`    expected ${label}`, expected)
-                        console.log(`    actual ${label}  `, actual)
                         newError = true
                         headerShown = true
                         if (args.vimdiff) {
@@ -1592,6 +1644,44 @@ const knowledgeModuleImpl = async ({
                         result.hasError = true
                       }
                     }
+                    if (!deepEqual(result.expected.summaries, result.actual.summaries)) {
+                      /*
+                      const suggestion = suggestAssociationsFix(result.expected.summaries, result.actual.summaries)
+                      let suggestedFix
+                      if (suggestion) {
+                        suggestedFix = suggestion
+                        console.log(`Try adding this to the associations: { context: ${JSON.stringify(getSuggestion(suggestedFix))}, choose: <indexOfMainElement> }`)
+                      }
+                      */
+
+                      const checkForFix = () => {
+                        for (let iDiff = 0; iDiff < result.expected.paraphrases.length; ++iDiff) {
+                          if (result.expected.paraphrases[iDiff] !== result.actual.paraphrases[iDiff]) {
+                            // const context = result.expected.contexts[iDiff]
+                            let lengths = 0
+                            // just check where the results differ since earlier summaries can be different but dont matter if the result is the same
+                            for (let iSummaries = 0; iSummaries < result.expected.summaries.length; ++iSummaries) {
+                              const summary = result.expected.summaries[iSummaries]
+                              lengths += summary.length
+                              if (lengths > iDiff) {
+                                const suggestion = suggestAssociationsFix(result.expected.summaries[iSummaries].summaries, result.actual.summaries[iSummaries].summaries)
+                                if (suggestion) {
+                                  suggestedFix = suggestion
+                                  // console.log(`Try adding this to the associations: { context: ${JSON.stringify(getSuggestion(suggestedFix))}, choose: <indexOfMainElement> },`)
+                                  console.log(getSuggestionMessage(suggestedFix))
+                                  return
+                                }
+                              }
+                            }
+                          }
+                        }
+                      }
+                      checkForFix()
+
+                      // find the context where the different is
+                      // select the first diff that has a different operator in the range
+                      // show that
+                    }
                     show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
                     if (!args.testNoParenthesized) {
                       show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
@@ -1755,7 +1845,6 @@ const knowledgeModuleImpl = async ({
     const initConfig = async (config) => {
       if (template) {
         if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess }).needsRebuild) {
-          debugger
           config.needsRebuild(template.template, template.instance, { isModule: !isProcess })
           const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
           throw new Error(error)

package/package.json

@@ -67,6 +67,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "8.9.1-beta.27",
+  "version": "8.9.1-beta.29",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -50,7 +50,8 @@ const debugPriority = (priority) => {
   }
   if (global.entodictonDebugPriority) {
     if (helpers.subPriority(entodictonDebugPriority, priority)) {
-      debugger // debug hierarchy hit
+      // debug hierarchy hit
+      debugger // eslint-disable-line no-debugger
     }
   }
 }
@@ -61,7 +62,8 @@ const debugAssociation = (association) => {
   }
   if (global.entodictonDebugAssociation) {
     if (helpers.safeEquals(global.entodictonDebugAssociation, association)) {
-      debugger // debug association hit
+      // debug association hit
+      debugger // eslint-disable-line no-debugger
     }
   }
 }
@@ -72,7 +74,8 @@ const debugWord = (word) => {
   }
   if (global.entodictonDebugWord) {
     if (helpers.safeEquals(global.entodictonDebugWord, word)) {
-      debugger // debug word hit
+      // debug word hit
+      debugger // eslint-disable-line no-debugger
     }
   }
 }
@@ -83,7 +86,8 @@ const debugHierarchy = (pair) => {
   }
   if (global.entodictonDebugHierarchy) {
     if (helpers.safeEquals(global.entodictonDebugHierarchy, pair)) {
-      debugger // debug hierarchy hit
+      // debug hierarchy hit
+      debugger // eslint-disable-line no-debugger
     }
   }
 }
@@ -94,7 +98,8 @@ const debugBridge = (bridge) => {
   }
   if (global.entodictonDebugBridge) {
     if (global.entodictonDebugBridge == bridge.id) {
-      debugger // debug hierarchy hit
+      // debug hierarchy hit
+      debugger // eslint-disable-line no-debugger
     }
   }
 }
@@ -105,7 +110,8 @@ const debugOperator = (operator) => {
   }
   if (global.entodictonDebugOperator) {
     if ((operator.pattern || operator) === global.entodictonDebugOperator) {
-      debugger // debug operator hit
+      // debug operator hit
+      debugger // eslint-disable-line no-debugger
     }
   }
 }
@@ -119,19 +125,35 @@ const debugConfigProps = (config) => {
   }
   const checkProps = [
     { property: 'priorities', check: (v) => debugPriority(v) },
-    { property: 'association', check: (v) => debugAssociation(v) },
-    { property: 'words', check: (v) => debugAssociation(v) },
+    { property: 'associations', check: (v) => debugAssociation(v) },
+    // TODO implement debugWords
+    { property: 'words', check: (v) => debugWords(v) },
     { property: 'hierarchy', check: (v) => debugHierarchy(v) },
     { property: 'operators', check: (v) => debugOperator(v) },
     { property: 'bridges', check: (v) => debugBridge(v) }
   ]
   for (const { property, check } of checkProps) {
-    if (property == 'words') {
+    if (property == 'associations') {
+      if (config[property]) {
+        if (config[property].negative) {
+          for (const value of config[property].negative) {
+            check(value)
+          }
+        }
+        if (config[property].positive) {
+          for (const value of config[property].positive) {
+            check(value)
+          }
+        }
+      }
+    } else if (property == 'words') {
+      /*
       if (config[property]) {
-        for (const value in config[property].literals) {
+        for (const value of config[property].literals) {
           check(value)
         }
       }
+      */
     } else if (config[property]) {
       for (const value of config[property]) {
         check(value)
@@ -173,6 +195,13 @@ const validConfigProps = (config) => {
     'eqClasses'
   ]
   helpers.validProps(valid, config, 'config')
+
+  // TODO add more type checks
+  if (config.associations) {
+    if (!config.associations.positive && !config.associations.negative) {
+      throw new Error('Expected the \'associations\' property to be a hash with \'positive\' and or \'negative\' properties')
+    }
+  }
 }
 
 const setupInitializerFNArgs = (config, args) => {
@@ -261,7 +290,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       if (bridge.development && config.isModule) {
         return
       }
-      if (!bridge.bridge) {
+      if (false && !bridge.bridge) {
         bridge.bridge = '{ ...next(operator) }'
       }
       if (!bridge.level) {
@@ -561,7 +590,7 @@ const normalizeConfig = (config) => {
         if (!bridge.level) {
           bridge.level = 0
         }
-        if (!bridge.bridge) {
+        if (false && !bridge.bridge) {
           bridge.bridge = '{ ...next(operator) }'
         }
       }
@@ -1172,7 +1201,7 @@ class Config {
             delete bridge.evaluators
             delete bridge.semantic
             delete bridge.semantics
-            if (!bridge.bridge) {
+            if (false && !bridge.bridge) {
               bridge.bridge = '{ ...next(operator) }'
             }
             return bridge
@@ -1305,7 +1334,7 @@ class Config {
       // TODO fix beforeQuery
       template = { fragments: [], configs: [], ...template }
       template.fragments = template.fragments.concat(this.dynamicFragments)
-      await rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
+      await rebuildTemplate({ config: this, instance, target: this.name, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options, previousResultss: options.previousResultss || instance.resultss })
     } else {
       // no change
       // this.initInstances.push({ ...instance, name: config.name })
@@ -1984,9 +2013,6 @@ class Config {
       Object.assign(args, addedArgs)
       if (getUUIDScoped) {
         const currentGetUUIDScoped = args.getUUIDScoped
-        if (!currentGetUUIDScoped) {
-          debugger
-        }
         args.getUUIDScoped = (uuid) => Object.assign(currentGetUUIDScoped(uuid), getUUIDScoped(uuid))
       }
     }
@@ -2114,7 +2140,6 @@ class Config {
   // TODO add more details
   equal (config) {
     if (JSON.stringify(this.config) != JSON.stringify(config.config)) {
-      debugger
       return false
     }
     return true
@@ -2171,9 +2196,6 @@ class Config {
     }
     cp.mapUUIDs(map)
 
-    if (cp._uuid == 'concept2') {
-      // debugger
-    }
     if (options.callInitializers) {
       await cp.rebuild(options) // in copy
     } else {
@@ -2579,8 +2601,6 @@ class Config {
         args.currentConfig.wasInitialized = true
       }
       */
-      // debugger
-      // greg55
       config.initializerFn(args, { dontCallFn: true })
       initAfterApis.unshift({ config, args })
       if (config._api) {

package/src/configHelpers.js

@@ -256,7 +256,7 @@ const setupContexts = (rawContexts) => {
     context.context_index = _index
     ++_index
   }
-  contexts.forEach( id )
+  contexts.forEach(id)
   return contexts
 }
 

package/src/digraph.js

@@ -17,7 +17,6 @@ class Digraph {
 
   // BFS
   path (from, to) {
-    debugger
     const frontier = { [from]: [[]] }
     const done = new Set()
     while (Object.keys(frontier).length > 0) {
@@ -155,7 +154,6 @@ class Digraph {
     if (nodes.length === 0) {
       return new Set([])
     }
-    debugger
     nodes = Array.from(nodes)
     let common = this.ancestors(nodes[0], { includeSelf: true })
 

package/src/generators.js

@@ -66,7 +66,8 @@ class Generator {
     const matches = await this.match(args)
     if ((matches && (options.debug || {}).match) ||
         callId == this.callId) {
-      debugger // next line is the matcher
+      // next line is the matcher
+      debugger // eslint-disable-line no-debugger
       await this.match(args)
     }
     return matches
@@ -118,16 +119,9 @@ class Generator {
     if (this.property == 'generatorp') {
       args.g = args.gp
     }
-    // if (this.callId) {
-    // greg
-    /*
-    if (callId == 'call11' && this.callId) {
-      debugger;
-    }
-    */
     if ((options.debug || {}).apply ||
         callId == this.callId) {
-      debugger
+      debugger // eslint-disable-line no-debugger
     }
     return await this._apply(args)
   }

package/src/helpers.js

@@ -27,6 +27,28 @@ function where (goUp = 2) {
   }
 }
 
+function suggestAssociationsFix (esummary, asummary) {
+  for (let isummary = 0; isummary < esummary.length; ++isummary) {
+    if (!deepEqual(esummary[isummary], asummary[isummary])) {
+      return esummary[isummary].operators
+    }
+  }
+  return []
+}
+
+function suggestAssociationsFixFromSummaries (esummaries, asummaries) {
+  for (let isummaries = 0; isummaries < esummaries.length; ++isummaries) {
+    const esummary = esummaries[isummaries].summaries
+    const asummary = asummaries[isummaries].summaries
+    for (let isummary = 0; isummary < esummary.length; ++isummary) {
+      if (!deepEqual(esummary[isummary], asummary[isummary])) {
+        return esummary[isummary].operators
+      }
+    }
+  }
+  return []
+}
+
 function w (func) {
   func.where = where(3)
   return func
@@ -437,5 +459,7 @@ module.exports = {
   sortJson,
   subPriority,
   where,
-  w
+  w,
+  suggestAssociationsFix,
+  suggestAssociationsFixFromSummaries
 }

package/src/semantics.js

@@ -73,7 +73,8 @@ class Semantic {
     this.fixUpArgs(args, context)
     const matches = await this.matcher(args)
     if (matches && (options.debug || {}).match || args.callId == this.callId) {
-      debugger // next line is the matcher
+      // next line is the matcher
+      debugger // eslint-disable-line no-debugger
       await this.matcher(args)
     }
     return matches
@@ -92,10 +93,10 @@ class Semantic {
     this.fixUpArgs(args, contextPrime)
 
     if ((options.debug || {}).apply || args.callId == this.callId) {
-      debugger
+      debugger // eslint-disable-line no-debugger
     }
     if (args.breakOnSemantics) {
-      debugger
+      debugger // eslint-disable-line no-debugger
     }
     await this._apply(args)
     return contextPrime
@@ -166,9 +167,6 @@ class Semantics {
     args.log = (message) => { this.logs.push(message) }
     for (const isemantic in this.semantics) {
       const semantic = this.semantics[isemantic]
-      if (!semantic) {
-        debugger
-      }
       // only one question at a time
       if (semantic.isQuestion && seenQuestion) {
         continue