theprogrammablemind 7.5.0 → 7.5.2

package/client.js

@@ -78,7 +78,15 @@ const listable = (hierarchy) => (c, type) => {
   return false
 }
 
-const isA = (hierarchy) => (child, parent) => hierarchy.isA(child, parent)
+const isA = (hierarchy) => (child, parent) => {
+  if (child.marker) {
+    child = child.marker
+  }
+  if (parent.marker) {
+    parent = parent.marker
+  }
+  return hierarchy.isA(child, parent)
+}
 
 const asList = (context) => {
   if (context.marker === 'list') {
@@ -518,7 +526,7 @@ const processInstance = (config, instance) => {
   global.transitoryMode = transitoryMode
 }
 
-const _process = async (config, query, { credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
+const _process = async (config, query, { commandLineArgs, credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -539,6 +547,9 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
   }
 
   let { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config, allowDelta: true })
+  if (commandLineArgs && commandLineArgs.checkForLoop) {
+    data.checkForLoop = true
+  }
   let queries = query.split('\\n')
 
   try {
@@ -670,9 +681,12 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
     defaultErrorHandler(error)
   }
 
-  const objects = getObjects(config.config.objects)(config.uuid)
+  let objects = getObjects(config.config.objects)(config.uuid)
+  if (testConfig.testModuleName) {
+    objects = getObjects(config.config.objects)(config.getConfigs()[testConfig.testModuleName].uuid)
+  }
   config.beforeQuery({ query: test, isModule: false, objects })
-  config.resetMotivations()
+  // config.resetMotivations()
     try {
       const result = await _process(config, test, { errorHandler, isTest: true })
       result.query = test
@@ -692,10 +706,23 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
       const failed_responses = !matching(result.responses, expected.responses)
       const failed_contexts = !matching(result.contexts, expected.contexts)
       const failed_objects = !matching(actual_objects, expected_objects)
+
+      const pickEm = (objects) => {
+        const picked = {}
+        for (let prop of (testConfig.check || [])) {
+          picked[prop] = objects[prop]
+        }
+        return picked
+      }
+      const expected_checked = sortJson(pickEm(expected.objects.namespaced[0]), { depth: 25 })
+      const actual_checked = sortJson(pickEm(objects), { depth: 25 })
+      const failed_checked = !matching(actual_objects, expected_objects)
+
+      const failed_checks = !matching(actual_objects, expected_objects)
       const actual_config = sortJson(convertToStable(getConfigForTest(config, testConfig)), { depth: 25 })
       const expected_config = sortJson(convertToStable(expected.config), { depth: 25 })
       const failed_config = !matching(actual_config, expected_config)
-      let failed = failed_paraphrases || failed_responses || failed_contexts || failed_objects || failed_config
+      let failed = failed_paraphrases || failed_responses || failed_contexts || failed_objects || failed_config || failed_checked
       if (!failed) {
         if (config.afterTest) {
           failed = config.afterTest({ query: test, expected, actual: result, config })
@@ -703,8 +730,8 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
             return {
               utterance: test,
               errorFromAfterTest: failed,
-              expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, objects: expected_objects, config: expected.config },
-              actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, objects: actual_objects, config: actual_config }
+              expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, checked: expected_checked, objects: expected_objects, config: expected.config },
+              actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, checked: actual_checked, objects: actual_objects, config: actual_config }
             }
           }
         }
@@ -720,8 +747,8 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
       if (failed) {
         return {
           utterance: test,
-          expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, objects: expected_objects, config: expected.config },
-          actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, objects: actual_objects, config: actual_config }
+          expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, checked: expected_checked, objects: expected_objects, config: expected.config },
+          actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, checked: actual_checked, objects: actual_objects, config: actual_config }
         }
       }
     } catch(error) {
@@ -770,7 +797,7 @@ const runTests = async (config, testFile, juicyBits) => {
 const saveTest = async (testFile, config, test, expected, testConfig, saveDeveloper) => {
   config.rebuild()
   const objects = getObjects(config.config.objects)(config.uuid)
-  config.resetMotivations()
+  //config.resetMotivations()
   config.beforeQuery({ query: test, isModule: false, objects })
   console.log(test)
   const result = await _process(config, test, { isTest: true })
@@ -845,6 +872,7 @@ const submitBugToAPI = async (subscription_id, subscription_password, config) =>
     })
 }
 
+/*
 const submitBug = async (subscription_id, subscription_password, config, utterance, retries = 2) => {
   // TODO remove these from the config
   const properties = ['expected_contexts', 'expected_generated']
@@ -881,6 +909,7 @@ const submitBug = async (subscription_id, subscription_password, config, utteran
       throw error
     })
 }
+*/
 
 const defaultErrorHandler = async (error) => {
   if (error.logs) {
@@ -1035,7 +1064,6 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
       if (typeof queryOrExtraConfig === 'string') {
         query = { query }
       }
-      console.log('query', query.query)
       config.config.skipSemantics = skipSemantics
       const transitoryMode = global.transitoryMode
       if (property == 'fragments') {
@@ -1059,6 +1087,11 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
           // TODO pass in the error handler like the other ones
           defaultInnerProcess(config, defaultErrorHandler, results)
         }
+        if (results.contexts.length > 1) {
+          console.log(`query ${query.query}. There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
+        } else {
+          console.log(`query ${query.query}`)
+        }
         global.transitoryMode = transitoryMode
         config.config.skipSemantics = null
         results.query = query.query
@@ -1239,7 +1272,7 @@ const knowledgeModule = async ({
       description: 'Entodicton knowledge module'
     })
 
-    parser.add_argument('-tfn', '--testFileName', { help: 'Override the test file for the module when the tests are being run' })
+    parser.add_argument('-tmn', '--testModuleName', { help: 'When running tests instead of using the current modules tests use the specified modules tests' })
     parser.add_argument('-t', '--test', { action: 'store_true', help: 'Run the tests. Create tests by running with the --query + --save flag' })
     parser.add_argument('-tv', '--testVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. Create tests by running with the --query or --loop with the --save flag' })
     parser.add_argument('-tva', '--testAllVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. All the tests will be run instead of stopping at first failure. Create tests by running with the --query or --loop with the --save flag' })
@@ -1250,11 +1283,12 @@ const knowledgeModule = async ({
     parser.add_argument('-i', '--info', { action: 'store_true', help: 'Print meta-data for the module' })
     parser.add_argument('-v', '--vimdiff', { action: 'store_true', help: 'For failures run vimdiff' })
     parser.add_argument('-g', '--greg', { action: 'store_true', help: 'Set the server to be localhost so I can debug stuff' })
+    parser.add_argument('-cl', '--checkForLoop', { action: 'store_true', help: 'Check for loops in the priorities' })
     parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
     parser.add_argument('-q', '--query', { help: 'Run the specified query' })
     parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
     parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
-    parser.add_argument('-td', '--testDelete', { help: 'Delete the specified query from the tests file.' })
+    parser.add_argument('-dt', '--deleteTest', { help: 'Delete the specified query from the tests file.' })
     parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
     parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
     parser.add_argument('-daa', '--dontAddAssociations', { action: 'store_true', help: 'Do not add associations from the tests.' })
@@ -1293,11 +1327,11 @@ const knowledgeModule = async ({
       return
     }
 
-    if (args.testDelete) {
+    if (args.deleteTest) {
       let tests = JSON.parse(runtime.fs.readFileSync(testConfig.name))
-      tests = tests.filter( (test) => test.query !== args.testDelete );
+      tests = tests.filter( (test) => test.query !== args.deleteTest );
       writeTestFile(testConfig.name, tests)
-      console.log(`Remove the test for "${args.testDelete}"`)
+      console.log(`Remove the test for "${args.deleteTest}"`)
       return
     }
 
@@ -1453,7 +1487,15 @@ const knowledgeModule = async ({
             }
             return
           }
-          runTests(config, args.testFileName ? `${args.testFileName}.test.json` : test, { debug: args.debug, testConfig: testConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
+          let useTestConfig = testConfig
+          if (args.testModuleName) {
+            useTestConfig = config.getConfigs()[args.testModuleName].getTestConfig()
+            useTestConfig.testModuleName = args.testModuleName
+            test = useTestConfig.name
+
+          }
+          // runTests(config, args.testFileName ? `${args.testFileName}.test.json` : test, { debug: args.debug, testConfig: testConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
+          runTests(config, test, { debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
             if (results.length > 0 && args.vimdiff) {
               for (const result of results) {
                 vimdiff(result.expected, result.actual)
@@ -1462,34 +1504,69 @@ const knowledgeModule = async ({
             let newError = false
             if (results.length > 0) {
               let headerShown = false
-              console.log('**************************** ERRORS ************************')
+
+              let hasError = false
               for (const result of results) {
-                console.log('Utterance: ', result.utterance)
                 if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
-                  if (!headerShown) {
-                    console.log('  Failure')
-                  }
-                  console.log('    expected paraphrases', result.expected.paraphrases)
-                  console.log('    actual paraphrases  ', result.actual.paraphrases)
-                  newError = true
-                  headerShown = true
+                  hasError = true
                 }
                 if (JSON.stringify(result.expected.responses) !== JSON.stringify(result.actual.responses)) {
-                  if (!headerShown) {
-                    console.log('  Failure')
+                  hasError = true
+                }
+                if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
+                  hasError = true
+                }
+              }
+
+              if (hasError) {
+                console.log('**************************** ERRORS ************************')
+                for (const result of results) {
+                  console.log('Utterance: ', result.utterance)
+                  if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
+                    if (!headerShown) {
+                      console.log('  Failure')
+                    }
+                    console.log('    expected paraphrases', result.expected.paraphrases)
+                    console.log('    actual paraphrases  ', result.actual.paraphrases)
+                    newError = true
+                    headerShown = true
+                  }
+                  if (JSON.stringify(result.expected.responses) !== JSON.stringify(result.actual.responses)) {
+                    if (!headerShown) {
+                      console.log('  Failure')
+                    }
+                    console.log('    expected responses  ', result.expected.responses)
+                    console.log('    actual responses    ', result.actual.responses)
+                    newError = true
+                    headerShown = true
+                  }
+                  if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
+                    if (!headerShown) {
+                      console.log('  Failure')
+                    }
+                    const widths = [4, 18, 72]
+                    const lines = new Lines(widths)
+                    lines.setElement(1, 1, 'expected checked')
+                    lines.setElement(2, 2, JSON.stringify(result.expected.checked, null, 2))
+                    lines.log()
+                    lines.setElement(1, 1, 'actual checked')
+                    lines.setElement(2, 2, JSON.stringify(result.actual.checked, null, 2))
+                    lines.log()
+                    newError = true
+                    headerShown = true
                   }
-                  console.log('    expected responses  ', result.expected.responses)
-                  console.log('    actual responses    ', result.actual.responses)
-                  newError = true
-                  headerShown = true
                 }
               }
               if (!headerShown) {
-                console.log('There are failures due to things other than paraphrases and response being different. They are not shown because you ran -tv or -tva which only shows difference in paraphrase and results. Usually what I do is -s and do a diff to make sure there are no other problems. If the paraphrases or results were different they would have shown here.')
+                if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
+                  console.log('There are failures due to things other than paraphrases, responses and checked properties being different. They are not shown because you ran -tv or -tva which only shows difference in paraphrase and results. Usually what I do is -s and do a diff to make sure there are no other problems. If the paraphrases or results were different they would have shown here.')
+                }
+              }
+              if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
+                console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
+                // console.log(JSON.stringify(contexts))
+                console.log('**************************** ERRORS ************************')
               }
-              console.log('use -v arg to write files expected.json and actual.json in the current directory for detailed comparison. Or do -s and then git diff the changes.')
-              // console.log(JSON.stringify(contexts))
-              console.log('**************************** ERRORS ************************')
             }
             // const contexts = { failures: results }
             l(n - 1, hasError || newError)
@@ -1540,13 +1617,14 @@ const knowledgeModule = async ({
       }
       config.beforeQuery({ query: args.query, isModule: false, objects })
       try {
-        processResults(_process(config, args.query, { dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
+        processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
       } catch( error ) {
         console.log('Error', error);
       }
     }
   } else {
     config.addAssociationsFromTests(config.tests);
+    config.setTestConfig(testConfig)
     //for (let query in config.tests) {
     //  config.addAssociations(config.tests[query].associations || []);
     //}
@@ -1596,7 +1674,7 @@ module.exports = {
   process: _process,
   where,
   w,
-  submitBug,
+  // submitBug,
   ensureTestFile,
   build,
   processContext,

package/lines.js

@@ -12,6 +12,9 @@ class Lines {
   // will wrap to next line within the column
   setElement (row, column, value) {
     const values = value.toString().split('\n')
+    if (column >= this.widths.length) {
+      throw "Column out of range."
+    }
     const width = this.widths[column]
     let index = 0
     for (value of values) {

package/package.json

@@ -61,6 +61,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.5.0",
+  "version": "7.5.2",
   "license": "ISC"
 }

package/src/config.js

@@ -330,6 +330,17 @@ class Config {
     return `${maybeName}${counter}`
   }
 
+  setTestConfig(testConfig) {
+    if (this.name == 'ui') {
+      console.log('ui setting testConfig')
+    }
+    this.testConfig = testConfig
+  }
+
+  getTestConfig() {
+    return this.testConfig
+  }
+
   defaultConfig () {
     this.config = {
       operators: [], // TODO
@@ -412,11 +423,11 @@ class Config {
   }
 
   getSemantics (logs = []) {
-    return new Semantics(this.config.semantics || [], logs, { km: this.name })
+    return new Semantics(this.config.semantics, logs, { km: this.name })
   }
 
   getGenerators (logs = []) {
-    return new Generators(this.config.generators || [], logs, { km: this.name })
+    return new Generators(this.config.generators, logs, { km: this.name })
   }
 
   warningNotEvaluated (log, value) {
@@ -531,9 +542,23 @@ class Config {
       client.build({ config: this, target: this.name, beforeQuery: () => {}, template, ...options })
     } else {
       // no change
-      this.initInstances.push(instance)
-      this.instances.push(instance)
-      client.processInstance(this, instance)
+      // this.initInstances.push({ ...instance, name: config.name })
+      const isEmpty = ( instance ) => {
+        const properties = [
+          "queries",
+          "resultss",
+          "fragments",
+          "semantics",
+          "associations",
+        ]
+        return !properties.find( (property) => instance[property].length > 0 )
+      }
+      if (!isEmpty(instance)) {
+        instance.name = this.name
+        this.initInstances.push(instance)
+        this.instances.push(instance)
+        client.processInstance(this, instance)
+      }
     }
   }
 
@@ -593,18 +618,21 @@ class Config {
     }
   }
 
-  addHierarchyProperties (properties) {
-    if (typeof properties.child !== 'string') {
+  addHierarchyProperties (edge) {
+    const { child, parent } = edge
+    if (typeof child !== 'string') {
       throw `addHierarchy expected child property to be a string. got ${JSON.stringify(child)}`
     }
-    if (typeof properties.parent !== 'string') {
-      throw `addHierarchy expected parent properties to be a string. got ${JSON.stringify(parent)}`
+    if (typeof parent !== 'string') {
+      throw `addHierarchy expected parent property to be a string. got ${JSON.stringify(parent)}`
     }
-    if (!this.config.hierarchy) {
-      this.config.hierarchy = []
+    if (global.entodictonDebugHierarchy) {
+      if (deepEqual(global.entodictonDebugHierarchy, [child, parent])) {
+        debugger; // debug hierarchy hit
+      }
     }
-    // this.config.hierarchy.push([properties.child, properties.parent])
-    this.config.hierarchy.push(properties)
+    this.config.hierarchy.push(edge)
+    // TODO greg11 this.hierarchy.addEdge(edge)
     this._delta.json.hierarchy.push([child, parent])
   }
 
@@ -615,14 +643,13 @@ class Config {
     if (typeof parent !== 'string') {
       throw `addHierarchy expected parent to be a string. got ${JSON.stringify(parent)}`
     }
-    if (!this.config.hierarchy) {
-      this.config.hierarchy = []
-    }
+
     if (global.entodictonDebugHierarchy) {
       if (deepEqual(global.entodictonDebugHierarchy, [child, parent])) {
         debugger; // debug hierarchy hit
       }
     }
+
     if (this.config.hierarchy.find( (element) => {
       const hc = hierarchyCanonical(element) 
       if (child == hc.child && parent == hc.parent) {
@@ -633,6 +660,7 @@ class Config {
     }
 
     this.config.hierarchy.push([child, parent])
+    // this.hierarchy.addEdge([child, parent])
     this._delta.json.hierarchy.push([child, parent])
   }
 
@@ -953,17 +981,11 @@ class Config {
     if (!config) {
       return
     }
-    config.operators = config.operators || []
-    config.bridges = config.bridges || []
-    config.words = config.words || {}
-    config.generators = config.generators || []
-    config.semantics = config.semantics || []
-
     config.operators = config.operators.filter( (element) => !element.development )
     config.bridges = config.bridges.filter( (element) => !element.development )
     config.generators = config.generators.filter( (element) => !element.development )
     config.semantics = config.semantics.filter( (element) => !element.development )
-    config.hierarchy = (config.hierarchy || []).filter( (element) => !element.development )
+    config.hierarchy = (config.hierarchy).filter( (element) => !element.development )
     for (const word in config.words) {
       const defs = config.words[word] || []
       config.words[word] = defs.filter( (def) => !def.development )
@@ -979,6 +1001,44 @@ class Config {
       throw 'Excepted the config argument to be a hash not a Config object'
     }
 
+    if (config) {
+      const valid = [ 
+        'hierarchy', 
+        'objects', 
+        'bridges', 
+        'operators', 
+        'words', 
+        'priorities', 
+        'associations', 
+        'name', 
+        'version', 
+        'generators', 
+        'semantics', 
+        'floaters', 
+        'debug',
+
+        // TODO Fix these from the test app
+        'implicit',
+        'expected_generated',
+        'expected_results',
+        'skipSemantics',
+        'description',
+        'contexts',
+        'utterances',
+        'flatten',
+      ]
+      helpers.validProps(valid, config, 'config')
+
+      config.operators = config.operators || []
+      config.bridges = config.bridges || []
+      config.words = config.words || {}
+      config.generators = config.generators || []
+      config.semantics = config.semantics || []
+      config.hierarchy = config.hierarchy || []
+      config.associations = config.associations || { negative: [], positive: [] }
+      config.priorities = config.priorities || []
+    }
+
     this.allowDelta = false
     this.resetDelta()
 
@@ -1030,7 +1090,8 @@ class Config {
     normalizeConfig(config)
 
     // set the default server so stuff just works
-    this.server('http://184.67.27.82:3000', '6804954f-e56d-471f-bbb8-08e3c54d9321')
+    // this.server('https://184.67.27.82:3000', '6804954f-e56d-471f-bbb8-08e3c54d9321')
+    this.server('https://thinktelligence.com:3000', '6804954f-e56d-471f-bbb8-08e3c54d9321')
 
     this.defaultConfig()
     this.initializerFn = ({ currentConfig }) => {
@@ -1053,25 +1114,10 @@ class Config {
     if (config) {
       config = _.cloneDeep(config)
       this.config = config
-      if (!this.config.generators) {
-        this.config.generators = []
-      }
-      if (!this.config.semantics) {
-        this.config.semantics = []
-      }
-      if (!this.config.words) {
-        // this.config.words = {}
-      }
-      for (let bridge of (this.config.bridges || [])) {
-        const valid = [ 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generators', 'hierarchy', 'id', 'inverted', 'isA',
-                        'level', 'optional', 'selector', 'semantic',  'uuid', 'words' ]
-        for (let prop of Object.keys(bridge)) {
-          if (!valid.includes(prop)) {
-            if (!prop.endsWith("Bridge")) {
-              throw `Unknown property "${prop}" in the bridge. Valid properties are ${valid}. The bridge is ${JSON.stringify(bridge)}`
-            }
-          }
-        }
+      for (let bridge of this.config.bridges) {
+        const valid = [ 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generators', 'id', 'implicit', 'inverted', 'isA', 'children', 'parents',
+                        'level', 'optional', 'selector', 'semantic',  'words', /Bridge$/, 'localHierarchy' ]
+        helpers.validProps(valid, bridge, 'bridge')
         /*
         if (bridge.generator) {
           this.config.generators.push({ 
@@ -1080,6 +1126,16 @@ class Config {
           })
         }
         */
+        if (bridge.children) {
+          for (let child of bridge.children) {
+            this.addHierarchy(child, bridge.id)
+          }
+        }
+        if (bridge.parents) {
+          for (let parent of bridge.parents) {
+            this.addHierarchy(bridge.id, parent)
+          }
+        }
         if (bridge.isA) {
           for (let parent of bridge.isA) {
             this.addHierarchy(bridge.id, parent)
@@ -1087,6 +1143,9 @@ class Config {
         }
         if (bridge.before) {
           for (let after of bridge.before) {
+            if (typeof after == 'string') {
+              after = [after, 0]
+            }
             this.addPriorities([after, [bridge.id, bridge.level]])
           }
         }
@@ -1151,6 +1210,7 @@ class Config {
         })
       }
     }
+    this.hierarchy = new Digraph(this.config.hierarchy)
     this.initConfig = _.cloneDeep(this.config)
     this.configs.push(new KM({ config: this.config, getCounter: (name) => this.config.getCounter(name), uuid: this._uuid }))
 
@@ -1238,6 +1298,10 @@ class Config {
   }
 
   set api (value) {
+    if (!value.initialize) {
+      throw `Expected the API to have an initialize function for ${this.name}.`
+    }
+
     if (this._api && this._api.multiApi) {
       this._api.add(this, this._api, value)
     } else {
@@ -1350,12 +1414,13 @@ class Config {
     cp.name = this.name
     cp.description = this.description
     cp.tests = this.tests
-    cp.motivations = this.motivations
+    cp.motivations = [...this.motivations]
     cp.isModule = this.isModule
     cp.loadedForTesting = this.loadedForTesting
     cp.initInstances = this.initInstances.slice()
     cp.instances = this.instances.slice()
     cp.configCounter = this.configCounter
+    cp.testConfig = this.testConfig
 
     cp.initConfig = _.cloneDeep(this.initConfig)
     cp.defaultConfig()
@@ -1401,7 +1466,7 @@ class Config {
     this.config.bridges && this.config.bridges.forEach((bridge) => { bridge.uuid = this._uuid })
     this.config.words && setWordsUUIDs(this.config.words, this._uuid)
     this.config.operators && this.config.operators.forEach((operator) => { operator.uuid = this._uuid })
-    const ids = Array.from(new Set((this.config.bridges && this.config.bridges.map((bridge) => bridge.id)) || []))
+    const ids = Array.from(new Set(this.config.bridges.map((bridge) => bridge.id)))
     ids.sort()
     this.config.namespaces = {}
     // if (true || ids.length > 0) {
@@ -1482,7 +1547,8 @@ class Config {
   }
 
   initializeFromConfigs () {
-    this.configs.forEach(({ config, namespace, uuid }) => {
+    // [...this.configs].reverse().forEach(({ config, namespace, uuid }) => {
+    [...this.configs].reverse().forEach(({ config, namespace, uuid }) => {
       /*
       let objects = this.get('objects')
       if (namespace) {
@@ -1617,6 +1683,13 @@ class Config {
       return false
     }
 
+    /* TODO greg11
+    if (!this.hierarchy) {
+      debugBreak()
+      return false
+    }
+    */
+
     for (const key in this.config.words) {
       const values = this.config.words[key]
       if (values.some((word) => (Object.keys(word).includes('uuid') && !word.uuid))) {
@@ -1626,7 +1699,7 @@ class Config {
     }
 
     const kmsUuids = this.configs.map((km) => km.uuid)
-    const bridgesUuids = (this.config.bridges && this.config.bridges.map((bridge) => bridge.uuid).filter((uuid) => uuid)) || []
+    const bridgesUuids = this.config.bridges.map((bridge) => bridge.uuid).filter((uuid) => uuid)
     let result = true
     bridgesUuids.forEach((buuid) => {
       if (!kmsUuids.includes(buuid)) {
@@ -1696,6 +1769,7 @@ class Config {
     this.resetDelta()
     const debug = this.config.debug;
     this.config = _.cloneDeep(this.initConfig)
+    this.hierarchy = new Digraph(this.config.hierarchy)
     if (debug) {
       this.config.debug = debug
     }
@@ -1710,6 +1784,7 @@ class Config {
     }
     this.config.objects.namespaced = {}
     this.resetWasInitialized()
+    this.resetMotivations()
 
     // reorder configs base on load ordering
     {
@@ -1725,8 +1800,17 @@ class Config {
       this.configs = [...oconfigs]
     }
 
+    const addInternals = []
     const inits = []
     const initAfterApis = []
+    if (false && this.config.hierarchy.find( (pair) => JSON.stringify(pair) === JSON.stringify(["equipable2","property"]))) {
+      debugger // happened
+    }
+    if (false && this.name == 'countable') {
+      debugger // in rebuild
+    }
+    const reverseIt = true
+    const interleaved = true
     this.configs.forEach((km) => {
       const namespace = km.namespace
       this.config.objects.namespaced[km._uuid] = {}
@@ -1755,10 +1839,16 @@ class Config {
       config.wasInitialized = false
       // TODO change name of config: to baseConfig:
       const kmFn = (name) => this.getConfig(name)
-      const args = { isModule, addWord: aw, km: kmFn, config, baseConfig: this, currentConfig: config, uuid: config._uuid, objects: namespacedObjects, namespace, api: config.api }
+      // const hierarchy = new Digraph((config.config || {}).hierarchy)
+      const args = { isModule, addWord: aw, km: kmFn, hierarchy: this.hierarchy, config, baseConfig: this, currentConfig: config, uuid: config._uuid, objects: namespacedObjects, namespace, api: config.api }
       config.initializerFn(args)
       if (config.initAfterApi) {
-        initAfterApis.push({ config, args })
+        // reverse the list
+        initAfterApis.unshift({ config, args })
+      } else {
+        if (interleaved) {
+          initAfterApis.unshift(null)
+        }
       }
       // greg
       if (config._api) {
@@ -1766,27 +1856,112 @@ class Config {
           // reverse the list
           inits.unshift( () => config._api.initialize({ config: this, km: kmFn, api: config._api }) )
           // config._api.initialize({ config, api: config._api })
+        } else {
+          if (interleaved) {
+            inits.unshift(null)
+          }
         }
         config._api.objects = namespacedObjects
         config._api.config = () => this
         config._api.uuid = config._uuid
+      } else {
+        if (interleaved) {
+          inits.unshift(null)
+        }
       }
       config.setUUIDs()
       config.applyNamespace(config.config, namespace, config.uuid)
       if (!isSelf) {
-        this.addInternal(config, true, false, false, true)
+        if (!reverseIt) {
+          addInternals.push(config)
+        } else {
+          addInternals.unshift(config)
+        }
+        // this.addInternal(config, true, false, false, true)
+      } else {
+        if (interleaved) {
+          addInternals.unshift(null)
+        }
       }
       km.valid()
     })
-    // console.log('inits from config', inits)
-    for (const init of inits) {
-      init()
+
+    if (addInternals.length !== inits.length || addInternals.length !== initAfterApis.length) {
+      debugger // bug
     }
-    for (let init of initAfterApis) {
-      // init.args.isAfterApi = true
-      init.config.initializerFn({ ...init.args, kms: this.getConfigs(), isAfterApi: true })
+
+    const generators = this.config.generators
+    const semantics = this.config.semantics
+    if (reverseIt) {
+      this.config.generators = []
+      this.config.semantics = []
     }
-    this.instances.forEach((instance) => client.processInstance(this, instance))
+
+    if (!interleaved) {
+      for (const config of addInternals) {
+        if (!reverseIt) {
+          this.addInternal(config, true, false, false, true)
+        } else {
+          this.addInternalR(config, true, false, false, true)
+        }
+      }
+      // console.log('inits from config', inits)
+      for (const init of inits) {
+        init()
+      }
+      for (let init of initAfterApis) {
+        // init.args.isAfterApi = true
+        init.config.initializerFn({ ...init.args, kms: this.getConfigs(), isAfterApi: true })
+      }
+      this.instances.forEach((instance) => client.processInstance(this, instance))
+    } else {
+      const base = {
+        operators: this.config.operators,
+        bridges: this.config.bridges,
+        hierarchy: this.config.hierarchy,
+        priorities: this.config.priorities,
+        associations: this.config.associations,
+        words: this.config.words
+      }
+
+      this.config.operators = []
+      this.config.bridges = []
+      this.config.hierarchy = []
+      this.config.priorities = []
+      this.config.associations = { positive: [], negative: [] }
+      this.config.words = {}
+
+      for (let i = 0; i < addInternals.length; ++i) {
+        let name;
+        if (addInternals[i]) {
+          this.addInternalR(addInternals[i], true, false, false, true)
+          name = addInternals[i].name
+        } else{
+          this.addInternalR(base, true, false, false, true)
+          name = this.name
+        }
+        // console.log('name -------------', name)
+        if (inits[i]) {
+          inits[i]()
+        }
+        if (initAfterApis[i]) {
+          const init = initAfterApis[i]
+          init.config.initializerFn({ ...init.args, kms: this.getConfigs(), isAfterApi: true })
+        }
+        const instance = this.instances.find((instance) => instance.name == name)
+        if (instance) {
+          client.processInstance(this, instance)
+        }
+        this.hierarchy.edges = this.config.hierarchy
+      }
+      // this.instances.forEach((instance) => client.processInstance(this, instance))
+    }
+
+    if (reverseIt) {
+      this.config.generators = generators.concat(this.config.generators)
+      this.config.semantics = semantics.concat(this.config.semantics)
+    }
+    this.hierarchy.edges = this.config.hierarchy
     this.valid()
     this.checkBridges()
   }
@@ -1952,8 +2127,7 @@ class Config {
     }
 
     if (config.hierarchy) {
-      let hierarchy = config.hierarchy
-      hierarchy = hierarchy.map((h) => {
+      helpers.mapInPlace(config.hierarchy, (h) => {
         if (Array.isArray(h)) {
           return h.map((id) => toNS(id))
         } else {
@@ -1962,7 +2136,6 @@ class Config {
           return Object.assign({}, h, { child: toNS(h.child), parent: toNS(h.parent) })
         }
       })
-      config.hierarchy = hierarchy
     }
 
     if (config.priorities) {
@@ -2155,6 +2328,13 @@ class Config {
     this.configs.forEach((km) => {
       this.instances = (km._config.instances || this.initInstances.slice()).concat(this.instances)
     })
+    let noDups = []
+    for (let instance of this.instances) {
+      if (!noDups.find( (existing) => existing.name == instance.name )) {
+        noDups.push(instance)
+      }
+    }
+    this.instances = noDups
 
     this.rebuild()
     this.valid()
@@ -2272,6 +2452,128 @@ class Config {
     }
     return this
   }
+
+  // assumes this is called in reverse order
+  addInternalR (more, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false) {
+    if (more instanceof Config) {
+      more.initialize({ force: false })
+      if (useOldVersion) {
+        more = more.config
+      } else {
+        // more = more.initConfig
+        more = _.cloneDeep(more.initConfig)
+      }
+    }
+    for (const key of Object.keys(more)) {
+      const value = more[key]
+      // TODO remove name and description on the config bag
+      const noOverwrite = ['name', 'namespaced']
+      if (!this.config[key]) {
+        if (allowNameToBeNull) {
+          if (noOverwrite.includes(key)) {
+            continue
+          }
+        } else if (this.config[key] && noOverwrite.includes(key)) {
+          continue
+        }
+        this.config[key] = value
+        continue
+      }
+      if (key === 'words') {
+        const configWords = this.config.words
+        const moreWords = more.words
+        for (const word of Object.keys(moreWords)) {
+          if (!configWords[word]) {
+            configWords[word] = []
+          }
+          // configWords[word] = configWords[word].concat(moreWords[word])
+          configWords[word] = moreWords[word].concat(configWords[word])
+        }
+      } else if (key === 'name') {
+        /*
+        if (this.config[key] === '') {
+          this.config[key] = more[key]
+        }
+        */
+      } else if (key === 'namespaces') {
+        if (includeNamespaces) {
+          Object.assign(this.config[key], more[key])
+        }
+      } else if (key === 'debug') {
+        this.config[key] = this.config[key] || more[key]
+      } else if (key === 'description') {
+        // this.config[key] += ' ' + more[key].trim()
+        this.config[key] = more[key].trim() + ' ' + this.config[key]
+      } else if (key === 'objects') {
+        if (!skipObjects) {
+          // namespaced is special
+          const namespaced = this.config.objects.namespaced
+          Object.assign(this.config[key], more[key])
+          this.config.objects.namespaced = namespaced
+        }
+      } else if (key === 'associations') {
+        const configAssociations = this.config.associations
+        const moreAssociations = more.associations
+        if (moreAssociations.positive) {
+          // configAssociations.positive = configAssociations.positive.concat(moreAssociations.positive)
+          configAssociations.positive = moreAssociations.positive.concat(configAssociations.positive)
+        }
+        if (moreAssociations.negative) {
+          // configAssociations.negative = configAssociations.negative.concat(moreAssociations.negative)
+          configAssociations.negative = moreAssociations.negative.concat(configAssociations.negative)
+        }
+      } else if (Array.isArray(value)) {
+        // handle allowDups
+        if (key == 'operators') {
+          // TODO what about other props
+          const isDup = (op1, op2) => op1.pattern == op2.pattern
+          for (const newOne of more[key]) {
+            for (let iOldOne = 0; iOldOne < this.config[key].length; ++iOldOne) {
+              const oldOne = this.config[key][iOldOne];
+              if (isDup(newOne, oldOne)) {
+                if (oldOne.allowDups) {
+                  // the old one takes precedence to match what would happen during the original load
+                  debugger
+                  this.config[key].splice(iOldOne, 1)
+                  break;
+                }
+              }
+            }
+          }
+        }
+        if (key == 'bridges') {
+          // TODO what about other props
+          const idDup = (b1, b2) => b1.id == b2.id && b1.level == b2.level && b1.bridge == b2.bridge
+          for (const newOne of more[key]) {
+            for (let iOldOne = 0; iOldOne < this.config[key].length; ++iOldOne) {
+              const oldOne = this.config[key][iOldOne];
+              if (newOne.id == oldOne.id) {
+                if (oldOne.allowDups) {
+                  // the old one takes precedence to match what would happen during the original load
+                  this.config[key].splice(iOldOne, 1)
+                  break;
+                }
+              }
+            }
+          }
+        }
+        // console.log('key', key, 'XXX')
+        // console.log('more', JSON.stringify(more, null, 2))
+        // this.config[key] = this.config[key].concat(more[key])
+        if (key == '2enerators') {
+          debugger
+        }
+        // this.config[key] = this.config[key].concat(more[key])
+        this.config[key] = more[key].concat(this.config[key])
+      } else {
+        if (!(key in this.config)) {
+          throw `Unexpected property in config ${key}`
+        }
+        this.config[key] = more[key]
+      }
+    }
+    return this
+  }
 }
 
 module.exports = {

package/src/digraph.js

@@ -9,21 +9,42 @@ const toA = (edge) => {
 class Digraph {
   // edges maybe either [child, parent] or { child, parent }
   constructor (edges = []) {
-    this.edges = edges
+    // dont make a copy of edges. this is shared and that breaks stuff. TODO fix this
+    this._edges = edges
   }
 
-  /*
-  edges () {
-    return this.edges
+  addEdges(edges) {
+    for (let edge of edges) {
+      this.addEdge(edge)
+    }
+  }
+
+  addEdge(edge) {
+    edge = toA(edge)
+    this._edges.push(edge)
+  }
+
+  get edges() {
+    return this._edges
+  }
+
+  set edges(edges) {
+    this._edges = edges
+  }
+
+/*
+  set edges(edges) {
+    this._edges = edges.map( toA )
   }
 */
+
   acdcs (s, from, to) {
     const todo = [s]
     const seen = new Set([s])
     const acdcs = new Set([])
     while (todo.length > 0) {
       const n = todo.pop()
-      this.edges.forEach((e) => {
+      this._edges.forEach((e) => {
         e = toA(e)
         if (e[from] === n) {
           acdcs.add(e[to])
@@ -84,12 +105,12 @@ class Digraph {
   }
 
   add (child, parent) {
-    this.edges.push([child, parent])
+    this._edges.push([child, parent])
   }
 
   addList (l) {
     for (let i = 1; i < l.length; ++i) {
-      this.edges.push([l[i - 1], l[i]])
+      this._edges.push([l[i - 1], l[i]])
     }
   }
 

package/src/generators.js

@@ -195,10 +195,12 @@ class Generators {
           } catch( e ) {
             // the next if handle this
             generated = null
+            e.retryCall = () => generator.apply(args, objects, g, args.gs, context, hierarchy, config, response, log)
+            const help = 'The error has a retryCall property that will recall the function that failed.'
             if (e.stack && e.message) {
-              errorMessage = `Error applying generator '${generator.notes}'. Error is ${e.toString()} stack is ${e.stack}. Generator is ${generator.toString()}`
+              errorMessage = `Error applying generator '${generator.notes}'. Error is ${e.toString()} stack is ${e.stack}. Generator is ${generator.toString()}. ${help}`
             } else if (e.error) {
-              errorMessage = `Error applying generator '${generator.notes}'. Error is ${e.error.join()}. Generator is ${generator.toString()}`
+              errorMessage = `Error applying generator '${generator.notes}'. Error is ${e.error.join()}. Generator is ${generator.toString()}. ${help}`
             } else {
 							errorMessage = e.toString()
 						}

package/src/helpers.js

@@ -217,4 +217,42 @@ const sortJson = (json) => {
   return json
 }
 
-module.exports = { args, safeEquals, appendNoDups, hashIndexesGet, hashIndexesSet, translationMapping, normalizeGenerator, normalizeSemantic, isArray, isObject, isCompound, InitCalls, hashCode, sortJson }
+const validProps = (valids, object, type) => {
+  for (let prop of Object.keys(object)) {
+    let okay = false
+    for (valid of valids) {
+      if (prop.match(valid)) {
+        okay = true
+        break
+      }
+    }
+    if (!okay) {
+      throw `Unknown property "${prop}" in the ${type}. Valid properties are ${valids}. The ${type} is ${JSON.stringify(object)}`
+    }
+  }
+}
+
+const mapInPlace = (list, fn) => {
+  for (let i = 0; i < list.length; ++i) {
+    list[i] =fn(list[i])
+  }
+}
+
+module.exports = { 
+  mapInPlace,
+  validProps, 
+  args, 
+  safeEquals, 
+  appendNoDups, 
+  hashIndexesGet, 
+  hashIndexesSet, 
+  translationMapping, 
+  normalizeGenerator, 
+  normalizeSemantic, 
+  isArray, 
+  isObject, 
+  isCompound, 
+  InitCalls, 
+  hashCode, 
+  sortJson 
+}

package/src/semantics.js

@@ -189,10 +189,12 @@ class Semantics {
         } catch( e ) {
           contextPrime = null
           let errorMessage
+          e.retryCall = () => semantic.apply(args, context, s, log, options)
+          const help = 'The error has a retryCall property that will recall the function that failed.'
           if (e.stack && e.message) {
-            errorMessage = `Error applying semantics '${semantic.notes}'. Error is ${e.toString()} stack is ${e.stack}. Semantic is ${semantic.toString()}`
+            errorMessage = `Error applying semantics '${semantic.notes}'. Error is ${e.toString()} stack is ${e.stack}. Semantic is ${semantic.toString()}. ${help}`
           } else if (e.error) {
-            errorMessage = `Error applying semantics '${semantic.notes}'. Error is ${e.error.join()}. Semantic is ${semantic.toString()}`
+            errorMessage = `Error applying semantics '${semantic.notes}'. Error is ${e.error.join()}. Semantic is ${semantic.toString()}. ${help}`
           } else {
  						errorMessage = e.toString();
 					}