theprogrammablemind 7.5.8-beta.73 → 7.5.8-beta.76

package/client.js

@@ -10,12 +10,12 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, InitCalls, updateQueries } = require('./src/helpers')
+const { appendNoDups, InitCalls, updateQueries, safeNoDups } = require('./src/helpers')
 const runtime = require('./runtime')
 const sortJson = runtime.sortJson
 
-const getConfig_getObjectCheck = (testConfig) => {
-  return (testConfig.checks && testConfig.checks.objects) || testConfig.check || []
+const getConfig_getObjectsCheck = (testConfig) => {
+  return (testConfig.checks && testConfig.checks.objects) || []
 }
 
 const getConfig_getContextCheck = (testConfig) => {
@@ -26,6 +26,10 @@ const pickContext = (testConfig) => (context) => {
   return project(context, getConfig_getContextCheck(testConfig))
 }
 
+const pickObjects = (testConfig, objects) => {
+  return project(objects, getConfig_getObjectsCheck(testConfig))
+}
+
 const getAsk = (config) => (uuid) => (asks) => {
   for (let ask of asks) {
     config.addSemantic({
@@ -64,6 +68,12 @@ const getAsk = (config) => (uuid) => (asks) => {
   })
 }
 
+const sameJSON = (json1, json2) => {
+  const sjson1 = sortJson(json1, { depth: 25 })
+  const sjson2 = sortJson(json2, { depth: 25 })
+  return JSON.stringify(sjson1) == JSON.stringify(sjson2)
+}
+
 const vimdiff = (actualJSON, expectedJSON) => {
   const path = '.'
   const actual = sortJson(actualJSON, { depth: 25 })
@@ -424,7 +434,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
           if (isInstance) {
             console.log('error', e.error)
           }
-          contextPrime = semantics.apply(args, { marker: 'error', context, reason: e.reason })
+          contextPrime = semantics.apply(args, { marker: 'error', context, reason: e.reason, error: e.stack })
         }
       }
       if (contextPrime.controlRemove) {
@@ -570,10 +580,22 @@ const setupProcessB = ({ config, initializer, allowDelta=false } = {}) => {
   }
 }
 
-// instance template
+// instance template loadTemplate
 const loadInstance = (config, instance) => {
   const transitoryMode = global.transitoryMode
   global.transitoryMode = false
+
+  if (instance && (instance.associations || instance.learned_contextual_priorities)) {
+    if (!config.config.retrain) {
+      if (instance.associations) {
+        config.addAssociations(instance.associations)
+      }
+      if (instance.learned_contextual_priorities && instance.learned_contextual_priorities.length > 0) {
+        config.addContextualPriorities(instance.learned_contextual_priorities)
+      }
+    }
+  }
+
   const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
   // for (const results of (instance.resultss || [])) {
   for (const i in (instance.resultss || [])) {
@@ -583,7 +605,13 @@ const loadInstance = (config, instance) => {
       // config.addInternal(config.template.queries[i], { handleCalculatedProps: true } )
       config.addInternal(instance.template.queries[i], { addFirst: true, handleCalculatedProps: true } )
     } else {
+      if (results.skipSemantics) {
+        config.config.skipSemantics = results.skipSemantics
+      }
       processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}`, instance: instance.queries[i] })
+      if (results.skipSemantics) {
+        config.config.skipSemantics = null
+      }
     }
   }
   global.transitoryMode = transitoryMode
@@ -670,6 +698,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
                                   processContextsB({ isTest, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         response.associations = json.associations
+        response.learned_contextual_priorities = json.learned_contextual_priorities
         response.hierarchy = json.hierarchy
         response.load_cache_time += json.load_cache_time
         appendNoDups(response.logs, json.logs)
@@ -797,33 +826,13 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug }) =
       const pickedExpectedContexts = expected.contexts.map(pickContext(testConfig))
       const failedCheckedContexts = !matching(pickedResultContexts, pickedExpectedContexts)
 
-      const pickEm = (getObjects) => {
-        const picked = {}
-        for (let prop of getConfig_getObjectCheck(testConfig)) {
-          if (prop.km) {
-            c = config.getConfig(prop.km)
-            o = getObjects(prop.km)
-            picked[prop.km] = {}
-            for (let p of getConfig_getObjectCheck(c.testConfig)) {
-              if (p.km) {
-                continue
-              }
-              picked[p] = o[p]
-            }
-          } else {
-            picked[prop] = getObjects(testConfigName)[prop]
-          }
-        }
-        return picked
-      }
       const expectedGetObjects = (name) => {
         if (!name) {
           name = config.name
         }
         return expected.objects.namespaced[expected.objects.nameToUUID[name]]
       }
-      sortJson(pickEm(expectedGetObjects), { depth: 25 })
-      const expected_checked = sortJson(pickEm(expectedGetObjects), { depth: 25 })
+      const expected_checked = sortJson(pickObjects(testConfig, objects), { depth: 25 })
       const actualGetObjects = (name) => {
         if (!name) {
           name = config.name
@@ -831,7 +840,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug }) =
         const km = config.configs.find( (km) => km.name == name )
         return config.config.objects.namespaced[km.uuid]
       }
-      const actual_checked = sortJson(pickEm(actualGetObjects), { depth: 25 })
+      const actual_checked = sortJson(pickObjects(testConfig, actualGetObjects(testConfigName)), { depth: 25 })
       const failed_checked = !matching(actual_objects, expected_objects)
 
       const failed_checks = !matching(actual_objects, expected_objects)
@@ -1155,7 +1164,8 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
       console.log(`    ${JSON.stringify(inputss)} reason: ${reason}`)
     }
   }
-  const picked = pickEm()
+  const objects = config.get('objects').namespaced[config.uuid]
+  const picked = sortJson(pickObjects(config.testConfig, objects), { depth: 25 })
   if (!_.isEmpty(picked)) {
     console.log('--- Object showing only the checked values ---')
     console.log(JSON.stringify(picked, null, 2))
@@ -1207,13 +1217,14 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
   }
 }
 
-// loadTemplate BuiltTemplate
+// builtTemplate saveInstance
 const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
   const accumulators = {
     resultss: [],
     fragments: [],
     semantics: [],
     associations: [],
+    learned_contextual_priorities: [],
   }
   const looper = async (queries) => {
     if (queries.length === 0) {
@@ -1241,7 +1252,6 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
           }
         }
       }
-
       try {
         const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
         if (config.config.debug) {
@@ -1258,10 +1268,12 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
         global.transitoryMode = transitoryMode
         config.config.skipSemantics = null
         results.query = query.query
+        results.skipSemantics = skipSemantics
         results.development = query.development
         results.key = { query: query.query, hierarchy }
         accumulators[property].push(results)
         accumulators.associations = accumulators.associations.concat(results.associations)
+        accumulators.learned_contextual_priorities = accumulators.learned_contextual_priorities.concat(results.learned_contextual_priorities)
         await looper(queries)
       } catch(e) {
         const error = { errors: [e], query: query.query };
@@ -1306,6 +1318,7 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
             delete result.version
             result.hierarchy.sort()
             stabilizeAssociations(result.associations)
+            result.learned_contextual_priorities = safeNoDups(result.learned_contextual_priorities)
           }
         }
       }
@@ -1326,14 +1339,14 @@ const rebuildTemplate = async ({ config, target, template, errorHandler = defaul
 
   const toProperties = (queryStringOrProperties) => {
     if (typeof queryStringOrProperties == 'string') {
-      return { query: queryStringOrProperties}
+      return { query: queryStringOrProperties }
     } else {
       return queryStringOrProperties
     }
   }
   let todo = []
-  todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false } }))
-  todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false } }))
+  todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
+  todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false || query.skipSemantics} }))
   todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
   todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
   await looper(Object.assign([], todo))
@@ -1439,7 +1452,7 @@ const knowledgeModule = async ({
     parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
     parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
     parser.add_argument('-daa', '--dontAddAssociations', { action: 'store_true', help: 'Do not add associations from the tests.' })
-    parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a == associations j == JSON sent to server. for example --print wb' })
+    parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a == associations j == JSON sent to server x == contextual priorities. for example --print wb' })
     parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
     parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
     parser.add_argument('-dl', '--debugLoops', { action: 'store_true', help: 'When running with the --debugLoops flag the logs calls to semantics and generators will be immediately written to the console '})
@@ -1552,26 +1565,6 @@ const knowledgeModule = async ({
 
     config.config.debugIncludeConvolutions = args.debugIncludeConvolutions || process.argv.includes('--debugIncludeConvolutions') || process.argv.includes('-dic') 
 
-    if (template) {
-      const needsRebuild = config.needsRebuild(template.template, template.instance, options)
-      if (needsRebuild) {
-        console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
-
-      }
-      config.load(template.template, template.instance, { rebuild: needsRebuild })
-      if (config.name == 'ui') {
-        console.log('config.COUNTER', config.COUNTER)
-        debugger
-      }
-      if (needsRebuild) {
-        return
-      }
-    }
-
-    if (!args.save && !args.rebuildTemplate && !args.dontAddAssociations) {
-      config.addAssociationsFromTests(config.tests);
-    }
-
     let configPrinted = false
     const printConfig = () => {
       if (configPrinted) {
@@ -1636,6 +1629,11 @@ const knowledgeModule = async ({
             console.log(JSON.stringify(priority))
           }
         }
+        if (args.print.includes('x')) {
+          for (let contextual_priority of config.config.contextual_priorities) {
+            console.log(JSON.stringify(contextual_priority))
+          }
+        }
         if (args.print.includes('h')) {
           for (let edge of config.config.hierarchy) {
             console.log(JSON.stringify(edge))
@@ -1663,6 +1661,24 @@ const knowledgeModule = async ({
       }
     }
 
+    if (template) {
+      const needsRebuild = config.needsRebuild(template.template, template.instance, options)
+      if (needsRebuild) {
+        console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
+        options.rebuild = true
+        config.config.rebuild = true
+      }
+      config.load(template.template, template.instance, { rebuild: needsRebuild })
+      printConfig()
+      if (needsRebuild) {
+        return
+      }
+    }
+
+    if (!args.save && !args.rebuildTemplate && !args.dontAddAssociations) {
+      config.addAssociationsFromTests(config.tests);
+    }
+
     if (args.retrain) {
       config.config.retrain = true
     }
@@ -1718,7 +1734,7 @@ const knowledgeModule = async ({
                 if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
                   hasError = true
                 }
-                if (JSON.stringify(result.expected.checkedContexts) !== JSON.stringify(result.actual.checkedContexts)) {
+                if (!sameJSON(result.expected.checkedContexts, result.actual.checkedContexts)) {
                   hasError = true
                 }
               }
@@ -1784,7 +1800,7 @@ const knowledgeModule = async ({
                     newError = true
                     headerShown = true
                   }
-                  if (JSON.stringify(result.expected.checkedContexts) !== JSON.stringify(result.actual.checkedContexts)) {
+                  if (!sameJSON(result.expected.checkedContexts, result.actual.checkedContexts)) {
                     if (!headerShown) {
                       console.log('  Failure')
                     }

package/package.json

@@ -64,6 +64,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.5.8-beta.73",
+  "version": "7.5.8-beta.76",
   "license": "ISC"
 }

package/src/config.js

@@ -23,11 +23,13 @@ const indent = (string, indent) => {
 }
 
 const config_toServer = (config) => {
+  /*
   if (config.contextual_priorities) {
     config.contextual_priorities = config.contextual_priorities.map((cp) => {
       return [cp.context, cp.choose]
     })
   }
+  */
 }
 
 const debugPriority = (priority) => {
@@ -48,7 +50,7 @@ const debugAssociation = (association) => {
 
 const debugContextualPriority = (contextual_priority) => {
   if (global.entodictonDebugContextualPriority) {
-    if (helpers.safeEquals(entodictonDebugContextualPriority, contextual_priorities)) {
+    if (helpers.safeEquals(entodictonDebugContextualPriority, contextual_priority)) {
       debugger; // debug hierarchy hit
     }
   }
@@ -79,6 +81,9 @@ const debugOperator = (operator) => {
 }
 
 const debugConfigProps = (config) => {
+  if (!config) {
+    return
+  }
   const checkProps = [
     { property: 'priorities', check: (v) => debugPriority(v) },
     { property: 'association', check: (v) => debugAssociation(v) },
@@ -149,20 +154,6 @@ const setupInitializerFNArgs = (config, args) => {
   }
 }
 
-const contextual_priorities_toServer = (cp) => {
-  if (cp.context && cp.choose) {
-    return [cp.context, cp.choose]
-  }
-  return cp
-}
-
-const contextual_priorities_toClient = (cp) => {
-  if (cp.context && cp.choose) {
-    return cp
-  }
-  return { context: cp[0], choose: cp[1] }
-}
-
 const operatorKey_valid = (key) => {
   if (
         !_.isArray(key) ||
@@ -254,7 +245,7 @@ const handleBridgeProps = (config, bridge, addFirst) => {
           if (typeof after == 'string') {
             after = [after, 0]
           }
-          config.addPriorities([[bridge.id, bridge.level], after])
+          config.addPriorities({ "context": [[bridge.id, bridge.level], after], "choose": [0] })
         }
       }
       if (bridge.after) {
@@ -262,7 +253,7 @@ const handleBridgeProps = (config, bridge, addFirst) => {
           if (typeof before == 'string') {
             before = [before, 0]
           }
-          config.addPriorities([before, [bridge.id, bridge.level]])
+          config.addPriorities({ "context": [before, [bridge.id, bridge.level]], "choose": [0] })
         }
       }
       if (bridge.words) {
@@ -407,6 +398,7 @@ if (runtime.process.env.DEBUG_BRIDGE) {
   global.entodictonDebugBridge = runtime.process.env.DEBUG_BRIDGE.split('/')
   if (global.entodictonDebugBridge.length !== 2) {
     console.log('Expected DEBUG_BRIDGE to be of the form "id/level"');
+    process.exit(-1)
   }
   global.entodictonDebugBridge[1] = parseInt(global.entodictonDebugBridge[1])
 }
@@ -762,6 +754,7 @@ class Config {
       addArgs: (...args) => this.addArgs(...args),
       getBridge: (...args) => this.getBridge(...args),
       fragment: (...args) => this.fragment(...args),
+      exists: (...args) => this.exists(...args),
       addAPI: (...args) => this.addAPI(...args),
     }
   }
@@ -1060,9 +1053,16 @@ class Config {
     this.validifyTemplate(template)
     instance.template = template
     this.logs.push(`loading template for ${this.name}`)
-    if (instance && instance.associations && !options.rebuild) {
-      this.addAssociations(instance.associations)
+    /*
+    if (instance && (instance.associations || instance.learned_contextual_priorities) && !options.rebuild) {
+      if (instance.associations) {
+        this.addAssociations(instance.associations)
+      }
+      if (instance.learned_contextual_priorities && instance.learned_contextual_priorities.length > 0) {
+        this.addContextualPriorities(instance.learned_contextual_priorities)
+      }
     }
+    */
     if (options.rebuild) {
       // TODO fix beforeQuery
       template = { fragments: [], queries: [], ...template }
@@ -1105,13 +1105,18 @@ class Config {
     }
   }
 
-
   addAssociations (associations) {
     for (let association of associations) {
       this.addAssociation(association)
     }
   }
 
+  addContextualPriorities (cps) {
+    for (let cp of cps) {
+      this.addContextualPriority(cp)
+    }
+  }
+
   debugConfig() {
   }
 
@@ -1146,8 +1151,7 @@ class Config {
     debugContextualPriority(contextual_priority)
     contextual_priority_valid(contextual_priority)
     this.config.contextual_priorities.push(contextual_priority)
-    const cpServer = contextual_priorities_toServer(contextual_priority)
-    this._delta.json.contextual_priorities.push({ action: 'add', contextual_priority: cpServer })
+    this._delta.json.contextual_priorities.push({ action: 'add', contextual_priority: contextual_priority })
   }
 
   addHierarchy (child, parent) {
@@ -1212,7 +1216,8 @@ class Config {
  
     debugBridge(bridge)
     if (bridge.allowDups) {
-      if (bridges.find( (b) => b.id == bridge.id && b.level == bridge.level && b.bridge == bridge.bridge )) {
+      // if (bridges.find( (b) => b.id == bridge.id && b.level == bridge.level && b.bridge == bridge.bridge )) {
+      if (bridges.find( (b) => b.id == bridge.id && b.level == bridge.level)) {
         return;
       }
     }
@@ -1641,6 +1646,7 @@ class Config {
     }
     this.get('objects').namespaced[this._uuid] = {}
     this.valid()
+    debugConfigProps(this.config)
   }
 
   addArgs(moreArgs) {
@@ -2535,9 +2541,7 @@ class Config {
     if (config.priorities) {
       let priorities = config.priorities
       priorities = priorities.map((p) => {
-        return p.map((id) => {
-          return [toNS(id[0]), id[1]]
-        })
+        return { ...p, context: p.context.map( (id) => [toNS(id[0]), id[1]] )}
       })
       config.priorities = priorities
     }
@@ -2545,12 +2549,13 @@ class Config {
     if (config.contextual_priorities) {
       let contextual_priorities = config.contextual_priorities
       contextual_priorities = contextual_priorities.map((cp) => {
-        const { context, choose } = cp
+        const { context, choose, ordered } = cp
         return {
           context: context.map((id) => {
             return [toNS(id[0]), id[1]]
           }),
-          choose
+          choose,
+          ordered
         }
       })
       config.contextual_priorities = contextual_priorities

package/src/helpers.js

@@ -43,6 +43,16 @@ const appendNoDups = (l1, l2) => {
   }
 }
 
+const safeNoDups = (list) => {
+  noDups = []
+  for (const element of list) {
+    if (!noDups.find((e) => safeEquals(e, element))) {
+      noDups.push(element)
+    }
+  }
+  return noDups
+}
+
 const safeEquals = (v1, v2) => {
   if (typeof v1 !== typeof v2) {
     return false
@@ -331,6 +341,7 @@ module.exports = {
   mapInPlace,
   validProps, 
   args, 
+  safeNoDups,
   safeEquals, 
   appendNoDups, 
   hashIndexesGet, 

package/src/project.js

@@ -5,6 +5,12 @@ const project = (object, filter) => {
   }
 
   let projection = {}
+  const set = (property, value) => {
+    if (value === null || value === undefined) {
+      return
+    }
+    projection[property] = value
+  }
   if (Array.isArray(filter)) {
     if (Array.isArray(object)) {
       return object.map( element => project(element, filter) )
@@ -15,7 +21,7 @@ const project = (object, filter) => {
             for (const propertyList in properties.propertyLists) {
               if (object[propertyList]) {
                 for (const property of object[propertyList]) {
-                  projection[property] = project(object[property], properties.propertyLists[propertyList])
+                  set(property, project(object[property], properties.propertyLists[propertyList]))
                 }
               }
             }
@@ -24,16 +30,32 @@ const project = (object, filter) => {
             for (const listProperty in properties.valueLists) {
               const old = object[listProperty]
               if (Array.isArray(old)) {
-                projection[listProperty] = old.map((element) => project(element, properties.valueLists[listProperty]))
+                set(listProperty, old.map((element) => project(element, properties.valueLists[listProperty])))
               } else {
-                projection[listProperty] = project(old, properties.valueLists[listProperty])
+                set(listProperty, project(old, properties.valueLists[listProperty]))
               }
             }
           }
           if (properties.properties) {
             for (const property in properties.properties) {
               const old = object[property]
-              projection[property] = project(old, properties.properties[property])
+              set(property, project(old, properties.properties[property]))
+            }
+          }
+          if (properties.property) {
+            const property = properties.property
+            if (properties.isPropertyList) {
+              debugger
+              if (!Array.isArray(object[property])) {
+                return projection
+              }
+              for (const propertyRef of object[property]) {
+                const old = object[propertyRef]
+                set(propertyRef, project(old, properties.filter))
+              }
+            } else {
+              const old = object[property]
+              set(property, project(old, properties.filter))
             }
           }
         } else {
@@ -41,14 +63,14 @@ const project = (object, filter) => {
             properties = [properties]
           }
           for (const property of properties) {
-            projection[property] = object[property]
+            set(property, object[property])
           }
         }
       }
     }
   } else if (typeof filter == 'object') {
     for (const property of Object.keys(filter)) {
-      projection[property] = project(object[property], filter[property])
+      set(property, project(object[property], filter[property]))
     }
   }
   return projection