theprogrammablemind_4wp 9.3.0-beta.9 → 9.4.0

package/client.js

@@ -4,6 +4,7 @@ const { Config } = require('./src/config')
 const { loadInstance, setupArgs, gs, processContext, getObjects, setupProcessB, processContextsB } = require('./src/configHelpers')
 const Digraph = require('./src/digraph')
 const { project } = require('./src/project')
+const { project:project2 } = require('./src/project2')
 const fetch = require('node-fetch')
 const base64 = require('base-64')
 const deepEqual = require('deep-equal')
@@ -49,8 +50,8 @@ const getConfig_getContextCheck = (testConfig) => {
   return (testConfig.checks && testConfig.checks.context) || []
 }
 
-const pickContext = (testConfig) => (context) => {
-  return project(context, getConfig_getContextCheck(testConfig))
+const pickContext = (contextChecks) => (context) => {
+  return project2(context, contextChecks)
 }
 
 const pickObjects = (config, testConfig, getObjects) => {
@@ -68,7 +69,11 @@ const pickObjects = (config, testConfig, getObjects) => {
     if (!objects) {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
-    projection[km] = project(objects, checks[km])
+    if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
+      projection[km] = project2(objects, checks[km])
+    } else {
+      projection[km] = project(objects, checks[km])
+    }
   }
   return projection
 }
@@ -260,7 +265,7 @@ const throwErrorHandler = (error) => {
   throw error
 }
 
-const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
+const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -315,7 +320,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       if (queries.length === 0) {
         break
       }
-
+      config.updateData(data)
       data.utterance = queries[0]
       data.start_counter = startCounter
       let json = await doWithRetries(retries, url, queryParams, data)
@@ -352,7 +357,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ isTest, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+                                  await processContextsB({ isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         if (isTest) {
           const end = runtime.performance.performance.now()
           clientSideTime = end - start
@@ -485,7 +490,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       setupArgs(args, config)
       await testConfig.initializer(args)
     }
-    const result = await _process(config, test, { errorHandler, isTest: true })
+    const result = await _process(config, test, { errorHandler, isTest: true, isProcess: true, isModule: false })
     result.query = test
     if (debug) {
       defaultInnerProcess(config, errorHandler, result)
@@ -515,8 +520,9 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       failed_generatedParenthesized = false
     }
 
-    const pickedResultContexts = result.contexts.map(pickContext(testConfig))
-    const pickedExpectedContexts = expected.contexts.map(pickContext(testConfig))
+    const contextChecks = config.getContextChecks()
+    const pickedResultContexts = result.contexts.map(pickContext(contextChecks))
+    const pickedExpectedContexts = expected.contexts.map(pickContext(contextChecks))
     const failedCheckedContexts = !matching(pickedResultContexts, pickedExpectedContexts)
 
     const expectedGetObjects = (name) => {
@@ -624,7 +630,7 @@ const saveTest = async (testFile, config, test, expected, testConfig, saveDevelo
   await config.rebuild()
   const objects = getObjects(config.config.objects)(config.uuid)
   console.log(test)
-  const result = await _process(config, test, { isTest: true })
+  const result = await _process(config, test, { isTest: true, isProcess: true, isModule: true })
   // const actualObjects = config.config.objects
   const actualConfig = getConfigForTest(config, testConfig)
   const args = {
@@ -880,7 +886,8 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     console.log(JSON.stringify(picked, null, 2))
   }
 
-  const pickedResultContexts = responses.contexts.map(pickContext(config.testConfig))
+  const contextChecks = config.getContextChecks()
+  const pickedResultContexts = responses.contexts.map(pickContext(contextChecks))
   if (pickedResultContexts.some((context) => Object.keys(context).length > 0)) {
     console.log('--- Contexts showing only the checked values ---')
     console.log(JSON.stringify(pickedResultContexts, null, 2))
@@ -1880,7 +1887,7 @@ const knowledgeModuleImpl = async ({
           global.beforeObjects = _.cloneDeep(objects)
         }
         try {
-          await processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
+          await processResults(_process(config, args.query, { commandLineArgs: args, isProcess, isModule: !isProcess, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
         } catch (error) {
           console.log('Error', error)
         }
@@ -1909,8 +1916,18 @@ const knowledgeModuleImpl = async ({
           loadForTesting = true
         }
       }
+      if (template) {
+        try {
+          await config.load(rebuildTemplate, template.template, template.instance)
+        } catch (e) {
+          errorHandler(e)
+        }
+      }
+
       // remove test only stuff
       if (!isProcess && !loadForTesting) {
+        config.removeDevelopmentElements(config.config)
+        /*
         config.config.operators = config.config.operators.filter((operator) => {
           if (operator.development) {
             return false
@@ -1925,21 +1942,14 @@ const knowledgeModuleImpl = async ({
             return true
           }
         })
-      }
-
-      if (template) {
-        try {
-          await config.load(rebuildTemplate, template.template, template.instance)
-        } catch (e) {
-          errorHandler(e)
-        }
+        */
       }
     }
 
     // no cache 21 minutes + rebuild fails "node tester_rebuild -m colors"
     // cache okay
-    const createConfigExport = async () => {
-      if (createConfig.cached) {
+    const createConfigExport = async (useCache = true) => {
+      if (useCache && createConfig.cached) {
         return createConfig.cached
       }
       const config = await createConfig()

package/package.json

@@ -53,6 +53,7 @@
     "src/digraph_internal.js",
     "src/generators.js",
     "src/project.js",
+    "src/project2.js",
     "src/semantics.js"
   ],
   "author": "dev@thinktelligence.com",
@@ -69,6 +70,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.3.0-beta.9",
+  "version": "9.4.0",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -335,6 +335,22 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
           config.addPriority({ context: [[bridge.id, bridge.level], after], choose: [0] })
         }
       }
+      if (bridge.check) {
+        if (!config.testConfig) {
+          config.testConfig = {}
+        }
+        if (!config.testConfig.checks) {
+          config.testConfig.checks = {}
+        }
+        if (!config.testConfig?.checks?.context) {
+          config.testConfig.checks.context = []
+        }
+        config.testConfig.checks.context.push({
+          match: ({context}) => context.marker == bridge.id,
+          exported: true,
+          apply: ({context}) => bridge.check,
+        })
+      }
       if (bridge.after) {
         for (let before of bridge.after) {
           if (typeof before === 'string') {
@@ -473,7 +489,7 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
   if (moreConfig.bridges) {
     moreConfig.bridges = moreConfig.bridges.map((bridge) => {
       bridge = { ...bridge }
-      const valid = ['after', 'conditional', 'associations', 'before', 'bridge', 'development', 'skipable', 'return_type_selector', 'evaluator', 'evaluators', 'generatorp', 'generatorr', 'generatorpr', 'generators', 'operator', 'id', 'convolution', 'inverted', 'isA', 'children', 'parents',
+      const valid = ['after', 'conditional', 'associations', 'before', 'bridge', 'check', 'disabled', 'development', 'skipable', 'return_type_selector', 'evaluator', 'evaluators', 'generatorp', 'generatorr', 'generatorpr', 'generators', 'operator', 'id', 'convolution', 'inverted', 'isA', 'children', 'parents',
         'level', 'optional', 'selector', 'semantic', 'semantics', 'words', /Bridge$/, 'localHierarchy', 'levelSpecificHierarchy', 'where', 'uuid']
       helpers.validProps(valid, bridge, 'bridge')
       handleBridgeProps(baseConfig, bridge, { addFirst, uuid })
@@ -878,6 +894,7 @@ class Config {
   getInfo () {
     const name = this.name
     const includes = this.configs.slice(1).map((km) => km.config.name)
+    includes.sort()
     const visibleExamples = []
     for (const test of this.tests) {
       if (!test.developerTest) {
@@ -1055,8 +1072,8 @@ class Config {
 
   // value is in response field
   // TODO maybe generalize out query+evaluate along the lines of set value and set reference
-  async getEvaluator (s, calls, log, context) {
-    const instance = await s({ ...context, evaluate: true })
+  async getEvaluator (s, calls, log, context, args) {
+    const instance = await s({ ...args, ...context, evaluate: true })
     calls.touch(instance)
     if (!instance.evalue && !instance.verbatim && !instance.value) {
       this.warningNotEvaluated(log, context)
@@ -1214,43 +1231,6 @@ class Config {
             }
             return bridge
           })
-        } else {
-          /* done in updateQueries now
-          config.generators = (config.generators || []).map((generator) => {
-            generator = { ...generator }
-            delete generator.where
-            generator.match = generator.match.toString()
-            generator.apply = generator.apply.toString()
-            return generator
-          })
-          config.semantics = (config.semantics || []).map((semantic) => {
-            semantic = { ...semantic }
-            delete semantic.where
-            semantic.match = semantic.match.toString()
-            semantic.apply = semantic.apply.toString()
-            return semantic
-          })
-          config.bridges = (config.bridges || []).map((bridge) => {
-            bridge = { ...bridge }
-            delete bridge.where
-            if (bridge.generatorp) {
-              bridge.generatorp = bridge.generatorp.toString()
-            }
-            if (bridge.generatorr) {
-              bridge.generatorr = bridge.generatorr.toString()
-            }
-            if (bridge.generatorpr) {
-              bridge.generatorpr = bridge.generatorpr.toString()
-            }
-            if (bridge.evaluator) {
-              bridge.evaluator = bridge.evaluator.toString()
-            }
-            if (bridge.semantic) {
-              bridge.semantic = bridge.semantic.toString()
-            }
-            return bridge
-          })
-          */
         }
         return config
       }
@@ -1328,6 +1308,15 @@ class Config {
     }
   }
 
+  addEnable (key) {
+    this._enable.push(key)
+  }
+
+  updateData (data) {
+    data.enable = this._enable
+    this._enable = []
+  }
+
   toData (data) {
     Object.assign(data, this.config)
     // greg99 delete data.objects
@@ -1577,8 +1566,14 @@ class Config {
   }
 
   removeSemantic (deleteSemantic) {
-    const id = deleteSemantic.id || deleteSemantic
-    const todo = [id]
+    if (!Array.isArray(deleteSemantic)) {
+      deleteSemantic = [deleteSemantic]
+    }
+    const todo = []
+    for (const ds of deleteSemantic) {
+      const id = ds.id || ds
+      todo.push(id)
+    }
     const seen = new Set()
 
     while (todo.length > 0) {
@@ -1591,6 +1586,7 @@ class Config {
       if (index === -1) {
         continue
       }
+      // TODO change to tiedIs(s)
       for (const tied_id of this.config.semantics[index].tied_ids || []) {
         if (!seen.has(tied_id)) {
           todo.push(tied_id)
@@ -1951,6 +1947,7 @@ class Config {
       config.priorities = config.priorities || []
     }
 
+    this._enable = []
     this._apiKMs = apiKMs
 
     this.clientProcess = clientProcess
@@ -2616,6 +2613,24 @@ class Config {
     })
   }
 
+  getContextChecks() {
+    const allChecks = []
+    for (const name of this.loadOrdering) {
+      const checks = this.kms[name].testConfig?.checks?.context || []
+      for (const check of checks) {
+        if (check.exported) {
+          allChecks.push(check)
+        }
+      }
+    }
+    for (const check of this.testConfig?.checks?.context || []) {
+      if (!check.exported) {
+        allChecks.push(check)
+      }
+    }
+    return allChecks
+  }
+
   // rebuild ({ isModule: mainIsModule = false } = {}) {
   async rebuild ({ isModule: mainIsModule } = {}) {
     if (this._stop_auto_rebuild) {
@@ -2643,6 +2658,7 @@ class Config {
     // reorder configs base on load ordering
     {
       const ordering = this.loadOrder.order(this.configs.map((km) => km.name || km.uuid))
+      this.loadOrdering = ordering
       const oconfigs = []
       for (const nameOrUUID of ordering) {
         for (const km of this.configs) {
@@ -2801,6 +2817,10 @@ class Config {
       }
     }
 
+    if (this.isModule) {
+      this.removeDevelopmentElements(this.config)
+    }
+
     this.valid()
     this.checks()
   }
@@ -3186,7 +3206,7 @@ class Config {
   }
 
   // TODO get rid of useOldVersion arg
-  addInternal (more, { uuid, addFirst = false, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false, handleCalculatedProps: hcps = false } = {}) {
+  addInternal (more, { uuid, km, addFirst = false, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false, handleCalculatedProps: hcps = false } = {}) {
     validConfigProps(more)
     if (more instanceof Config) {
       more.initialize({ force: false })
@@ -3203,6 +3223,20 @@ class Config {
       handleCalculatedProps(this, more, { addFirst, uuid })
       applyUUID(more, uuid || this._uuid)
     }
+    if (km) {
+      if (more.semantics) {
+        more.semantics.map((s) => s.km = km)
+      }
+      if (more.generators) {
+        more.generators.map((g) => g.km = km)
+      }
+    }
+    const normalizeIsA = (edge) => {
+      if (edge.length == 3 || edge.parent || edge.child) {
+        return edge
+      }
+      return [...edge, false]
+    }
     for (const key of Object.keys(more)) {
       const value = more[key]
       // TODO remove name and description on the config bag
@@ -3246,9 +3280,9 @@ class Config {
           const hierarchy = this.config.words.hierarchy
           const moreHierarchy = more.words.hierarchy
           if (addFirst) {
-            this.config.words.hierarchy = moreHierarchy.concat(hierarchy)
+            this.config.words.hierarchy = moreHierarchy.concat(hierarchy.map(normalizeIsA))
           } else {
-            this.config.words.hierarchy = hierarchy.concat(moreHierarchy)
+            this.config.words.hierarchy = hierarchy.concat(moreHierarchy.map(normalizeIsA))
           }
         }
       } else if (key === 'name') {
@@ -3320,7 +3354,9 @@ class Config {
         // console.log('more', JSON.stringify(more, null, 2))
         // hierarchy must update in place and does not care about the list order
         if (key === 'hierarchy') {
-          this.config[key].push(...more[key])
+          const moreEdges = more[key].map(normalizeIsA)
+          this.config[key].push(...moreEdges)
+          this.hierarchy.addEdges(moreEdges)
         } else {
           if (addFirst) {
             this.config[key] = more[key].concat(this.config[key])

package/src/configHelpers.js

@@ -86,6 +86,9 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
     args.objects = config.get('objects')
     args.getObjects = getObjects(args.objects)
   }
+  args.enable = (key) => {
+    config.addEnable(key)
+  }
   if (args.uuid) {
     args.objects = args.getObjects(args.uuid)
   }
@@ -140,7 +143,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
       if (!c) {
         return
       }
-      return config.getEvaluator(args.s, args.calls, logs, c)
+      return config.getEvaluator(args.s, args.calls, logs, c, { isModule: args.isModule, isProcess: args.isProcess })
     }
     args.gs = gs(args.g)
     args.gsp = gs(args.gp)
@@ -257,18 +260,18 @@ const setupProcessB = ({ config, initializer, allowDelta = false, rebuildingTemp
 const setupContexts = (rawContexts) => {
   let first = true
   const contexts = []
-  contexts.push({ marker: 'controlStart', controlRemove: true })
+  contexts.push({ marker: 'controlStart', controlRemove: true, isControl: true })
   let previous
   for (const context of rawContexts) {
     if (first) {
       first = false
     } else {
-      contexts.push({ marker: 'controlBetween', controlRemove: true, previous })
+      contexts.push({ marker: 'controlBetween', controlRemove: true, isControl: true, previous })
     }
     contexts.push(context)
     previous = context
   }
-  contexts.push({ marker: 'controlEnd', controlRemove: true, previous })
+  contexts.push({ marker: 'controlEnd', controlRemove: true, isControl: true, previous })
 
   let _index = 0
   const id = (context) => {
@@ -279,7 +282,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate }) => {
+const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -290,7 +293,7 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
   const contexts = setupContexts(json.contexts)
 
   const objects = config.get('objects')
-  const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance, contexts }
+  const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance, contexts, isProcess, isModule }
   if (!json.logs) {
     json.logs = []
   }
@@ -441,14 +444,14 @@ const loadInstance = async (config, instance) => {
   */
 
   // const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
-  const hierarchy = config.hierarchy
   for (const i in (instance.resultss || [])) {
+    const hierarchy = config.hierarchy
     const results = instance.resultss[i]
     if (results.extraConfig) {
       // config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
       const uuid = config.nameToUUID(instance.name)
       // used to do a CLONE
-      config.addInternal(instance.template.configs[i], { uuid, addFirst: true, handleCalculatedProps: true })
+      config.addInternal(instance.template.configs[i], { uuid, km: instance.name, addFirst: true, handleCalculatedProps: true })
     } else if (results.apply) {
       const objects = getObjects(config.get('objects'))(config.uuid)
       const args = { objects, getObjects: getObjects(objects) }
@@ -468,6 +471,8 @@ const loadInstance = async (config, instance) => {
       const args = { config, hierarchy, json: results, commandLineArgs: {}, forTemplate: true }
       args.isInstance = `instance${i}`
       args.instance = ''
+      args.isProcess = !config.isModule
+      args.isModule = !!config.isModule
       await processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null

package/src/debug.js

@@ -31,12 +31,12 @@ const wasHit = (name) => {
   return hits[name]
 }
 
-const counter = (name, breakAt = [], debugBreak = true) => {
+const counter = (name, { breakAt = [], debugBreak = true, tag = '' } = {}) => {
   if (!counters[name]) {
     counters[name] = 0
   }
   counters[name] += 1
-  console.log(`counters[${name}] = ${counters[name]}`)
+  console.log(`counters[${name}] = ${counters[name]} ${tag}`)
   unhit(name)
   if (Array.isArray(breakAt) && breakAt.includes(counters[name])) {
     if (debugBreak) {

package/src/helpers.js

@@ -310,6 +310,7 @@ const validProps = (valids, object, type) => {
       }
     }
     if (!okay) {
+      valids.sort()
       throw new Error(`Unknown property "${prop}" in the ${type}. Valid properties are ${valids.join(', ')}. The ${type} is ${JSON.stringify(object)}`)
     }
   }

package/src/project.js

@@ -1,15 +1,38 @@
+function isObject(value) {
+  return typeof value === "object" && value !== null;
+}
+
+const merge = (value1, value2) => {
+  if (Array.isArray(value1) && Array.isArray(value2)) {
+    const merged = []
+    for (let i = 0; i < value1.length; ++i) {
+      merged.push(merge(value1[i], value2[i]))
+    }
+    return merged
+  }
+  if (isObject(value1) && isObject(value2)) {
+    return Object.assign({}, value1, value2)
+  }
+}
 
 const project = (object, filter) => {
   if (!object) {
     return
   }
+  if (typeof object === 'string' || typeof object === 'number') {
+    return object
+  }
 
   const projection = {}
   const set = (property, value) => {
     if (value === null || value === undefined) {
       return
     }
-    projection[property] = value
+    if (projection[property]) {
+      projection[property] = merge(value, projection[property])
+    } else {
+      projection[property] = value
+    }
   }
   if (Array.isArray(filter)) {
     if (Array.isArray(object)) {
@@ -78,4 +101,4 @@ const project = (object, filter) => {
   return projection
 }
 
-module.exports = { project }
+module.exports = { project, merge }

package/src/project2.js

@@ -0,0 +1,54 @@
+function project(source, filters) {
+  if (['string', 'number'].includes(typeof source)) {
+    return source
+  }
+
+  if (Object.keys(source).length === 0 && filters.length === 0) {
+    return {};
+  }
+
+  // Find the applicable filter for the current context
+  const filter = filters.find(f => f.match({ context: source }));
+  if (!filter) {
+    if (Array.isArray(source)) {
+      return source.map((element) => project(element, filters))
+    }
+    return {};
+  }
+
+  // Get the properties to include from the apply function
+  let properties = filter.apply(source);
+
+  // update 
+  const updatedProperties = []
+  for (const property of properties) {
+    if (property.properties) {
+      for (const moreProperty of source[property.properties] || []) {
+        updatedProperties.push(moreProperty)
+      }
+    } else {
+      updatedProperties.push(property)
+    }
+  }
+  properties = updatedProperties
+
+  // Build the result object
+  const result = {};
+  properties.forEach(prop => {
+    if (source.hasOwnProperty(prop)) {
+      // If the property is an object and not null, recursively project it
+      if (typeof source[prop] === 'object' && source[prop] !== null) {
+        result[prop] = project(source[prop], filters);
+      } else {
+        // Copy primitive or null properties directly
+        result[prop] = source[prop];
+      }
+    }
+  });
+
+  return result;
+}
+
+module.exports = {
+  project
+}