theprogrammablemind 9.5.1-beta.1 → 9.5.1-beta.11

package/client.js

@@ -316,6 +316,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     }
 
     let startCounter = 0
+    let contextIdCounter = 0
     while (true) {
       if (queries.length === 0) {
         break
@@ -356,8 +357,9 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         }
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
-        const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
+                                  await processContextsB({ contextIdCounter, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
           clientSideTime = end - start
@@ -475,6 +477,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
     return
   }
   // initialize in between test so state is not preserved since the test was adding without state
+  config.testConfig.testModuleName = testConfig.testModuleName
   await config.rebuild()
   const errorHandler = (error) => {
     if (error.metadata) {
@@ -2008,9 +2011,15 @@ const ensureTestFile = (module, name, type) => {
 }
 
 const knowledgeModule = async (...args) => {
-  await knowledgeModuleImpl(...args).catch((e) => {
+  await knowledgeModuleImpl(...args).catch(async (e) => {
     console.error(e)
-    process.exit(-1)
+    function sleep(ms) {
+      return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+      });
+    }
+    await sleep(1) // get the stderr to flush
+    await process.exit(-1);   // tiny trick: empty write forces flush of console.error buffer
   })
 }
 

package/lines.js

@@ -5,6 +5,13 @@ class Lines {
     this.rows = []
   }
 
+  static SCREEN_WIDTH = 132
+
+  static addRemainder(widths) {
+    const sum = widths.reduce((a, b) => a + b)
+    return [...widths, Lines.SCREEN_WIDTH - sum]
+  }
+
   addLine () {
     this.lines.push(this.widths.map((width) => ''.padEnd(width)))
   }

package/package.json

@@ -3,14 +3,14 @@
     "@eslint/js": "^9.21.0",
     "@typescript-eslint/eslint-plugin": "^4.28.4",
     "@typescript-eslint/parser": "^4.28.4",
+    "argparse": "^2.0.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^16.0.3",
     "eslint-plugin-import": "^2.23.4",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
     "globals": "^16.0.0",
-    "jest": "^29.7.0",
-    "argparse": "^2.0.1"
+    "jest": "^30.2.0"
   },
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
@@ -45,6 +45,7 @@
     "runtime.js",
     "src/helpers.js",
     "src/flatten.js",
+    "src/fragments.js",
     "src/unflatten.js",
     "src/config.js",
     "src/configHelpers.js",
@@ -61,6 +62,7 @@
   "dependencies": {
     "base-64": "^1.0.0",
     "deep-equal": "^2.0.4",
+    "flatted": "^3.3.3",
     "fs": "0.0.1-security",
     "json-diff": "^1.0.3",
     "json-stable-stringify": "^1.0.1",
@@ -71,6 +73,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.5.1-beta.1",
+  "version": "9.5.1-beta.11",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -10,6 +10,7 @@ const { ecatch } = require('./helpers')
 const runtime = require('../runtime')
 const _ = require('lodash')
 const db = require('./debug')
+const { fragmentInstantiator, fragmentMapperInstantiator } = require('./fragments')
 
 const debugBreak = () => {
   // debugger
@@ -289,11 +290,9 @@ const priority_valid = (cp) => {
 const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
   ecatch(`While processing the bridge for ${bridge.id}#${bridge.level}`,
     () => {
-      /*
-      if (bridge.development && config.isModule) {
+      if (bridge.scope && config.isModule) {
         return
       }
-      */
       if (false && !bridge.bridge) {
         bridge.bridge = '{ ...next(operator) }'
       }
@@ -904,6 +903,10 @@ class Config {
     return config_toServer(config)
   }
 
+  async run(handler) {
+    return configHelpers.run(this, handler)
+  }
+
   async fixtures () {
     if (this.testConfig?.fixtures) {
       const args = {}
@@ -1109,59 +1112,42 @@ class Config {
     return instance
   }
 
-  fragmentInstantiator (args, contexts) {
-    return new Object({
-      contexts: () => contexts,
-      instantiate: async (mappings) => {
-        const instantiated = _.cloneDeep(contexts)
-        // const todo = [...instantiated]
-        // const todo = [...instantiated]
-        const todo = _.clone(instantiated)
-        args = { ...args }
-        while (todo.length > 0) {
-          const context = todo.pop()
-          args.context = context
-          for (const mapping of mappings) {
-            if (await mapping.match(args)) {
-              await mapping.apply(args)
-            }
-          }
-          for (const key of Object.keys(context)) {
-            // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
-            if (!helpers.isCompound(context[key])) {
-              continue
-            }
-            if (context[key].instantiated) {
-              continue
-            }
-            todo.push(context[key])
-          }
-        }
-        return instantiated
-      }
-    })
-  }
-
-  fragment (args, query) {
+  getFragment(query) {
     for (const instance of (this.instances || [])) {
       for (const fragment of (instance.fragments || [])) {
         if (fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
       for (const fragment of (instance.resultss || [])) {
         if (fragment.isFragment && fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
       for (const fragment of (this.fragmentsBeingBuilt || [])) {
         if (fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
     }
   }
 
+  fragment (args, query) {
+    const fragment = this.getFragment(query)
+    if (fragment) {
+      return fragmentInstantiator(args, fragment.contexts)
+    }
+  }
+
+  fragmentMapper (args, values, fromModelQuery, toModelQuery) {
+    const fromModelFragment = this.getFragment(fromModelQuery)
+    console.dir(fromModelFragment)
+    const toModelFragment = this.getFragment(toModelQuery)
+    console.dir(toModelFragment)
+    const mapper = fragmentMapperInstantiator(values, fromModelFragment.contexts, toModelFragment.contexts)
+    return mapper
+  }
+
   // { rebuild: false, isModule: false }
   needsRebuild (template, instance, options) {
     if (options.rebuild) {
@@ -1866,6 +1852,16 @@ class Config {
     }
   }
 
+  getObjects () {
+    const configs = {}
+    const ns = this.config.objects.namespaced
+    configs[this.name] = this
+    for (const config of this.configs) {
+      configs[config._name] = ns[config._uuid]
+    }
+    return configs
+  }
+
   getConfig (name) {
     if (this.name === name) {
       return this
@@ -1913,12 +1909,15 @@ class Config {
       } else if (this.scope == 'development') {
         new_result = !(element.scope === 'testing')
       }
+      /*
       if (global.GORDO && old_result !== new_result) {
         global.GORDO = false
         console.log("THERE WAS A DIFFERENCE ------------------------------------------------")
         debugger // greg23old
       }
-      return old_result
+      */
+      // return old_result
+      return new_result
     }
 
     config.operators = config.operators.filter((element) => keep(element))

package/src/configHelpers.js

@@ -121,6 +121,9 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.fragments = (query) => {
     return config.fragment(args, query)
   }
+  args.fragmentMapper = (values, fromModelQuery, toModelQuery) => {
+    return config.fragmentMapper(args, values, fromModelQuery, toModelQuery)
+  }
   args.breakOnSemantics = false
   args.theDebugger = {
     breakOnSemantics: (value) => args.breakOnSemantics = value
@@ -133,6 +136,9 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.addPattern = (pattern, def) => config.addPattern(pattern, args.uuid)
   args.addGenerator = (generator) => config.addGenerator(generator, args.uuid, config.name)
 
+  if (config.testConfig?.testModuleName) {
+    args.testModuleName = config.testConfig.testModuleName
+  }
   args.addAssumedScoped = (args, assumed) => {
     const addAssumed = (args, ...moreAssumed) => {
       return { ...args, assumed: Object.assign({}, assumed, (args.assumed || {}), ...moreAssumed) }
@@ -199,6 +205,23 @@ const getObjects = (objects) => {
   }
 }
 
+const run = async (config, handler) => {
+  // map to hash
+  config = config || {}
+  if (config.config) {
+    config = config
+  }
+
+  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+
+  debugger
+  const objects = config.config.objects.namespaced[config.uuid]
+  const logs = []
+  const args = {}
+  setupArgs(args, config, logs, hierarchy)
+  return handler(args)
+}
+
 const processContext = async (context, { objects = {}, config, logs = [] }) => {
   const generators = config.getGenerators(logs)
   const semantics = config.getSemantics(logs)
@@ -291,7 +314,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate }) => {
+const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -311,14 +334,13 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
   args.insert = (context) => toDo.unshift(context)
   let overlap, lastRange
   config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
-  let context_id_counter = 0
   while (toDo.length > 0) {
     const context = toDo.shift()
     args.calls.next()
     let contextPrime = context
     context.topLevel = true
-    context_id_counter += 1
-    context.context_id = context_id_counter
+    contextIdCounter += 1
+    context.context_id = contextIdCounter
     try {
       if (json.has_errors) {
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
@@ -328,6 +350,9 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
         const semantics = config.getSemantics(json.logs)
         try {
           contextPrime = await semantics.apply(args, context)
+          // contextPrime.greg = 'yes'
+//          console.log("context_id", context.context_id)
+//          console.log("semantics.apply", JSON.stringify(contextPrime, null, 2))
         } catch (e) {
           if (e.message == 'Maximum call stack size exceeded') {
             const mostCalled = semantics.getMostCalled()
@@ -421,7 +446,7 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
       throw e
     }
   }
-  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
+  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime, updatedContextIdCounter: contextIdCounter }
 }
 
 // instance template loadTemplate
@@ -479,6 +504,9 @@ const loadInstance = async (config, instance) => {
         args.isModule = true
         args.isProcess = false
       }
+      if (instance.name == config.testConfig.testModuleName) {
+        args.isTesting = true
+      }
       await results.apply(args)
     } else if (results.isFragment) {
     } else {
@@ -490,6 +518,9 @@ const loadInstance = async (config, instance) => {
       args.instance = ''
       args.isProcess = !config.isModule
       args.isModule = !!config.isModule
+      if (instance.name == config.testConfig.testModuleName) {
+        args.isTesting = true
+      }
       await processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null
@@ -505,6 +536,7 @@ module.exports = {
   // listable,
   setupArgs,
   processContext,
+  run,
   getObjects,
   gs,
   processContextsB,

package/src/fragments.js

@@ -0,0 +1,77 @@
+const _ = require('lodash')
+const helpers = require('./helpers')
+
+function fragmentInstantiator (args, contexts) {
+  return new Object({ 
+    contexts: () => contexts,
+    instantiate: async (mappings) => {
+      const instantiated = _.cloneDeep(contexts)
+      const todo = [{ context: instantiated, path: [] }]
+      args = { ...args }
+      while (todo.length > 0) {
+        const { context, path } = todo.pop()
+        args.context = context
+        args.path = path
+        for (const mapping of mappings) {
+          if (await mapping.match(args)) {
+            await mapping.apply(args)
+          }
+        }
+        for (const key of Object.keys(context)) {
+          // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
+          if (!helpers.isCompound(context[key])) {
+            continue
+          }
+          if (context[key].instantiated) {
+            continue
+          }
+          todo.push({ context: context[key], path: [...path, key] })
+        }
+      }
+      return instantiated
+    }
+  })
+}
+
+async function fragmentMapperInstantiator(values, modelFrom, modelTo) {
+  const paths = {}
+  for (const value of values) {
+    paths[value] = { value }
+  }
+
+  {
+    const fi = fragmentInstantiator({paths}, modelFrom)
+    await fi.instantiate([
+      {
+        match: ({context, path}) => values.includes(context.value),
+        apply: ({context, path}) => paths[context.value].from = path
+      },
+    ])
+  }
+
+  {
+    const fi = fragmentInstantiator({paths}, modelTo)
+    await fi.instantiate([
+      {
+        match: ({context, path}) => values.includes(context.value),
+        apply: ({context, path}) => paths[context.value].to = path
+      },
+    ])
+  }
+  return {
+    instantiate: (actualFrom) => {
+      const actualTo = structuredClone(modelTo)
+      for (const value in paths) {
+        const { from, to } = paths[value]
+        const actualValue = helpers.getByPath(actualFrom, from, null)
+        helpers.setByPath(actualTo, to, actualValue)
+      }
+      return actualTo
+    }
+  }
+}
+
+module.exports = {
+  fragmentInstantiator,
+  fragmentMapperInstantiator,
+}

package/src/generators.js

@@ -1,3 +1,4 @@
+const { stringify } = require('flatted');
 const { args: contextArgs, normalizeGenerator } = require('./helpers')
 const Lines = require('../lines')
 const helpers = require('./helpers')
@@ -182,7 +183,7 @@ class Generators {
       const generator = this.generators[igenerator]
       if (await generator.matches(args, objects, context, hierarchy, config, options)) {
         const log = (message) => { this.logs.push(message) }
-        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
+        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${stringify(context)}`)
         let errorMessage = 'The apply function did not return a value'
         try {
           generated = await generator.apply(args, objects, context, hierarchy, config, response, log)
@@ -202,7 +203,7 @@ class Generators {
           }
         }
         if (!generated && generated !== '') {
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
           const source = `${generator.km}/#${generator.index}`
@@ -212,7 +213,7 @@ class Generators {
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
-          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          lines.setElement(1, 2, stringify(helpers.sortJson(context, { depth: 10 }), null, 2))
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
@@ -223,14 +224,14 @@ class Generators {
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
           this.logs.push(lines.toString())
-          const message = `ERROR while applying (${source}) ${generator.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
+          const message = `ERROR while applying (${source}) ${generator.toLabel()}\n to\n ${stringify(context, null, 2)}.\n${errorMessage}'`
           // this.logs.push(message)
           // return [message]
           args.calls.pop()
           throw { error: [message], logs: this.logs }
         }
         if (((config || {}).config || {}).debug) {
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
           if (generator.index > -1 && generator.km) {
@@ -254,7 +255,7 @@ class Generators {
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
-          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          lines.setElement(1, 2, stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           this.logs.push(lines.toString())
         }
         applied = true
@@ -263,7 +264,7 @@ class Generators {
     }
     args.calls.pop()
     if (!applied && ((config || {}).config || {}).debug) {
-      const widths = [10, 10, 90]
+      const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Generator')
       lines.setElement(0, 2, 'No generator applied')
@@ -272,7 +273,7 @@ class Generators {
       lines.setElement(0, 2, stack)
       lines.newRow()
       lines.setElement(0, 1, 'TO')
-      lines.setElement(0, 2, JSON.stringify(context, null, 2))
+      lines.setElement(0, 2, stringify(context, null, 2))
       this.logs.push(lines.toString())
     }
     return ((config || {}).parenthesized ? '(' + generated + ')' : generated)

package/src/helpers.js

@@ -436,6 +436,63 @@ const stableId = (tag) => {
   return id
 }
 
+function getByPath(obj, path, defaultValue) {
+  let current = obj;
+  for (const key of path) {
+    if (current === null || current === undefined) return defaultValue;
+    if (typeof current !== 'object') return defaultValue;
+    current = current[key];
+  }
+  return current === undefined ? defaultValue : current;
+}
+
+/**
+ * Set a value in an object by path array.
+ * Automatically creates missing objects {} or arrays [] as needed.
+ *
+ * @param {Object} obj - The root object to modify
+ * @param {Array<string|number>} path - Array of keys/indices
+ * @param {*} value - Value to set
+ * @returns {*} The set value (for chaining)
+ */
+function setByPath(obj, path, value) {
+  if (!Array.isArray(path) || path.length === 0) {
+    throw new Error('Path must be a non-empty array');
+  }
+
+  let current = obj;
+
+  for (let i = 0; i < path.length; i++) {
+    const key = path[i];
+    const isLast = i === path.length - 1;
+
+    if (isLast) {
+      // Final step — just assign
+      current[key] = value;
+    } else {
+      // Not last — ensure next level exists
+      const nextKey = path[i + 1];
+
+      if (current[key] == null) {
+        // Auto-create: array if next key is number, otherwise object
+        current[key] = typeof nextKey === 'number' || String(nextKey >>> 0) === nextKey
+          ? []
+          : {};
+      } else if (Array.isArray(current[key]) && typeof nextKey !== 'number') {
+        // Safety: if current is array but next key isn't a valid index → convert to object
+        current[key] = { ...current[key] };
+      } else if (!Array.isArray(current[key]) && typeof nextKey === 'number') {
+        // If next expects array but current is object → convert
+        current[key] = Object.values(current[key]);
+      }
+
+      current = current[key];
+    }
+  }
+
+  return value;
+}
+
 module.exports = {
   stableId,
   ecatch,
@@ -463,4 +520,6 @@ module.exports = {
   w,
   suggestAssociationsFix,
   suggestAssociationsFixFromSummaries,
+  getByPath,
+  setByPath,
 }

package/src/semantics.js

@@ -218,7 +218,7 @@ class Semantics {
  						errorMessage = e.toString()
           }
 
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
           const source = `${semantic.km}/#${semantic.index}`
@@ -248,7 +248,7 @@ class Semantics {
         args.calls.touch(contextPrime)
         // this.logs.push(`Semantics: applied ${semantic.toString()}\n to\n ${JSON.stringify(context)}\n the result was ${JSON.stringify(contextPrime)}\n`)
         if (((config || {}).config || {}).debug) {
-          const widths = [10, 10, 132]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
           if (semantic.index > -1 && semantic.km) {
@@ -295,7 +295,7 @@ class Semantics {
     }
     args.calls.pop()
     if (!applied && ((config || {}).config || {}).debug) {
-      const widths = [10, 10, 90]
+      const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Semantic')
       lines.setElement(0, 2, 'No semantic applied')