theprogrammablemind 9.5.1-beta.3 → 9.5.1-beta.30

package/client.js

@@ -12,9 +12,8 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
+const { sortJson, appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
 const runtime = require('./runtime')
-const sortJson = runtime.sortJson
 const debug = require('./src/debug')
 
 const getConfig_getObjectsCheck = (config, testConfig) => {
@@ -69,6 +68,7 @@ const pickObjects = (config, testConfig, getObjects) => {
     if (!objects) {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
+
     if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
       projection[km] = project2(objects, checks[km])
     } else {
@@ -316,6 +316,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     }
 
     let startCounter = 0
+    let contextIdCounter = 0
     while (true) {
       if (queries.length === 0) {
         break
@@ -356,8 +357,9 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         }
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
-        const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
+                                  await processContextsB({ contextIdCounter, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
           clientSideTime = end - start
@@ -475,6 +477,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
     return
   }
   // initialize in between test so state is not preserved since the test was adding without state
+  config.testConfig.testModuleName = testConfig.testModuleName
   await config.rebuild()
   const errorHandler = (error) => {
     if (error.metadata) {
@@ -913,7 +916,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   console.log('')
   const screen_width = process.stdout.columns
   // || 0 for when running without a console
-  const widths = [70, 10, Math.max(80, screen_width - 71 || 0)]
+  const widths = Lines.addRemainder([70, 10])
   const lines = new Lines(widths)
   lines.setElement(0, 0, '--- The paraphrases are ----------')
   lines.setElement(0, 2, '--- The response strings are ----------')
@@ -970,7 +973,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     if (instance.fragments) {
       pr = instance.fragments[index]
     }
-    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: false })
+    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: true })
   }
 
   const looper = async (configs) => {
@@ -2008,9 +2011,15 @@ const ensureTestFile = (module, name, type) => {
 }
 
 const knowledgeModule = async (...args) => {
-  await knowledgeModuleImpl(...args).catch((e) => {
+  await knowledgeModuleImpl(...args).catch(async (e) => {
     console.error(e)
-    process.exit(-1)
+    function sleep(ms) {
+      return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+      });
+    }
+    await sleep(1) // get the stderr to flush
+    await process.exit(-1);   // tiny trick: empty write forces flush of console.error buffer
   })
 }
 

package/lines.js

@@ -5,6 +5,13 @@ class Lines {
     this.rows = []
   }
 
+  static SCREEN_WIDTH = 164
+
+  static addRemainder(widths) {
+    const sum = widths.reduce((a, b) => a + b)
+    return [...widths, Lines.SCREEN_WIDTH - sum]
+  }
+
   addLine () {
     this.lines.push(this.widths.map((width) => ''.padEnd(width)))
   }

package/package.json

@@ -3,14 +3,14 @@
     "@eslint/js": "^9.21.0",
     "@typescript-eslint/eslint-plugin": "^4.28.4",
     "@typescript-eslint/parser": "^4.28.4",
+    "argparse": "^2.0.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^16.0.3",
     "eslint-plugin-import": "^2.23.4",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
     "globals": "^16.0.0",
-    "jest": "^29.7.0",
-    "argparse": "^2.0.1"
+    "jest": "^30.2.0"
   },
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
@@ -45,6 +45,7 @@
     "runtime.js",
     "src/helpers.js",
     "src/flatten.js",
+    "src/fragments.js",
     "src/unflatten.js",
     "src/config.js",
     "src/configHelpers.js",
@@ -61,6 +62,7 @@
   "dependencies": {
     "base-64": "^1.0.0",
     "deep-equal": "^2.0.4",
+    "flatted": "^3.3.3",
     "fs": "0.0.1-security",
     "json-diff": "^1.0.3",
     "json-stable-stringify": "^1.0.1",
@@ -68,9 +70,8 @@
     "node-fetch": "^2.6.1",
     "readline": "^1.3.0",
     "scriptjs": "^2.5.9",
-    "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.5.1-beta.3",
+  "version": "9.5.1-beta.30",
   "license": "UNLICENSED"
 }

package/runtime.js

@@ -13,7 +13,6 @@ module.exports = {
   ArgumentParser,
   readline: require('readline'),
   jsonDiff: require('json-diff'),
-  sortJson: require('sort-json'),
   util: require('util'),
   performance: require('perf_hooks')
 }

package/src/config.js

@@ -10,6 +10,7 @@ const { ecatch } = require('./helpers')
 const runtime = require('../runtime')
 const _ = require('lodash')
 const db = require('./debug')
+const { fragmentInstantiator, fragmentMapperInstantiator } = require('./fragments')
 
 const debugBreak = () => {
   // debugger
@@ -20,8 +21,6 @@ const bags = [
   'semantics'
 ]
 
-global.GORDO = true
-
 const indent = (string, indent) => {
   return string.replace(/^/gm, ' '.repeat(indent))
 }
@@ -401,11 +400,15 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       if (bridge.generatorp) {
         const match = bridge.generatorp.match || (() => true)
         const apply = typeof bridge.generatorp === 'function' ? bridge.generatorp : bridge.generatorp.apply || bridge.generatorp
-        const level = bridge.generatorp.level >= 0 ? bridge.generatorp.level : bridge.level + 1
+        let level = bridge.generatorp.level >= 0 ? bridge.generatorp.level : bridge.level + 1
+        if (!bridge.bridge) {
+          level = 0
+        }
 
         const generator = {
           where: bridge.generatorp.where || bridge.where || helpers.where(4),
-          match: async (args) => bridge.id === args.context.marker && args.context.level === level && args.context.paraphrase && await match(args),
+          // match: async (args) => bridge.id === args.context.marker && args.context.level === level && args.context.paraphrase && await match(args),
+          match: async (args) => args.isA(args.context.marker, bridge.id) && args.context.level === level && args.context.paraphrase && await match(args),
           apply: (args) => apply(args),
           applyWrapped: apply,
           property: 'generatorp'
@@ -422,7 +425,8 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         const level = bridge.generatorr.level >= 0 ? bridge.generatorr.level : bridge.level + 1
         const generator = {
           where: bridge.generatorr.where || bridge.where || helpers.where(4),
-          match: async (args) => bridge.id === args.context.marker && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
+          // match: async (args) => bridge.id === args.context.marker && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
+          match: async (args) => args.isA(args.context.marker, bridge.id) && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
           apply: (args) => apply(args),
           applyWrapped: apply,
           property: 'generatorr'
@@ -509,8 +513,43 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
   if (moreConfig.bridges) {
     moreConfig.bridges = moreConfig.bridges.map((bridge) => {
       bridge = { ...bridge }
-      const valid = ['after', 'conditional', 'associations', 'before', 'bridge', 'check', 'disabled', 'scope', 'skipable', 'return_type_selector', 'evaluator', 'evaluators', 'generatorp', 'generatorr', 'generatorpr', 'generators', 'operator', 'id', 'convolution', 'inverted', 'isA', 'children', 'parents',
-        'level', 'optional', 'selector', 'separators', 'semantic', 'semantics', 'words', /Bridge$/, 'localHierarchy', 'levelSpecificHierarchy', 'where', 'uuid']
+      const valid = [
+        'after', 
+        'associations', 
+        'before', 
+        'bridge', 
+        'check', 
+        'children', 
+        'conditional', 
+        'convolution', 
+        'disabled', 
+        'enhanced_associations', 
+        'evaluator', 
+        'evaluators', 
+        'generatorp', 
+        'generatorpr', 
+        'generatorr', 
+        'generators', 
+        'id', 
+        'inverted', 
+        'isA', 
+        'level', 
+        'levelSpecificHierarchy', 
+        'localHierarchy', 
+        'operator', 
+        'optional', 
+        'parents',
+        'return_type_selector', 
+        'scope', 
+        'selector', 
+        'semantic', 
+        'semantics', 
+        'separators', 
+        'skipable', 
+        'uuid',
+        'where', 
+        'words', /Bridge$/, 
+      ]
       helpers.validProps(valid, bridge, 'bridge')
       handleBridgeProps(baseConfig, bridge, { addFirst, uuid })
       return bridge
@@ -673,7 +712,7 @@ function setWordsUUIDs (words, uuid) {
   for (const key in literals) {
     literals[key] = literals[key].map((o) => Object.assign(o, { uuid }))
   }
-  const patterns = words.patterns
+  const patterns = words.patterns || []
   for (const pattern of patterns) {
     pattern.defs.map((def) => Object.assign(def, { uuid }))
   }
@@ -902,6 +941,10 @@ class Config {
     return config_toServer(config)
   }
 
+  async run(handler) {
+    return configHelpers.run(this, handler)
+  }
+
   async fixtures () {
     if (this.testConfig?.fixtures) {
       const args = {}
@@ -935,6 +978,7 @@ class Config {
 
   getPseudoConfig (uuid, config) {
     return {
+      pseudo: true,
       description: 'this is a pseudo config that has limited functionality due to being available in the initializer and fixtures function context',
       addAssociation: (...args) => this.addAssociation(...args),
       addAssociations: (...args) => this.addAssociations(...args),
@@ -948,6 +992,14 @@ class Config {
       removeSemantic: (...args) => this.removeSemantic(...args, uuid, config.name),
       addWord: (...args) => this.addWord(...args, uuid),
       addPattern: (...args) => this.addPattern(...args, uuid),
+      updateBridge: (...args) => this.updateBridge(...args),
+      processContext: (...args) => this.processContext(...args),
+
+      semantics: () => this.config.semantics,
+      getConfigs: () => this.configs,
+      getTests: () => this.getTests(),
+      getName: () => this.getName(),
+      getDescription: () => this.getDescription(),
 
       getHierarchy: (...args) => this.config.hierarchy,
       getBridges: (...args) => this.config.bridges,
@@ -957,10 +1009,33 @@ class Config {
       fragment: (...args) => this.fragment(...args),
       server: (...args) => this.server(...args),
       exists: (...args) => this.exists(...args),
-      addAPI: (...args) => this.addAPI(...args)
+      addAPI: (...args) => this.addAPI(...args),
+
+      getParenthesized: () => this.getParenthesized(),
+      setParenthesized: (...args) => this.setParenthesized(...args),
     }
   }
 
+  getName() {
+    return this.name
+  }
+
+  getDescription() {
+    return this.description
+  }
+
+  getTests() {
+    return this.tests
+  }
+
+  getParenthesized(value) {
+    return this.parenthesized
+  }
+
+  setParenthesized(value) {
+    this.parenthesized = value
+  }
+
   inDevelopmentMode (call) {
     config.developmentModeOn += 1
     try {
@@ -1107,59 +1182,52 @@ class Config {
     return instance
   }
 
-  fragmentInstantiator (args, contexts) {
-    return new Object({
-      contexts: () => contexts,
-      instantiate: async (mappings) => {
-        const instantiated = _.cloneDeep(contexts)
-        // const todo = [...instantiated]
-        // const todo = [...instantiated]
-        const todo = _.clone(instantiated)
-        args = { ...args }
-        while (todo.length > 0) {
-          const context = todo.pop()
-          args.context = context
-          for (const mapping of mappings) {
-            if (await mapping.match(args)) {
-              await mapping.apply(args)
-            }
-          }
-          for (const key of Object.keys(context)) {
-            // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
-            if (!helpers.isCompound(context[key])) {
-              continue
-            }
-            if (context[key].instantiated) {
-              continue
-            }
-            todo.push(context[key])
-          }
-        }
-        return instantiated
-      }
-    })
-  }
-
-  fragment (args, query) {
+  getFragment(query) {
     for (const instance of (this.instances || [])) {
       for (const fragment of (instance.fragments || [])) {
         if (fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
       for (const fragment of (instance.resultss || [])) {
         if (fragment.isFragment && fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
       for (const fragment of (this.fragmentsBeingBuilt || [])) {
         if (fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
     }
   }
 
+  // mappings are optional
+  async fragment (args, query, mappings) {
+    const fragment = this.getFragment(query)
+    if (fragment) {
+      let fi = fragmentInstantiator(args, fragment.contexts)
+      if (mappings) {
+        fi = await fi
+        return fi.instantiate([{
+          match: ({context}) => !!mappings[context.value], // is the value in the mappings
+          apply: ({context}) => Object.assign(context, mappings[context.value]),
+        }])
+      } else {
+        return fi
+      }
+    }
+  }
+
+  fragmentMapper (args, values, fromModelQuery, toModelQuery) {
+    const fromModelFragment = this.getFragment(fromModelQuery)
+    console.dir(fromModelFragment)
+    const toModelFragment = this.getFragment(toModelQuery)
+    console.dir(toModelFragment)
+    const mapper = fragmentMapperInstantiator(values, fromModelFragment.contexts, toModelFragment.contexts)
+    return mapper
+  }
+
   // { rebuild: false, isModule: false }
   needsRebuild (template, instance, options) {
     if (options.rebuild) {
@@ -1518,6 +1586,11 @@ class Config {
     }
   }
 
+  updateBridge(id, updater) {
+    const bridge = this.config.bridges.find((b) => b.id === id)
+    updater({ config: this, bridge })
+  }
+
   addBridge (bridge, uuid) {
     if (!this.config.bridges) {
       this.config.bridges = []
@@ -1819,7 +1892,7 @@ class Config {
 
   // set the args in the api's
   setArgs (args) {
-    const setArgs = (config) => {
+    const setConfigArgs = (config) => {
       if (!config._api) {
         return
       }
@@ -1832,10 +1905,10 @@ class Config {
       }
     }
 
-    setArgs(this)
+    setConfigArgs(this)
     for (const config of this.configs) {
       if (config.config instanceof Config) {
-        setArgs(config.config)
+        setConfigArgs(config.config)
       }
     }
   }
@@ -1864,6 +1937,16 @@ class Config {
     }
   }
 
+  getObjects () {
+    const configs = {}
+    const ns = this.config.objects.namespaced
+    configs[this.name] = this
+    for (const config of this.configs) {
+      configs[config._name] = ns[config._uuid]
+    }
+    return configs
+  }
+
   getConfig (name) {
     if (this.name === name) {
       return this
@@ -1911,12 +1994,7 @@ class Config {
       } else if (this.scope == 'development') {
         new_result = !(element.scope === 'testing')
       }
-      if (global.GORDO && old_result !== new_result) {
-        global.GORDO = false
-        console.log("THERE WAS A DIFFERENCE ------------------------------------------------")
-        debugger // greg23old
-      }
-      return old_result
+      return new_result
     }
 
     config.operators = config.operators.filter((element) => keep(element))

package/src/configHelpers.js

@@ -112,14 +112,18 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
     throw new ErrorReason(context)
   }
   args.kms = config.getConfigs()
-  args.config = config
+  args.config = config.getPseudoConfig(uuidForScoping, config)
   args.hierarchy = hierarchy
   args.isA = isA(hierarchy)
   // args.listable = listable(hierarchy)
   // args.asList = asList
   args.retry = () => { throw new RetryError() }
-  args.fragments = (query) => {
-    return config.fragment(args, query)
+  // mappings are optional
+  args.fragments = (query, mappings) => {
+    return config.fragment(args, query, mappings)
+  }
+  args.fragmentMapper = (values, fromModelQuery, toModelQuery) => {
+    return config.fragmentMapper(args, values, fromModelQuery, toModelQuery)
   }
   args.breakOnSemantics = false
   args.theDebugger = {
@@ -133,20 +137,35 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.addPattern = (pattern, def) => config.addPattern(pattern, args.uuid)
   args.addGenerator = (generator) => config.addGenerator(generator, args.uuid, config.name)
 
+  if (config.testConfig?.testModuleName) {
+    args.testModuleName = config.testConfig.testModuleName
+  }
   args.addAssumedScoped = (args, assumed) => {
     const addAssumed = (args, ...moreAssumed) => {
       return { ...args, assumed: Object.assign({}, assumed, (args.assumed || {}), ...moreAssumed) }
     }
 
-    args.s = (c) => config.getSemantics(logs).apply(args, c)
-    args.g = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
+    args.s = (c, options = {}) => config.getSemantics(logs).apply(args, c, options)
+    args.g = (c, rest = {}) => {
+      // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
+      //   debugger
+      // }
+      const { assumed = {}, options = {} } = rest
+      return config.getGenerators(logs).apply(addAssumed(args, assumed), c, assumed, options)
     }
-    args.gp = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: true, isResponse: false, response: false }), c, { paraphrase: true, isResponse: false, response: false })
+    args.gp = (c, rest = {}) => {
+      // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
+      //   debugger
+      // }
+      const { assumed = {}, options = {} } = rest
+      return config.getGenerators(logs).apply(addAssumed(args, assumed, { paraphrase: true, isResponse: false, response: false }), c, { paraphrase: true, isResponse: false, response: false }, options)
     }
-    args.gr = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true })
+    args.gr = (c, rest  = {}) => {
+      // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
+      //   debugger
+      // }
+      const { assumed = {}, options = {} } = rest
+      return config.getGenerators(logs).apply(addAssumed(args, assumed, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true }, options)
     }
     args.e = (c) => {
       if (!c) {
@@ -181,6 +200,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
 
   Object.assign(args, args.getUUIDScoped(uuidForScoping || config.uuid))
   args.apis = args.apis || ((name) => config.getConfig(name).api)
+  // args.apis = (name) => config.getAPIs(name)
   /*
   if (uuidForScoping) {
     Object.assign(args, args.getUUIDScoped(uuidForScoping))
@@ -199,6 +219,22 @@ const getObjects = (objects) => {
   }
 }
 
+const run = async (config, handler) => {
+  // map to hash
+  config = config || {}
+  if (config.config) {
+    config = config
+  }
+
+  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+
+  const objects = config.config.objects.namespaced[config.uuid]
+  const logs = []
+  const args = {}
+  setupArgs(args, config, logs, hierarchy)
+  return handler(args)
+}
+
 const processContext = async (context, { objects = {}, config, logs = [] }) => {
   const generators = config.getGenerators(logs)
   const semantics = config.getSemantics(logs)
@@ -291,7 +327,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate }) => {
+const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -311,14 +347,13 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
   args.insert = (context) => toDo.unshift(context)
   let overlap, lastRange
   config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
-  let context_id_counter = 0
   while (toDo.length > 0) {
     const context = toDo.shift()
     args.calls.next()
     let contextPrime = context
     context.topLevel = true
-    context_id_counter += 1
-    context.context_id = context_id_counter
+    contextIdCounter += 1
+    context.context_id = contextIdCounter
     try {
       if (json.has_errors) {
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
@@ -328,6 +363,9 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
         const semantics = config.getSemantics(json.logs)
         try {
           contextPrime = await semantics.apply(args, context)
+          // contextPrime.greg = 'yes'
+//          console.log("context_id", context.context_id)
+//          console.log("semantics.apply", JSON.stringify(contextPrime, null, 2))
         } catch (e) {
           if (e.message == 'Maximum call stack size exceeded') {
             const mostCalled = semantics.getMostCalled()
@@ -368,21 +406,21 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
         const generated = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
         let generatedParenthesized = []
         if (generateParenthesized) {
-          config.parenthesized = true
+          config.setParenthesized(true)
           generatedParenthesized = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
-          config.parenthesized = false
+          config.setParenthesized(false)
         }
         // assumed = { paraphrase: true, response: false };
         assumed = { paraphrase: true, isResponse: false, response: false }
         if (generateParenthesized) {
-          config.parenthesized = false
+          config.setParenthesized(false)
         }
         const paraphrases = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
         let paraphrasesParenthesized = []
         if (generateParenthesized) {
-          config.parenthesized = true
+          config.setParenthesized(true)
           paraphrasesParenthesized = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
-          config.parenthesized = false
+          config.setParenthesized(false)
         }
         contextsPrime.push(contextPrime)
         generatedPrime.push(generated)
@@ -421,7 +459,7 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
       throw e
     }
   }
-  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
+  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime, updatedContextIdCounter: contextIdCounter }
 }
 
 // instance template loadTemplate
@@ -479,6 +517,9 @@ const loadInstance = async (config, instance) => {
         args.isModule = true
         args.isProcess = false
       }
+      if (instance.name == config.testConfig.testModuleName) {
+        args.isTesting = true
+      }
       await results.apply(args)
     } else if (results.isFragment) {
     } else {
@@ -490,6 +531,9 @@ const loadInstance = async (config, instance) => {
       args.instance = ''
       args.isProcess = !config.isModule
       args.isModule = !!config.isModule
+      if (instance.name == config.testConfig.testModuleName) {
+        args.isTesting = true
+      }
       await processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null
@@ -505,6 +549,7 @@ module.exports = {
   // listable,
   setupArgs,
   processContext,
+  run,
   getObjects,
   gs,
   processContextsB,

package/src/fragments.js

@@ -0,0 +1,83 @@
+const _ = require('lodash')
+const helpers = require('./helpers')
+
+function fragmentInstantiator (args, contexts) {
+  return new Object({ 
+    contexts: () => {
+      return _.cloneDeep(contexts)
+    },
+    instantiate: async (mappings) => {
+      const instantiated = _.cloneDeep(contexts)
+      const todo = [{ context: instantiated, path: [] }]
+      args = { ...args }
+      while (todo.length > 0) {
+        const { context, path } = todo.pop()
+        args.context = context
+        args.path = path
+        for (const mapping of mappings) {
+          if (await mapping.match(args)) {
+            await mapping.apply(args)
+          }
+        }
+        for (const key of Object.keys(context)) {
+          // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
+          if (!helpers.isCompound(context[key])) {
+            continue
+          }
+          if (context[key].instantiated) {
+            continue
+          }
+          todo.push({ context: context[key], path: [...path, key] })
+        }
+      }
+      if (contexts.length == 1 && instantiated.length == 1) {
+        return instantiated[0]
+      } else {
+        return instantiated
+      }
+    }
+  })
+}
+
+async function fragmentMapperInstantiator(values, modelFrom, modelTo) {
+  const paths = {}
+  for (const value of values) {
+    paths[value] = { value }
+  }
+
+  {
+    const fi = fragmentInstantiator({paths}, modelFrom)
+    await fi.instantiate([
+      {
+        match: ({context, path}) => values.includes(context.value),
+        apply: ({context, path}) => paths[context.value].from = path
+      },
+    ])
+  }
+
+  {
+    const fi = fragmentInstantiator({paths}, modelTo)
+    await fi.instantiate([
+      {
+        match: ({context, path}) => values.includes(context.value),
+        apply: ({context, path}) => paths[context.value].to = path
+      },
+    ])
+  }
+  return {
+    instantiate: (actualFrom) => {
+      const actualTo = structuredClone(modelTo)
+      for (const value in paths) {
+        const { from, to } = paths[value]
+        const actualValue = helpers.getByPath(actualFrom, from, null)
+        helpers.setByPath(actualTo, to, actualValue)
+      }
+      return actualTo
+    }
+  }
+}
+
+module.exports = {
+  fragmentInstantiator,
+  fragmentMapperInstantiator,
+}

package/src/generators.js

@@ -182,10 +182,10 @@ class Generators {
       const generator = this.generators[igenerator]
       if (await generator.matches(args, objects, context, hierarchy, config, options)) {
         const log = (message) => { this.logs.push(message) }
-        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
+        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${stringify(context)}`)
         let errorMessage = 'The apply function did not return a value'
         try {
-          generated = await generator.apply(args, objects, context, hierarchy, config, response, log)
+          generated = await generator.apply(args, objects, context, hierarchy, config, response, log, options)
         } catch (e) {
           // the next if handle this
           generated = null
@@ -202,7 +202,7 @@ class Generators {
           }
         }
         if (!generated && generated !== '') {
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
           const source = `${generator.km}/#${generator.index}`
@@ -212,7 +212,7 @@ class Generators {
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
-          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 10 }), null, 2))
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
@@ -230,7 +230,7 @@ class Generators {
           throw { error: [message], logs: this.logs }
         }
         if (((config || {}).config || {}).debug) {
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
           if (generator.index > -1 && generator.km) {
@@ -263,7 +263,7 @@ class Generators {
     }
     args.calls.pop()
     if (!applied && ((config || {}).config || {}).debug) {
-      const widths = [10, 10, 90]
+      const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Generator')
       lines.setElement(0, 2, 'No generator applied')
@@ -275,7 +275,11 @@ class Generators {
       lines.setElement(0, 2, JSON.stringify(context, null, 2))
       this.logs.push(lines.toString())
     }
-    return ((config || {}).parenthesized ? '(' + generated + ')' : generated)
+    let parenthesized = false
+    if (config && config.getParenthesized()) {
+      parenthesized = true
+    }
+    return parenthesized ? '(' + generated + ')' : generated
   }
 }
 

package/src/helpers.js

@@ -292,8 +292,53 @@ const hashCode = (str) => {
   return hash
 }
 
-const sortJson = (json) => {
-  return json
+/**
+ * Recursively sorts object keys alphabetically.
+ * Fully handles arrays (including objects inside arrays) and preserves Date/other non-plain objects.
+ *
+ * @param {any} input - The value to sort (object, array, primitive, etc.)
+ * @param {Object} [options]
+ *   - ignoreCase: boolean (default false) – case-insensitive sort
+ *   - reverse: boolean (default false) – reverse alphabetical order
+ * @returns {any} New sorted value
+ */
+function sortJson(input, options = {}) {
+  const { ignoreCase = false, reverse = false } = options;
+
+  // Helper: is this a plain object {} (not Array, Date, Map, null, etc.)
+  function isPlainObject(value) {
+    return value !== null && typeof value === 'object' && value.constructor === Object;
+  }
+
+  // Handle arrays: map over elements and recurse
+  if (Array.isArray(input)) {
+    return input.map(item => sortJson(item, options));
+  }
+
+  // If not a plain object, return unchanged (preserves Date, Map, Set, primitives, etc.)
+  if (!isPlainObject(input)) {
+    return input;
+  }
+
+  // Sorter for keys
+  const sorter = (a, b) => {
+    const A = ignoreCase ? a.toLowerCase() : a;
+    const B = ignoreCase ? b.toLowerCase() : b;
+    return reverse ? B.localeCompare(A) : A.localeCompare(B);
+  };
+
+  // Build new sorted object
+  const sorted = {};
+
+  Object.keys(input)
+    .sort(sorter)
+    .forEach(key => {
+      const value = input[key];
+      // Always recurse: handles nested objects and arrays properly
+      sorted[key] = sortJson(value, options);
+    });
+
+  return sorted;
 }
 
 const validProps = (valids, object, type) => {
@@ -436,6 +481,63 @@ const stableId = (tag) => {
   return id
 }
 
+function getByPath(obj, path, defaultValue) {
+  let current = obj;
+  for (const key of path) {
+    if (current === null || current === undefined) return defaultValue;
+    if (typeof current !== 'object') return defaultValue;
+    current = current[key];
+  }
+  return current === undefined ? defaultValue : current;
+}
+
+/**
+ * Set a value in an object by path array.
+ * Automatically creates missing objects {} or arrays [] as needed.
+ *
+ * @param {Object} obj - The root object to modify
+ * @param {Array<string|number>} path - Array of keys/indices
+ * @param {*} value - Value to set
+ * @returns {*} The set value (for chaining)
+ */
+function setByPath(obj, path, value) {
+  if (!Array.isArray(path) || path.length === 0) {
+    throw new Error('Path must be a non-empty array');
+  }
+
+  let current = obj;
+
+  for (let i = 0; i < path.length; i++) {
+    const key = path[i];
+    const isLast = i === path.length - 1;
+
+    if (isLast) {
+      // Final step — just assign
+      current[key] = value;
+    } else {
+      // Not last — ensure next level exists
+      const nextKey = path[i + 1];
+
+      if (current[key] == null) {
+        // Auto-create: array if next key is number, otherwise object
+        current[key] = typeof nextKey === 'number' || String(nextKey >>> 0) === nextKey
+          ? []
+          : {};
+      } else if (Array.isArray(current[key]) && typeof nextKey !== 'number') {
+        // Safety: if current is array but next key isn't a valid index → convert to object
+        current[key] = { ...current[key] };
+      } else if (!Array.isArray(current[key]) && typeof nextKey === 'number') {
+        // If next expects array but current is object → convert
+        current[key] = Object.values(current[key]);
+      }
+
+      current = current[key];
+    }
+  }
+
+  return value;
+}
+
 module.exports = {
   stableId,
   ecatch,
@@ -463,4 +565,6 @@ module.exports = {
   w,
   suggestAssociationsFix,
   suggestAssociationsFixFromSummaries,
+  getByPath,
+  setByPath,
 }

package/src/project2.js

@@ -3,6 +3,13 @@ function project(source, filters) {
     return source
   }
 
+  function isPlainObject(obj) {
+    return Object.prototype.toString.call(obj) === '[object Object]';
+  }
+  if (!isPlainObject(source) && !Array.isArray(source)) {
+    return source
+  }
+
   if (Object.keys(source).length === 0 && filters.length === 0) {
     return {};
   }

package/src/semantics.js

@@ -1,6 +1,7 @@
 const { args: contextArgs, normalizeGenerator, normalizeSemantic } = require('./helpers')
 const Lines = require('../lines')
 const helpers = require('./helpers')
+const debug = require('./debug')
 
 class Semantic {
   // constructor ({match, apply, uuid, index, km, notes}) {
@@ -70,6 +71,7 @@ class Semantic {
   }
 
   async matches (args, context, options = {}) {
+    args = {...args}
     this.fixUpArgs(args, context)
     const matches = await this.matcher(args)
     if (matches && (options.debug || {}).match || args.callId === this.callId) {
@@ -81,6 +83,7 @@ class Semantic {
   }
 
   async apply (args, context, s, options = {}) {
+    args = {...args}
     const { config } = args
     if (config && config.debugLoops) {
       console.log('apply', this.toLabel())
@@ -218,7 +221,7 @@ class Semantics {
  						errorMessage = e.toString()
           }
 
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
           const source = `${semantic.km}/#${semantic.index}`
@@ -248,7 +251,7 @@ class Semantics {
         args.calls.touch(contextPrime)
         // this.logs.push(`Semantics: applied ${semantic.toString()}\n to\n ${JSON.stringify(context)}\n the result was ${JSON.stringify(contextPrime)}\n`)
         if (((config || {}).config || {}).debug) {
-          const widths = [10, 10, 132]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
           if (semantic.index > -1 && semantic.km) {
@@ -295,7 +298,7 @@ class Semantics {
     }
     args.calls.pop()
     if (!applied && ((config || {}).config || {}).debug) {
-      const widths = [10, 10, 90]
+      const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Semantic')
       lines.setElement(0, 2, 'No semantic applied')