theprogrammablemind 9.5.1-beta.3 → 9.5.1-beta.31

package/client.js

@@ -12,9 +12,8 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
+const { sortJson, appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
 const runtime = require('./runtime')
-const sortJson = runtime.sortJson
 const debug = require('./src/debug')
 
 const getConfig_getObjectsCheck = (config, testConfig) => {
@@ -46,33 +45,33 @@ const getSuggestionMessage = (suggestion) => {
   return `Try adding this to the associations: { context: ${JSON.stringify(getSuggestion(suggestion))}, choose: <indexOfMainElement> },\n If that does not work look at the logs and check when the operators become wrong during an interation. Deduce the change based on the previous iteration and what operator was applied.`
 }
 
-const getConfig_getContextCheck = (testConfig) => {
-  return (testConfig.checks && testConfig.checks.context) || []
-}
-
 const pickContext = (contextChecks) => (context) => {
   return project2(context, contextChecks)
 }
 
 const pickObjects = (config, testConfig, getObjects) => {
-  /*
-  let testConfigName = config.name
-  if (testConfig.testModuleName) {
-    objects = getObjects(config.config.objects)(config.getConfigs()[testConfig.testModuleName].uuid)
-    testConfigName = testConfig.testModuleName
-  }
-  */
   const checks = getConfig_getObjectsCheck(config, testConfig)
+  const contextChecks = config.getContextChecks()
   const projection = {}
   for (const km in checks) {
     const objects = getObjects(km)
     if (!objects) {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
-    if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
-      projection[km] = project2(objects, checks[km])
+
+    if (false) {
+      if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
+        projection[km] = project2(objects, checks[km])
+      } else {
+        projection[km] = project(objects, checks[km])
+      }
     } else {
-      projection[km] = project(objects, checks[km])
+      const lastIndex = contextChecks.length - 1
+      const last = contextChecks[lastIndex]
+      contextChecks[lastIndex] = { match: last.match, apply: () => [...new Set([...(checks[km] || []), ...last.apply()])] }
+
+      // projection[km] = project2(objects, [...checks[km], ...contextChecks])
+      projection[km] = project2(objects, contextChecks)
     }
   }
   return projection
@@ -316,6 +315,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     }
 
     let startCounter = 0
+    let contextIdCounter = 0
     while (true) {
       if (queries.length === 0) {
         break
@@ -356,8 +356,9 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         }
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
-        const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
+                                  await processContextsB({ contextIdCounter, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
           clientSideTime = end - start
@@ -475,6 +476,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
     return
   }
   // initialize in between test so state is not preserved since the test was adding without state
+  config.testConfig.testModuleName = testConfig.testModuleName
   await config.rebuild()
   const errorHandler = (error) => {
     if (error.metadata) {
@@ -838,7 +840,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
   console.log(responses.trace)
 
-  if (true) {
+  if (false) {
     if (global.beforeObjects) {
       console.log('objects', runtime.jsonDiff.diffString(global.beforeObjects, config.get('objects')))
     } else {
@@ -913,7 +915,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   console.log('')
   const screen_width = process.stdout.columns
   // || 0 for when running without a console
-  const widths = [70, 10, Math.max(80, screen_width - 71 || 0)]
+  const widths = Lines.addRemainder([70, 10])
   const lines = new Lines(widths)
   lines.setElement(0, 0, '--- The paraphrases are ----------')
   lines.setElement(0, 2, '--- The response strings are ----------')
@@ -970,7 +972,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     if (instance.fragments) {
       pr = instance.fragments[index]
     }
-    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: false })
+    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: true })
   }
 
   const looper = async (configs) => {
@@ -1324,13 +1326,32 @@ const knowledgeModuleImpl = async ({
       parser.add_argument('-cl', '--checkForLoop', { nargs: '?', help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
       parser.add_argument('-r', '--reset', { action: 'store_true', help: 'Get the server to bypass the cache and rebuild everything' })
       parser.add_argument('-q', '--query', { help: 'Run the specified query' })
+      parser.add_argument('-f', '--filter', { help: 'for -pd only the data for the knowledge modules that start with this string will be shown' })
       parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
       parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
       parser.add_argument('-dt', '--deleteTest', { help: 'Delete the specified query from the tests file.' })
       parser.add_argument('--parenthesized', { action: 'store_true', help: 'Show the generated phrases with parenthesis.' })
       parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
       parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
-      parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, ha === hierarchy ancestors g === generators, s === semantics, l === load t=tests ordering p === priorities a === associations j === JSON sent to server. for example --print wb' })
+      parser.add_argument('-p', '--print', { help: 
+`Print the specified elements 
+    a === associations 
+    b === bridges, 
+    c === config,
+    cc === test checks,
+    d === objects (d for data), 
+    g === generators, 
+    h === hierarchy, 
+    ha === hierarchy ancestors,
+    j === JSON sent to server, 
+    l === load ordering,
+    o === operators,
+    p === priorities,
+    s === semantics, 
+    t === tests ordering,
+    w === words, 
+ for example --print wb' })
+` })
       parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
       parser.add_argument('-fr', '--failRebuild', { action: 'store_true', help: 'If a rebuild is required fail out.' })
       parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
@@ -1491,6 +1512,12 @@ const knowledgeModuleImpl = async ({
               counter += 1
             }
           }
+          if (hasArg('cc')) {
+            for (const cc of config.getContextChecks()) {
+              const printable = { ...cc, match: cc.match.toString(), apply: cc.apply.toString() }
+              console.log(JSON.stringify(printable, null, 2))
+            }
+          }
           if (hasArg('c')) {
             const { data } = setupProcessB({ config })
             console.log('Config as sent to server')
@@ -1554,8 +1581,19 @@ const knowledgeModuleImpl = async ({
           }
 
           if (hasArg('d')) {
-            console.log('objects (data) ================')
-            console.log(JSON.stringify(config.config.objects, null, 2))
+            if (args.filter) {
+              console.log(`objects (data) filtered by ${args.filter} ================`)
+              const projection = { namespaced: {} }
+              for (const key of Object.keys(config.config.objects.namespaced)) {
+                if (key.startsWith(args.filter)) {
+                  projection.namespaced[key] = config.config.objects.namespaced[key]
+                }
+              }
+              console.log(JSON.stringify(projection, null, 2))
+            } else {
+              console.log('objects (data) ================')
+              console.log(JSON.stringify(config.config.objects, null, 2))
+            }
           }
 
           if (hasArg('p')) {
@@ -2008,9 +2046,15 @@ const ensureTestFile = (module, name, type) => {
 }
 
 const knowledgeModule = async (...args) => {
-  await knowledgeModuleImpl(...args).catch((e) => {
+  await knowledgeModuleImpl(...args).catch(async (e) => {
     console.error(e)
-    process.exit(-1)
+    function sleep(ms) {
+      return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+      });
+    }
+    await sleep(1) // get the stderr to flush
+    await process.exit(-1);   // tiny trick: empty write forces flush of console.error buffer
   })
 }
 
@@ -2035,5 +2079,6 @@ module.exports = {
   gs,
   flattens,
   writeTest,
-  getConfigForTest
+  getConfigForTest,
+  debug,
 }

package/lines.js

@@ -5,6 +5,13 @@ class Lines {
     this.rows = []
   }
 
+  static SCREEN_WIDTH = 164
+
+  static addRemainder(widths) {
+    const sum = widths.reduce((a, b) => a + b)
+    return [...widths, Lines.SCREEN_WIDTH - sum]
+  }
+
   addLine () {
     this.lines.push(this.widths.map((width) => ''.padEnd(width)))
   }

package/package.json

@@ -3,14 +3,14 @@
     "@eslint/js": "^9.21.0",
     "@typescript-eslint/eslint-plugin": "^4.28.4",
     "@typescript-eslint/parser": "^4.28.4",
+    "argparse": "^2.0.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^16.0.3",
     "eslint-plugin-import": "^2.23.4",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
     "globals": "^16.0.0",
-    "jest": "^29.7.0",
-    "argparse": "^2.0.1"
+    "jest": "^30.2.0"
   },
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
@@ -45,6 +45,7 @@
     "runtime.js",
     "src/helpers.js",
     "src/flatten.js",
+    "src/fragments.js",
     "src/unflatten.js",
     "src/config.js",
     "src/configHelpers.js",
@@ -61,6 +62,7 @@
   "dependencies": {
     "base-64": "^1.0.0",
     "deep-equal": "^2.0.4",
+    "flatted": "^3.3.3",
     "fs": "0.0.1-security",
     "json-diff": "^1.0.3",
     "json-stable-stringify": "^1.0.1",
@@ -68,9 +70,8 @@
     "node-fetch": "^2.6.1",
     "readline": "^1.3.0",
     "scriptjs": "^2.5.9",
-    "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.5.1-beta.3",
+  "version": "9.5.1-beta.31",
   "license": "UNLICENSED"
 }

package/runtime.js

@@ -13,7 +13,6 @@ module.exports = {
   ArgumentParser,
   readline: require('readline'),
   jsonDiff: require('json-diff'),
-  sortJson: require('sort-json'),
   util: require('util'),
   performance: require('perf_hooks')
 }

package/src/config.js

@@ -10,6 +10,7 @@ const { ecatch } = require('./helpers')
 const runtime = require('../runtime')
 const _ = require('lodash')
 const db = require('./debug')
+const { fragmentInstantiator, fragmentMapperInstantiator } = require('./fragments')
 
 const debugBreak = () => {
   // debugger
@@ -20,8 +21,6 @@ const bags = [
   'semantics'
 ]
 
-global.GORDO = true
-
 const indent = (string, indent) => {
   return string.replace(/^/gm, ' '.repeat(indent))
 }
@@ -347,6 +346,8 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
           config.testConfig.checks.context = []
         }
         config.testConfig.checks.context.push({
+          'bridge.id': bridge.id,
+          'bridge.check': bridge.check,
           match: ({context}) => context.marker == bridge.id,
           exported: true,
           apply: ({context}) => bridge.check,
@@ -401,11 +402,15 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       if (bridge.generatorp) {
         const match = bridge.generatorp.match || (() => true)
         const apply = typeof bridge.generatorp === 'function' ? bridge.generatorp : bridge.generatorp.apply || bridge.generatorp
-        const level = bridge.generatorp.level >= 0 ? bridge.generatorp.level : bridge.level + 1
+        let level = bridge.generatorp.level >= 0 ? bridge.generatorp.level : bridge.level + 1
+        if (!bridge.bridge) {
+          level = 0
+        }
 
         const generator = {
           where: bridge.generatorp.where || bridge.where || helpers.where(4),
-          match: async (args) => bridge.id === args.context.marker && args.context.level === level && args.context.paraphrase && await match(args),
+          // match: async (args) => bridge.id === args.context.marker && args.context.level === level && args.context.paraphrase && await match(args),
+          match: async (args) => args.isA(args.context.marker, bridge.id) && args.context.level === level && args.context.paraphrase && await match(args),
           apply: (args) => apply(args),
           applyWrapped: apply,
           property: 'generatorp'
@@ -422,7 +427,8 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         const level = bridge.generatorr.level >= 0 ? bridge.generatorr.level : bridge.level + 1
         const generator = {
           where: bridge.generatorr.where || bridge.where || helpers.where(4),
-          match: async (args) => bridge.id === args.context.marker && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
+          // match: async (args) => bridge.id === args.context.marker && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
+          match: async (args) => args.isA(args.context.marker, bridge.id) && args.context.level === level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
           apply: (args) => apply(args),
           applyWrapped: apply,
           property: 'generatorr'
@@ -509,8 +515,43 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
   if (moreConfig.bridges) {
     moreConfig.bridges = moreConfig.bridges.map((bridge) => {
       bridge = { ...bridge }
-      const valid = ['after', 'conditional', 'associations', 'before', 'bridge', 'check', 'disabled', 'scope', 'skipable', 'return_type_selector', 'evaluator', 'evaluators', 'generatorp', 'generatorr', 'generatorpr', 'generators', 'operator', 'id', 'convolution', 'inverted', 'isA', 'children', 'parents',
-        'level', 'optional', 'selector', 'separators', 'semantic', 'semantics', 'words', /Bridge$/, 'localHierarchy', 'levelSpecificHierarchy', 'where', 'uuid']
+      const valid = [
+        'after', 
+        'associations', 
+        'before', 
+        'bridge', 
+        'check', 
+        'children', 
+        'conditional', 
+        'convolution', 
+        'disabled', 
+        'enhanced_associations', 
+        'evaluator', 
+        'evaluators', 
+        'generatorp', 
+        'generatorpr', 
+        'generatorr', 
+        'generators', 
+        'id', 
+        'inverted', 
+        'isA', 
+        'level', 
+        'levelSpecificHierarchy', 
+        'localHierarchy', 
+        'operator', 
+        'optional', 
+        'parents',
+        'return_type_selector', 
+        'scope', 
+        'selector', 
+        'semantic', 
+        'semantics', 
+        'separators', 
+        'skipable', 
+        'uuid',
+        'where', 
+        'words', /Bridge$/, 
+      ]
       helpers.validProps(valid, bridge, 'bridge')
       handleBridgeProps(baseConfig, bridge, { addFirst, uuid })
       return bridge
@@ -673,7 +714,7 @@ function setWordsUUIDs (words, uuid) {
   for (const key in literals) {
     literals[key] = literals[key].map((o) => Object.assign(o, { uuid }))
   }
-  const patterns = words.patterns
+  const patterns = words.patterns || []
   for (const pattern of patterns) {
     pattern.defs.map((def) => Object.assign(def, { uuid }))
   }
@@ -902,6 +943,10 @@ class Config {
     return config_toServer(config)
   }
 
+  async run(handler) {
+    return configHelpers.run(this, handler)
+  }
+
   async fixtures () {
     if (this.testConfig?.fixtures) {
       const args = {}
@@ -935,6 +980,7 @@ class Config {
 
   getPseudoConfig (uuid, config) {
     return {
+      pseudo: true,
       description: 'this is a pseudo config that has limited functionality due to being available in the initializer and fixtures function context',
       addAssociation: (...args) => this.addAssociation(...args),
       addAssociations: (...args) => this.addAssociations(...args),
@@ -948,6 +994,14 @@ class Config {
       removeSemantic: (...args) => this.removeSemantic(...args, uuid, config.name),
       addWord: (...args) => this.addWord(...args, uuid),
       addPattern: (...args) => this.addPattern(...args, uuid),
+      updateBridge: (...args) => this.updateBridge(...args),
+      processContext: (...args) => this.processContext(...args),
+
+      semantics: () => this.config.semantics,
+      getConfigs: () => this.configs,
+      getTests: () => this.getTests(),
+      getName: () => this.getName(),
+      getDescription: () => this.getDescription(),
 
       getHierarchy: (...args) => this.config.hierarchy,
       getBridges: (...args) => this.config.bridges,
@@ -957,10 +1011,33 @@ class Config {
       fragment: (...args) => this.fragment(...args),
       server: (...args) => this.server(...args),
       exists: (...args) => this.exists(...args),
-      addAPI: (...args) => this.addAPI(...args)
+      addAPI: (...args) => this.addAPI(...args),
+
+      getParenthesized: () => this.getParenthesized(),
+      setParenthesized: (...args) => this.setParenthesized(...args),
     }
   }
 
+  getName() {
+    return this.name
+  }
+
+  getDescription() {
+    return this.description
+  }
+
+  getTests() {
+    return this.tests
+  }
+
+  getParenthesized(value) {
+    return this.parenthesized
+  }
+
+  setParenthesized(value) {
+    this.parenthesized = value
+  }
+
   inDevelopmentMode (call) {
     config.developmentModeOn += 1
     try {
@@ -1107,59 +1184,52 @@ class Config {
     return instance
   }
 
-  fragmentInstantiator (args, contexts) {
-    return new Object({
-      contexts: () => contexts,
-      instantiate: async (mappings) => {
-        const instantiated = _.cloneDeep(contexts)
-        // const todo = [...instantiated]
-        // const todo = [...instantiated]
-        const todo = _.clone(instantiated)
-        args = { ...args }
-        while (todo.length > 0) {
-          const context = todo.pop()
-          args.context = context
-          for (const mapping of mappings) {
-            if (await mapping.match(args)) {
-              await mapping.apply(args)
-            }
-          }
-          for (const key of Object.keys(context)) {
-            // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
-            if (!helpers.isCompound(context[key])) {
-              continue
-            }
-            if (context[key].instantiated) {
-              continue
-            }
-            todo.push(context[key])
-          }
-        }
-        return instantiated
-      }
-    })
-  }
-
-  fragment (args, query) {
+  getFragment(query) {
     for (const instance of (this.instances || [])) {
       for (const fragment of (instance.fragments || [])) {
         if (fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
       for (const fragment of (instance.resultss || [])) {
         if (fragment.isFragment && fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
       for (const fragment of (this.fragmentsBeingBuilt || [])) {
         if (fragment.query === query) {
-          return this.fragmentInstantiator(args, fragment.contexts)
+          return fragment
         }
       }
     }
   }
 
+  // mappings are optional
+  async fragment (args, query, mappings) {
+    const fragment = this.getFragment(query)
+    if (fragment) {
+      let fi = fragmentInstantiator(args, fragment.contexts)
+      if (mappings) {
+        fi = await fi
+        return fi.instantiate([{
+          match: ({context}) => !!mappings[context.value], // is the value in the mappings
+          apply: ({context}) => Object.assign(context, mappings[context.value]),
+        }])
+      } else {
+        return fi
+      }
+    }
+  }
+
+  fragmentMapper (args, values, fromModelQuery, toModelQuery) {
+    const fromModelFragment = this.getFragment(fromModelQuery)
+    console.dir(fromModelFragment)
+    const toModelFragment = this.getFragment(toModelQuery)
+    console.dir(toModelFragment)
+    const mapper = fragmentMapperInstantiator(values, fromModelFragment.contexts, toModelFragment.contexts)
+    return mapper
+  }
+
   // { rebuild: false, isModule: false }
   needsRebuild (template, instance, options) {
     if (options.rebuild) {
@@ -1518,6 +1588,11 @@ class Config {
     }
   }
 
+  updateBridge(id, updater) {
+    const bridge = this.config.bridges.find((b) => b.id === id)
+    updater({ config: this, bridge })
+  }
+
   addBridge (bridge, uuid) {
     if (!this.config.bridges) {
       this.config.bridges = []
@@ -1819,7 +1894,7 @@ class Config {
 
   // set the args in the api's
   setArgs (args) {
-    const setArgs = (config) => {
+    const setConfigArgs = (config) => {
       if (!config._api) {
         return
       }
@@ -1832,10 +1907,10 @@ class Config {
       }
     }
 
-    setArgs(this)
+    setConfigArgs(this)
     for (const config of this.configs) {
       if (config.config instanceof Config) {
-        setArgs(config.config)
+        setConfigArgs(config.config)
       }
     }
   }
@@ -1864,6 +1939,16 @@ class Config {
     }
   }
 
+  getObjects () {
+    const configs = {}
+    const ns = this.config.objects.namespaced
+    configs[this.name] = this
+    for (const config of this.configs) {
+      configs[config._name] = ns[config._uuid]
+    }
+    return configs
+  }
+
   getConfig (name) {
     if (this.name === name) {
       return this
@@ -1911,12 +1996,7 @@ class Config {
       } else if (this.scope == 'development') {
         new_result = !(element.scope === 'testing')
       }
-      if (global.GORDO && old_result !== new_result) {
-        global.GORDO = false
-        console.log("THERE WAS A DIFFERENCE ------------------------------------------------")
-        debugger // greg23old
-      }
-      return old_result
+      return new_result
     }
 
     config.operators = config.operators.filter((element) => keep(element))
@@ -2707,19 +2787,37 @@ class Config {
 
   getContextChecks() {
     const allChecks = []
-    for (const name of this.loadOrdering) {
-      const checks = this.kms[name].testConfig?.checks?.context || []
-      for (const check of checks) {
-        if (check.exported) {
-          allChecks.push(check)
+    let defaults = () => []
+    if (this.loadOrdering) {
+      for (const name of this.loadOrdering) {
+        const checks = this.kms[name].testConfig?.checks?.context || []
+        const oldDefaults = defaults
+        for (const check of checks) {
+          if (!check.match) {
+            const oldDefaults = defaults
+            defaults = () => [...new Set([...check.apply(), ...oldDefaults()])]
+            continue
+          }
+          if (check.exported) {
+            allChecks.push(check)
+          }
         }
       }
     }
     for (const check of this.testConfig?.checks?.context || []) {
+      if (!check.match) {
+        const oldDefaults = defaults
+        defaults = () => [...new Set([...check.apply(), ...oldDefaults()])]
+        continue
+      }
       if (!check.exported) {
         allChecks.push(check)
       }
     }
+    allChecks.push({
+      match: () => true,
+      apply: defaults
+    })
     return allChecks
   }
 

package/src/configHelpers.js

@@ -112,14 +112,18 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
     throw new ErrorReason(context)
   }
   args.kms = config.getConfigs()
-  args.config = config
+  args.config = config.getPseudoConfig(uuidForScoping, config)
   args.hierarchy = hierarchy
   args.isA = isA(hierarchy)
   // args.listable = listable(hierarchy)
   // args.asList = asList
   args.retry = () => { throw new RetryError() }
-  args.fragments = (query) => {
-    return config.fragment(args, query)
+  // mappings are optional
+  args.fragments = (query, mappings) => {
+    return config.fragment(args, query, mappings)
+  }
+  args.fragmentMapper = (values, fromModelQuery, toModelQuery) => {
+    return config.fragmentMapper(args, values, fromModelQuery, toModelQuery)
   }
   args.breakOnSemantics = false
   args.theDebugger = {
@@ -133,20 +137,35 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.addPattern = (pattern, def) => config.addPattern(pattern, args.uuid)
   args.addGenerator = (generator) => config.addGenerator(generator, args.uuid, config.name)
 
+  if (config.testConfig?.testModuleName) {
+    args.testModuleName = config.testConfig.testModuleName
+  }
   args.addAssumedScoped = (args, assumed) => {
     const addAssumed = (args, ...moreAssumed) => {
       return { ...args, assumed: Object.assign({}, assumed, (args.assumed || {}), ...moreAssumed) }
     }
 
-    args.s = (c) => config.getSemantics(logs).apply(args, c)
-    args.g = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
+    args.s = (c, options = {}) => config.getSemantics(logs).apply(args, c, options)
+    args.g = (c, rest = {}) => {
+      // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
+      //   debugger
+      // }
+      const { assumed = {}, options = {} } = rest
+      return config.getGenerators(logs).apply(addAssumed(args, assumed), c, assumed, options)
     }
-    args.gp = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: true, isResponse: false, response: false }), c, { paraphrase: true, isResponse: false, response: false })
+    args.gp = (c, rest = {}) => {
+      // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
+      //   debugger
+      // }
+      const { assumed = {}, options = {} } = rest
+      return config.getGenerators(logs).apply(addAssumed(args, assumed, { paraphrase: true, isResponse: false, response: false }), c, { paraphrase: true, isResponse: false, response: false }, options)
     }
-    args.gr = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true })
+    args.gr = (c, rest  = {}) => {
+      // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
+      //   debugger
+      // }
+      const { assumed = {}, options = {} } = rest
+      return config.getGenerators(logs).apply(addAssumed(args, assumed, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true }, options)
     }
     args.e = (c) => {
       if (!c) {
@@ -181,6 +200,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
 
   Object.assign(args, args.getUUIDScoped(uuidForScoping || config.uuid))
   args.apis = args.apis || ((name) => config.getConfig(name).api)
+  // args.apis = (name) => config.getAPIs(name)
   /*
   if (uuidForScoping) {
     Object.assign(args, args.getUUIDScoped(uuidForScoping))
@@ -199,6 +219,22 @@ const getObjects = (objects) => {
   }
 }
 
+const run = async (config, handler) => {
+  // map to hash
+  config = config || {}
+  if (config.config) {
+    config = config
+  }
+
+  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+
+  const objects = config.config.objects.namespaced[config.uuid]
+  const logs = []
+  const args = {}
+  setupArgs(args, config, logs, hierarchy)
+  return handler(args)
+}
+
 const processContext = async (context, { objects = {}, config, logs = [] }) => {
   const generators = config.getGenerators(logs)
   const semantics = config.getSemantics(logs)
@@ -291,7 +327,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate }) => {
+const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -311,14 +347,13 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
   args.insert = (context) => toDo.unshift(context)
   let overlap, lastRange
   config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
-  let context_id_counter = 0
   while (toDo.length > 0) {
     const context = toDo.shift()
     args.calls.next()
     let contextPrime = context
     context.topLevel = true
-    context_id_counter += 1
-    context.context_id = context_id_counter
+    contextIdCounter += 1
+    context.context_id = contextIdCounter
     try {
       if (json.has_errors) {
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
@@ -328,6 +363,9 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
         const semantics = config.getSemantics(json.logs)
         try {
           contextPrime = await semantics.apply(args, context)
+          // contextPrime.greg = 'yes'
+//          console.log("context_id", context.context_id)
+//          console.log("semantics.apply", JSON.stringify(contextPrime, null, 2))
         } catch (e) {
           if (e.message == 'Maximum call stack size exceeded') {
             const mostCalled = semantics.getMostCalled()
@@ -368,21 +406,21 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
         const generated = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
         let generatedParenthesized = []
         if (generateParenthesized) {
-          config.parenthesized = true
+          config.setParenthesized(true)
           generatedParenthesized = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
-          config.parenthesized = false
+          config.setParenthesized(false)
         }
         // assumed = { paraphrase: true, response: false };
         assumed = { paraphrase: true, isResponse: false, response: false }
         if (generateParenthesized) {
-          config.parenthesized = false
+          config.setParenthesized(false)
         }
         const paraphrases = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
         let paraphrasesParenthesized = []
         if (generateParenthesized) {
-          config.parenthesized = true
+          config.setParenthesized(true)
           paraphrasesParenthesized = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
-          config.parenthesized = false
+          config.setParenthesized(false)
         }
         contextsPrime.push(contextPrime)
         generatedPrime.push(generated)
@@ -421,7 +459,7 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
       throw e
     }
   }
-  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
+  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime, updatedContextIdCounter: contextIdCounter }
 }
 
 // instance template loadTemplate
@@ -479,6 +517,9 @@ const loadInstance = async (config, instance) => {
         args.isModule = true
         args.isProcess = false
       }
+      if (instance.name == config.testConfig.testModuleName) {
+        args.isTesting = true
+      }
       await results.apply(args)
     } else if (results.isFragment) {
     } else {
@@ -490,6 +531,9 @@ const loadInstance = async (config, instance) => {
       args.instance = ''
       args.isProcess = !config.isModule
       args.isModule = !!config.isModule
+      if (instance.name == config.testConfig.testModuleName) {
+        args.isTesting = true
+      }
       await processContextsB(args)
       if (results.skipSemantics) {
         config.config.skipSemantics = null
@@ -505,6 +549,7 @@ module.exports = {
   // listable,
   setupArgs,
   processContext,
+  run,
   getObjects,
   gs,
   processContextsB,

package/src/fragments.js

@@ -0,0 +1,83 @@
+const _ = require('lodash')
+const helpers = require('./helpers')
+
+function fragmentInstantiator (args, contexts) {
+  return new Object({ 
+    contexts: () => {
+      return _.cloneDeep(contexts)
+    },
+    instantiate: async (mappings) => {
+      const instantiated = _.cloneDeep(contexts)
+      const todo = [{ context: instantiated, path: [] }]
+      args = { ...args }
+      while (todo.length > 0) {
+        const { context, path } = todo.pop()
+        args.context = context
+        args.path = path
+        for (const mapping of mappings) {
+          if (await mapping.match(args)) {
+            await mapping.apply(args)
+          }
+        }
+        for (const key of Object.keys(context)) {
+          // if (['number', 'string', 'boolean'].includes(typeof (context[key]))) {
+          if (!helpers.isCompound(context[key])) {
+            continue
+          }
+          if (context[key].instantiated) {
+            continue
+          }
+          todo.push({ context: context[key], path: [...path, key] })
+        }
+      }
+      if (contexts.length == 1 && instantiated.length == 1) {
+        return instantiated[0]
+      } else {
+        return instantiated
+      }
+    }
+  })
+}
+
+async function fragmentMapperInstantiator(values, modelFrom, modelTo) {
+  const paths = {}
+  for (const value of values) {
+    paths[value] = { value }
+  }
+
+  {
+    const fi = fragmentInstantiator({paths}, modelFrom)
+    await fi.instantiate([
+      {
+        match: ({context, path}) => values.includes(context.value),
+        apply: ({context, path}) => paths[context.value].from = path
+      },
+    ])
+  }
+
+  {
+    const fi = fragmentInstantiator({paths}, modelTo)
+    await fi.instantiate([
+      {
+        match: ({context, path}) => values.includes(context.value),
+        apply: ({context, path}) => paths[context.value].to = path
+      },
+    ])
+  }
+  return {
+    instantiate: (actualFrom) => {
+      const actualTo = structuredClone(modelTo)
+      for (const value in paths) {
+        const { from, to } = paths[value]
+        const actualValue = helpers.getByPath(actualFrom, from, null)
+        helpers.setByPath(actualTo, to, actualValue)
+      }
+      return actualTo
+    }
+  }
+}
+
+module.exports = {
+  fragmentInstantiator,
+  fragmentMapperInstantiator,
+}

package/src/generators.js

@@ -182,10 +182,10 @@ class Generators {
       const generator = this.generators[igenerator]
       if (await generator.matches(args, objects, context, hierarchy, config, options)) {
         const log = (message) => { this.logs.push(message) }
-        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
+        // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${stringify(context)}`)
         let errorMessage = 'The apply function did not return a value'
         try {
-          generated = await generator.apply(args, objects, context, hierarchy, config, response, log)
+          generated = await generator.apply(args, objects, context, hierarchy, config, response, log, options)
         } catch (e) {
           // the next if handle this
           generated = null
@@ -202,7 +202,7 @@ class Generators {
           }
         }
         if (!generated && generated !== '') {
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
           const source = `${generator.km}/#${generator.index}`
@@ -212,7 +212,7 @@ class Generators {
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
-          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
+          lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 10 }), null, 2))
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
@@ -230,7 +230,7 @@ class Generators {
           throw { error: [message], logs: this.logs }
         }
         if (((config || {}).config || {}).debug) {
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
           if (generator.index > -1 && generator.km) {
@@ -263,7 +263,7 @@ class Generators {
     }
     args.calls.pop()
     if (!applied && ((config || {}).config || {}).debug) {
-      const widths = [10, 10, 90]
+      const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Generator')
       lines.setElement(0, 2, 'No generator applied')
@@ -275,7 +275,11 @@ class Generators {
       lines.setElement(0, 2, JSON.stringify(context, null, 2))
       this.logs.push(lines.toString())
     }
-    return ((config || {}).parenthesized ? '(' + generated + ')' : generated)
+    let parenthesized = false
+    if (config && config.getParenthesized()) {
+      parenthesized = true
+    }
+    return parenthesized ? '(' + generated + ')' : generated
   }
 }
 

package/src/helpers.js

@@ -292,8 +292,53 @@ const hashCode = (str) => {
   return hash
 }
 
-const sortJson = (json) => {
-  return json
+/**
+ * Recursively sorts object keys alphabetically.
+ * Fully handles arrays (including objects inside arrays) and preserves Date/other non-plain objects.
+ *
+ * @param {any} input - The value to sort (object, array, primitive, etc.)
+ * @param {Object} [options]
+ *   - ignoreCase: boolean (default false) – case-insensitive sort
+ *   - reverse: boolean (default false) – reverse alphabetical order
+ * @returns {any} New sorted value
+ */
+function sortJson(input, options = {}) {
+  const { ignoreCase = false, reverse = false } = options;
+
+  // Helper: is this a plain object {} (not Array, Date, Map, null, etc.)
+  function isPlainObject(value) {
+    return value !== null && typeof value === 'object' && value.constructor === Object;
+  }
+
+  // Handle arrays: map over elements and recurse
+  if (Array.isArray(input)) {
+    return input.map(item => sortJson(item, options));
+  }
+
+  // If not a plain object, return unchanged (preserves Date, Map, Set, primitives, etc.)
+  if (!isPlainObject(input)) {
+    return input;
+  }
+
+  // Sorter for keys
+  const sorter = (a, b) => {
+    const A = ignoreCase ? a.toLowerCase() : a;
+    const B = ignoreCase ? b.toLowerCase() : b;
+    return reverse ? B.localeCompare(A) : A.localeCompare(B);
+  };
+
+  // Build new sorted object
+  const sorted = {};
+
+  Object.keys(input)
+    .sort(sorter)
+    .forEach(key => {
+      const value = input[key];
+      // Always recurse: handles nested objects and arrays properly
+      sorted[key] = sortJson(value, options);
+    });
+
+  return sorted;
 }
 
 const validProps = (valids, object, type) => {
@@ -436,6 +481,63 @@ const stableId = (tag) => {
   return id
 }
 
+function getByPath(obj, path, defaultValue) {
+  let current = obj;
+  for (const key of path) {
+    if (current === null || current === undefined) return defaultValue;
+    if (typeof current !== 'object') return defaultValue;
+    current = current[key];
+  }
+  return current === undefined ? defaultValue : current;
+}
+
+/**
+ * Set a value in an object by path array.
+ * Automatically creates missing objects {} or arrays [] as needed.
+ *
+ * @param {Object} obj - The root object to modify
+ * @param {Array<string|number>} path - Array of keys/indices
+ * @param {*} value - Value to set
+ * @returns {*} The set value (for chaining)
+ */
+function setByPath(obj, path, value) {
+  if (!Array.isArray(path) || path.length === 0) {
+    throw new Error('Path must be a non-empty array');
+  }
+
+  let current = obj;
+
+  for (let i = 0; i < path.length; i++) {
+    const key = path[i];
+    const isLast = i === path.length - 1;
+
+    if (isLast) {
+      // Final step — just assign
+      current[key] = value;
+    } else {
+      // Not last — ensure next level exists
+      const nextKey = path[i + 1];
+
+      if (current[key] == null) {
+        // Auto-create: array if next key is number, otherwise object
+        current[key] = typeof nextKey === 'number' || String(nextKey >>> 0) === nextKey
+          ? []
+          : {};
+      } else if (Array.isArray(current[key]) && typeof nextKey !== 'number') {
+        // Safety: if current is array but next key isn't a valid index → convert to object
+        current[key] = { ...current[key] };
+      } else if (!Array.isArray(current[key]) && typeof nextKey === 'number') {
+        // If next expects array but current is object → convert
+        current[key] = Object.values(current[key]);
+      }
+
+      current = current[key];
+    }
+  }
+
+  return value;
+}
+
 module.exports = {
   stableId,
   ecatch,
@@ -463,4 +565,6 @@ module.exports = {
   w,
   suggestAssociationsFix,
   suggestAssociationsFixFromSummaries,
+  getByPath,
+  setByPath,
 }

package/src/project2.js

@@ -1,7 +1,32 @@
-function project(source, filters) {
+function areFirstNEqual(arr1, arr2, n) {
+  if (n <= 0) return true;
+  if (arr1.length < n || arr2.length < n) return false;
+
+  for (let i = 0; i < n; i++) {
+    if (arr1[i] !== arr2[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function project(source, filters, path=[]) {
   if (['string', 'number'].includes(typeof source)) {
     return source
   }
+  if (Array.isArray(source)) {
+    const result = []
+    for (const value of source) {
+      result.push(project(value, filters, [...path, '*']))
+    }
+    return result
+  }
+  function isPlainObject(obj) {
+    return Object.prototype.toString.call(obj) === '[object Object]';
+  }
+  if (!isPlainObject(source) && !Array.isArray(source)) {
+    return source
+  }
 
   if (Object.keys(source).length === 0 && filters.length === 0) {
     return {};
@@ -11,7 +36,7 @@ function project(source, filters) {
   const filter = filters.find(f => f.match({ context: source }));
   if (!filter) {
     if (Array.isArray(source)) {
-      return source.map((element) => project(element, filters))
+      return source.map((element) => project(element, filters, [...path, '*']))
     }
     return {};
   }
@@ -22,6 +47,7 @@ function project(source, filters) {
   // update 
   const updatedProperties = []
   for (const property of properties) {
+    // property that contains a list of properties to be checked
     if (property.properties) {
       for (const moreProperty of source[property.properties] || []) {
         updatedProperties.push(moreProperty)
@@ -35,10 +61,34 @@ function project(source, filters) {
   // Build the result object
   const result = {};
   properties.forEach(prop => {
-    if (source.hasOwnProperty(prop)) {
+    if (prop.path && (prop.path.length === path.length + 1) && areFirstNEqual(path, prop.path, path.length) && source.hasOwnProperty(prop.path[path.length])) {
+      const endProp = prop.path[path.length]
+      if (Array.isArray(source[endProp])) {
+        result[endProp] = []
+        for (const key in source[endProp]) {
+          result[endProp].push(project(source[endProp][key], filters, [...path, endProp, key]))
+        }
+      } else {
+        result[endProp] = {}
+        for (const key in source[endProp]) {
+          result[endProp][key] = project(source[endProp][key], filters, [...path, endProp, key])
+        }
+      }
+    } else if (source.hasOwnProperty(prop)) {
       // If the property is an object and not null, recursively project it
       if (typeof source[prop] === 'object' && source[prop] !== null) {
-        result[prop] = project(source[prop], filters);
+        result[prop] = project(source[prop], filters, [...path, prop]);
+      } else {
+        // Copy primitive or null properties directly
+        result[prop] = source[prop];
+      }
+    } else if (prop.property && source.hasOwnProperty(prop.property)) {
+      // If the property is an object and not null, recursively project it
+      if (typeof source[prop.property] === 'object' && source[prop.property] !== null) {
+        result[prop.property] = {}
+        for (const key of prop.check) {
+          result[prop.property][key] = project(source[prop.property][key], filters, [...path, prop.property, key]);
+        }
       } else {
         // Copy primitive or null properties directly
         result[prop] = source[prop];

package/src/semantics.js

@@ -1,6 +1,7 @@
 const { args: contextArgs, normalizeGenerator, normalizeSemantic } = require('./helpers')
 const Lines = require('../lines')
 const helpers = require('./helpers')
+const debug = require('./debug')
 
 class Semantic {
   // constructor ({match, apply, uuid, index, km, notes}) {
@@ -70,6 +71,7 @@ class Semantic {
   }
 
   async matches (args, context, options = {}) {
+    args = {...args}
     this.fixUpArgs(args, context)
     const matches = await this.matcher(args)
     if (matches && (options.debug || {}).match || args.callId === this.callId) {
@@ -81,6 +83,7 @@ class Semantic {
   }
 
   async apply (args, context, s, options = {}) {
+    args = {...args}
     const { config } = args
     if (config && config.debugLoops) {
       console.log('apply', this.toLabel())
@@ -218,7 +221,7 @@ class Semantics {
  						errorMessage = e.toString()
           }
 
-          const widths = [10, 10, 90]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
           const source = `${semantic.km}/#${semantic.index}`
@@ -248,7 +251,7 @@ class Semantics {
         args.calls.touch(contextPrime)
         // this.logs.push(`Semantics: applied ${semantic.toString()}\n to\n ${JSON.stringify(context)}\n the result was ${JSON.stringify(contextPrime)}\n`)
         if (((config || {}).config || {}).debug) {
-          const widths = [10, 10, 132]
+          const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
           if (semantic.index > -1 && semantic.km) {
@@ -295,7 +298,7 @@ class Semantics {
     }
     args.calls.pop()
     if (!applied && ((config || {}).config || {}).debug) {
-      const widths = [10, 10, 90]
+      const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Semantic')
       lines.setElement(0, 2, 'No semantic applied')