theprogrammablemind 9.8.0 → 9.9.0

package/client.js

@@ -12,7 +12,7 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { helpDebugHierarchy, sortJson, appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps, OverrideCheck, } = require('./src/helpers')
+const { helpDebugHierarchy, sortJson, appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps, OverrideCheck, removeProp, } = require('./src/helpers')
 const runtime = require('./runtime')
 const db = require('./src/debug')
 
@@ -59,20 +59,11 @@ const pickObjects = (config, testConfig, getObjects) => {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
 
-    if (false) {
-      if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
-        projection[km] = project2(objects, checks[km])
-      } else {
-        projection[km] = project(objects, checks[km])
-      }
-    } else {
-      const lastIndex = contextChecks.length - 1
-      const last = contextChecks[lastIndex]
-      contextChecks[lastIndex] = { match: last.match, apply: () => [...new Set([...(checks[km] || []), ...last.apply()])] }
+    const lastIndex = contextChecks.length - 1
+    const last = contextChecks[lastIndex]
+    contextChecks[lastIndex] = { match: last.match, apply: () => [...new Set([...(checks[km] || []), ...last.apply()])] }
 
-      // projection[km] = project2(objects, [...checks[km], ...contextChecks])
-      projection[km] = project2(objects, contextChecks)
-    }
+    projection[km] = project2(objects, contextChecks)
   }
   return projection
 }
@@ -167,13 +158,6 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   if (runtime.fs.existsSync(fn)) {
     tests = JSON.parse(runtime.fs.readFileSync(fn))
   }
-  /*
-  for (const association of associations) {
-    association.sort()
-  }
-  associations.sort()
-  */
-  // tests[query] = sortJson({ paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
   const results = sortJson({
     query,
     summaries,
@@ -198,7 +182,6 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   if (!wasSet) {
     tests.push(results)
   }
-  // fs.writeFileSync(fn, stringify(tests, { space: 2 }), { encoding: 'utf8', flag: 'w+' })
   writeTestFile(fn, tests)
 }
 
@@ -267,7 +250,7 @@ const throwErrorHandler = (error) => {
   throw error
 }
 
-const _process = async (config, query, { queryNumber, initializer, commandLineArgs, credentials, writeTests, logs, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
+const _process = async (config, query, { isFragment, queryNumber, initializer, commandLineArgs, credentials, writeTests, logs, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -373,7 +356,7 @@ const _process = async (config, query, { queryNumber, initializer, commandLineAr
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
         const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ contextIdCounter, logs, calls, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+                                  await processContextsB({ skipSemantics: isFragment, isFragment, contextIdCounter, logs, calls, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
@@ -390,9 +373,9 @@ const _process = async (config, query, { queryNumber, initializer, commandLineAr
         response.trace = response.trace.concat(json.trace)
         response.version = json.version
         response.explain_priorities = json.explain_priorities
-        response.explain_associations = json.explain_associations
+        response.explain_associations_details = (response.explain_associations_details || []).concat(json.explain_associations_details)
+        response.explain_associations_trie = json.explain_associations_trie
         response.contextual_priorities_ambiguities = json.contextual_priorities_ambiguities
-        // response.rtf_associations = json.rtf_associations
 
         response.contexts = response.contexts.concat(contextsPrime)
         response.generated = response.generated.concat(generatedPrime)
@@ -895,8 +878,11 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     }
   }
 
-  if (responses.explain_associations) {
-    console.log(responses.explain_associations)
+  if (responses.explain_associations_details) {
+    console.log(responses.explain_associations_trie)
+    for (const l of responses.explain_associations_details) {
+      console.log(l)
+    }
   }
 
   // const objects = config.get('objects').namespaced[config.uuid]
@@ -907,6 +893,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     const km = config.configs.find((km) => km.name === name)
     return config.config.objects.namespaced[km.uuid] || {}
   }
+
   const picked = sortJson(pickObjects(config, config.testConfig, actualGetObjects), { depth: 25 })
   if (!_.isEmpty(picked)) {
     console.log('--- Object showing only the checked values ---')
@@ -1041,7 +1028,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
           await loadInstance(config, { resultss: [results] })
         } else {
           try {
-            results = await _process(config, query.query, { initializer, rebuildingTemplate: true })
+            results = await _process(config, query.query, { initializer, rebuildingTemplate: true, isFragment })
           } catch (e) {
             if (e.summaries && e.summaries.length > 0) {
               suggestFix(e.summaries)
@@ -1069,7 +1056,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
         results.isFragment = isFragment
         results.skipSemantics = skipSemantics
         results.development = query.development
-        results.key = { query: query.query, hierarchy }
+        results.key = { query: query.query, hierarchy, preserve: query.preserve }
         accumulators[property].push(results)
         if (isFragment) {
           config.fragmentsBeingBuilt.push({ query: query.query, contexts: results.contexts })
@@ -1135,6 +1122,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     const stabilizeOutput = (template) => {
       // stabilizeAssociations(template.associations)
       const stabilize = (results) => {
+        removeProp(results, (_, k) => k === "context_id")
         for (let i = 0; i < results.length; ++i) {
           const result = results[i]
           if (result.apply) {

package/index.js

@@ -1,7 +1,7 @@
 const { Semantics, Semantic } = require('./src/semantics')
 const { Generators, Generator } = require('./src/generators')
 const { Config } = require('./src/config')
-const { w, where, OverrideCheck, setByPath, getByPath } = require('./src/helpers')
+const { w, where, OverrideCheck, setByPath, getByPath, removeProp } = require('./src/helpers')
 const Digraph = require('./src/digraph')
 const client = require('./client')
 const flattens = require('./src/flatten')
@@ -32,4 +32,5 @@ module.exports = {
   unflatten: unflatten.unflatten,
   debug,
   OverrideCheck,
+  removeProp,
 }

package/lines.js

@@ -40,7 +40,7 @@ class Lines {
     this.newRow()
     let result = ''
     for (const row of this.rows) {
-      result += row
+      result += row.trimEnd()
       result += '\n'
     }
     return result

package/package.json

@@ -18,9 +18,11 @@
     "test:debug": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
     "td": "node inspect node_modules/.bin/jest --runInBand --config ./jest.config.json",
     "tod": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
+    "toda": "node inspect node_modules/.bin/jest --runInBand -t NEO23A",
     "lint:fix": "eslint \"**/*.js\" --fix",
     "lint": "eslint \"**/*.js\"",
     "to": "node node_modules/.bin/jest --runInBand -t NEO23",
+    "toa": "node node_modules/.bin/jest --runInBand -t NEO23A",
     "tos": "node node_modules/.bin/jest --runInBand -t NEOS23",
     "test": "jest --config ./jest.config.json",
     "test:python": "node node_modules/.bin/jest --runInBand -t PYTHON",
@@ -73,6 +75,6 @@
     "scriptjs": "^2.5.9",
     "uuid": "^8.3.2"
   },
-  "version": "9.8.0",
+  "version": "9.9.0",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -295,9 +295,6 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       if (bridge.scope && config.isModule) {
         return
       }
-      if (false && !bridge.bridge) {
-        bridge.bridge = '{ ...next(operator) }'
-      }
       if (!bridge.level) {
         bridge.level = 0
       }
@@ -391,6 +388,13 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       }
       */
 
+      if (!bridge.where) {
+        if (bridge.generator || bridge.generatorp || bridge.generatorr || bridge.generator ||
+            bridge.semantic || bridge.semantics) {
+          throw new Error(`In the KM "${config.name}", for the bridge ${bridge.id} the where property is missing. Use where: where().`)
+        }
+      }
+
       const addUUID = (obj) => { return { ...obj, uuid: uuid || config.uuid } }
 
       if (bridge.generators) {
@@ -469,7 +473,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         }
       }
 
-      const addSemantic = (semantic, evaluate) => {
+      const addSemantic = (semantic, evaluate, property) => {
         const match = semantic.match || (() => true)
         let apply = semantic
         // if I do apply === semantic.apply or semantic there is one function that has apply defined for some reason even though not explicitly set
@@ -491,13 +495,13 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
           }
         } else {
           semanticDef = {
-            match: (args) => bridge.id === args.context.marker && !!args.context.evaluate === evaluate && match(args),
+            match: (args) => bridge.id === args.context.marker && match(args),
             apply: (args) => apply(args),
             applyWrapped: semantic,
           }
         }
         semanticDef.where = semantic.where || bridge.where || helpers.where(4)
-        semanticDef.property = evaluate ? 'evaluator' : 'semantic'
+        semanticDef.property = property
         if (semantic.priority) {
           semanticDef.priority = semantic.priority
         }
@@ -509,22 +513,8 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         }
       }
 
-      if (bridge.evaluator) {
-        addSemantic(bridge.evaluator, true)
-      }
-
       if (bridge.semantic) {
-        addSemantic(bridge.semantic, false)
-      }
-
-      if (bridge.evaluators) {
-        const evaluators = [...bridge.evaluators]
-        evaluators.reverse()
-        if (evaluators) {
-          for (const evaluator of evaluators) {
-            addSemantic(evaluator, true)
-          }
-        }
+        addSemantic(bridge.semantic, false, 'semantic')
       }
 
       if (bridge.semantics) {
@@ -556,8 +546,6 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
         'convolution', 
         'disabled', 
         'enhanced_associations', 
-        'evaluator', 
-        'evaluators', 
         'generatorp', 
         'generatorpr', 
         'generatorr', 
@@ -996,9 +984,10 @@ class Config {
 
   async fixtures () {
     if (this.testConfig?.fixtures) {
-      const args = {}
+      const args = {
+      }
       args.uuid = this._uuid
-      configHelpers.setupArgs(args, this)
+      configHelpers.setupArgs(args, this, this.logs)
       return this.testConfig.fixtures(args)
     }
   }
@@ -1029,6 +1018,7 @@ class Config {
     return {
       pseudo: true,
       description: 'this is a pseudo config that has limited functionality due to being available in the initializer and fixtures function context',
+      nextFragmentContextId: () => this.nextFragmentContextId(),
       addAssociation: (...args) => this.addAssociation(...args),
       addAssociations: (...args) => this.addAssociations(...args),
       addBridge: (...args) => this.addBridge(...args, uuid),
@@ -1043,6 +1033,7 @@ class Config {
       addPattern: (...args) => this.addPattern(...args, uuid),
       updateBridge: (...args) => this.updateBridge(...args),
       processContext: (...args) => this.processContext(...args),
+      logs: this.logs,
 
       semantics: () => this.config.semantics,
       getConfigs: () => this.configs,
@@ -1242,7 +1233,7 @@ class Config {
 
   warningNotEvaluated (log, value) {
     const description = 'WARNING: for semantics, implement an evaluations handler, set "value" property of the operator to the value.'
-    const match = `({context}) => context.marker === '${value.marker}' && context.evaluate && <other conditions as you like>`
+    const match = `({context}) => context.marker === '${value.marker}' && <other conditions as you like>`
     const apply = '({context}) => <do stuff...>; context.value = <value>'
     const input = indent(JSON.stringify(value, null, 2), 2)
     const message = `${description}\nThe semantic would be\n  match: ${match}\n  apply: ${apply}\nThe input context would be:\n${input}\n`
@@ -1251,18 +1242,15 @@ class Config {
 
   // value is in response field
   // TODO maybe generalize out query+evaluate along the lines of set value and set reference
-  async getEvaluator (s, calls, log, context, args) {
-    const instance = await s({ ...args, ...context, evaluate: true })
-    calls.touch(instance)
-    if (instance.evalue == null && !instance.verbatim && !instance.value) {
-      this.warningNotEvaluated(log, context)
+  async evaluate (args, logs, context) {
+    const apply = (c) => this.getSemantics(logs).apply(args, c)
+    const instance = await apply(context)
+    if (typeof instance !== 'string') {
+      args.calls.touch(instance)
     }
-    if (instance.evalue == null) {
-      instance.evalue = instance.value
-      instance.edefault = true
+    if (instance.evalue == null && !instance.verbatim && !instance.value) {
+      this.warningNotEvaluated(logs, context)
     }
-    delete instance.evaluate
-    instance.instance = true
     return instance
   }
 
@@ -1293,10 +1281,14 @@ class Config {
       let fi = fragmentInstantiator(args, fragment.contexts)
       if (mappings) {
         fi = await fi
-        return fi.instantiate([{
+        const instantiated = await fi.instantiate([{
           match: ({context}) => !!mappings[context.value], // is the value in the mappings
           apply: ({context}) => Object.assign(context, mappings[context.value]),
-        }])
+        }], fragment.key?.preserve)
+        if (process.env.DEBUG_CONTEXT_ID && process.env.DEBUG_CONTEXT_ID == instantiated.context_id) {
+          debugger
+        }
+        return instantiated
       } else {
         return fi
       }
@@ -1305,6 +1297,7 @@ class Config {
     }
   }
 
+  /*
   fragmentMapper (args, values, fromModelQuery, toModelQuery) {
     const fromModelFragment = this.getFragment(fromModelQuery)
     console.dir(fromModelFragment)
@@ -1313,6 +1306,7 @@ class Config {
     const mapper = fragmentMapperInstantiator(values, fromModelFragment.contexts, toModelFragment.contexts)
     return mapper
   }
+  */
 
   // { rebuild: false, isModule: false }
   needsRebuild (template, instance, options) {
@@ -1384,6 +1378,7 @@ class Config {
           bridge = { ...bridge },
           bridge.level = bridge.level || 0
           delete bridge.uuid
+          delete bridge.where
           return bridge
         })
         if (options.isModule) {
@@ -1396,8 +1391,6 @@ class Config {
             delete bridge.generatorp
             delete bridge.generatorr
             delete bridge.generatorpr
-            delete bridge.evaluator
-            delete bridge.evaluators
             delete bridge.semantic
             delete bridge.semantics
             if (false && !bridge.bridge) {
@@ -2179,6 +2172,12 @@ class Config {
     // await this.demoInitializer(this)
   }
 
+  nextFragmentContextId() {
+    const id = this.fragmentNextContextId
+    this.fragmentNextContextId = this.fragmentNextContextId + 1
+    return id
+  }
+
   // configs = [ { config, namespace } ... ]
   constructor (config, module, clientProcess, apiKMs, rootIsProcess, testingModuleName) {
     if (config instanceof Config) {
@@ -2189,6 +2188,8 @@ class Config {
       apiKMs = []
     }
 
+    this.fragmentNextContextId = 1000
+
     if (config) {
       validConfigProps(config)
 

package/src/configHelpers.js

@@ -1,7 +1,24 @@
-const { InitCalls } = require('./helpers')
+const { InitCalls, where } = require('./helpers')
 const DigraphInternal = require('./digraph_internal')
 const debug = require('./debug')
 
+class ErrorReason extends Error {
+  constructor (context, stack) {
+    super(JSON.stringify(context))
+    this.reason = context
+    this.stack = stack
+  }
+
+  toString() {
+    if (this.reason) {
+      return this.reason
+    } else {
+      return super.toString()
+    }
+  }
+}
+
+
 const gs = (g) => async (contexts, separator, lastSeparator) => {
   if (!Array.isArray(contexts)) {
     throw new Error('Expected a list')
@@ -63,13 +80,6 @@ const isA = (hierarchy) => (child, parent, { strict = false } = {}) => {
   }
 }
 
-class ErrorReason extends Error {
-  constructor (context) {
-    super(JSON.stringify(context))
-    this.reason = context
-  }
-}
-
 const cleanAssign = (dest, ...srcs) => {
   for (const key in dest) {
     let found = false
@@ -93,13 +103,25 @@ class HandlerStack {
   }
 
   push(handler) {
-    this.handlers.push(handler)
+    this.handlers.push(handler.where || handler)
   }
 
   pop() {
     this.handlers.pop()
   }
 
+  async wrap(call, w) {
+    this.push(w)
+    const p = await call()
+    if (p && typeof p.finally === 'function') {
+      return p.finally(() => {
+        this.pop()
+      })
+    }
+    this.pop()
+  return p
+}
+
   toString() {
     let str = ''
     for (let i = this.handlers.length - 1; i >= 0; --i) {
@@ -185,7 +207,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.km = (name) => config.getConfig(name)
   args.api = (name) => config.getConfig(name).api
   args.error = (context) => {
-    throw new ErrorReason(context)
+    throw new ErrorReason(context, [...args.stack.handlers, where()])
   }
   args.kms = config.getConfigs()
   args.config = config.getPseudoConfig(uuidForScoping, config)
@@ -198,15 +220,15 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.fragments = (query, mappings) => {
     return config.fragment(args, query, mappings)
   }
+  /*
   args.fragmentMapper = (values, fromModelQuery, toModelQuery) => {
     return config.fragmentMapper(args, values, fromModelQuery, toModelQuery)
   }
+  */
   args.breakOnSemantics = false
   args.theDebugger = {
     breakOnSemantics: (value) => args.breakOnSemantics = value
   }
-  if (!logs) {
-  }
   args.log = (message) => logs.push(message)
 
   args.addWord = (word, def) => config.addWord(word, def, args.uuid)
@@ -221,7 +243,6 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
       return { ...args, assumed: Object.assign({}, assumed, (args.assumed || {}), ...moreAssumed) }
     }
 
-    args.s = (c, options = {}) => config.getSemantics(logs).apply(args, c, options)
     args.g = (c, rest = {}) => {
       // if (JSON.stringify(rest) !== '{}' && !(rest.assumed || rest.options)) {
       //   debugger
@@ -256,12 +277,18 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
       const { assumed = {}, options = {} } = rest
       return config.getGenerators(logs).apply(addAssumed(args, assumed, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true }, options)
     }
-    args.e = (c) => {
+    
+    args.e = (c, where) => {
       if (!c) {
         return
       }
-      return config.getEvaluator(args.s, args.calls, logs, c, { isModule: args.isModule, isProcess: args.isProcess })
+      if (where) {
+        return args.stack.wrap(() => config.evaluate(args, logs, c), where)
+      } else {
+        return config.evaluate(args, logs, c)
+      }
     }
+
     args.gs = gs(args.g)
     args.gsp = gs(args.gp)
     args.gsr = gs(args.gr)
@@ -346,7 +373,7 @@ const processContext = async (context, { objects = {}, config, logs = [] }) => {
   return { context, generated, paraphrases, responses }
 }
 
-const setupProcessB = ({ config, initializer, allowDelta = false, rebuildingTemplate = false } = {}) => {
+const setupProcessB = ({ config, initializer, isFragment, allowDelta = false, rebuildingTemplate = false } = {}) => {
   const key = config._key
   const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
   /*
@@ -411,7 +438,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, calls, hierarchy, logs, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
+const processContextsB = async ({ skipSemantics, isFragment, config, calls, hierarchy, logs, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -419,6 +446,7 @@ const processContextsB = async ({ config, calls, hierarchy, logs, semantics, gen
   const paraphrasesParenthesizedPrime = []
   const generatedParenthesizedPrime = []
   const responsesPrime = []
+
   const contexts = setupContexts(json.contexts)
 
   const objects = config.get('objects')
@@ -440,7 +468,7 @@ const processContextsB = async ({ config, calls, hierarchy, logs, semantics, gen
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
       }
       const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
-      if (!config.get('skipSemantics')) {
+      if (!config.get('skipSemantics') && !skipSemantics) {
         const semantics = config.getSemantics(logs)
         try {
           contextPrime = await semantics.apply(args, context)
@@ -455,17 +483,18 @@ const processContextsB = async ({ config, calls, hierarchy, logs, semantics, gen
           if (isInstance) {
             console.log('error', e.error)
           }
+        
           let reason = e.reason
           if (!reason) {
             if (e.error) {
               reason = e.error[0]
             }
           }
+
           contextPrime = await semantics.apply(args, {
             marker: 'error',
             context,
-            text: e ? e.toString() : 'not available',
-            // reason: e.reason,
+            text: e.toString(),
             reason,
             error: e.stack || e.error
           })
@@ -578,14 +607,14 @@ const loadInstance = async (config, instance) => {
       // used to do a CLONE
       config.addInternal(instance.template.configs[i], { uuid, km: instance.name, addFirst: true, handleCalculatedProps: true })
     } else if (results.apply) {
-      const objects = getObjects(config.get('objects'))(config.uuid)
+      const uuid = config.nameToUUID(instance.name)
+      const objects = getObjects(config.get('objects'))(uuid)
       const args = { objects, getObjects: getObjects(objects) }
       if (instance.configs) {
         args.isInstance = `instance${i}`
         args.instance = instance.configs[i]
       }
 
-      const uuid = config.nameToUUID(instance.name)
       setupArgs(args, config, config.logs, hierarchy, uuid)
       if (config.name == instance.name) {
         args.isModule = !!config.isModule
@@ -609,6 +638,7 @@ const loadInstance = async (config, instance) => {
       args.instance = ''
       args.isProcess = !config.isModule
       args.isModule = !!config.isModule
+      args.logs = config.logs
       if (instance.name == config.testConfig.testModuleName) {
         args.isTesting = true
       }

package/src/fragments.js

@@ -1,4 +1,5 @@
 const _ = require('lodash')
+const debug = require('./debug')
 const helpers = require('./helpers')
 
 function createChangeTracker(obj) {
@@ -23,17 +24,28 @@ function createChangeTracker(obj) {
   };
 }
 
-
 function fragmentInstantiator (args, contexts) {
   return new Object({ 
     contexts: () => {
       return _.cloneDeep(contexts)
     },
-    instantiate: async (mappings) => {
+    // preserve property not to be changed. 
+    instantiate: async (mappings, preserve) => {
+      preserve = preserve || []
       const instantiated = _.cloneDeep(contexts)
       const todo = [{ context: instantiated, path: [] }]
       args = { ...args }
       args.pathEquals = helpers.pathEquals
+      const preserved = preserve.map((path) => {
+        if (contexts.length == 1) {
+          path = ['0', path]
+        }
+        const value = helpers.getByPath(contexts, path)
+        return { path, value }
+      })
+      for (const context of contexts) {
+        context.context_id = args.config.nextFragmentContextId()
+      }
       while (todo.length > 0) {
         const { context, path } = todo.pop()
         const { proxy, getChanges } = createChangeTracker(context);
@@ -58,6 +70,9 @@ function fragmentInstantiator (args, contexts) {
           }
         }
       }
+      for (const { path, value } of preserved) {
+        helpers.setByPath(instantiated, path, value)
+      }
       if (contexts.length == 1 && instantiated.length == 1) {
         return instantiated[0]
       } else {

package/src/generators.js

@@ -125,7 +125,7 @@ class Generator {
       args.g = args.gp
     }
     return await debug.hitScoped(args.callId, async () => {
-      if ((options.debug || {}).apply || process.env.DEBUG_CALLID == args.callId || callId === this.callId) {
+      if ((options.debug || {}).apply || process.env.DEBUG_BREAKAT == args.callId || callId === this.callId) {
         debugger // eslint-disable-line no-debugger
       }
       return await this._apply(args)  // DEBUG: This is the line you want to step into
@@ -184,6 +184,9 @@ class Generators {
     let generated = ''
     let applied = false
     const stack = args.calls.push()
+    const pOrRTag = assumed.isResponse ? "(RESPONSE)" : "(PARAPHRASE)"
+    const finallyList = []; // called after the generator is processed
+    args._finally = (call) => finallyList.unshift(call)
     for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
       const generator = this.generators[igenerator]
       if (await generator.matches(args, objects, context, hierarchy, config, options)) {
@@ -194,7 +197,7 @@ class Generators {
           console.error("TPMKMS_TRACE", debug.counter("TPMKMS_TRACE"))
           console.error(generator.toLabel())
           console.error(generator.toString())
-          console.log(`To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_CALLID="${args.calls.current()}"`)
+          console.log(`To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_BREAKAT="${args.calls.current()}"`)
         }
 
         try {
@@ -218,21 +221,22 @@ class Generators {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           const source = `${generator.km}/#${generator.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
           lines.newRow()
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_BREAKAT="${args.calls.current()}"`)
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
           lines.setElement(0, 2, generator.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 10 }), null, 2))
           lines.newRow()
-          lines.setElement(0, 1, 'STACK')
-          lines.setElement(0, 2, stack)
-          lines.newRow()
-          lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_CALLID="${args.calls.current()}"`)
-          lines.newRow()
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
           this.logs.push(lines.toString())
@@ -246,24 +250,25 @@ class Generators {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Generator')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           if (generator.index > -1 && generator.km) {
             // lines.setElement(0, 2, `KM '${generator.km}' ordinal: ${generator.index}`)
             lines.setElement(0, 2, generator.toLabel())
           }
           lines.newRow()
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
+          lines.newRow()
           lines.setElement(0, 1, 'APPLIED')
           lines.setElement(0, 2, generator.toLabel())
           lines.newRow()
-          lines.setElement(0, 2, generator.toString())
-          lines.newRow()
           lines.setElement(0, 1, 'RESULT')
-          lines.setElement(0, 2, generated)
+          lines.setElement(0, 2, generated + " " + pOrRTag)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
           lines.newRow()
-          lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
+          lines.setElement(0, 2, generator.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
@@ -274,11 +279,15 @@ class Generators {
         break
       }
     }
+    for (const f of finallyList) {
+      await f()
+    }
     args.calls.pop()
     if (!applied && config.getDebug()) {
       const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Generator')
+      lines.setElement(0, 1, `#${context.context_id || "?"}`)
       lines.setElement(0, 2, 'No generator applied')
       lines.newRow()
       lines.setElement(0, 1, 'STACK')

package/src/helpers.js

@@ -22,12 +22,30 @@ class OverrideCheck {
   }
 }
 
-function pathEquals (p1, p2) {
+function pathEquals (p1, p2, n) {
+  if (n !== undefined) {
+    if (n < 0) {
+      p1 = p1.slice(n)
+      p2 = p2.slice(n)
+    } else {
+      p1 = p1.slice(0, n)
+      p2 = p2.slice(0, n)
+    }
+  }
+
   if (p1.length !== p2.length) {
     return false
   }
   for (let i = 0; i < p1.length; ++i) {
-    if (typeof p2[i] == 'function') {
+    if (p1[i].integer) {
+      if (!Number.isInteger(p2[i])) {
+        return false
+      }
+    } else if (p2[i].integer) {
+      if (!Number.isInteger(p1[i])) {
+        return false
+      }
+    } else if (typeof p2[i] == 'function') {
       if (!p2[i](p1[i])) {
         return false
       }
@@ -72,7 +90,14 @@ function where (goUp = 2) {
   let match
   for (line of lines.slice(1)) {
     // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('<anonymous>'))) {
-    if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('helpers.js:'))) {
+    const ignore = [
+      'config.js:',
+      'client.js:',
+      'helpers.js:',
+      'configHelpers.js:',
+    ]
+    // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('helpers.js:'))) {
+    if(!ignore.some(p => line.includes(p))) {
       match = regexForm1.exec(line) || regexForm2.exec(line)
       if (!match) {
         continue
@@ -477,12 +502,6 @@ const functionsToStrings = (config) => {
       if (bridge.semantics) {
         bridge.semantics = bridge.semantics.toString()
       }
-      if (bridge.evaluator) {
-        bridge.evaluator = bridge.evaluator.toString()
-      }
-      if (bridge.evaluators) {
-        bridge.evaluators = bridge.evaluators.toString()
-      }
       return bridge
     })
   }
@@ -627,7 +646,60 @@ function setByPath(obj, path, value) {
 
 const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'["cat", "mammel"]\''
 
+// removeProp.js
+function removeProp(obj, testFn, { maxDepth = Infinity, seen = new WeakSet() } = {}) {
+  if (!obj || typeof obj !== 'object' || maxDepth <= 0) return obj;
+  if (seen.has(obj)) return obj;
+  seen.add(obj);
+
+  if (Array.isArray(obj)) {
+    // ---- ARRAY: process each element (but don't remove elements unless testFn says so)
+    let writeIdx = 0;
+    for (let i = 0; i < obj.length; i++) {
+      const element = obj[i];
+      const shouldRemoveElement = testFn(element, i, obj);
+
+      if (shouldRemoveElement) {
+        // Remove the whole array element
+        if (element && typeof element === 'object') {
+          // Still walk inside it in case testFn wants side effects
+          removeProp(element, testFn, { maxDepth: maxDepth - 1, seen });
+        }
+        // Skip writing it back
+      } else {
+        // Keep element, but walk into it to remove inner props
+        removeProp(element, testFn, { maxDepth: maxDepth - 1, seen });
+        if (writeIdx !== i) {
+          obj[writeIdx] = element;
+        }
+        writeIdx++;
+      }
+    }
+    obj.length = writeIdx;
+    return obj;
+  }
+
+  // ---- OBJECT: iterate over own keys
+  const keys = Reflect.ownKeys(obj);
+  for (const key of keys) {
+    const val = obj[key];
+    const shouldRemove = testFn(val, key, obj);
+
+    if (shouldRemove) {
+      delete obj[key];
+      if (val && typeof val === 'object') {
+        removeProp(val, testFn, { maxDepth: maxDepth - 1, seen });
+      }
+    } else {
+      removeProp(val, testFn, { maxDepth: maxDepth - 1, seen });
+    }
+  }
+
+  return obj;
+}
+
 module.exports = {
+  removeProp,
   helpDebugHierarchy,
 
   stableId,
@@ -662,4 +734,5 @@ module.exports = {
   watchProperty,
   pathEquals,
   OverrideCheck,
+  normalizePath,
 }

package/src/project2.js

@@ -1,18 +1,6 @@
 const debug = require('./debug')
 const helpers = require('./helpers')
 
-function areFirstNEqual(arr1, arr2, n) {
-  if (n <= 0) return true;
-  if (arr1.length < n || arr2.length < n) return false;
-
-  for (let i = 0; i < n; i++) {
-    if (arr1[i] !== arr2[i]) {
-      return false;
-    }
-  }
-  return true;
-}
-
 function project(source, filters, path=[]) {
   if (['boolean', 'string', 'number'].includes(typeof source)) {
     return source
@@ -49,12 +37,16 @@ function project(source, filters, path=[]) {
   let properties = []
   for (const filter of selectedFilters) {
     for (const property of filter.apply(source)) {
-      properties.push(property)
+      if (!properties.includes(property)) {
+        properties.push(property)
+      }
     }
   }
   if (source?.checks) {
     for (const check of source.checks) {
-      properties.push(check)
+      if (!properties.includes(check)) {
+        properties.push(check)
+      }
     }
   }
 
@@ -64,10 +56,14 @@ function project(source, filters, path=[]) {
     // property that contains a list of properties to be checked
     if (property.properties) {
       for (const moreProperty of source[property.properties] || []) {
-        updatedProperties.push(moreProperty)
+        if (!updatedProperties.includes(property)) {
+          updatedProperties.push(moreProperty)
+        }
       }
     } else {
-      updatedProperties.push(property)
+      if (!updatedProperties.includes(property)) {
+        updatedProperties.push(property)
+      }
     }
   }
   properties = updatedProperties
@@ -75,7 +71,7 @@ function project(source, filters, path=[]) {
   // Build the result object
   const result = {};
   properties.forEach(prop => {
-    if (prop.path && (prop.path.length === path.length + 1) && areFirstNEqual(path, prop.path, path.length) && source.hasOwnProperty(prop.path[path.length])) {
+    if (prop.path && (prop.path.length === path.length + 1) && helpers.pathEquals(path, prop.path, path.length) && source.hasOwnProperty(prop.path[path.length])) {
       const endProp = prop.path[path.length]
       if (['boolean', 'string', 'number'].includes(typeof source[endProp])) {
         result[endProp] = source[endProp]
@@ -109,8 +105,18 @@ function project(source, filters, path=[]) {
         const instantiatedCheck = []
         for (const check of prop.check) {
           if (typeof check.property == 'function') {
-            for (const sourceKey of source[prop.property].keys()) {
+            const keys = (value) => {
+              if (Array.isArray(value)) {
+                return value.keys()
+              } else {
+                return Object.keys(value)
+              }
+            }
+            for (const sourceKey of keys(source[prop.property])) {
               if (check.property(sourceKey)) {
+                if (!check.check) {
+                  throw new Error(`For the property check of the property '${check.property.toString()}', the 'check' (singular) property is missing.`)
+                }
                 instantiatedCheck.push({ property: sourceKey, check: check.check })
               }
             }

package/src/semantics.js

@@ -78,7 +78,7 @@ class Semantic {
     return await debug.hitScoped(args.callId, async () => {
       const matches = await this.matcher(args)
       if (matches) {
-        if ((options.debug || {}).match || args.callId === this.callId || process.env.DEBUG_CALLID == args.callId) {
+        if ((options.debug || {}).match || args.callId === this.callId || process.env.DEBUG_BREAKAT == args.callId) {
           debugger // eslint-disable-line no-debugger
           await this.matcher(args) // DEBUG: This is the line you step into
         }
@@ -101,7 +101,7 @@ class Semantic {
     this.fixUpArgs(args, contextPrime)
 
     return await debug.hitScoped(args.callId, async () => {
-      if ((options.debug || {}).apply || args.callId === this.callId || process.env.DEBUG_CALLID == args.callId) {
+      if ((options.debug || {}).apply || args.callId === this.callId || process.env.DEBUG_BREAKAT == args.callId) {
         debugger // eslint-disable-line no-debugger  // STEP ABOUT THREE TIMES TO THE DEBUG LINE
       }
       if (args.breakOnSemantics) {
@@ -226,6 +226,7 @@ class Semantics {
           }
           contextPrime = await semantic.apply(args, context, s, options)
           if (continueWasCalled) {
+            context = contextPrime
             continue
           }
           if (deferWasCalled) {
@@ -255,27 +256,31 @@ class Semantics {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           const source = `${semantic.km}/#${semantic.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${semantic.toLabel()}`)
           lines.newRow()
-          lines.setElement(0, 2, semantic.toString())
-          lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_CALLID="${args.calls.current()}"`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_BREAKAT="${args.calls.current()}"`)
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
+          lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           lines.newRow()
-          lines.setElement(0, 1, 'STACK')
-          lines.setElement(0, 2, stack)
-          lines.newRow()
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
           this.logs.push(lines.toString())
           const message = `ERROR while applying (${source}) ${semantic.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
           // this.logs.push(message)
           // return [message]
+          if (e.reason) {
+            throw e
+          }
           args.calls.pop()
           throw { error: [message], logs: this.logs, reason: e.reason }
         }
@@ -285,6 +290,7 @@ class Semantics {
           const widths = Lines.addRemainder([10, 10])
           const lines = new Lines(widths)
           lines.setElement(0, 0, 'Semantic')
+          lines.setElement(0, 1, `#${context.context_id || "?"}`)
           if (semantic.index > -1 && semantic.km) {
             // lines.setElement(0, 2, `KM '${semantic.km}' ordinal: ${semantic.index} ${ semantic.notes ? semantic.nodes : '' }`)
             lines.setElement(0, 2, semantic.toLabel())
@@ -293,18 +299,18 @@ class Semantics {
           lines.setElement(0, 1, 'APPLIED')
           lines.setElement(0, 2, semantic.toLabel())
           lines.newRow()
-          lines.setElement(0, 2, semantic.toString())
-          lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_CALLID="${args.calls.current()}"`)
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_BREAKAT="${args.calls.current()}"`)
+          lines.newRow()
+          lines.setElement(0, 1, 'STACK')
+          lines.setElement(0, 2, stack)
+          lines.newRow()
+          lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           lines.newRow()
-          lines.setElement(0, 1, 'STACK')
-          lines.setElement(0, 2, stack)
-          lines.newRow()
           lines.setElement(0, 1, 'RESULT')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(contextPrime, null, 2))
@@ -330,13 +336,16 @@ class Semantics {
     for (const f of finallyList) {
       await f()
     }
-    args.calls.pop()
     if (!applied && debug) {
       const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
       lines.setElement(0, 0, 'Semantic')
+      lines.setElement(0, 1, `#${context.context_id || "?"}`)
       lines.setElement(0, 2, 'No semantic applied')
       lines.newRow()
+      lines.setElement(0, 1, 'DEBUG')
+      lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_BREAKAT="${args.calls.current()}"`)
+      lines.newRow()
       lines.setElement(0, 1, 'STACK')
       lines.setElement(0, 2, stack)
       lines.newRow()
@@ -345,6 +354,7 @@ class Semantics {
       lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
       this.logs.push(lines.toString())
     }
+    args.calls.pop()
     return contextPrime
   }