theprogrammablemind 9.8.0-beta.10 → 9.8.0-beta.11

package/client.js

@@ -158,13 +158,6 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   if (runtime.fs.existsSync(fn)) {
     tests = JSON.parse(runtime.fs.readFileSync(fn))
   }
-  /*
-  for (const association of associations) {
-    association.sort()
-  }
-  associations.sort()
-  */
-  // tests[query] = sortJson({ paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
   const results = sortJson({
     query,
     summaries,
@@ -189,7 +182,6 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   if (!wasSet) {
     tests.push(results)
   }
-  // fs.writeFileSync(fn, stringify(tests, { space: 2 }), { encoding: 'utf8', flag: 'w+' })
   writeTestFile(fn, tests)
 }
 
@@ -258,7 +250,7 @@ const throwErrorHandler = (error) => {
   throw error
 }
 
-const _process = async (config, query, { queryNumber, initializer, commandLineArgs, credentials, writeTests, logs, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
+const _process = async (config, query, { isFragment, queryNumber, initializer, commandLineArgs, credentials, writeTests, logs, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -364,7 +356,7 @@ const _process = async (config, query, { queryNumber, initializer, commandLineAr
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
         const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ contextIdCounter, logs, calls, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+                                  await processContextsB({ skipSemantics: isFragment, isFragment, contextIdCounter, logs, calls, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
@@ -384,7 +376,6 @@ const _process = async (config, query, { queryNumber, initializer, commandLineAr
         response.explain_associations_details = (response.explain_associations_details || []).concat(json.explain_associations_details)
         response.explain_associations_trie = json.explain_associations_trie
         response.contextual_priorities_ambiguities = json.contextual_priorities_ambiguities
-        // response.rtf_associations = json.rtf_associations
 
         response.contexts = response.contexts.concat(contextsPrime)
         response.generated = response.generated.concat(generatedPrime)
@@ -1036,7 +1027,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
           await loadInstance(config, { resultss: [results] })
         } else {
           try {
-            results = await _process(config, query.query, { initializer, rebuildingTemplate: true })
+            results = await _process(config, query.query, { initializer, rebuildingTemplate: true, isFragment })
           } catch (e) {
             if (e.summaries && e.summaries.length > 0) {
               suggestFix(e.summaries)
@@ -1064,7 +1055,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
         results.isFragment = isFragment
         results.skipSemantics = skipSemantics
         results.development = query.development
-        results.key = { query: query.query, hierarchy }
+        results.key = { query: query.query, hierarchy, preserve: query.preserve }
         accumulators[property].push(results)
         if (isFragment) {
           config.fragmentsBeingBuilt.push({ query: query.query, contexts: results.contexts })

package/lines.js

@@ -40,7 +40,7 @@ class Lines {
     this.newRow()
     let result = ''
     for (const row of this.rows) {
-      result += row
+      result += row.trimEnd()
       result += '\n'
     }
     return result

package/package.json

@@ -73,6 +73,6 @@
     "scriptjs": "^2.5.9",
     "uuid": "^8.3.2"
   },
-  "version": "9.8.0-beta.10",
+  "version": "9.8.0-beta.11",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -390,7 +390,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
 
       if (!bridge.where) {
         if (bridge.generator || bridge.generatorp || bridge.generatorr || bridge.generator ||
-            bridge.semantic || bridge.evaluator || bridge.evaluator2 || bridge.semantics) {
+            bridge.semantic || bridge.semantics) {
           throw new Error(`In the KM "${config.name}", for the bridge ${bridge.id} the where property is missing. Use where: where().`)
         }
       }
@@ -494,18 +494,10 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
             applyWrapped: semantic,
           }
         } else {
-          if (property == 'evaluator2') {
-            semanticDef = {
-              match: (args) => bridge.id === args.context.marker && match(args),
-              apply: (args) => apply(args),
-              applyWrapped: semantic,
-            }
-          } else {
-            semanticDef = {
-              match: (args) => bridge.id === args.context.marker && !!args.context.evaluate === evaluate && match(args),
-              apply: (args) => apply(args),
-              applyWrapped: semantic,
-            }
+          semanticDef = {
+            match: (args) => bridge.id === args.context.marker && match(args),
+            apply: (args) => apply(args),
+            applyWrapped: semantic,
           }
         }
         semanticDef.where = semantic.where || bridge.where || helpers.where(4)
@@ -521,28 +513,10 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         }
       }
 
-      if (bridge.evaluator) {
-        addSemantic(bridge.evaluator, true, 'evaluator')
-      }
-
-      if (bridge.evaluator2) {
-        addSemantic(bridge.evaluator2, false, 'evaluator2')
-      }
-
       if (bridge.semantic) {
         addSemantic(bridge.semantic, false, 'semantic')
       }
 
-      if (bridge.evaluators) {
-        const evaluators = [...bridge.evaluators]
-        evaluators.reverse()
-        if (evaluators) {
-          for (const evaluator of evaluators) {
-            addSemantic(evaluator, true, 'evaluator')
-          }
-        }
-      }
-
       if (bridge.semantics) {
         const semantics = [...bridge.semantics]
         semantics.reverse()
@@ -572,9 +546,6 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
         'convolution', 
         'disabled', 
         'enhanced_associations', 
-        'evaluator', 
-        'evaluator2', 
-        'evaluators', 
         'generatorp', 
         'generatorpr', 
         'generatorr', 
@@ -1047,6 +1018,7 @@ class Config {
     return {
       pseudo: true,
       description: 'this is a pseudo config that has limited functionality due to being available in the initializer and fixtures function context',
+      nextFragmentContextId: () => this.nextFragmentContextId(),
       addAssociation: (...args) => this.addAssociation(...args),
       addAssociations: (...args) => this.addAssociations(...args),
       addBridge: (...args) => this.addBridge(...args, uuid),
@@ -1261,7 +1233,7 @@ class Config {
 
   warningNotEvaluated (log, value) {
     const description = 'WARNING: for semantics, implement an evaluations handler, set "value" property of the operator to the value.'
-    const match = `({context}) => context.marker === '${value.marker}' && context.evaluate && <other conditions as you like>`
+    const match = `({context}) => context.marker === '${value.marker}' && <other conditions as you like>`
     const apply = '({context}) => <do stuff...>; context.value = <value>'
     const input = indent(JSON.stringify(value, null, 2), 2)
     const message = `${description}\nThe semantic would be\n  match: ${match}\n  apply: ${apply}\nThe input context would be:\n${input}\n`
@@ -1270,18 +1242,15 @@ class Config {
 
   // value is in response field
   // TODO maybe generalize out query+evaluate along the lines of set value and set reference
-  async getEvaluator (s, calls, log, context, args) {
-    const instance = await s({ ...args, ...context })
-    calls.touch(instance)
-    if (instance.evalue == null && !instance.verbatim && !instance.value) {
-      this.warningNotEvaluated(log, context)
+  async evaluate (args, logs, context) {
+    const apply = (c) => this.getSemantics(logs).apply(args, c)
+    const instance = await apply(context)
+    if (typeof instance !== 'string') {
+      args.calls.touch(instance)
     }
-    if (instance.evalue == null) {
-      instance.evalue = instance.value
-      instance.edefault = true
+    if (instance.evalue == null && !instance.verbatim && !instance.value) {
+      this.warningNotEvaluated(logs, context)
     }
-    delete instance.evaluate
-    instance.instance = true
     return instance
   }
 
@@ -1312,10 +1281,14 @@ class Config {
       let fi = fragmentInstantiator(args, fragment.contexts)
       if (mappings) {
         fi = await fi
-        return fi.instantiate([{
+        const instantiated = await fi.instantiate([{
           match: ({context}) => !!mappings[context.value], // is the value in the mappings
           apply: ({context}) => Object.assign(context, mappings[context.value]),
-        }])
+        }], fragment.key?.preserve)
+        if (process.env.DEBUG_CONTEXT_ID && process.env.DEBUG_CONTEXT_ID == instantiated.context_id) {
+          debugger
+        }
+        return instantiated
       } else {
         return fi
       }
@@ -1415,9 +1388,6 @@ class Config {
             delete bridge.generatorp
             delete bridge.generatorr
             delete bridge.generatorpr
-            delete bridge.evaluator
-            delete bridge.evaluator2
-            delete bridge.evaluators
             delete bridge.semantic
             delete bridge.semantics
             if (false && !bridge.bridge) {
@@ -2199,6 +2169,12 @@ class Config {
     // await this.demoInitializer(this)
   }
 
+  nextFragmentContextId() {
+    const id = this.fragmentNextContextId
+    this.fragmentNextContextId = this.fragmentNextContextId + 1
+    return id
+  }
+
   // configs = [ { config, namespace } ... ]
   constructor (config, module, clientProcess, apiKMs, rootIsProcess, testingModuleName) {
     if (config instanceof Config) {
@@ -2209,6 +2185,8 @@ class Config {
       apiKMs = []
     }
 
+    this.fragmentNextContextId = 1000
+
     if (config) {
       validConfigProps(config)
 

package/src/configHelpers.js

@@ -1,7 +1,24 @@
-const { InitCalls } = require('./helpers')
+const { InitCalls, where } = require('./helpers')
 const DigraphInternal = require('./digraph_internal')
 const debug = require('./debug')
 
+class ErrorReason extends Error {
+  constructor (context, stack) {
+    super(JSON.stringify(context))
+    this.reason = context
+    this.stack = stack
+  }
+
+  toString() {
+    if (this.reason) {
+      return this.reason
+    } else {
+      return super.toString()
+    }
+  }
+}
+
+
 const gs = (g) => async (contexts, separator, lastSeparator) => {
   if (!Array.isArray(contexts)) {
     throw new Error('Expected a list')
@@ -63,13 +80,6 @@ const isA = (hierarchy) => (child, parent, { strict = false } = {}) => {
   }
 }
 
-class ErrorReason extends Error {
-  constructor (context) {
-    super(JSON.stringify(context))
-    this.reason = context
-  }
-}
-
 const cleanAssign = (dest, ...srcs) => {
   for (const key in dest) {
     let found = false
@@ -93,7 +103,7 @@ class HandlerStack {
   }
 
   push(handler) {
-    this.handlers.push(handler)
+    this.handlers.push(handler.where || handler)
   }
 
   pop() {
@@ -210,7 +220,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   args.km = (name) => config.getConfig(name)
   args.api = (name) => config.getConfig(name).api
   args.error = (context) => {
-    throw new ErrorReason(context)
+    throw new ErrorReason(context, [...args.stack.handlers, where()])
   }
   args.kms = config.getConfigs()
   args.config = config.getPseudoConfig(uuidForScoping, config)
@@ -278,24 +288,18 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
       const { assumed = {}, options = {} } = rest
       return config.getGenerators(logs).apply(addAssumed(args, assumed, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true }, options)
     }
-    if (true) {
-      args.s = (c, options = {}) => config.getSemantics(logs).apply(args, c, options)
-      args.e = (c) => {
-        if (!c) {
-          return
-        }
-        return config.getEvaluator(args.s, args.calls, logs, c, { isModule: args.isModule, isProcess: args.isProcess })
+    
+    args.e = (c, where) => {
+      if (!c) {
+        return
       }
-    } else {
-      args.e = (c) => {
-        if (!c) {
-          return
-        }
-        const s = (c, options = {}) => config.getSemantics(logs).apply(args, c, options)
-        return config.getEvaluator(s, args.calls, logs, c, { isModule: args.isModule, isProcess: args.isProcess })
+      if (where) {
+        return args.stack.wrap(() => config.evaluate(args, logs, c), where)
+      } else {
+        return config.evaluate(args, logs, c)
       }
-      args.s = args.e
     }
+
     args.gs = gs(args.g)
     args.gsp = gs(args.gp)
     args.gsr = gs(args.gr)
@@ -380,7 +384,7 @@ const processContext = async (context, { objects = {}, config, logs = [] }) => {
   return { context, generated, paraphrases, responses }
 }
 
-const setupProcessB = ({ config, initializer, allowDelta = false, rebuildingTemplate = false } = {}) => {
+const setupProcessB = ({ config, initializer, isFragment, allowDelta = false, rebuildingTemplate = false } = {}) => {
   const key = config._key
   const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
   /*
@@ -445,7 +449,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = async ({ config, calls, hierarchy, logs, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
+const processContextsB = async ({ skipSemantics, isFragment, config, calls, hierarchy, logs, semantics, generators, json, isTest, isProcess, isModule, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs, forTemplate, contextIdCounter }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -453,6 +457,7 @@ const processContextsB = async ({ config, calls, hierarchy, logs, semantics, gen
   const paraphrasesParenthesizedPrime = []
   const generatedParenthesizedPrime = []
   const responsesPrime = []
+
   const contexts = setupContexts(json.contexts)
 
   const objects = config.get('objects')
@@ -474,7 +479,7 @@ const processContextsB = async ({ config, calls, hierarchy, logs, semantics, gen
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
       }
       const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
-      if (!config.get('skipSemantics')) {
+      if (!config.get('skipSemantics') && !skipSemantics) {
         const semantics = config.getSemantics(logs)
         try {
           contextPrime = await semantics.apply(args, context)
@@ -489,17 +494,18 @@ const processContextsB = async ({ config, calls, hierarchy, logs, semantics, gen
           if (isInstance) {
             console.log('error', e.error)
           }
+        
           let reason = e.reason
           if (!reason) {
             if (e.error) {
               reason = e.error[0]
             }
           }
+
           contextPrime = await semantics.apply(args, {
             marker: 'error',
             context,
-            text: e ? e.toString() : 'not available',
-            // reason: e.reason,
+            text: e.toString(),
             reason,
             error: e.stack || e.error
           })

package/src/fragments.js

@@ -1,4 +1,5 @@
 const _ = require('lodash')
+const debug = require('./debug')
 const helpers = require('./helpers')
 
 function createChangeTracker(obj) {
@@ -28,11 +29,23 @@ function fragmentInstantiator (args, contexts) {
     contexts: () => {
       return _.cloneDeep(contexts)
     },
-    instantiate: async (mappings) => {
+    // preserve property not to be changed. 
+    instantiate: async (mappings, preserve) => {
+      preserve = preserve || []
       const instantiated = _.cloneDeep(contexts)
       const todo = [{ context: instantiated, path: [] }]
       args = { ...args }
       args.pathEquals = helpers.pathEquals
+      const preserved = preserve.map((path) => {
+        if (contexts.length == 1) {
+          path = ['0', path]
+        }
+        const value = helpers.getByPath(contexts, path)
+        return { path, value }
+      })
+      for (const context of contexts) {
+        context.context_id = args.config.nextFragmentContextId()
+      }
       while (todo.length > 0) {
         const { context, path } = todo.pop()
         const { proxy, getChanges } = createChangeTracker(context);
@@ -57,6 +70,9 @@ function fragmentInstantiator (args, contexts) {
           }
         }
       }
+      for (const { path, value } of preserved) {
+        helpers.setByPath(instantiated, path, value)
+      }
       if (contexts.length == 1 && instantiated.length == 1) {
         return instantiated[0]
       } else {

package/src/generators.js

@@ -225,18 +225,18 @@ class Generators {
           const source = `${generator.km}/#${generator.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${generator.toLabel()}`)
           lines.newRow()
-          lines.setElement(0, 2, generator.toString())
+          lines.setElement(0, 1, 'DEBUG')
+          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_BREAKAT="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
           lines.newRow()
+          lines.setElement(0, 2, generator.toString())
+          lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 10 }), null, 2))
           lines.newRow()
-          lines.setElement(0, 1, 'DEBUG')
-          lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') or DEBUG_BREAKAT="${args.calls.current()}"`)
-          lines.newRow()
           lines.setElement(0, 1, 'ERROR')
           lines.setElement(0, 2, errorMessage)
           this.logs.push(lines.toString())
@@ -259,14 +259,14 @@ class Generators {
           lines.setElement(0, 1, 'APPLIED')
           lines.setElement(0, 2, generator.toLabel())
           lines.newRow()
-          lines.setElement(0, 2, generator.toString())
-          lines.newRow()
           lines.setElement(0, 1, 'RESULT')
           lines.setElement(0, 2, generated + " " + pOrRTag)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
           lines.newRow()
+          lines.setElement(0, 2, generator.toString())
+          lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
           lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}')`)
           lines.newRow()

package/src/helpers.js

@@ -72,7 +72,14 @@ function where (goUp = 2) {
   let match
   for (line of lines.slice(1)) {
     // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('<anonymous>'))) {
-    if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('helpers.js:'))) {
+    const ignore = [
+      'config.js:',
+      'client.js:',
+      'helpers.js:',
+      'configHelpers.js:',
+    ]
+    // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('helpers.js:'))) {
+    if(!ignore.some(p => line.includes(p))) {
       match = regexForm1.exec(line) || regexForm2.exec(line)
       if (!match) {
         continue
@@ -477,15 +484,6 @@ const functionsToStrings = (config) => {
       if (bridge.semantics) {
         bridge.semantics = bridge.semantics.toString()
       }
-      if (bridge.evaluator) {
-        bridge.evaluator = bridge.evaluator.toString()
-      }
-      if (bridge.evaluator2) {
-        bridge.evaluator2 = bridge.evaluator2.toString()
-      }
-      if (bridge.evaluators) {
-        bridge.evaluators = bridge.evaluators.toString()
-      }
       return bridge
     })
   }
@@ -718,4 +716,5 @@ module.exports = {
   watchProperty,
   pathEquals,
   OverrideCheck,
+  normalizePath,
 }

package/src/semantics.js

@@ -259,14 +259,14 @@ class Semantics {
           const source = `${semantic.km}/#${semantic.index}`
           lines.setElement(0, 2, `ERROR while applying (${source}) ${semantic.toLabel()}`)
           lines.newRow()
-          lines.setElement(0, 2, semantic.toString())
-          lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
           lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_BREAKAT="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
           lines.newRow()
+          lines.setElement(0, 2, semantic.toString())
+          lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
@@ -277,6 +277,9 @@ class Semantics {
           const message = `ERROR while applying (${source}) ${semantic.toLabel()}\n to\n ${JSON.stringify(context, null, 2)}.\n${errorMessage}'`
           // this.logs.push(message)
           // return [message]
+          if (e.reason) {
+            throw e
+          }
           args.calls.pop()
           throw { error: [message], logs: this.logs, reason: e.reason }
         }
@@ -295,14 +298,14 @@ class Semantics {
           lines.setElement(0, 1, 'APPLIED')
           lines.setElement(0, 2, semantic.toLabel())
           lines.newRow()
-          lines.setElement(0, 2, semantic.toString())
-          lines.newRow()
           lines.setElement(0, 1, 'DEBUG')
           lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_BREAKAT="${args.calls.current()}"`)
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
           lines.setElement(0, 2, stack)
           lines.newRow()
+          lines.setElement(0, 2, semantic.toString())
+          lines.newRow()
           lines.setElement(0, 1, 'TO')
           lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
@@ -332,7 +335,6 @@ class Semantics {
     for (const f of finallyList) {
       await f()
     }
-    args.calls.pop()
     if (!applied && debug) {
       const widths = Lines.addRemainder([10, 10])
       const lines = new Lines(widths)
@@ -340,6 +342,9 @@ class Semantics {
       lines.setElement(0, 1, `#${context.context_id || "?"}`)
       lines.setElement(0, 2, 'No semantic applied')
       lines.newRow()
+      lines.setElement(0, 1, 'DEBUG')
+      lines.setElement(0, 2, `To debug this use debug.breakAt('${args.calls.current()}') in the code or DEBUG_BREAKAT="${args.calls.current()}"`)
+      lines.newRow()
       lines.setElement(0, 1, 'STACK')
       lines.setElement(0, 2, stack)
       lines.newRow()
@@ -348,6 +353,7 @@ class Semantics {
       lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
       this.logs.push(lines.toString())
     }
+    args.calls.pop()
     return contextPrime
   }