theprogrammablemind 8.0.0-beta.41 → 8.0.0-beta.43

package/client.js

@@ -1,5 +1,7 @@
 const { Semantics, Semantic } = require('./src/semantics')
 const { Generators, Generator } = require('./src/generators')
+const { Config } = require('./src/config')
+const { loadInstance, ErrorReason, listable, setupArgs, gs, processContext, getObjects, setupProcessB, processContextsB  } = require('./src/configHelpers')
 const DigraphInternal = require('./src/digraph_internal')
 const Digraph = require('./src/digraph')
 const { project } = require('./src/project')
@@ -10,36 +12,10 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId } = require('./src/helpers')
+const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where } = require('./src/helpers')
 const runtime = require('./runtime')
 const sortJson = runtime.sortJson
 
-function where (goUp = 2) {
-  const e = new Error()
-  const regexForm1 = /\((.*):(\d+):(\d+)\)$/
-  const regexForm2 = /at (.*):(\d+):(\d+)$/
-  const lines = e.stack.split('\n')
-  let line
-  let match
-  for (line of lines.slice(1)) {
-    // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('<anonymous>'))) {
-    if (!(line.includes('config.js:') || line.includes('client.js:'))) {
-      match = regexForm1.exec(line) || regexForm2.exec(line)
-      if (!match) {
-        continue
-      }
-      break
-    }
-  }
-  // const line = e.stack.split("\n")[goUp];
-  // const match = regexForm1.exec(line) || regexForm2.exec(line)
-  if (match) {
-    return `${match[1]}:${match[2]}`
-  } else {
-    return 'running in browser'
-  }
-}
-
 const getConfig_getObjectsCheck = (config, testConfig) => {
   let testConfigName = config.name
   if (testConfig.testModuleName) {
@@ -108,163 +84,6 @@ const vimdiff = (actualJSON, expectedJSON, title) => {
   }
 }
 
-const listable = (hierarchy) => (c, type) => {
-  if (!c) {
-    return false
-  }
-  if (hierarchy.isA(c.marker, type)) {
-    return true
-  }
-  if (c.marker === 'list') {
-    for (const t of c.types) {
-      if (hierarchy.isA(t, type)) {
-        return true
-      }
-    }
-  }
-  return false
-}
-
-const isA = (hierarchy) => (child, parent) => {
-  if (!child || !parent) {
-    return false
-  }
-  if (child.marker) {
-    child = child.marker
-  }
-  if (parent.marker) {
-    parent = parent.marker
-  }
-  return hierarchy.isA(child, parent)
-}
-
-const asList = (context) => {
-  if (context.marker === 'list') {
-    return context
-  }
-  return {
-    marker: 'list',
-    types: [context.marker],
-    value: [context]
-  }
-}
-
-class ErrorReason extends Error {
-  constructor (context) {
-    super(JSON.stringify(context))
-    this.reason = context
-  }
-}
-
-const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
-
-  // callId
-  args.calls = new InitCalls(args.isInstance ? `${args.isInstance}#${config.name}` : config.name)
-  if (global.theprogrammablemind && global.theprogrammablemind.loadForTesting) {
-    args.calls = new InitCalls(Object.keys(global.theprogrammablemind.loadForTesting)[0])
-  }
-  args.km = (name) => config.getConfig(name)
-  args.api = (name) => config.getConfig(name).api
-  args.error = (context) => {
-    throw new ErrorReason(context)
-  }
-  args.kms = config.getConfigs()
-  args.config = config
-  args.hierarchy = hierarchy
-  args.isA = isA(hierarchy)
-  args.listable = listable(hierarchy)
-  args.asList = asList
-  args.retry = () => { throw new RetryError() }
-  args.fragments = (query) => config.fragment(query)
-  args.breakOnSemantics = false
-  args.theDebugger = {
-    breakOnSemantics: (value) => args.breakOnSemantics = value
-  }
-  if (!logs) {
-  }
-  args.log = (message) => logs.push(message)
-
-  args.addAssumedScoped = (args, assumed) => {
-    const addAssumed = (args, ...moreAssumed) => {
-      return { ...args, assumed: Object.assign({}, assumed, (args.assumed || {}), ...moreAssumed) }
-    }
-
-    args.s = (c) => config.getSemantics(logs).apply(args, c)
-    args.g = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
-    }
-    args.gp = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: true, isResponse: false, response: false }), c, { paraphrase: true, isResponse: false, response: false })
-    }
-    args.gr = (c, a = {}) => {
-      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true })
-    }
-    args.e = (c) => {
-      return config.getEvaluator(args.s, args.calls, logs, c)
-    }
-    args.gs = gs(args.g)
-    args.gsp = gs(args.gp)
-    args.gsr = gs(args.gr)
-  }
-  // for semantics
-  args.addAssumedScoped(args, {})
-
-  const getAPI = (uuid) => {
-    if (config && config.getAPI) {
-      return config.getAPI(uuid)
-    }
-  }
-  const getAPIs = (uuid) => {
-    if (config && config.getAPIs) {
-      return config.getAPIs(uuid)
-    }
-  }
-  args.getUUIDScoped = (uuid) => {
-    return {
-      api: getAPI(uuid),
-      apis: getAPIs(uuid)
-    }
-  }
-  config.getAddedArgs(args)
-
-  Object.assign(args, args.getUUIDScoped(uuidForScoping || config.uuid))
-  /*
-  if (uuidForScoping) {
-    Object.assign(args, args.getUUIDScoped(uuidForScoping))
-  }
-  */
-  // sets args for all the API. that make a copy so the args must be fully setup by here except for scoped
-  config.setArgs(args)
-}
-
-const gs = (g) => async (contexts, separator, lastSeparator) => {
-  if (!Array.isArray(contexts)) {
-    throw new Error('Expected a list')
-  }
-
-  let s = ''
-  if (!separator) {
-    separator = ' '
-  }
-  if (!lastSeparator) {
-    lastSeparator = separator
-  }
-  let nextSeparator = ''
-  for (let i = 0; i < contexts.length; ++i) {
-    const context = contexts[i]
-    const value = await g(context)
-    if (i > 0) {
-      if (i === contexts.length - 1) {
-        nextSeparator = lastSeparator
-      } else {
-        nextSeparator = separator
-      }
-    }
-    s += nextSeparator + value
-  }
-  return s
-}
-
 const matching = (actual, expected) => {
   if (!deepEqual(stringify(sortJson(actual, { depth: 25 })), stringify(sortJson(expected, { depth: 25 })))) {
     return false
@@ -301,44 +120,6 @@ const processContexts = async (contexts, params) => {
   return { contexts: contextsPrime, generated, logs }
 }
 
-const getObjects = (objects) => {
-  return (uuid) => {
-    if (objects && objects.namespaced) {
-      return objects.namespaced[uuid]
-    }
-    return objects
-  }
-}
-
-const processContext = async (context, { objects = {}, config, logs = [] }) => {
-  const generators = config.getGenerators(logs)
-  const semantics = config.getSemantics(logs)
-
-  // map to hash
-  config = config || {}
-  if (config.config) {
-    config = config
-  }
-
-  const response = {} // NA but passed in
-  // generators = new Generators(generators.map((g) => new Generator(normalizeGenerator(g))))
-  // semantics = new Semantics(semantics.map((g) => new Semantic(normalizeSemantic(g))))
-  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
-
-  const args = { objects, response, getObjects: getObjects(objects) }
-  setupArgs(args, config, logs, hierarchy)
-
-  context = await semantics.apply(args, context)
-  const generated = await generators.apply(args, context)
-  const assumed = { paraphrase: true, response: false, isResponse: false }
-  const paraphrases = await generators.apply({ ...args, assumed }, context, { paraphrase: true, response: false, isResponse: false })
-  let responses = []
-  if (context.isResponse) {
-    responses = generated
-  }
-  return { context, generated, paraphrases, responses }
-}
-
 const convertToStable = (objects) => {
   if (true) {
     return objects
@@ -430,139 +211,6 @@ const overlaps = (r1, context) => {
   return false
 }
 
-const setupContexts = (rawContexts) => {
-  let first = true
-  const contexts = []
-  contexts.push({ marker: 'controlStart', controlRemove: true })
-  for (const context of rawContexts) {
-    if (first) {
-      first = false
-    } else {
-      contexts.push({ marker: 'controlBetween', controlRemove: true })
-    }
-    contexts.push(context)
-  }
-  contexts.push({ marker: 'controlEnd', controlRemove: true })
-  return contexts
-}
-
-const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs }) => {
-  // TODO fix this name to contextsPrime
-  const contextsPrime = []
-  const generatedPrime = []
-  const paraphrasesPrime = []
-  const paraphrasesParenthesizedPrime = []
-  const generatedParenthesizedPrime = []
-  const responsesPrime = []
-  const contexts = setupContexts(json.contexts)
-
-  const objects = config.get('objects')
-  const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance }
-  if (!json.logs) {
-    json.logs = []
-  }
-  setupArgs(args, config, json.logs, hierarchy)
-  const toDo = [...contexts]
-  args.insert = (context) => toDo.unshift(context)
-  let overlap, lastRange
-  config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
-  while (toDo.length > 0) {
-    const context = toDo.shift()
-    args.calls.next()
-    let contextPrime = context
-    context.topLevel = true
-    try {
-      if (json.has_errors) {
-        throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
-      }
-      const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
-      if (!config.get('skipSemantics')) {
-        const semantics = config.getSemantics(json.logs)
-        try {
-          contextPrime = await semantics.apply(args, context)
-        } catch (e) {
-          if (e.message == 'Maximum call stack size exceeded') {
-            const mostCalled = semantics.getMostCalled()
-            e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
-          }
-          // contextPrime = semantics.apply(args, { marker: 'error', context, error: e })
-          if (isInstance) {
-            console.log('error', e.error)
-          }
-          contextPrime = await semantics.apply(args, {
-            marker: 'error',
-            context,
-            text: e ? e.toString() : 'not available',
-            reason: e.reason,
-            error: e.stack || e.error
-          })
-          if (rebuildingTemplate) {
-            throw e
-          }
-        }
-      }
-      if (contextPrime.controlRemove) {
-        continue
-      }
-      let assumed = { isResponse: true }
-      const generated = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
-      let generatedParenthesized = []
-      if (generateParenthesized) {
-        config.parenthesized = true
-        generatedParenthesized = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
-        config.parenthesized = false
-      }
-      // assumed = { paraphrase: true, response: false };
-      assumed = { paraphrase: true, isResponse: false, response: false }
-      if (generateParenthesized) {
-        config.parenthesized = false
-      }
-      const paraphrases = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
-      let paraphrasesParenthesized = []
-      if (generateParenthesized) {
-        config.parenthesized = true
-        paraphrasesParenthesized = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
-        config.parenthesized = false
-      }
-      contextsPrime.push(contextPrime)
-      generatedPrime.push(generated)
-      paraphrasesPrime.push(paraphrases)
-      if (generateParenthesized) {
-        paraphrasesParenthesizedPrime.push(paraphrasesParenthesized)
-        generatedParenthesizedPrime.push(generatedParenthesized)
-      }
-      if (contextPrime.isResponse) {
-        responsesPrime.push(generated)
-      } else {
-        responsesPrime.push('')
-      }
-
-      // add results to processed list
-      config.config.objects.processed = config.config.objects.processed || []
-      config.config.objects.processed = config.config.objects.processed.slice(0, 5)
-      config.config.objects.processed.unshift({ context: contextPrime, paraphrases: paraphrases, paraphrasesParenthesized, generatedParenthesized, responses: responsesPrime })
-    } catch (e) {
-      if (Array.isArray(e)) {
-        e = {
-          errors: e
-        }
-      }
-      e.context = contextPrime
-      if (e.logs) {
-        e.logs = e.logs.concat(json.logs)
-      } else {
-        e.logs = json.logs
-      }
-      e.metadata = json.metadata
-      if (json.trace) {
-        e.trace = json.trace
-      }
-      throw e
-    }
-  }
-  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
-}
-
 const doWithRetries = async (n, url, queryParams, data) => {
   if (!queryParams) {
     queryParams = ''
@@ -597,93 +245,6 @@ const doWithRetries = async (n, url, queryParams, data) => {
   }
 }
 
-const setupProcessB = ({ config, initializer, allowDelta = false } = {}) => {
-  const key = config._key
-
-  const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
-  if (allowDelta && config.allowDelta && config.hasDelta()) {
-    // console.log('config', config)
-    data.delta = config.delta()
-  } else {
-    config.toData(data)
-    // Object.assign(data, config.config)
-  }
-
-  // config.toServer(data)
-
-  if (data.namespaces) {
-    for (const uuid of Object.keys(data.namespaces)) {
-      const km = config.configs.find((km) => km.uuid === uuid)
-      data.namespaces[uuid].name = km.name
-    }
-  }
-
-  // const generators = new Generators((data.generators || []).map((g) => new Generator(normalizeGenerator(g))))
-  delete data.generators
-  // const semantics = new Semantics((data.semantics || []).map((g) => new Semantic(normalizeSemantic(g))))
-  delete data.semantics
-  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
-
-  return {
-    data,
-    // generators,
-    // semantics,
-    hierarchy
-  }
-}
-
-// instance template loadTemplate
-const loadInstance = async (config, instance) => {
-  const transitoryMode = global.transitoryMode
-  global.transitoryMode = false
-
-  if (instance && (instance.associations || instance.learned_contextual_priorities)) {
-    if (!config.config.retrain) {
-      if (instance.associations) {
-        config.addAssociations(instance.associations)
-      }
-      if (instance.learned_contextual_priorities && instance.learned_contextual_priorities.length > 0) {
-        config.addPriorities(instance.learned_contextual_priorities)
-      }
-    }
-  }
-
-  const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
-  // for (const results of (instance.resultss || [])) {
-  for (const i in (instance.resultss || [])) {
-    const results = instance.resultss[i]
-    if (results.extraConfig) {
-      // config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
-      const uuid = config.nameToUUID(instance.name)
-      // used to do a CLONE
-      config.addInternal(instance.template.configs[i], { uuid, addFirst: true, handleCalculatedProps: true })
-    } else if (results.apply) {
-      const objects = config.get('objects')
-      const args = { objects, getObjects: getObjects(objects) }
-      if (instance.configs) {
-        args.isInstance = `instance${i}`
-        args.instance = instance.configs[i]
-      }
-
-      const uuid = config.nameToUUID(instance.name)
-      setupArgs(args, config, config.logs, hierarchy, uuid)
-      await results.apply(args)
-    } else {
-      if (results.skipSemantics) {
-        config.config.skipSemantics = results.skipSemantics
-      }
-      const args = { config, hierarchy, json: results, commandLineArgs: {} }
-      args.isInstance = `instance${i}`
-      args.instance = ''
-      await processContextsB(args)
-      if (results.skipSemantics) {
-        config.config.skipSemantics = null
-      }
-    }
-  }
-  global.transitoryMode = transitoryMode
-}
-
 const throwErrorHandler = (error) => {
   throw error
 }
@@ -1508,11 +1069,15 @@ const checkTest = (testConfig) => {
 }
 
 const knowledgeModuleImpl = async ({
+  includes,
+  config : configStruct,
+  api,
+  initializer,
+  multiApiInitializer,
+
   module: moduleFromJSFile,
   description,
   section,
-  // config, createConfig,
-  createConfig,
   newWay,
   demo,
   test,
@@ -1520,7 +1085,6 @@ const knowledgeModuleImpl = async ({
   errorHandler = defaultErrorHandler,
   process: processResults = defaultProcess,
   stopAtFirstFailure = true,
-  acceptsAdditionalConfig = false,
   ...rest
 } = {}) => {
   const unknownArgs = Object.keys(rest)
@@ -1533,10 +1097,28 @@ const knowledgeModuleImpl = async ({
   if (!moduleFromJSFile) {
     throw new Error("'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module.")
   }
-  // if (!config && !createConfig) {
-  if (!createConfig) {
+
+  if (!configStruct) {
     throw new Error("'config' or 'createConfig' is a required parameter. The value should the config that defines the knowledge module.")
   }
+
+  const createConfig = async () => {
+    const config = new Config(configStruct, moduleFromJSFile)
+    config.stop_auto_rebuild()
+    await config.add(...(includes || []))
+    if (api) {
+      config.setApi(api())
+    }
+    if (multiApiInitializer) {
+      await config.setMultiApi(multiApiInitializer)
+    }
+    if (initializer) {
+      config.initializer(initializer)
+    }
+    await config.restart_auto_rebuild()
+    return config
+  }
+
   if (!description) {
     throw new Error("'description' is a required parameter. The value should the description of the knowledge module.")
   }
@@ -1863,7 +1445,7 @@ const knowledgeModuleImpl = async ({
         config.config.rebuild = true
       }
       try {
-        await config.load(template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
+        await config.load(rebuildTemplate, template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
       } catch (e) {
         console.error(`Error loading template for ${config.name}. ${e.error ? e.error : e}${e.stack ? e.stack : ''}`)
         runtime.process.exit(-1)
@@ -2114,6 +1696,8 @@ const knowledgeModuleImpl = async ({
     const initConfig = async (config) => {
       if (template) {
         if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess }).needsRebuild) {
+          debugger
+          config.needsRebuild(template.template, template.instance, { isModule: !isProcess })
           const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
           throw new Error(error)
         }
@@ -2146,15 +1730,17 @@ const knowledgeModuleImpl = async ({
 
       if (template) {
         try {
-          await config.load(template.template, template.instance)
+          await config.load(rebuildTemplate, template.template, template.instance)
         } catch (e) {
           errorHandler(e)
         }
       }
     }
 
+    // no cache 21 minutes + rebuild fails "node tester_rebuild -m colors"
+    // cache okay
     createConfigExport = async () => {
-      if (createConfig.cached) {
+      if (false && createConfig.cached) {
         return createConfig.cached
       }
       const config = await createConfig()
@@ -2187,11 +1773,6 @@ const ensureTestFile = (module, name, type) => {
   }
 }
 
-function w (func) {
-  func.where = where(3)
-  return func
-}
-
 const knowledgeModule = async (...args) => {
   await knowledgeModuleImpl(...args).catch((e) => {
     console.error(e)
@@ -2202,9 +1783,6 @@ const knowledgeModule = async (...args) => {
 module.exports = {
   process: _process,
   stableId,
-  where,
-  w,
-  // submitBug,
   ensureTestFile,
   rebuildTemplate,
   processContext,

package/index.js

@@ -1,6 +1,7 @@
 const { Semantics, Semantic } = require('./src/semantics')
 const { Generators, Generator } = require('./src/generators')
 const { Config } = require('./src/config')
+const { w, where } = require('./src/helpers')
 const Digraph = require('./src/digraph')
 const client = require('./client')
 const flattens = require('./src/flatten')
@@ -14,9 +15,9 @@ module.exports = {
   runTests: client.runTests,
   knowledgeModule: client.knowledgeModule,
   ensureTestFile: client.ensureTestFile,
-  where: client.where,
+  where,
   stableId: client.stableId,
-  w: client.w,
+  w,
   Config,
   Semantics,
   Semantic,

package/package.json

@@ -64,6 +64,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "8.0.0-beta.41",
+  "version": "8.0.0-beta.43",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -2,9 +2,10 @@
 const { Semantics, normalizeGenerator } = require('./semantics')
 const { Generators } = require('./generators')
 const { v4: uuidv4 } = require('uuid')
-const client = require('../client')
+const configHelpers = require('./configHelpers')
 const DigraphInternal = require('./digraph_internal')
 const helpers = require('./helpers')
+const { InitCalls } = require('./helpers')
 const { ecatch } = require('./helpers')
 const runtime = require('../runtime')
 const _ = require('lodash')
@@ -351,7 +352,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         const level = bridge.generatorp.level >= 0 ? bridge.generatorp.level : bridge.level + 1
 
         const generator = {
-          where: bridge.generatorp.where || bridge.where || client.where(4),
+          where: bridge.generatorp.where || bridge.where || helpers.where(4),
           match: async (args) => bridge.id == args.context.marker && args.context.level == level && args.context.paraphrase && await match(args),
           apply: (args) => apply(args),
           applyWrapped: apply,
@@ -368,7 +369,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
         const apply = typeof bridge.generatorr === 'function' ? bridge.generatorr : bridge.generatorr.apply || bridge.generatorr
         const level = bridge.generatorr.level >= 0 ? bridge.generatorr.level : bridge.level + 1
         const generator = {
-          where: bridge.generatorr.where || bridge.where || client.where(4),
+          where: bridge.generatorr.where || bridge.where || helpers.where(4),
           match: async (args) => bridge.id == args.context.marker && args.context.level == level && !args.context.paraphrase && (args.context.response || args.context.isResponse) && await match(args),
           apply: (args) => apply(args),
           applyWrapped: apply,
@@ -382,7 +383,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       }
       if (bridge.evaluator) {
         const semantic = {
-          where: bridge.evaluator.where || bridge.where || client.where(3),
+          where: bridge.evaluator.where || bridge.where || helpers.where(3),
           match: ({ context }) => bridge.id == context.marker && context.evaluate,
           apply: (args) => bridge.evaluator(args),
           applyWrapped: bridge.evaluator,
@@ -396,7 +397,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       }
       if (bridge.semantic) {
         const semantic = {
-          where: bridge.semantic.where || bridge.where || client.where(3),
+          where: bridge.semantic.where || bridge.where || helpers.where(3),
           match: ({ context }) => bridge.id == context.marker && !context.evaluate,
           apply: (args) => bridge.semantic(args),
           applyWrapped: bridge.semantic,
@@ -990,21 +991,6 @@ class Config {
 
   // value is in response field
   // TODO maybe generalize out query+evaluate along the lines of set value and set reference
-  /*
-  getEvaluator (s, log, context) {
-    const instance = s({ ...context, evaluate: true })
-    if (!instance.evalue && !instance.verbatim && !instance.value) {
-      this.warningNotEvaluated(log, context);
-    }
-    if (!instance.evalue) {
-      instance.evalue = instance.value
-      instance.edefault = true
-    }
-    delete instance.evaluate
-    instance.instance = true;
-    return instance
-  }
-  */
   async getEvaluator (s, calls, log, context) {
     const instance = await s({ ...context, evaluate: true })
     calls.touch(instance)
@@ -1095,6 +1081,27 @@ class Config {
       } else {
         const config = { ...queryOrConfig }
         delete config.where
+
+        if (config.words && config.words.hierarchy) {
+          config.words.hierarchy = config.words.hierarchy.map( (value) => {
+            value = {...value}
+            delete value.uuid
+            return value
+          })
+        }
+
+        if (config.words && config.words.patterns) {
+          config.words.patterns = config.words.patterns.map( (value) => {
+            value = {...value}
+            value.defs = value.defs.map( (value) => {
+              value = {...value}
+              delete value.uuid
+              return value
+            })
+            return value
+          })
+        }
+
         config.operators = (config.operators || []).map((operator) => {
           if (typeof operator === 'string') {
             return { pattern: operator }
@@ -1122,6 +1129,9 @@ class Config {
             delete bridge.generatorpr
             delete bridge.evaluator
             delete bridge.semantic
+            if (!bridge.bridge) {
+              bridge.bridge = "{ ...next(operator) }"
+            }
             return bridge
           })
         } else {
@@ -1226,7 +1236,7 @@ class Config {
   }
 
   // loadTemplate
-  async load (template, instance, options = { rebuild: false, previousResultss: undefined, startOfChanges: undefined }) {
+  async load (rebuildTemplate, template, instance, options = { rebuild: false, previousResultss: undefined, startOfChanges: undefined }) {
     this.validifyTemplate(template)
     instance.template = template
     this.logs.push(`loading template for ${this.name}`)
@@ -1234,7 +1244,7 @@ class Config {
       // TODO fix beforeQuery
       template = { fragments: [], configs: [], ...template }
       template.fragments = template.fragments.concat(this.dynamicFragments)
-      client.rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
+      rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
     } else {
       // no change
       // this.initInstances.push({ ...instance, name: config.name })
@@ -1259,7 +1269,7 @@ class Config {
         instance.name = this.name
         this.initInstances.push(instance)
         this.instances.push(instance)
-        await client.loadInstance(this, instance)
+        await configHelpers.loadInstance(this, instance)
       }
     }
   }
@@ -1408,7 +1418,7 @@ class Config {
     }
 
     if (!generator.where) {
-      generator.where = client.where(3)
+      generator.where = helpers.where(3)
     }
 
     const generators = this.config.generators
@@ -1431,7 +1441,7 @@ class Config {
     }
 
     if (!semantic.where) {
-      semantic.where = client.where(3)
+      semantic.where = helpers.where(3)
     }
 
     const semantics = this.config.semantics
@@ -1619,11 +1629,7 @@ class Config {
   }
 
   async processContext (context) {
-    return await client.processContext(context, this.getParams())
-  }
-
-  process (query, options) {
-    return client.process(this, query, options)
+    return await configHelpers.processContext(context, this.getParams())
   }
 
   query (query, options) {
@@ -2529,7 +2535,7 @@ class Config {
         }
         const instance = this.instances.find((instance) => instance.name == name)
         if (instance) {
-          await client.loadInstance(this, instance)
+          await configHelpers.loadInstance(this, instance)
         }
         this.hierarchy.edges = this.config.hierarchy
       }

package/src/helpers.js

@@ -1,6 +1,37 @@
 const deepEqual = require('deep-equal')
 const stringify = require('json-stable-stringify')
 
+function where (goUp = 2) {
+  const e = new Error()
+  const regexForm1 = /\((.*):(\d+):(\d+)\)$/
+  const regexForm2 = /at (.*):(\d+):(\d+)$/
+  const lines = e.stack.split('\n')
+  let line
+  let match
+  for (line of lines.slice(1)) {
+    // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('<anonymous>'))) {
+    if (!(line.includes('config.js:') || line.includes('client.js:'))) {
+      match = regexForm1.exec(line) || regexForm2.exec(line)
+      if (!match) {
+        continue
+      }
+      break
+    }
+  }
+  // const line = e.stack.split("\n")[goUp];
+  // const match = regexForm1.exec(line) || regexForm2.exec(line)
+  if (match) {
+    return `${match[1]}:${match[2]}`
+  } else {
+    return 'running in browser'
+  }
+}
+
+function w (func) {
+  func.where = where(3)
+  return func
+}
+
 // properties - the properties that correspond to types
 // types - the expected types of the properties
 // returns list of properties found matching order of types
@@ -403,5 +434,7 @@ module.exports = {
   InitCalls,
   hashCode,
   sortJson,
-  subPriority
+  subPriority,
+  where,
+  w,
 }

package/src/semantics.js

@@ -49,76 +49,49 @@ class Semantic {
     }
   }
 
-  async matches (baseArgs, context, options = {}) {
-    const hierarchy = baseArgs.hierarchy
-    const config = baseArgs.config
-
-    const objects = baseArgs.getObjects(this.uuid)
-    // const ask = baseArgs.getAsk(this.uuid)
-
-    // return this.matcher(Object.assign({}, argsBase, {args: contextArgs(context, hierarchy), objects: objects, global: objects, context: context, hierarchy: hierarchy, api: this.getAPI(config)})
-    const callId = baseArgs.calls.current()
-    const moreArgs = {
-      uuid: this.uuid,
-      args: contextArgs(context, hierarchy),
-      objects,
-      global: objects,
-      context: context,
-      // hierarchy: hierarchy,
-      callId,
-      api: this.getAPI(config),
-      apis: this.getAPIs(config)
+  fixUpArgs (args, context) {
+    args.uuid = this.uuid
+    args.callId = args.calls.current()
+    const objects = args.getObjects(this.uuid)
+    args.objects = objects
+    args.global = objects
+    const config = args.config
+    args.api = this.getAPI(config)
+    args.apis = this.getAPIs(config)
+    args.args = contextArgs(context, args.hierarchy)
+    args.context = context
+    let n = (id) => id
+    if (config && 'nsToString' in config) {
+      n = (id) => config.nsToString(id)
     }
-    const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
+    args.n = n
+    args.uuid = this.uuid
+    Object.assign(args, (args.getUUIDScoped || (() => { return {} }))(this.uuid))
+  }
 
+  async matches (args, context, options = {}) {
+    this.fixUpArgs(args, context)
     const matches = await this.matcher(args)
-    if (matches && (options.debug || {}).match ||
-        callId == this.callId) {
+    if (matches && (options.debug || {}).match || args.callId == this.callId) {
       debugger // next line is the matcher
       await this.matcher(args)
     }
     return matches
   }
 
-  async apply (baseArgs, context, s, log, options = {}) {
-    const { hierarchy, config, response } = baseArgs
-    const objects = baseArgs.getObjects(this.uuid)
+  async apply (args, context, s, options = {}) {
+    const { config } = args
     if (config && config.debugLoops) {
       console.log('apply', this.toLabel())
     }
-    if (baseArgs.calls && config && baseArgs.calls.stack.length > config.maxDepth) {
+    if (args.calls && config && args.calls.stack.length > config.maxDepth) {
       throw new Error(`Max depth of ${config.maxDepth} for calls has been exceeded. maxDepth can be set on the config object. To see the calls run with the --dl or set the debugLoops property on the config`)
     }
 
-    // const ask = baseArgs.getAsk(this.uuid)
-    if (!log) {
-      console.trace()
-      throw new Error('log is a required argument')
-    }
     const contextPrime = Object.assign({}, context)
-    let n = (id) => id
-    if (config && 'nsToString' in config) {
-      n = (id) => config.nsToString(id)
-    }
-    const callId = baseArgs.calls.current()
-    const moreArgs = {
-      uuid: this.uuid,
-      callId,
-      args: contextArgs(context, hierarchy),
-      objects,
-      log,
-      global: objects,
-      n,
-      context: contextPrime,
-      uuid: this.uuid,
-      // config,
-      response,
-      api: this.getAPI(config),
-      apis: this.getAPIs(config)
-    }
-    const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
-    if ((options.debug || {}).apply ||
-        callId == this.callId) {
+    this.fixUpArgs(args, contextPrime)
+
+    if ((options.debug || {}).apply || args.callId == this.callId) {
       debugger
     }
     if (args.breakOnSemantics) {
@@ -181,6 +154,7 @@ class Semantics {
     if (!(context instanceof Array || context instanceof Object)) {
       return context
     }
+    args = { ...args }
     const config = args.config
     let contextPrime = Object.assign({}, context)
     const s = (context, options) => this.apply(args, context, options)
@@ -188,6 +162,8 @@ class Semantics {
     const stack = args.calls.push()
     let counter = 0
     let seenQuestion = false
+    const deferred = []
+    args.log = (message) => { this.logs.push(message) }
     for (const isemantic in this.semantics) {
       const semantic = this.semantics[isemantic]
       if (!semantic) {
@@ -202,17 +178,28 @@ class Semantics {
           this.calls[counter] = 0
         }
         this.calls[counter] += 1
-        const log = (message) => { this.logs.push(message) }
         try {
-          contextPrime = await semantic.apply(args, context, s, log, options)
+          let deferWasCalled = false
+          const defer = (listener) => {
+            deferred.push({ semantic, listener })
+            deferWasCalled = true 
+          }
+          args.defer = defer
+          contextPrime = await semantic.apply(args, context, s, options)
+          if (deferWasCalled) {
+            continue
+          }
           if (!contextPrime.controlKeepMotivation && semantic.oneShot) {
             // semantic.tied_ids.forEach((tied_id) => args.config.removeSemantic(tied_id))
             args.config.removeSemantic(semantic)
           }
+          for (const { listener } of deferred) {
+            listener(args)
+          }
         } catch (e) {
           contextPrime = null
           let errorMessage
-          e.retryCall = () => semantic.apply(args, context, s, log, options)
+          e.retryCall = () => semantic.apply(args, context, s, options)
           const help = 'The error has a retryCall property that will recall the function that failed.'
           if (e.stack && e.message) {
             const info = `${semantic.notes ? semantic.notes : ''}${semantic.where ? semantic.where : ''}`
@@ -280,6 +267,13 @@ class Semantics {
           lines.setElement(0, 1, 'RESULT')
           lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(contextPrime, { depth: 25 })))})`)
           lines.setElement(1, 2, JSON.stringify(contextPrime, null, 2))
+          for (const { semantic } of deferred) {
+            lines.setElement(0, 1, 'DEFERRED')
+            lines.setElement(0, 2, semantic.toLabel())
+            lines.newRow()
+            lines.setElement(0, 2, semantic.toString())
+            lines.newRow()
+          }
           this.logs.push(lines.toString())
         }
         applied = true