theprogrammablemind_4wp 7.12.8 → 8.0.0-beta.1

package/client.js

@@ -47,7 +47,13 @@ const getConfig_getObjectsCheck = (config, testConfig) => {
   }
   const checks = (testConfig.checks && testConfig.checks.objects) || []
   if (Array.isArray(checks)) {
-    return { [testConfigName]: checks }
+    const kmToChecks = { [testConfigName]: checks.filter( (check) => !check.km ) }
+    for (const check of checks) {
+      if (check.km) {
+        kmToChecks[check.km] = config.km(check.km).testConfig.checks.objects
+      }
+    }
+    return kmToChecks
   } else {
     return checks
   }
@@ -81,104 +87,6 @@ const pickObjects = (config, testConfig, getObjects) => {
   return projection
 }
 
-// move ask to the KM's since verbatim is called probably in dialogues?
-const getAsk = (config) => (uuid) => {
-    // if (!uuid) {
-    //  debugger
-    //}
-    return (asks) => {
-    const ask = (ask) => {
-      let oneShot = true // default
-      if (ask.oneShot === false) {
-        oneShot = false
-      }
-
-      const id_q = stableId('semantic')
-      const id_rs = []
-      let wasAsked = false
-      let wasApplied = false
-      const getWasAsked = () => {
-        return wasAsked
-      }
-      const setWasAsked = (value) => {
-        wasAsked = value
-      }
-      const getWasApplied = () => {
-        return wasApplied
-      }
-      const setWasApplied = (value) => {
-        wasApplied = value
-      }
-
-      const semanticsr = ask.semanticsr || []
-      if (semanticsr.length == 0) {
-        semanticsr.push({ match: ask.matchr, apply: ask.applyr })
-      }
-      for (const semantic of semanticsr) {
-        const id_r = stableId('semantic')
-        id_rs.push(id_r)
-        config.addSemantic({
-          uuid,
-          id: id_r,
-          tied_ids: [id_q],
-          oneShot,
-          where: semantic.where || ask.where || where(2),
-          source: 'response',
-          match: (args) => semantic.match(args),
-          apply: (args) => {
-            setWasApplied(true)
-            semantic.apply(args)
-          },
-        })
-      }
-      
-      config.addSemantic({
-        uuid,
-        oneShot,
-        id: id_q,
-        tied_ids: id_rs,
-        where: ask.where,
-        isQuestion: true,  // do one question at a time
-        getWasAsked,
-        getWasApplied,
-        onNevermind: ask.onNevermind,
-        source: 'question',
-        match: ({ context }) => context.marker == 'controlEnd' || context.marker == 'controlBetween',
-        apply: (args) => {
-          let matchq = ask.matchq
-          let applyq = ask.applyq
-          if (!matchq) {
-            let wasAsked = false
-            matchq = () => !wasAsked,
-            applyq = (args) => {
-              wasAsked = true
-              return ask.applyq(args)
-            }
-          }
-          if (matchq(args)) {
-            setWasAsked(true)
-            setWasApplied(false)
-            // args.context.motivationKeep = true
-            args.verbatim(applyq(args))
-            /*
-              args.context.verbatim = applyq(args)
-              args.context.isResponse = true;
-              delete args.context.controlRemove;
-              */
-            args.context.controlKeepMotivation = true
-          }
-          args.context.cascade = true
-        }
-      })
-    }
-    if (!Array.isArray(asks)) {
-      asks = [asks]
-    }
-
-    [...asks].reverse().forEach( (a) => ask(a) )
-  }
-}
-
 const sameJSON = (json1, json2) => {
   const sjson1 = sortJson(json1, { depth: 25 })
   const sjson2 = sortJson(json2, { depth: 25 })
@@ -300,7 +208,6 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   }
   // for semantics
   args.addAssumedScoped(args, {})
-  config.getAddedArgs(args)
 
   const getAPI = (uuid) => {
     if (config && config.getAPI) {
@@ -312,14 +219,14 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
       return config.getAPIs(uuid)
     }
   }
-  const scopedAsk = getAsk(config)
   args.getUUIDScoped = (uuid) => {
     return {
-      ask: scopedAsk(uuid),
       api: getAPI(uuid),
       apis: getAPIs(uuid)
     }
   }
+  config.getAddedArgs(args)
+
   Object.assign(args, args.getUUIDScoped(uuidForScoping || config.uuid))
   /*
   if (uuidForScoping) {
@@ -540,7 +447,7 @@ const setupContexts = (rawContexts) => {
   return contexts
 }
 
-const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, isInstance, instance, query, data, retries, url, commandLineArgs }) => {
+const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs }) => {
   // TODO fix this name to contextsPrime
   const contextsPrime = []
   const generatedPrime = []
@@ -590,6 +497,9 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
             reason: e.reason,
             error: e.stack || e.error
           })
+          if (rebuildingTemplate) {
+            throw e
+          }
         }
       }
       if (contextPrime.controlRemove) {
@@ -745,17 +655,15 @@ const loadInstance = (config, instance) => {
     const results = instance.resultss[i]
     if (results.extraConfig) {
       // config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
-      // config.addInternal(config.template.queries[i], { handleCalculatedProps: true } )
       const uuid = config.nameToUUID(instance.name)
       // used to do a CLONE
-      // config.addInternal(_.cloneDeep(instance.template.queries[i]), { uuid, addFirst: true, handleCalculatedProps: true })
-      config.addInternal(instance.template.queries[i], { uuid, addFirst: true, handleCalculatedProps: true })
+      config.addInternal(instance.template.configs[i], { uuid, addFirst: true, handleCalculatedProps: true })
     } else if (results.apply) {
       const objects = config.get('objects')
       const args = { objects, getObjects: getObjects(objects) }
-      if (instance.queries) {
+      if (instance.configs) {
         args.isInstance = `instance${i}`
-        args.instance = instance.queries[i]
+        args.instance = instance.configs[i]
       }
 
       const uuid = config.nameToUUID(instance.name)
@@ -865,7 +773,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
           start = runtime.performance.performance.now()
         }
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  processContextsB({ isTest, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+                                  processContextsB({ isTest, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         if (isTest) {
           end = runtime.performance.performance.now()
           clientSideTime = end - start
@@ -923,15 +831,27 @@ const getConfigForTest = (config, testConfig) => {
     // configForTest[key] = config.config[key]
     if (key === 'words') {
       const words = config.config.words
-      configForTest.words = {}
-      for (const key of Object.keys(words)) {
+      configForTest.words = {
+        literals: {},
+        patterns: [],
+        hierarchy: [],
+      }
+
+      const literals = config.config.words.literals
+      for (const key in literals) {
         const defs = []
-        for (const def of words[key]) {
+        for (const def of literals[key]) {
           // TODO handle thie uuids the right way
           defs.push(Object.assign({}, def, { uuid: undefined }))
         }
-        configForTest.words[key] = defs
+        configForTest.words.literals[key] = defs
       }
+
+      const patterns = config.config.words.patterns
+      configForTest.words.patterns = patterns.map((pattern) => Object.assign({}, pattern, { uuid: undefined }))
+
+      const hierarchy = config.config.words.hierarchy
+      configForTest.words.hierarchy = hierarchy.map((hierarchy) => Object.assign({}, hierarchy, { uuid: undefined }))
     } else if (key === 'operators') {
       configForTest.operators = config.config.operators.map((operator) => Object.assign({}, operator, { uuid: undefined }))
     } else if (key === 'bridges') {
@@ -1281,7 +1201,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     console.log('Errors')
     responses.errors.forEach((error) => console.log(`    ${error}`))
   }
-  console.log("KM's loaded", config.configs.map((c) => c.name))
+  console.log("KM's loaded (ordered)", config.configs.map((c) => c.name))
   console.log('This is the global objects from running semantics:\n', config.objects)
   if (!_.isEmpty(responses.learned_contextual_priorities)) {
     console.log('\nThe learned contextual priorties are :\n')
@@ -1404,12 +1324,12 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
     associations: [],
     learned_contextual_priorities: []
   }
-  const looper = async (queries) => {
-    if (queries.length === 0) {
+  const looper = async (configs) => {
+    if (configs.length === 0) {
       finish()
       return
     }
-    const { property, hierarchy, query: queryOrExtraConfig, previousResults, initializer, skipSemantics } = queries.shift()
+    const { property, hierarchy, query: queryOrExtraConfig, previousResults, initializer, skipSemantics } = configs.shift()
     // queries are strings or { query: "blah", development: true/false }
     if (typeof queryOrExtraConfig === 'string' || queryOrExtraConfig.query) {
       let query = queryOrExtraConfig
@@ -1461,7 +1381,7 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
         accumulators[property].push(results)
         accumulators.associations = accumulators.associations.concat(results.associations)
         accumulators.learned_contextual_priorities = accumulators.learned_contextual_priorities.concat(results.learned_contextual_priorities)
-        await looper(queries)
+        await looper(configs)
       } catch (e) {
         const error = { errors: [e], query: query.query }
         config.config.skipSemantics = null
@@ -1475,7 +1395,7 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
       setupArgs(args, config, config.logs, hierarchy)
       initFunction(args)
       accumulators[property].push({ apply: queryOrExtraConfig })
-      await looper(queries)
+      await looper(configs)
     } else {
       // extra config is def from a time like operators or bridges or words etc
       // it will just get added to the config
@@ -1491,7 +1411,7 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
           throw new Error(`Error processing extra config${where}: ${e.stack}}`)
         }
         accumulators[property].push({ extraConfig: true, ...extraConfig })
-        await looper(queries)
+        await looper(configs)
       }
     }
   }
@@ -1531,7 +1451,7 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
       return template
     }
     stabilizeOutput(accumulators)
-    runtime.fs.writeFileSync(instanceName, JSON.stringify(Object.assign({ queries: template.queries.map(updateQueries) }, accumulators), 0, 2))
+    runtime.fs.writeFileSync(instanceName, JSON.stringify(Object.assign({ configs: template.configs.map(updateQueries) }, accumulators), 0, 2))
 
     // km tests file
     const testsName = `./${target}.test.json`
@@ -1550,7 +1470,7 @@ const rebuildTemplate = async ({ config, target, previousResultss, startOfChange
   }
   let todo = []
   todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
-  todo = todo.concat((template.queries || []).map((query, index) => {
+  todo = todo.concat((template.configs || []).map((query, index) => {
     let pr
     if (index < startOfChanges) {
       pr = previousResultss[index]
@@ -1600,19 +1520,6 @@ const knowledgeModuleImpl = async ({
   acceptsAdditionalConfig = false,
   ...rest
 } = {}) => {
-  /*
-  if (description == 'fastfood related concepts') {
-    debugger
-    global.old = template.template.queries[91].bridges[0]
-  }
-  let old
-  if (template && template.template && template.template.queries) {
-    old = template.template.queries
-    template.template.queries = _.cloneDeep(template.template.queries)
-    template.wasCopied = true
-  }
-  */
-
   const unknownArgs = Object.keys(rest)
   if (unknownArgs.length > 0) {
     throw new Error(`Unknown arguments to knowledgeModule: ${unknownArgs.join()}`)
@@ -1659,19 +1566,16 @@ const knowledgeModuleImpl = async ({
   }
 
   if (isProcess) {
-    const config = createConfig()
-    setupConfig(config)
-    processResults = processResults({ config, errorHandler })
-    // setup();
     const parser = new runtime.ArgumentParser({
       description: 'Entodicton knowledge module'
     })
 
+    const helpDebugWord = 'In order to get a debug break when a specific word is created set the DEBUG_WORD environment variable to the JSON of the association to break on. For example DEBUG_WORD=\'"the"\''
     const helpDebugAssociation = 'In order to get a debug break when a specific association is created set the DEBUG_ASSOCIATION environment variable to the JSON of the association to break on. For example DEBUG_ASSOCIATION=\'[["the", 0], ["mammal", 1]]\''
     const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'[["cat", 1], ["mammel", 1]]\''
     const helpDebugPriority = 'In order to get a debug break when a specific set of priorities is created set set DEBUG_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_PRIORITY=\'[["verb", 0], ["article", 0]]\''
     const helpDebugContextualPriority = 'In order to get a debug break when a specific set of contextual priorities is created set set DEBUG_CONTEXTUAL_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_CONTEXTUAL_PRIORITY=\'{ context: [["verb", 0], ["article", 0], select: 1}\''
-    const helpDebugBridge = 'In order to get a debug break when a specific bridge is created set the DEBUG_BRIDGE environment variable to id/level to break on. For example DEBUG_BRIDGE=\'id#level\''
+    const helpDebugBridge = 'In order to get a debug break when a specific bridge is created set the DEBUG_BRIDGE environment variable to id to break on. For example DEBUG_BRIDGE=\'car\''
     const helpDebugOperator = 'In order to get a debug break when a specific hierarcy is created set the DEBUG_OPERATOR environment variable to debug any config loaded. For example DEBUG_OPERATOR=\'([operator] ([arg]))\''
 
     parser.add_argument('-tmn', '--testModuleName', { help: 'When running tests instead of using the current modules tests use the specified modules tests' })
@@ -1703,6 +1607,7 @@ const knowledgeModuleImpl = async ({
     parser.add_argument('-dl', '--debugLoops', { action: 'store_true', help: 'When running with the --debugLoops flag the logs calls to semantics and generators will be immediately written to the console ' })
     parser.add_argument('-d', '--debug', { action: 'store_true', help: 'When running with the --debug flag this set the debug flag in the config' })
     parser.add_argument('-da', '--debugAssociation', { action: 'store_true', help: helpDebugAssociation })
+    parser.add_argument('-dw', '--debugWord', { action: 'store_true', help: helpDebugWord })
     parser.add_argument('-dh', '--debugHierarchy', { action: 'store_true', help: helpDebugHierarchy })
     parser.add_argument('-dp', '--debugPriority', { action: 'store_true', help: helpDebugPriority })
     parser.add_argument('-dcp', '--debugContextualPriority', { action: 'store_true', help: helpDebugContextualPriority })
@@ -1718,6 +1623,21 @@ const knowledgeModuleImpl = async ({
       args.rebuildTemplate = true
     }
 
+    // dont debug the load of the KM's if rebuild template is on since we want to debug the template rebuild not the load
+    if (args.rebuildTemplate) {
+      global.pauseDebugging = true
+    }
+
+    const config = createConfig()
+    setupConfig(config)
+    processResults = processResults({ config, errorHandler })
+
+    if (args.rebuildTemplate) {
+      global.pauseDebugging = false
+    }
+
+    // setup();
+
     if (args.parenthesized) {
       config.parenthesized = true
     }
@@ -1754,6 +1674,10 @@ const knowledgeModuleImpl = async ({
       console.log(helpDebugAssociation)
       runtime.process.exit(-1)
     }
+    if (args.debugWord) {
+      console.log(helpDebugWord)
+      runtime.process.exit(-1)
+    }
     if (args.debugHierarchy) {
       console.log(helpDebugHierarchy)
       runtime.process.exit(-1)
@@ -1846,8 +1770,14 @@ const knowledgeModuleImpl = async ({
           }
         }
         if (args.print.includes('w')) {
-          for (const word in config.config.words) {
-            console.log(word.concat('          ', ...config.config.words[word].map((def) => JSON.stringify(def))))
+          // { literals: Object, patterns: Array(2), hierarchy: Array(97) }
+          console.log('literals')
+          for (const word in config.config.words.literals) {
+            console.log('  ' + word.concat('          ', ...config.config.words.literals[word].map((def) => JSON.stringify(def))))
+          }
+          console.log('patterns')
+          for (const pattern of config.config.words.patterns) {
+            console.log('  ' + JSON.stringify(pattern))
           }
         }
         if (args.print.includes('b')) {
@@ -2270,6 +2200,7 @@ const knowledgeModule = async (...args) => {
 
 module.exports = {
   process: _process,
+  stableId,
   where,
   w,
   // submitBug,

package/index.js

@@ -15,6 +15,7 @@ module.exports = {
   knowledgeModule: client.knowledgeModule,
   ensureTestFile: client.ensureTestFile,
   where: client.where,
+  stableId: client.stableId,
   w: client.w,
   Config,
   Semantics,

package/package.json

@@ -65,6 +65,6 @@
     "json-stable-stringify": "^1.0.1",
     "node-fetch": "^2.6.1"
   },
-  "version": "7.12.8",
+  "version": "8.0.0-beta.1",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -26,7 +26,23 @@ const config_toServer = (config) => {
   // cant change things because copy breaks something
 }
 
+const initWords = (words) => {
+  if (!words.literals) {
+    words.literals = {}
+  }
+  if (!words.patterns) {
+    words.patterns = []
+  }
+  if (!words.hierarchy) {
+    words.hierarchy = []
+  }
+  return words
+}
+
 const debugPriority = (priority) => {
+  if (global.pauseDebugging) {
+    return
+  }
   if (global.entodictonDebugPriority) {
     if (helpers.subPriority(entodictonDebugPriority, priority)) {
       debugger // debug hierarchy hit
@@ -35,6 +51,9 @@ const debugPriority = (priority) => {
 }
 
 const debugAssociation = (association) => {
+  if (global.pauseDebugging) {
+    return
+  }
   if (global.entodictonDebugAssociation) {
     if (helpers.safeEquals(global.entodictonDebugAssociation, association)) {
       debugger // debug association hit
@@ -42,7 +61,21 @@ const debugAssociation = (association) => {
   }
 }
 
+const debugWord = (word) => {
+  if (global.pauseDebugging) {
+    return
+  }
+  if (global.entodictonDebugWord) {
+    if (helpers.safeEquals(global.entodictonDebugWord, word)) {
+      debugger // debug word hit
+    }
+  }
+}
+
 const debugHierarchy = (pair) => {
+  if (global.pauseDebugging) {
+    return
+  }
   if (global.entodictonDebugHierarchy) {
     if (helpers.safeEquals(global.entodictonDebugHierarchy, pair)) {
       debugger // debug hierarchy hit
@@ -51,14 +84,20 @@ const debugHierarchy = (pair) => {
 }
 
 const debugBridge = (bridge) => {
+  if (global.pauseDebugging) {
+    return
+  }
   if (global.entodictonDebugBridge) {
-    if (global.entodictonDebugBridge[0] == bridge.id && global.entodictonDebugBridge[1] == bridge.level) {
+    if (global.entodictonDebugBridge == bridge.id) {
       debugger // debug hierarchy hit
     }
   }
 }
 
 const debugOperator = (operator) => {
+  if (global.pauseDebugging) {
+    return
+  }
   if (global.entodictonDebugOperator) {
     if ((operator.pattern || operator) === global.entodictonDebugOperator) {
       debugger // debug operator hit
@@ -67,18 +106,29 @@ const debugOperator = (operator) => {
 }
 
 const debugConfigProps = (config) => {
+  if (global.pauseDebugging) {
+    return
+  }
   if (!config) {
     return
   }
   const checkProps = [
     { property: 'priorities', check: (v) => debugPriority(v) },
     { property: 'association', check: (v) => debugAssociation(v) },
+    { property: 'words', check: (v) => debugAssociation(v) },
     { property: 'hierarchy', check: (v) => debugHierarchy(v) },
     { property: 'operators', check: (v) => debugOperator(v) },
     { property: 'bridges', check: (v) => debugBridge(v) }
   ]
   for (const { property, check } of checkProps) {
-    if (config[property]) {
+    if (property == 'words') {
+      if (config[property]) {
+        for (const value in config[property].literals) {
+          check(value)
+        }
+      }
+    }
+    else if (config[property]) {
       for (const value of config[property]) {
         check(value)
       }
@@ -123,6 +173,7 @@ const validConfigProps = (config) => {
 
 const setupInitializerFNArgs = (config, args) => {
   const aw = (word, def) => config.addWord(word, def, args.uuid)
+  const ap = (pattern, def) => config.addPattern(pattern, def, args.uuid)
   const ag = (generator) => config.addGenerator(generator, args.uuid, config.name)
   const km = (name) => config.getConfig(name)
   const apis = (name) => config.getConfig(name).api
@@ -130,6 +181,7 @@ const setupInitializerFNArgs = (config, args) => {
   return {
     ...args,
     addWord: aw,
+    addPattern: ap,
     addGenerator: ag,
     config: config.getPseudoConfig(args.uuid, args.currentConfig),
     km,
@@ -213,6 +265,9 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
           config.addHierarchy(child, bridge.id)
         }
       }
+      if (bridge.operator) {
+        config.addOperator(bridge.operator, uuid || config.uuid)
+      }
       if (bridge.parents) {
         for (const parent of bridge.parents) {
           config.addHierarchy(bridge.id, parent)
@@ -242,7 +297,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       if (bridge.words) {
         for (let def of bridge.words) {
           if (typeof def === 'string') {
-            config.addWordInternal(def, { id: bridge.id, initial: `{ value: "${def}"}` })
+            config.addWordInternal(def, { id: bridge.id, initial: `{ value: "${bridge.id}"}` })
           } else {
             const word = def.word
             def = { initial: JSON.stringify(def), id: bridge.id, word }
@@ -329,7 +384,7 @@ const handleBridgeProps = (config, bridge, { addFirst, uuid } = {}) => {
       if (bridge.semantic) {
         const semantic = {
           where: bridge.semantic.where || bridge.where || client.where(3),
-          match: ({ context }) => bridge.id == context.marker,
+          match: ({ context }) => bridge.id == context.marker && !context.evaluate,
           apply: (args) => bridge.semantic(args),
           applyWrapped: bridge.semantic,
           property: 'semantic'
@@ -348,7 +403,7 @@ const handleCalculatedProps = (baseConfig, moreConfig, { addFirst, uuid } = {})
   if (moreConfig.bridges) {
     moreConfig.bridges = moreConfig.bridges.map((bridge) => {
       bridge = { ...bridge }
-      const valid = ['after', 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generatorpr', 'generators', 'id', 'convolution', 'inverted', 'isA', 'children', 'parents',
+      const valid = ['after', 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generatorpr', 'generators', 'operator', 'id', 'convolution', 'inverted', 'isA', 'children', 'parents',
         'level', 'optional', 'selector', 'semantic', 'words', /Bridge$/, 'localHierarchy', 'levelSpecificHierarchy', 'where', 'uuid']
       helpers.validProps(valid, bridge, 'bridge')
       handleBridgeProps(baseConfig, bridge, { addFirst, uuid })
@@ -386,13 +441,13 @@ if (runtime.process.env.DEBUG_ASSOCIATION) {
   global.entodictonDebugAssociation = JSON.parse(runtime.process.env.DEBUG_ASSOCIATION)
 }
 
+if (runtime.process.env.DEBUG_WORD) {
+  global.entodictonDebugWord = runtime.process.env.DEBUG_WORD
+}
+
 if (runtime.process.env.DEBUG_BRIDGE) {
-  // id/level
-  global.entodictonDebugBridge = runtime.process.env.DEBUG_BRIDGE.split('/')
-  if (global.entodictonDebugBridge.length !== 2) {
-    console.log('Expected DEBUG_BRIDGE to be of the form "id/level"')
-    process.exit(-1)
-  }
+  // id
+  global.entodictonDebugBridge = runtime.process.env.DEBUG_BRIDGE
   global.entodictonDebugBridge[1] = parseInt(global.entodictonDebugBridge[1])
 }
 
@@ -409,10 +464,13 @@ const hierarchyCanonical = (element) => {
   }
 }
 
-const isValidDef = (word, def, config) => {
+const isValidWordDef = (word, def, config) => {
+  // TODO trie
+  /*
   if (!def.id) {
     throw new Error(`In the KM "${config.name}", for the word ${word} the following definition is missing the "id" property: ${JSON.stringify(def)}`)
   }
+  */
   /*
   if (!def.initial) {
     throw `In the KM "${config.name}", for the word ${word} the following definition is missing the "initial" property: ${JSON.stringify(def)}`
@@ -429,16 +487,20 @@ const hierarchyToCanonical = (edge) => {
 
 const addWord = (config, uuid) => ({ word, id, initial }) => {
   if (!config.words) {
-    config.words = {}
+    config.words = {
+      literals: {},
+      patterns: [],
+      hierarchy: [],
+    }
   }
-  const words = config.words
+  const literals = config.words.literals
   const def = { id, initial, uuid }
-  if (words[word]) {
-    if (!words[word].some((e) => helpers.safeEquals(e, def))) {
-      words[word].unshift(def)
+  if (literals[word]) {
+    if (!literals[word].some((e) => helpers.safeEquals(e, def))) {
+      literals[word].unshift(def)
     }
   } else {
-    words[word] = [def]
+    literals[word] = [def]
   }
 }
 
@@ -453,6 +515,7 @@ const normalizeConfig = (config) => {
         }
       }
     }
+
     if (config.bridges) {
       for (const bridge of config.bridges) {
         if (!bridge.level) {
@@ -471,6 +534,14 @@ const normalizeConfig = (config) => {
         }
       }
     }
+
+    if (config.operators) {
+      for (let i = 0; i < config.operators.length; ++i) {
+        if (typeof config.operators[i] === 'string') {
+          config.operators[i] = { pattern: config.operators[i] }
+        }
+      }
+    }
   }
 }
 
@@ -488,8 +559,17 @@ function configDup (config, options) {
 }
 
 function setWordsUUIDs (words, uuid) {
-  for (const key in words) {
-    words[key] = words[key].map((o) => Object.assign(o, { uuid }))
+  const literals = words.literals
+  for (const key in literals) {
+    literals[key] = literals[key].map((o) => Object.assign(o, { uuid }))
+  }
+  const patterns = words.patterns
+  for (const pattern of patterns) {
+    pattern.defs.map((def) => Object.assign(def, { uuid }))
+  }
+  const hierarchy = words.hierarchy
+  for (const pair of hierarchy) {
+    pair.uuid = uuid 
   }
 }
 
@@ -740,7 +820,7 @@ class Config {
     }
     const templateQueries = []
     if (this.instances && this.instances.length > 0) {
-      for (const query of this.instances.slice(-1)[0].queries) {
+      for (const query of this.instances.slice(-1)[0].configs) {
         if (typeof query === 'string') {
           templateQueries.push(query)
         }
@@ -764,6 +844,7 @@ class Config {
       addSemantic: (...args) => this.addSemantic(...args, uuid, config.name),
       removeSemantic: (...args) => this.removeSemantic(...args, uuid, config.name),
       addWord: (...args) => this.addWord(...args, uuid),
+      addPattern: (...args) => this.addPattern(...args, uuid),
 
       getHierarchy: (...args) => this.config.hierarchy,
       getBridges: (...args) => this.config.bridges,
@@ -785,28 +866,6 @@ class Config {
     }
   }
 
-  // return the config with just the elements from the included KM's
-  baseConfig () {
-    const operators = this.config.operators.filter((operator) => {
-      return operator.uuid !== this.uuid
-    })
-    const bridges = this.config.bridges.filter((bridge) => {
-      return bridge.uuid !== this.uuid
-    })
-    const words = {}
-    for (const word in this.config.words) {
-      const defs = this.config.words[word].filter((def) => def.uuid !== this.uuid)
-      if (defs.length > 0) {
-        words[word] = defs
-      }
-    }
-    return {
-      operators,
-      bridges,
-      words
-    }
-  }
-
   getCounter (maybeName = '') {
     const counter = this.configCounter
     this.configCounter += 1
@@ -843,7 +902,11 @@ class Config {
         namespaced: {}
       },
       description: '',
-      words: {}, // Done
+      words: {
+        literals: {},
+        patterns: [],
+        hierarchy: [],
+      }, // Done
       floaters: [],
       implicits: [],
       flatten: [],
@@ -1102,8 +1165,8 @@ class Config {
       return elements.map(toCanonicalQuery)
     }
 
-    const templateQueries = toCanonicalQueries(template.queries || []).map(helpers.updateQueries)
-    const instanceQueries = toCanonicalQueries(instance.queries || [])
+    const templateQueries = toCanonicalQueries(template.configs || []).map(helpers.updateQueries)
+    const instanceQueries = toCanonicalQueries(instance.configs || [])
     let sameQueries = true
     let startOfChanges
     for (let iq = 0; iq < templateQueries.length; ++iq) {
@@ -1124,7 +1187,6 @@ class Config {
     if (templateQueries.length < instanceQueries.length) {
       startOfChanges = instanceQueries.length
     }
-    // const sameQueries = helpers.safeEquals(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), toCanonicalQueries(instance.queries || []))
 
     if (debug) {
       if (!(instance && sameQueries && sameFragments)) {
@@ -1133,8 +1195,6 @@ class Config {
         console.log('sameFragments', sameFragments)
         // console.log("templateFragments", templateFragments)
         // console.log("instanceFragments", instanceFragments)
-        // console.log('template.queries', JSON.stringify(toCanonicalQueries(template.queries || []).map(helpers.updateQueries), null, 2))
-        // console.log("instance.queries", JSON.stringify(toCanonicalQueries(instance.queries || []), null, 2))
       }
     }
     if (startOfChanges || instance.resultss) {
@@ -1145,13 +1205,13 @@ class Config {
   }
 
   validifyTemplate (template) {
-    if (!template.queries && !template.fragments) {
-      throw new Error(`Expected the template for ${this.name} to be an object that can have the properties: queries and fragments`)
+    if (!template.configs && !template.fragments) {
+      throw new Error(`Expected the template for ${this.name} to be an object that can have the properties: configs and fragments`)
     }
-    for (const query of template.queries || []) {
+    for (const query of template.configs || []) {
       if (typeof query === 'string') {
       } else if (query instanceof Config) {
-        throw new Error(`For the template for ${this.name}, each element in queries should be either a string or a structure with a config (not a Config object).`)
+        throw new Error(`For the template for ${this.name}, each element in configs should be either a string or a structure with a config (not a Config object).`)
       }
     }
   }
@@ -1168,7 +1228,7 @@ class Config {
     this.logs.push(`loading template for ${this.name}`)
     if (options.rebuild) {
       // TODO fix beforeQuery
-      template = { fragments: [], queries: [], ...template }
+      template = { fragments: [], configs: [], ...template }
       template.fragments = template.fragments.concat(this.dynamicFragments)
       client.rebuildTemplate({ config: this, target: this.name, previousResultss: options.previousResultss, startOfChanges: options.startOfChanges, beforeQuery: () => {}, template, ...options })
     } else {
@@ -1176,7 +1236,7 @@ class Config {
       // this.initInstances.push({ ...instance, name: config.name })
       const isEmpty = (instance) => {
         const properties = [
-          'queries',
+          'configs',
           'resultss',
           'fragments',
           'semantics',
@@ -1189,7 +1249,7 @@ class Config {
         for (let i = 0; i < instance.resultss.length; ++i) {
           const result = instance.resultss[i]
           if (result.apply) {
-            result.apply = template.queries[i]
+            result.apply = template.configs[i]
           }
         }
         instance.name = this.name
@@ -1429,21 +1489,42 @@ class Config {
 
   addWordInternal (word, def, uuid) {
     if (!this.config.words) {
-      this.config.words = {}
+      this.config.words = {
+        literals: {},
+        patterns: [],
+        hierarchy: [],
+      }
     }
-    const words = this.config.words
+    debugWord(word)
+
+    const literals = this.config.words.literals
     def = Object.assign({}, def, { uuid: uuid || this._uuid })
-    if (words[word]) {
-      if (!words[word].some((e) => helpers.safeEquals(e, def))) {
-        words[word].unshift(def)
+    if (literals[word]) {
+      if (!literals[word].some((e) => helpers.safeEquals(e, def))) {
+        literals[word].unshift(def)
       }
     } else {
-      words[word] = [def]
+      literals[word] = [def]
     }
 
     this._delta.json.words.push({ action: 'add', word, def })
   }
 
+  addPattern (pattern, def, uuid) {
+    if (!this.config.words) {
+      this.config.words = {
+        literals: {},
+        patterns: [],
+        hierarchy: [],
+      }
+    }
+
+    const patterns = this.config.words.patterns
+    def = Object.assign({}, def, { uuid: uuid || this._uuid })
+    patterns.unshift({ pattern, defs: [def] })
+    this._delta.json.words.push({ action: 'add', pattern, defs: [def] })
+  }
+
   getAPI (uuid) {
     if (this._uuid === uuid) {
       return this.api
@@ -1539,6 +1620,10 @@ class Config {
     return client.process(this, query, options)
   }
 
+  processQuery (query, options) {
+    return this.process(query, options)
+  }
+
   initDefaults () {
     const init = (config) => {
       if (!config.objects) {
@@ -1637,13 +1722,35 @@ class Config {
     config.generators = config.generators.filter((element) => !element.development)
     config.semantics = config.semantics.filter((element) => !element.development)
     config.hierarchy = (config.hierarchy).filter((element) => !element.development)
-    for (const word in config.words) {
-      const defs = config.words[word] || []
-      config.words[word] = defs.filter((def) => !def.development)
-      if (config.words[word].length == 0) {
-        delete config.words[word]
+
+    const literals = config.words.literals
+    for (const word in literals) {
+      const defs = literals[word] || []
+      literals[word] = defs.filter((def) => !def.development)
+      if (literals[word].length == 0) {
+        delete literals[word]
+      }
+    }
+
+    const patterns = config.words.patterns || []
+    const patternsPrime = []
+    for (const pattern of patterns) {
+      let defs = pattern.defs || []
+      defs = defs.filter((def) => !def.development)
+      if (defs.length !== 0) {
+        patternsPrime.push({...pattern, defs})
       }
     }
+    config.words.patterns = patternsPrime
+
+    const hierarchy = config.words.hierarchy || []
+    const hierarchyPrime = []
+    for (const pair of hierarchy) {
+      if (!pair.development) {
+        hierarchyPrime.push(pair)
+      }
+    }
+    config.words.hierarchy = hierarchyPrime
   }
 
   // configs = [ { config, namespace } ... ]
@@ -1657,7 +1764,7 @@ class Config {
 
       config.operators = config.operators || []
       config.bridges = config.bridges || []
-      config.words = config.words || {}
+      config.words = config.words || { literals: {}, patterns: [], hierarchy: [] }
       config.generators = config.generators || []
       config.semantics = config.semantics || []
       config.hierarchy = config.hierarchy || []
@@ -1687,7 +1794,7 @@ class Config {
     this.instances = []
     this.logs = []
     this.dynamicFragments = []
-    // when running queries any bridges added are marked as transitory so that the associated will ignore those op's
+    // when running configs any bridges added are marked as transitory so that the associated will ignore those op's
     this.transitoryMode = false
 
     // check for duplicate bridges
@@ -1709,11 +1816,8 @@ class Config {
     }
 
     if (config && config.words) {
-      for (const word of Object.keys(config.words)) {
-        for (const def of config.words[word]) {
-          isValidDef(word, def, config)
-        }
-      }
+      initWords(config.words)
+      isValidWordDef(config.words)
     }
 
     if (config && config.priorities) {
@@ -1778,7 +1882,16 @@ class Config {
   getAddedArgs (args) {
     for (let addedArgs of this.addedArgss) {
       addedArgs = addedArgs(args)
+      const getUUIDScoped = addedArgs.getUUIDScoped
+      delete addedArgs.getUUIDScoped
       Object.assign(args, addedArgs)
+      if (getUUIDScoped) {
+        const currentGetUUIDScoped = args.getUUIDScoped
+        if (!currentGetUUIDScoped) {
+          debugger
+        }
+        args.getUUIDScoped = (uuid) => Object.assign(currentGetUUIDScoped(uuid), getUUIDScoped(uuid))
+      }
     }
   }
 
@@ -2020,10 +2133,14 @@ class Config {
       this.config.namespaces = ns
     }
 
-    if (this.config.words) {
-      const words = this.config.words
-      for (const key in words) {
-        words[key].forEach((o) => {
+    if (!this.config.words) {
+      this.config.words = {}
+    }
+
+    if (this.config.words.literals) {
+      const literals = this.config.words.literals
+      for (const key in literals) {
+        literals[key].forEach((o) => {
           if (o.uuid) {
             if (map[o.uuid]) {
               o.uuid = map[o.uuid]
@@ -2035,6 +2152,36 @@ class Config {
       }
     }
 
+    const mapDefList = (list) => {
+      for (const element of list) {
+        element.defs.forEach((o) => {
+          if (o.uuid) {
+            if (map[o.uuid]) {
+              o.uuid = map[o.uuid]
+            }
+          } else {
+            o.uuid = this._uuid
+          }
+        })
+      }
+    }
+
+    if (this.config.words.patterns) {
+      mapDefList(this.config.words.patterns)
+    }
+
+    if (this.config.words.hierarchy) {
+      for (const o of this.config.words.hierarchy) {
+        if (o.uuid) {
+          if (map[o.uuid]) {
+            o.uuid = map[o.uuid]
+          }
+        } else {
+          o.uuid = this._uuid
+        }
+      }
+    }
+
     if (this.config.bridges) {
       this.config.bridges.forEach((bridge) => {
         if (bridge.uuid) {
@@ -2207,11 +2354,14 @@ class Config {
     }
     */
 
-    for (const key in this.config.words) {
-      const values = this.config.words[key]
-      if (values.some((word) => (Object.keys(word).includes('uuid') && !word.uuid))) {
-        debugBreak()
-        return false
+    const literals = this.config.words.literals
+    if (literals) {
+      for (const key in literals) {
+        const values = literals[key]
+        if (values.some((word) => (Object.keys(word).includes('uuid') && !word.uuid))) {
+          debugBreak()
+          return false
+        }
       }
     }
 
@@ -2425,7 +2575,7 @@ class Config {
       this.config.hierarchy = []
       this.config.priorities = []
       this.config.associations = { positive: [], negative: [] }
-      this.config.words = {}
+      this.config.words = initWords({})
 
       for (let i = 0; i < addInternals.length; ++i) {
         let name
@@ -2612,13 +2762,25 @@ class Config {
     }
 
     if (config.words) {
-      const words = {}
-      for (const word in config.words) {
-        words[word] = config.words[word].map((word) => {
+      const literals = {}
+      for (const word in config.words.literals) {
+        literals[word] = config.words.literals[word].map((word) => {
           return Object.assign({}, word, { id: toNS(word.id), uuid })
         })
       }
-      config.words = words
+      config.words.literals = literals
+
+      for (const pattern of config.words.patterns) {
+        pattern.defs.forEach((def) => {
+          def.id = toNS(def.id)
+          def.uuid = uuid
+        })
+      }
+
+      for (const pair of config.words.hierarchy) {
+        pair.child = toNS(pair.child)
+        pair.parent = toNS(pair.parent)
+      }
     }
 
     if (config.hierarchy) {
@@ -2739,6 +2901,7 @@ class Config {
       throw new Error(`Setting invalid property ${property}`)
     }
 
+    // TODO trie
     if (property == 'words') {
       for (const word in value) {
         for (const def of value[word]) {
@@ -2803,7 +2966,8 @@ class Config {
       more = more.copy()
       more.server(this._server, this._key, this._queryParams)
 
-      this.loadOrder.addList(more.configs.map((km) => km.name || km.uuid))
+      const moreConfigs = more.configs.map((km) => km.name || km.uuid)
+      this.loadOrder.addList(moreConfigs)
 
       // get the new ones
       // remove the dups
@@ -2885,16 +3049,36 @@ class Config {
         continue
       }
       if (key === 'words') {
-        const configWords = this.config.words
-        const moreWords = more.words
-        for (const word of Object.keys(moreWords)) {
-          if (!configWords[word]) {
-            configWords[word] = []
+        if (more.words.literals) {
+          const literals = this.config.words.literals
+          const moreLiterals = more.words.literals
+          for (const word of Object.keys(moreLiterals)) {
+            if (!literals[word]) {
+              literals[word] = []
+            }
+            if (addFirst) {
+              literals[word] = moreLiterals[word].concat(literals[word])
+            } else {
+              literals[word] = literals[word].concat(moreLiterals[word])
+            }
+          }
+        }
+        if (more.words.patterns) {
+          const patterns = this.config.words.patterns
+          const morePatterns = more.words.patterns
+          if (addFirst) {
+            this.config.words.patterns = morePatterns.concat(patterns)
+          } else {
+            this.config.words.patterns = patterns.concat(morePatterns)
           }
+        }
+        if (more.words.hierarchy) {
+          const hierarchy = this.config.words.hierarchy
+          const moreHierarchy = more.words.hierarchy
           if (addFirst) {
-            configWords[word] = moreWords[word].concat(configWords[word])
+            this.config.words.hierarchy = moreHierarchy.concat(hierarchy)
           } else {
-            configWords[word] = configWords[word].concat(moreWords[word])
+            this.config.words.hierarchy = hierarchy.concat(moreHierarchy)
           }
         }
       } else if (key === 'name') {
@@ -3006,14 +3190,25 @@ class Config {
         continue
       }
       if (key === 'words') {
-        const configWords = this.config.words
-        const moreWords = more.words
-        for (const word of Object.keys(moreWords)) {
-          if (!configWords[word]) {
-            configWords[word] = []
+        if (this.config.words.literals) {
+          const literals = this.config.words.literals
+          const moreLiterals = more.words.literals
+          for (const word of Object.keys(moreLiterals)) {
+            if (!literals[word]) {
+              literals[word] = []
+            }
+            literals[word] = moreLiterals[word].concat(literals[word])
           }
-          // configWords[word] = configWords[word].concat(moreWords[word])
-          configWords[word] = moreWords[word].concat(configWords[word])
+        }
+        if (this.config.words.patterns) {
+          const patterns = this.config.words.patterns
+          const morePatterns = more.words.patterns
+          this.config.words.patterns = morePatterns.concat(patterns)
+        }
+        if (this.config.words.hierarchy) {
+          const hierarchy = this.config.words.hierarchy
+          const moreHierarchy = more.words.hierarchy
+          this.config.words.hierarchy = moreHierarchy.concat(hierarchy)
         }
       } else if (key === 'name') {
         /*

package/src/digraph.js

@@ -113,19 +113,31 @@ class Digraph {
   }
 
   minima (nodes) {
-    let minima = new Set(nodes)
-    const ancestors = new Set([])
-    nodes.forEach((node) => {
-      this.ancestors(node).forEach((n) => ancestors.add(n))
-    })
-    ancestors.forEach((n) => minima.delete(n))
-    if (minima.size === 0) {
-      // all unrelated
-      minima = new Set(nodes)
+    nodes = new Set(nodes)
+    const nodeToDescendants = {}
+    for (const node of nodes) {
+      nodeToDescendants[node] = this.descendants(node)
+    }
+    let minima = new Set()
+    for (const node of nodes) {
+      let okay = true
+      for (const key of nodeToDescendants[node]) {
+        if (nodes.has(key)) {
+          if (key in nodeToDescendants && nodeToDescendants[key].has(node)) {
+            continue
+          }
+          okay = false
+          break
+        }
+      }
+      if (okay) {
+        minima.add(node)
+      }
     }
     return minima
   }
 
+  /*
   maxima (nodes) {
     const maxima = new Set(nodes)
     const descendants = new Set([])
@@ -135,14 +147,21 @@ class Digraph {
     descendants.forEach((n) => maxima.delete(n))
     return maxima
   }
+  */
 
   add (child, parent) {
     this._edges.push([child, parent])
   }
 
+  exists(child, parent) {
+    return this._edges.find((edge) => edge[0] == child && edge[1] == parent)
+  }
+
   addList (l) {
     for (let i = 1; i < l.length; ++i) {
-      this._edges.push([l[i - 1], l[i]])
+      if (!this.exists(l[i-1], l[i])) {
+        this._edges.push([l[i - 1], l[i]])
+      }
     }
   }
 

package/src/digraph_internal.js

@@ -81,19 +81,31 @@ class DigraphInternal {
   }
 
   minima (nodes) {
-    let minima = new Set(nodes)
-    const ancestors = new Set([])
-    nodes.forEach((node) => {
-      this.ancestors(node).forEach((n) => ancestors.add(n))
-    })
-    ancestors.forEach((n) => minima.delete(n))
-    if (minima.size === 0) {
-      // all unrelated
-      minima = new Set(nodes)
+    nodes = new Set(nodes)
+    const nodeToDescendants = {}
+    for (const node of nodes) {
+      nodeToDescendants[node] = this.descendants(node)
+    }
+    let minima = new Set()
+    for (const node of nodes) {
+      let okay = true
+      for (const key of nodeToDescendants[node]) {
+        if (nodes.has(key)) {
+          if (key in nodeToDescendants && nodeToDescendants[key].has(node)) {
+            continue
+          }
+          okay = false
+          break
+        }
+      }
+      if (okay) {
+        minima.add(node)
+      }
     }
     return minima
   }
 
+  /*
   maxima (nodes) {
     const maxima = new Set(nodes)
     const descendants = new Set([])
@@ -103,14 +115,21 @@ class DigraphInternal {
     descendants.forEach((n) => maxima.delete(n))
     return maxima
   }
+  */
 
   add (child, parent) {
     this._edges.push([child, parent])
   }
 
+  exists(child, parent) {
+    return this._edges.findIndex((edge) => edge[0] == child && edge[1] == parent) != -1
+  }
+
   addList (l) {
     for (let i = 1; i < l.length; ++i) {
-      this._edges.push([l[i - 1], l[i]])
+      if (!this.exists(l[i-1], l[i])) {
+        this._edges.push([l[i - 1], l[i]])
+      }
     }
   }