theprogrammablemind 8.0.0 → 8.1.0-beta.1

package/src/configHelpers.js

@@ -0,0 +1,455 @@
+const { InitCalls } = require('./helpers')
+const DigraphInternal = require('./digraph_internal')
+
+const gs = (g) => async (contexts, separator, lastSeparator) => {
+  if (!Array.isArray(contexts)) {
+    throw new Error('Expected a list')
+  }
+
+  let s = ''
+  if (!separator) {
+    separator = ' '
+  }
+  if (!lastSeparator) {
+    lastSeparator = separator
+  }
+  let nextSeparator = ''
+  for (let i = 0; i < contexts.length; ++i) {
+    const context = contexts[i]
+    const value = await g(context)
+    if (i > 0) {
+      if (i === contexts.length - 1) {
+        nextSeparator = lastSeparator
+      } else {
+        nextSeparator = separator
+      }
+    }
+    s += nextSeparator + value
+  }
+  return s
+}
+
+const asList = (context) => {
+  if (context.marker === 'list') {
+    return context
+  }
+  return {
+    marker: 'list',
+    types: [context.marker],
+    value: [context]
+  }
+}
+
+const isA = (hierarchy) => (child, parent) => {
+  if (!child || !parent) {
+    return false
+  }
+  if (child.marker) {
+    child = child.marker
+  }
+  if (parent.marker) {
+    parent = parent.marker
+  }
+  return hierarchy.isA(child, parent)
+}
+
+class ErrorReason extends Error {
+  constructor (context) {
+    super(JSON.stringify(context))
+    this.reason = context
+  }
+}
+
+const listable = (hierarchy) => (c, type) => {
+  if (!c) {
+    return false
+  }
+  if (hierarchy.isA(c.marker, type)) {
+    return true
+  }
+  if (c.marker === 'list') { 
+    for (const t of c.types) {
+      if (hierarchy.isA(t, type)) {
+        return true
+      }
+    }
+  }
+  return false
+}
+
+const cleanAssign = (dest, ...srcs) => {
+  for (const key in dest) {
+    let found = false
+    for (const src of srcs) {
+      if (src[key]) {
+        found = true
+        break
+      }
+    }
+    if (!found) {
+      delete dest[key]
+    }
+  }
+  Object.assign(dest, ...srcs)
+}
+
+const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
+
+  // callId
+  args.calls = new InitCalls(args.isInstance ? `${args.isInstance}#${config.name}` : config.name)
+  if (global.theprogrammablemind && global.theprogrammablemind.loadForTesting) {
+    args.calls = new InitCalls(Object.keys(global.theprogrammablemind.loadForTesting)[0])
+  }
+  args.cleanAssign = cleanAssign
+  args.km = (name) => config.getConfig(name)
+  args.api = (name) => config.getConfig(name).api
+  args.error = (context) => {
+    throw new ErrorReason(context)
+  }
+  args.kms = config.getConfigs()
+  args.config = config
+  args.hierarchy = hierarchy
+  args.isA = isA(hierarchy)
+  args.listable = listable(hierarchy)
+  args.asList = asList
+  args.retry = () => { throw new RetryError() }
+  args.fragments = (query) => {
+    return config.fragment(args, query)
+  }
+  args.breakOnSemantics = false
+  args.theDebugger = {
+    breakOnSemantics: (value) => args.breakOnSemantics = value
+  }
+  if (!logs) {
+  }
+  args.log = (message) => logs.push(message)
+
+  args.addAssumedScoped = (args, assumed) => {
+    const addAssumed = (args, ...moreAssumed) => {
+      return { ...args, assumed: Object.assign({}, assumed, (args.assumed || {}), ...moreAssumed) }
+    }
+
+    args.s = (c) => config.getSemantics(logs).apply(args, c)
+    args.g = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a), c, a)
+    }
+    args.gp = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: true, isResponse: false, response: false }), c, { paraphrase: true, isResponse: false, response: false })
+    }
+    args.gr = (c, a = {}) => {
+      return config.getGenerators(logs).apply(addAssumed(args, a, { paraphrase: false, isResponse: true }), { ...c, paraphrase: false, isResponse: true })
+    }
+    args.e = (c) => {
+      return config.getEvaluator(args.s, args.calls, logs, c)
+    }
+    args.gs = gs(args.g)
+    args.gsp = gs(args.gp)
+    args.gsr = gs(args.gr)
+  }
+  // for semantics
+  args.addAssumedScoped(args, {})
+
+  const getAPI = (uuid) => {
+    if (config && config.getAPI) {
+      return config.getAPI(uuid)
+    }
+  }
+  const getAPIs = (uuid) => {
+    if (config && config.getAPIs) {
+      return config.getAPIs(uuid)
+    }
+  }
+  args.getUUIDScoped = (uuid) => {
+    return {
+      api: getAPI(uuid),
+      apis: getAPIs(uuid)
+    }
+  }
+  config.getAddedArgs(args)
+
+  Object.assign(args, args.getUUIDScoped(uuidForScoping || config.uuid))
+  /*
+  if (uuidForScoping) {
+    Object.assign(args, args.getUUIDScoped(uuidForScoping))
+  }
+  */
+  // sets args for all the API. that make a copy so the args must be fully setup by here except for scoped
+  config.setArgs(args)
+}
+
+const getObjects = (objects) => {
+  return (uuid) => {
+    if (objects && objects.namespaced) {
+      return objects.namespaced[uuid]
+    }
+    return objects
+  }
+}
+
+const processContext = async (context, { objects = {}, config, logs = [] }) => {
+  const generators = config.getGenerators(logs)
+  const semantics = config.getSemantics(logs)
+
+  // map to hash
+  config = config || {}
+  if (config.config) {
+    config = config
+  }
+
+  const response = {} // NA but passed in
+  // generators = new Generators(generators.map((g) => new Generator(normalizeGenerator(g))))
+  // semantics = new Semantics(semantics.map((g) => new Semantic(normalizeSemantic(g))))
+  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+
+  const args = { objects, response, getObjects: getObjects(objects) }
+  setupArgs(args, config, logs, hierarchy)
+
+  context = await semantics.apply(args, context)
+  const generated = await generators.apply(args, context)
+  const assumed = { paraphrase: true, response: false, isResponse: false }
+  const paraphrases = await generators.apply({ ...args, assumed }, context, { paraphrase: true, response: false, isResponse: false })
+  let responses = []
+  if (context.isResponse) {
+    responses = generated
+  }
+  return { context, generated, paraphrases, responses }
+}
+
+const setupProcessB = ({ config, initializer, allowDelta = false } = {}) => {
+  const key = config._key
+
+  const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
+  if (allowDelta && config.allowDelta && config.hasDelta()) {
+    // console.log('config', config)
+    data.delta = config.delta()
+  } else {
+    config.toData(data)
+    // Object.assign(data, config.config)
+  }
+
+  // config.toServer(data)
+
+  if (data.namespaces) {
+    for (const uuid of Object.keys(data.namespaces)) {
+      const km = config.configs.find((km) => km.uuid === uuid)
+      data.namespaces[uuid].name = km.name
+    }
+  }
+
+  // const generators = new Generators((data.generators || []).map((g) => new Generator(normalizeGenerator(g))))
+  delete data.generators
+  // const semantics = new Semantics((data.semantics || []).map((g) => new Semantic(normalizeSemantic(g))))
+  delete data.semantics
+  const hierarchy = new DigraphInternal((config.config || {}).hierarchy || [])
+
+  return {
+    data,
+    // generators,
+    // semantics,
+    hierarchy
+  }
+}
+
+const setupContexts = (rawContexts) => {
+  let first = true
+  const contexts = []
+  contexts.push({ marker: 'controlStart', controlRemove: true })
+  for (const context of rawContexts) {
+    if (first) {
+      first = false
+    } else {
+      contexts.push({ marker: 'controlBetween', controlRemove: true })
+    }
+    contexts.push(context)
+  }
+  contexts.push({ marker: 'controlEnd', controlRemove: true })
+  return contexts
+}
+
+const processContextsB = async ({ config, hierarchy, semantics, generators, json, isTest, rebuildingTemplate, isInstance, instance, query, data, retries, url, commandLineArgs }) => {
+  // TODO fix this name to contextsPrime
+  const contextsPrime = []
+  const generatedPrime = []
+  const paraphrasesPrime = []
+  const paraphrasesParenthesizedPrime = []
+  const generatedParenthesizedPrime = []
+  const responsesPrime = []
+  const contexts = setupContexts(json.contexts)
+
+  const objects = config.get('objects')
+  const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance }
+  if (!json.logs) {
+    json.logs = []
+  }
+  setupArgs(args, config, json.logs, hierarchy)
+  const toDo = [...contexts]
+  args.insert = (context) => toDo.unshift(context)
+  let overlap, lastRange
+  config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
+  while (toDo.length > 0) {
+    const context = toDo.shift()
+    args.calls.next()
+    let contextPrime = context
+    context.topLevel = true
+    try {
+      if (json.has_errors) {
+        throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
+      }
+      const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
+      if (!config.get('skipSemantics')) {
+        const semantics = config.getSemantics(json.logs)
+        try {
+          contextPrime = await semantics.apply(args, context)
+        } catch (e) {
+          if (e.message == 'Maximum call stack size exceeded') {
+            const mostCalled = semantics.getMostCalled()
+            e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
+          }
+          // contextPrime = semantics.apply(args, { marker: 'error', context, error: e })
+          if (isInstance) {
+            console.log('error', e.error)
+          }
+          let reason = e.reason
+          if (!reason) {
+            if (e.error) {
+              reason = e.error[0]
+            }
+          }
+          contextPrime = await semantics.apply(args, {
+            marker: 'error',
+            context,
+            text: e ? e.toString() : 'not available',
+            // reason: e.reason,
+            reason,
+            error: e.stack || e.error
+          })
+          if (rebuildingTemplate) {
+            throw e
+          }
+        }
+      }
+      if (contextPrime.controlRemove) {
+        continue
+      }
+      let assumed = { isResponse: true }
+      const generated = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
+      let generatedParenthesized = []
+      if (generateParenthesized) {
+        config.parenthesized = true
+        generatedParenthesized = contextPrime.isResponse ? await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed) : ''
+        config.parenthesized = false
+      }
+      // assumed = { paraphrase: true, response: false };
+      assumed = { paraphrase: true, isResponse: false, response: false }
+      if (generateParenthesized) {
+        config.parenthesized = false
+      }
+      const paraphrases = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
+      let paraphrasesParenthesized = []
+      if (generateParenthesized) {
+        config.parenthesized = true
+        paraphrasesParenthesized = await config.getGenerators(json.logs).apply({ ...args, assumed }, contextPrime, assumed)
+        config.parenthesized = false
+      }
+      contextsPrime.push(contextPrime)
+      generatedPrime.push(generated)
+      paraphrasesPrime.push(paraphrases)
+      if (generateParenthesized) {
+        paraphrasesParenthesizedPrime.push(paraphrasesParenthesized)
+        generatedParenthesizedPrime.push(generatedParenthesized)
+      }
+      if (contextPrime.isResponse) {
+        responsesPrime.push(generated)
+      } else {
+        responsesPrime.push('')
+      }
+
+      // add results to processed list
+      config.config.objects.processed = config.config.objects.processed || []
+      config.config.objects.processed = config.config.objects.processed.slice(0, 5)
+      config.config.objects.processed.unshift({ context: contextPrime, paraphrases: paraphrases, paraphrasesParenthesized, generatedParenthesized, responses: responsesPrime })
+    } catch (e) {
+      if (Array.isArray(e)) {
+        e = {
+          errors: e
+        }
+      }
+      e.context = contextPrime
+      if (e.logs) {
+        e.logs = e.logs.concat(json.logs)
+      } else {
+        e.logs = json.logs
+      }
+      e.metadata = json.metadata
+      if (json.trace) {
+        e.trace = json.trace
+      }
+      throw e
+    }
+  }
+  return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
+}
+
+// instance template loadTemplate
+const loadInstance = async (config, instance) => {
+  const transitoryMode = global.transitoryMode
+  global.transitoryMode = false
+
+  if (instance && (instance.associations || instance.learned_contextual_priorities)) {
+    if (!config.config.retrain) {
+      if (instance.associations) {
+        config.addAssociations(instance.associations)
+      }
+      if (instance.learned_contextual_priorities && instance.learned_contextual_priorities.length > 0) {
+        config.addPriorities(instance.learned_contextual_priorities)
+      }
+    }
+  }
+
+  const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
+  for (const i in (instance.resultss || [])) {
+    const results = instance.resultss[i]
+    if (results.extraConfig) {
+      // config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
+      const uuid = config.nameToUUID(instance.name)
+      // used to do a CLONE
+      config.addInternal(instance.template.configs[i], { uuid, addFirst: true, handleCalculatedProps: true })
+    } else if (results.apply) {
+      const objects = getObjects(config.get('objects'))(config.uuid)
+      const args = { objects, getObjects: getObjects(objects) }
+      if (instance.configs) {
+        args.isInstance = `instance${i}`
+        args.instance = instance.configs[i]
+      }
+
+      const uuid = config.nameToUUID(instance.name)
+      setupArgs(args, config, config.logs, hierarchy, uuid)
+      await results.apply(args)
+    } else if (results.isFragment) {
+    } else {
+      if (results.skipSemantics) {
+        config.config.skipSemantics = results.skipSemantics
+      }
+      const args = { config, hierarchy, json: results, commandLineArgs: {} }
+      args.isInstance = `instance${i}`
+      args.instance = ''
+      await processContextsB(args)
+      if (results.skipSemantics) {
+        config.config.skipSemantics = null
+      }
+    }
+  }
+  global.transitoryMode = transitoryMode
+}
+
+module.exports = {
+  setupProcessB,
+  ErrorReason,
+  listable,
+  setupArgs,
+  processContext,
+  getObjects,
+  gs,
+  processContextsB,
+  loadInstance,
+}

package/src/generators.js

@@ -44,7 +44,7 @@ class Generator {
     return `Generator(${this.match}, ${this._applyWrapped || this._apply})${this.property ? `\nsee the ${this.property} property` : ''}`
   }
 
-  matches (baseArgs, objects, context, hierarchy, config, options = {}) {
+  async matches (baseArgs, objects, context, hierarchy, config, options = {}) {
     if (objects && objects.namespaced) {
       objects = objects.namespaced[this.uuid]
     }
@@ -63,17 +63,17 @@ class Generator {
     }
     const args = Object.assign({}, baseArgs, moreArgs, (baseArgs.getUUIDScoped || (() => { return {} }))(this.uuid))
     // return this.match(args)
-    const matches = this.match(args)
+    const matches = await this.match(args)
     if ((matches && (options.debug || {}).match) ||
         callId == this.callId) {
       debugger // next line is the matcher
-      this.match(args)
+      await this.match(args)
     }
     return matches
   }
 
   // apply (baseArgs, objects, g, gs, context, hierarchy, config, response, log, options = {}) {
-  apply (baseArgs, objects, context, hierarchy, config, response, log, options = {}) {
+  async apply (baseArgs, objects, context, hierarchy, config, response, log, options = {}) {
     if (!log) {
       throw new Error('generators.apply argument log is required')
     }
@@ -129,7 +129,7 @@ class Generator {
         callId == this.callId) {
       debugger
     }
-    return this._apply(args)
+    return await this._apply(args)
   }
 }
 
@@ -165,7 +165,7 @@ class Generators {
   };
 
   // assumed - properties added to context before the generators are called. For setting paraphrase property
-  apply (args, context, assumed = {}, options = {}) {
+  async apply (args, context, assumed = {}, options = {}) {
     if (Array.isArray(context)) {
       throw new Error('Expected a context not an array')
     }
@@ -186,12 +186,12 @@ class Generators {
     const stack = args.calls.push()
     for (let igenerator = 0; igenerator < this.generators.length; ++igenerator) {
       const generator = this.generators[igenerator]
-      if (generator.matches(args, objects, context, hierarchy, config, options)) {
+      if (await generator.matches(args, objects, context, hierarchy, config, options)) {
         const log = (message) => { this.logs.push(message) }
         // this.logs.push(`Generators: applied ${generator.toString()}\n to\n ${JSON.stringify(context)}`)
         let errorMessage = 'The apply function did not return a value'
         try {
-          generated = generator.apply(args, objects, context, hierarchy, config, response, log)
+          generated = await generator.apply(args, objects, context, hierarchy, config, response, log)
         } catch (e) {
           // the next if handle this
           generated = null

package/src/helpers.js

@@ -1,6 +1,37 @@
 const deepEqual = require('deep-equal')
 const stringify = require('json-stable-stringify')
 
+function where (goUp = 2) {
+  const e = new Error()
+  const regexForm1 = /\((.*):(\d+):(\d+)\)$/
+  const regexForm2 = /at (.*):(\d+):(\d+)$/
+  const lines = e.stack.split('\n')
+  let line
+  let match
+  for (line of lines.slice(1)) {
+    // if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('<anonymous>'))) {
+    if (!(line.includes('config.js:') || line.includes('client.js:') || line.includes('helpers.js:'))) {
+      match = regexForm1.exec(line) || regexForm2.exec(line)
+      if (!match) {
+        continue
+      }
+      break
+    }
+  }
+  // const line = e.stack.split("\n")[goUp];
+  // const match = regexForm1.exec(line) || regexForm2.exec(line)
+  if (match) {
+    return `${match[1]}:${match[2]}`
+  } else {
+    return 'running in browser or in an async call so the stack is broken.'
+  }
+}
+
+function w (func) {
+  func.where = where(3)
+  return func
+}
+
 // properties - the properties that correspond to types
 // types - the expected types of the properties
 // returns list of properties found matching order of types
@@ -403,5 +434,7 @@ module.exports = {
   InitCalls,
   hashCode,
   sortJson,
-  subPriority
+  subPriority,
+  where,
+  w,
 }

package/src/project.js

@@ -49,7 +49,7 @@ const project = (object, filter) => {
             const property = properties.property
             if (properties.isPropertyList) {
               if (!Array.isArray(object[property])) {
-                return projection
+                continue
               }
               for (const propertyRef of object[property]) {
                 const old = object[propertyRef]