theprogrammablemind_4wp 9.1.1-beta.1 → 9.1.1-beta.11

package/client.js

@@ -12,9 +12,10 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries } = require('./src/helpers')
+const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
 const runtime = require('./runtime')
 const sortJson = runtime.sortJson
+const debug = require('./src/debug')
 
 const getConfig_getObjectsCheck = (config, testConfig) => {
   let testConfigName = config.name
@@ -477,6 +478,11 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
 
   const objects = getObjects(config.config.objects)(config.uuid)
   try {
+    if (testConfig.initializer) {
+      const args = {}
+      setupArgs(args, config)
+      await testConfig.initializer(args)
+    }
     const result = await _process(config, test, { errorHandler, isTest: true })
     result.query = test
     if (debug) {
@@ -820,7 +826,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
   console.log(responses.trace)
 
-  if (false) {
+  if (true) {
     if (global.beforeObjects) {
       console.log('objects', runtime.jsonDiff.diffString(global.beforeObjects, config.get('objects')))
     } else {
@@ -931,6 +937,19 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     learned_contextual_priorities: []
   }
   config.fragmentsBeingBuilt = []
+
+  const toProperties = (queryStringOrProperties) => {
+    if (typeof queryStringOrProperties === 'string') {
+      return { query: queryStringOrProperties }
+    } else {
+      return queryStringOrProperties
+    }
+  }
+  const fragmentToTodo = (query, index) => {
+    const pr = instance.fragments[index]
+    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: false })
+  }
+
   const looper = async (configs) => {
     if (configs.length === 0) {
       finish()
@@ -1024,7 +1043,17 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
       const objects = config.get('objects')
       const args = { objects, getObjects: getObjects(objects) }
       setupArgs(args, config, config.logs, config.hierarchy)
+      // this watch stuff was for allowing fragment to be made in the template load. that is not a good idea
+      // because needs rebuild would need to run the rebuild to know if a rebuild was needed. the fragment creates
+      // need to go in the intializer
+      // const new_fragments = []
+      // config.watchNewFragments(new_fragments)
       await initFunction(args)
+      /*
+      if (new_fragments.length > 0) {
+        configs = configs.concat(new_fragments.map(fragmentToTodo))
+      }
+      */
       accumulators[property].push({ apply: queryOrExtraConfig })
       await looper(configs)
     } else {
@@ -1093,13 +1122,6 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     }
   }
 
-  const toProperties = (queryStringOrProperties) => {
-    if (typeof queryStringOrProperties === 'string') {
-      return { query: queryStringOrProperties }
-    } else {
-      return queryStringOrProperties
-    }
-  }
   let todo = []
   todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
   /*
@@ -1115,14 +1137,17 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     for (let configsIndex = 0; configsIndex < template.configs.length; ++configsIndex) {
       const query = template.configs[configsIndex]
       // account for things for example associations being added to the config while debugginer
-      const pr = previousResultss.find((pr) => pr.query == query)
+      const pr = previousResultss && previousResultss.find((pr) => pr.query == query)
       todo.push({ property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics })
     }
   }
+  todo = todo.concat((template.fragments || []).map(fragmentToTodo))
+  /*
   todo = todo.concat((template.fragments || []).map((query, index) => {
     const pr = instance.fragments[index]
     return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: false })
   }))
+  */
   todo = todo.concat((template.semantics || []).map((definition) => {
     return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true }
   }))
@@ -1136,6 +1161,7 @@ const checkTemplate = (template) => {
   if (template.checks) {
     throw new Error("The 'checks' property should be in the 'test' property not the 'template' property")
   }
+  validProps([ 'fragments', 'configs' ], template.template, 'template.template')
 }
 
 const checkTest = (testConfig) => {
@@ -1156,7 +1182,7 @@ const checkTest = (testConfig) => {
 const knowledgeModuleImpl = async ({
   includes,
   config: configStruct,
-  api,
+  api, apiKMs,
   initializer,
   terminator,
   multiApiInitializer,
@@ -1188,8 +1214,31 @@ const knowledgeModuleImpl = async ({
     throw new Error("'config' or 'createConfig' is a required parameter. The value should the config that defines the knowledge module.")
   }
 
+  const setupConfig = (config) => {
+    if (!config.name) {
+      throw new Error("config must have 'name' set to the knowledge module name.")
+    }
+
+    config.description = description
+    if (typeof testConfig === 'object') {
+      if (testConfig.contents) {
+        config.tests = testConfig.contents
+        test = testConfig.name
+      }
+    } else {
+      if (runtime.fs && runtime.fs.existsSync(test)) {
+        config.tests = JSON.parse(runtime.fs.readFileSync(test))
+      } else {
+        config.tests = []
+      }
+    }
+    config.setTestConfig(testConfig)
+  }
+
   const createConfig = async () => {
-    const config = new Config(configStruct, moduleFromJSFile, _process)
+    const config = new Config(configStruct, moduleFromJSFile, _process, apiKMs)
+    setupConfig(config)
+    config.expect_template = !!template
     config.setTerminator(terminator)
     config.stop_auto_rebuild()
     await config.add(...(includes || []))
@@ -1216,27 +1265,6 @@ const knowledgeModuleImpl = async ({
 
   const isProcess = require.main === moduleFromJSFile
 
-  const setupConfig = (config) => {
-    if (!config.name) {
-      throw new Error("config must have 'name' set to the knowledge module name.")
-    }
-
-    config.description = description
-    if (typeof testConfig === 'object') {
-      if (testConfig.contents) {
-        config.tests = testConfig.contents
-        test = testConfig.name
-      }
-    } else {
-      if (runtime.fs && runtime.fs.existsSync(test)) {
-        config.tests = JSON.parse(runtime.fs.readFileSync(test))
-      } else {
-        config.tests = []
-      }
-    }
-    config.setTestConfig(testConfig)
-  }
-
   if (isProcess) {
     let config
     try {
@@ -1298,7 +1326,6 @@ const knowledgeModuleImpl = async ({
       if (args.rebuildTemplateFull) {
         args.rebuildTemplate = true
       }
-
       config = await createConfig()
 
       // dont debug the load of the KM's if rebuild template is on since we want to debug the template rebuild not the load
@@ -1306,7 +1333,7 @@ const knowledgeModuleImpl = async ({
         global.pauseDebugging = true
       }
 
-      setupConfig(config)
+      // setupConfig(config)
       processResults = processResults({ config, errorHandler })
 
       if (args.rebuildTemplate) {
@@ -1542,6 +1569,7 @@ const knowledgeModuleImpl = async ({
         }
         try {
           await config.load(rebuildTemplate, template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
+          config.fixtures()
         } catch (e) {
           console.error(`Error loading template for ${config.name}. ${e.error ? e.error : e}${e.stack ? e.stack : ''}`)
           runtime.process.exit(-1)
@@ -1841,6 +1869,7 @@ const knowledgeModuleImpl = async ({
         config.terminate()
       }
     }
+    moduleFromJSFile.exports = createConfig
   } else {
     const initConfig = async (config) => {
       if (template) {
@@ -1910,6 +1939,11 @@ const test = (name) => {
 }
 */
 
+// for testing
+const getRequireMain = () => {
+  return require.main
+}
+
 const ensureTestFile = (module, name, type) => {
   const isProcess = require.main === module
   if (isProcess) {
@@ -1931,6 +1965,7 @@ const knowledgeModule = async (...args) => {
 module.exports = {
   process: _process,
   stableId,
+  getRequireMain, // for testing
   ensureTestFile,
   rebuildTemplate,
   processContext,

package/package.json

@@ -9,7 +9,8 @@
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
     "globals": "^16.0.0",
-    "jest": "^29.7.0"
+    "jest": "^29.7.0",
+    "argparse": "^2.0.1"
   },
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
@@ -47,6 +48,7 @@
     "src/config.js",
     "src/configHelpers.js",
     "src/copy.js",
+    "src/debug.js",
     "src/digraph.js",
     "src/digraph_internal.js",
     "src/generators.js",
@@ -67,6 +69,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.1.1-beta.1",
+  "version": "9.1.1-beta.11",
   "license": "UNLICENSED"
 }

package/src/config.js

@@ -9,6 +9,7 @@ const { InitCalls } = require('./helpers')
 const { ecatch } = require('./helpers')
 const runtime = require('../runtime')
 const _ = require('lodash')
+const db = require('./debug')
 
 const debugBreak = () => {
   // debugger
@@ -844,14 +845,6 @@ const multiApiImpl = (initializer) => {
       }
     },
 
-    /*
-    set objects (value) {
-      for (const key in Object.keys(this.apis)) {
-        this.apis[key].objects = value
-      }
-    },
-    */
-
     // "product1": apiInstance(testData1),
     apis: {
     },
@@ -869,6 +862,14 @@ class Config {
     return config_toServer(config)
   }
 
+  async fixtures () {
+    if (this.testConfig?.fixtures) {
+      const args = {}
+      configHelpers.setupArgs(args, this)
+      return this.testConfig.fixtures(args)
+    }
+  }
+
   getInfo () {
     const name = this.name
     const includes = this.configs.slice(1).map((km) => km.config.name)
@@ -1363,12 +1364,22 @@ class Config {
         this.instances.push(instance)
         await configHelpers.loadInstance(this, instance)
       }
+      this.expect_template = false
     }
   }
 
+  watchNewFragments (list) {
+    this.addFragmentWatcher = list
+  }
+
   addFragments (fragments) {
     // only run this if not loading as module write error if loading as module and different
     this.dynamicFragments = this.dynamicFragments.concat(fragments)
+    if (this.addFragmentWatcher) {
+      for (const fragment of fragments) {
+        this.addFragmentWatcher.push(fragment)
+      }
+    }
   }
 
   objects () {
@@ -1908,11 +1919,15 @@ class Config {
   }
 
   // configs = [ { config, namespace } ... ]
-  constructor (config, module, clientProcess) {
+  constructor (config, module, clientProcess, apiKMs) {
     if (config instanceof Config) {
       throw new Error('Excepted the config argument to be a hash not a Config object')
     }
 
+    if (!apiKMs) {
+      apiKMs = []
+    }
+
     if (config) {
       validConfigProps(config)
 
@@ -1926,6 +1941,8 @@ class Config {
       config.priorities = config.priorities || []
     }
 
+    this._apiKMs = apiKMs
+
     this.clientProcess = clientProcess
     this.maxDepth = 20 // for generators and semantics
     this.debugLoops = false // for generators and semantics
@@ -1934,7 +1951,7 @@ class Config {
     this.resetDelta()
 
     this.addedArgss = []
-    let isProcess = require.main === module
+    const isProcess = require.main === module
     if (global.theprogrammablemind && config) {
       if (global.theprogrammablemind.loadForTesting[config.name]) {
         isProcess = true
@@ -2094,6 +2111,7 @@ class Config {
     }
   }
 
+  // constructors is a constructor
   async setApi (constructor) {
     if (typeof constructor !== 'function') {
       throw new Error(`Expected the argument to be an API constructor for ${this.name}.`)
@@ -2115,6 +2133,12 @@ class Config {
     } else {
       this._api = value
       this._apiConstructor = constructor
+      for (const name of this._apiKMs) {
+        if (this.name == name) {
+          continue
+        }
+        this.km(name).setApi(() => this._api)
+      }
       await this.rebuild()
     }
   }
@@ -2182,10 +2206,10 @@ class Config {
     // update uuid here set the uuid in the objects and add error checking
     cp.initializerFn = this.initializerFn
     cp.terminatorFn = this.terminatorFn
-    // cp._api = _.cloneDeep(this._api)
     if (this._apiConstructor) {
       cp._api = this._apiConstructor(cp)
       cp._apiConstructor = this._apiConstructor
+      cp._apiKMs = this._apiKMs
     }
     cp._namespace = this._namespace
     cp._eqClasses = this._eqClasses
@@ -2603,6 +2627,7 @@ class Config {
         currentConfig: config,
         uuid: config._uuid,
         objects: namespacedObjects,
+        globals: this.config.objects,
         namespace,
         api: config.api
       }))
@@ -2697,6 +2722,13 @@ class Config {
     }
 
     this.hierarchy.edges = this.config.hierarchy
+
+    if (!this.isModule) {
+      if (this.template || !this.expect_template) {
+        await this.fixtures()
+      }
+    }
+
     this.valid()
     this.checks()
   }

package/src/configHelpers.js

@@ -81,6 +81,13 @@ const cleanAssign = (dest, ...srcs) => {
 }
 
 const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
+  if (!args.objects) {
+    args.objects = config.get('objects')
+    args.getObjects = getObjects(args.objects)
+  }
+  if (!hierarchy) {
+    hierarchy = config.hierarchy
+  }
   // callId
   args.calls = new InitCalls(args.isInstance ? `${args.isInstance}#${config.name}` : config.name)
   if (global.theprogrammablemind && global.theprogrammablemind.loadForTesting) {
@@ -154,6 +161,7 @@ const setupArgs = (args, config, logs, hierarchy, uuidForScoping) => {
   config.getAddedArgs(args)
 
   Object.assign(args, args.getUUIDScoped(uuidForScoping || config.uuid))
+  args.apis = args.apis || ((name) => config.getConfig(name).api)
   /*
   if (uuidForScoping) {
     Object.assign(args, args.getUUIDScoped(uuidForScoping))
@@ -241,15 +249,17 @@ const setupContexts = (rawContexts) => {
   let first = true
   const contexts = []
   contexts.push({ marker: 'controlStart', controlRemove: true })
+  let previous;
   for (const context of rawContexts) {
     if (first) {
       first = false
     } else {
-      contexts.push({ marker: 'controlBetween', controlRemove: true })
+      contexts.push({ marker: 'controlBetween', controlRemove: true, previous })
     }
     contexts.push(context)
+    previous = context
   }
-  contexts.push({ marker: 'controlEnd', controlRemove: true })
+  contexts.push({ marker: 'controlEnd', controlRemove: true, previous })
 
   let _index = 0
   const id = (context) => {
@@ -280,11 +290,14 @@ const processContextsB = async ({ config, hierarchy, semantics, generators, json
   args.insert = (context) => toDo.unshift(context)
   let overlap, lastRange
   config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
+  let context_id_counter = 0
   while (toDo.length > 0) {
     const context = toDo.shift()
     args.calls.next()
     let contextPrime = context
     context.topLevel = true
+    context_id_counter += 1
+    context.context_id = context_id_counter
     try {
       if (json.has_errors) {
         throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')

package/src/debug.js

@@ -0,0 +1,78 @@
+const counters = {}
+const hits = {}
+
+let db = () => {
+  debugger // eslint-disable-line no-debugger
+}
+
+const reset = () => {
+  Object.keys(counters).forEach(key => delete counters[key])
+  Object.keys(hits).forEach(key => delete hits[key])
+  db = () => { debugger } // eslint-disable-line no-debugger
+}
+
+const getDb = () => {
+  return db
+}
+
+const setDb = (value) => {
+  db = value
+}
+
+const hit = (name) => {
+  hits[name] = true
+}
+
+const unhit = (name) => {
+  hits[name] = false
+}
+
+const wasHit = (name) => {
+  return hits[name]
+}
+
+const counter = (name, breakAt = [], debugBreak = true) => {
+  if (!counters[name]) {
+    counters[name] = 0
+  }
+  counters[name] += 1
+  console.log(`counters[${name}] = ${counters[name]}`)
+  unhit(name)
+  if (Array.isArray(breakAt) && breakAt.includes(counters[name])) {
+    if (debugBreak) {
+      db()
+    }
+    hit(name)
+    return true
+  } else if (breakAt == counters[name]) {
+    if (debugBreak) {
+      db()
+    }
+    hit(name)
+    return true
+  }
+}
+
+const get = (name) => {
+  return counters[name]
+}
+
+const _break = (name) => {
+  if (wasHit(name)) {
+    db()
+  }
+}
+
+module.exports = {
+  counter,
+  get,
+  _break, // break is a keyword
+
+  // used for tests
+  reset,
+  hit,
+  unhit,
+  wasHit,
+  getDb,
+  setDb
+}

package/src/generators.js

@@ -211,7 +211,7 @@ class Generators {
           lines.setElement(0, 2, generator.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
-          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
@@ -253,7 +253,7 @@ class Generators {
           lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
           lines.newRow()
           lines.setElement(0, 1, 'TO')
-          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           this.logs.push(lines.toString())
         }

package/src/helpers.js

@@ -310,7 +310,7 @@ const validProps = (valids, object, type) => {
       }
     }
     if (!okay) {
-      throw new Error(`Unknown property "${prop}" in the ${type}. Valid properties are ${valids}. The ${type} is ${JSON.stringify(object)}`)
+      throw new Error(`Unknown property "${prop}" in the ${type}. Valid properties are ${valids.join(', ')}. The ${type} is ${JSON.stringify(object)}`)
     }
   }
 }

package/src/semantics.js

@@ -183,7 +183,16 @@ class Semantics {
             deferWasCalled = true
           }
           args.defer = defer
+
+          let continueWasCalled = false
+          const _continue = () => {
+            continueWasCalled = true
+          }
+          args._continue = _continue
           contextPrime = await semantic.apply(args, context, s, options)
+          if (continueWasCalled) {
+            continue
+          }
           if (deferWasCalled) {
             continue
           }
@@ -218,7 +227,7 @@ class Semantics {
           lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
-          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
@@ -253,7 +262,7 @@ class Semantics {
           lines.setElement(0, 2, semantic.toString())
           lines.newRow()
           lines.setElement(0, 1, 'TO')
-          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+          lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
           lines.newRow()
           lines.setElement(0, 1, 'STACK')
@@ -263,7 +272,7 @@ class Semantics {
           lines.setElement(0, 2, `To debug this use args.callId == '${args.calls.current()}'`)
           lines.newRow()
           lines.setElement(0, 1, 'RESULT')
-          lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(contextPrime, { depth: 25 })))})`)
+          lines.setElement(0, 2, `context_id: ${context.context_id}`)
           lines.setElement(1, 2, JSON.stringify(contextPrime, null, 2))
           for (const { semantic } of deferred) {
             lines.setElement(0, 1, 'DEFERRED')
@@ -295,7 +304,7 @@ class Semantics {
       lines.setElement(0, 2, stack)
       lines.newRow()
       lines.setElement(0, 1, 'TO')
-      lines.setElement(0, 2, `(HASHCODE ${helpers.hashCode(JSON.stringify(helpers.sortJson(context, { depth: 25 })))})`)
+      lines.setElement(0, 2, `context_id: ${context.context_id}`)
       lines.setElement(1, 2, JSON.stringify(helpers.sortJson(context, { depth: 25 }), null, 2))
       this.logs.push(lines.toString())
     }