theprogrammablemind 9.5.1 → 9.6.0-beta.1

package/client.js

@@ -12,10 +12,9 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
+const { sortJson, appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
 const runtime = require('./runtime')
-const sortJson = runtime.sortJson
-const debug = require('./src/debug')
+const db = require('./src/debug')
 
 const getConfig_getObjectsCheck = (config, testConfig) => {
   let testConfigName = config.name
@@ -46,33 +45,33 @@ const getSuggestionMessage = (suggestion) => {
   return `Try adding this to the associations: { context: ${JSON.stringify(getSuggestion(suggestion))}, choose: <indexOfMainElement> },\n If that does not work look at the logs and check when the operators become wrong during an interation. Deduce the change based on the previous iteration and what operator was applied.`
 }
 
-const getConfig_getContextCheck = (testConfig) => {
-  return (testConfig.checks && testConfig.checks.context) || []
-}
-
 const pickContext = (contextChecks) => (context) => {
   return project2(context, contextChecks)
 }
 
 const pickObjects = (config, testConfig, getObjects) => {
-  /*
-  let testConfigName = config.name
-  if (testConfig.testModuleName) {
-    objects = getObjects(config.config.objects)(config.getConfigs()[testConfig.testModuleName].uuid)
-    testConfigName = testConfig.testModuleName
-  }
-  */
   const checks = getConfig_getObjectsCheck(config, testConfig)
+  const contextChecks = config.getContextChecks()
   const projection = {}
   for (const km in checks) {
     const objects = getObjects(km)
     if (!objects) {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
-    if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
-      projection[km] = project2(objects, checks[km])
+
+    if (false) {
+      if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
+        projection[km] = project2(objects, checks[km])
+      } else {
+        projection[km] = project(objects, checks[km])
+      }
     } else {
-      projection[km] = project(objects, checks[km])
+      const lastIndex = contextChecks.length - 1
+      const last = contextChecks[lastIndex]
+      contextChecks[lastIndex] = { match: last.match, apply: () => [...new Set([...(checks[km] || []), ...last.apply()])] }
+
+      // projection[km] = project2(objects, [...checks[km], ...contextChecks])
+      projection[km] = project2(objects, contextChecks)
     }
   }
   return projection
@@ -237,9 +236,10 @@ const doWithRetries = async (n, url, queryParams, data) => {
       body: JSON.stringify(data),
       timeout: 1000 * 60 * 5, // it does not respect this timeout so that's why I have the retries
       headers: {
-        mode: 'no-cors',
+        // mode: 'no-cors',
         'Content-Type': 'application/json'
-      }
+      },
+      credentials: 'same-origin',
     })
     if (result.ok) {
       return JSON.parse(JSON.stringify(await result.json()))
@@ -316,6 +316,7 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     }
 
     let startCounter = 0
+    let contextIdCounter = 0
     while (true) {
       if (queries.length === 0) {
         break
@@ -356,8 +357,9 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         }
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
-        const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        const { updatedContextIdCounter, contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
+                                  await processContextsB({ contextIdCounter, isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+        contextIdCounter = updatedContextIdCounter
         if (isTest) {
           const end = runtime.performance.performance.now()
           clientSideTime = end - start
@@ -475,6 +477,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
     return
   }
   // initialize in between test so state is not preserved since the test was adding without state
+  config.testConfig.testModuleName = testConfig.testModuleName
   await config.rebuild()
   const errorHandler = (error) => {
     if (error.metadata) {
@@ -838,7 +841,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
   console.log(responses.trace)
 
-  if (true) {
+  if (false) {
     if (global.beforeObjects) {
       console.log('objects', runtime.jsonDiff.diffString(global.beforeObjects, config.get('objects')))
     } else {
@@ -913,7 +916,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   console.log('')
   const screen_width = process.stdout.columns
   // || 0 for when running without a console
-  const widths = [70, 10, Math.max(80, screen_width - 71 || 0)]
+  const widths = Lines.addRemainder([70, 10])
   const lines = new Lines(widths)
   lines.setElement(0, 0, '--- The paraphrases are ----------')
   lines.setElement(0, 2, '--- The response strings are ----------')
@@ -933,12 +936,15 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
   }
 }
 
-const defaultProcess = ({ config, errorHandler }) => async (promise) => {
+const defaultProcess = ({ config, errorHandler, print }) => async (promise) => {
   try {
     const responses = await promise
     defaultInnerProcess(config, errorHandler, responses)
   } catch (error) {
     error.config = config
+    if (print) {
+      print()
+    }
     defaultErrorHandler(error)
   }
 }
@@ -963,8 +969,11 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     }
   }
   const fragmentToTodo = (query, index) => {
-    const pr = instance.fragments[index]
-    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: false })
+    let pr = []
+    if (instance.fragments) {
+      pr = instance.fragments[index]
+    }
+    return Object.assign({}, toProperties(query), { property: 'fragments', previousResults: pr, skipSemantics: true })
   }
 
   const looper = async (configs) => {
@@ -973,7 +982,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
       return
     }
     const { property, hierarchy, query: queryOrExtraConfig, previousResults, initializer, skipSemantics } = configs.shift()
-    // queries are strings or { query: "blah", development: true/false }
+    // queries are strings or { query: "blah", scope: "development" | "testing" }
     if (typeof queryOrExtraConfig === 'string' || queryOrExtraConfig.query || queryOrExtraConfig.isFragment) {
       let query = queryOrExtraConfig
       const isFragment = queryOrExtraConfig.isFragment
@@ -1175,10 +1184,7 @@ const checkTemplate = (template) => {
   if (!template) {
     return
   }
-  if (template.checks) {
-    throw new Error("The 'checks' property should be in the 'test' property not the 'template' property")
-  }
-  validProps(['fragments', 'configs'], template.template, 'template.template')
+  validProps(['fragments', 'configs'], template, 'template.template')
 }
 
 const checkTest = (testConfig) => {
@@ -1212,6 +1218,7 @@ const knowledgeModuleImpl = async ({
   demo,
   test,
   template,
+  instance, // instance file
   errorHandler = defaultErrorHandler,
   process: processResults = defaultProcess,
   stopAtFirstFailure = true,
@@ -1253,8 +1260,8 @@ const knowledgeModuleImpl = async ({
     config.setTestConfig(testConfig)
   }
 
-  const createConfig = async () => {
-    const config = new Config(configStruct, moduleFromJSFile, _process, apiKMs)
+  const createConfig = async (rootIsProcess, testingModuleName) => {
+    const config = new Config(configStruct, moduleFromJSFile, _process, apiKMs, rootIsProcess, testingModuleName)
     if (sendObjectsToServer) {
       config.setSendObjectsToServer()
     }
@@ -1318,13 +1325,32 @@ const knowledgeModuleImpl = async ({
       parser.add_argument('-cl', '--checkForLoop', { nargs: '?', help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
       parser.add_argument('-r', '--reset', { action: 'store_true', help: 'Get the server to bypass the cache and rebuild everything' })
       parser.add_argument('-q', '--query', { help: 'Run the specified query' })
+      parser.add_argument('-f', '--filter', { help: 'for -pd only the data for the knowledge modules that start with this string will be shown' })
       parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
       parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
       parser.add_argument('-dt', '--deleteTest', { help: 'Delete the specified query from the tests file.' })
       parser.add_argument('--parenthesized', { action: 'store_true', help: 'Show the generated phrases with parenthesis.' })
       parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
       parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
-      parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, ha === hierarchy ancestors g === generators, s === semantics, l === load t=tests ordering p === priorities a === associations j === JSON sent to server. for example --print wb' })
+      parser.add_argument('-p', '--print', { help: 
+`Print the specified elements 
+    a === associations 
+    b === bridges, 
+    c === config,
+    cc === test checks,
+    d === objects (d for data), 
+    g === generators, 
+    h === hierarchy, 
+    ha === hierarchy ancestors,
+    j === JSON sent to server, 
+    l === load ordering,
+    o === operators,
+    p === priorities,
+    s === semantics, 
+    t === tests ordering,
+    w === words, 
+ for example --print wb' })
+` })
       parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
       parser.add_argument('-fr', '--failRebuild', { action: 'store_true', help: 'If a rebuild is required fail out.' })
       parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
@@ -1347,16 +1373,13 @@ const knowledgeModuleImpl = async ({
       if (args.rebuildTemplateFull) {
         args.rebuildTemplate = true
       }
-      config = await createConfig()
+      config = await createConfig(true, configStruct.name)
 
       // dont debug the load of the KM's if rebuild template is on since we want to debug the template rebuild not the load
       if (args.rebuildTemplate) {
         global.pauseDebugging = true
       }
 
-      // setupConfig(config)
-      processResults = processResults({ config, errorHandler })
-
       if (args.rebuildTemplate) {
         global.pauseDebugging = false
       }
@@ -1488,6 +1511,12 @@ const knowledgeModuleImpl = async ({
               counter += 1
             }
           }
+          if (hasArg('cc')) {
+            for (const cc of config.getContextChecks()) {
+              const printable = { ...cc, match: cc.match.toString(), apply: cc.apply.toString() }
+              console.log(JSON.stringify(printable, null, 2))
+            }
+          }
           if (hasArg('c')) {
             const { data } = setupProcessB({ config })
             console.log('Config as sent to server')
@@ -1549,14 +1578,29 @@ const knowledgeModuleImpl = async ({
               }
             }
           }
+
           if (hasArg('d')) {
-            console.log(JSON.stringify(config.config.objects, null, 2))
+            if (args.filter) {
+              console.log(`objects (data) filtered by ${args.filter} ================`)
+              const projection = { namespaced: {} }
+              for (const key of Object.keys(config.config.objects.namespaced)) {
+                if (key.startsWith(args.filter)) {
+                  projection.namespaced[key] = config.config.objects.namespaced[key]
+                }
+              }
+              console.log(JSON.stringify(projection, null, 2))
+            } else {
+              console.log('objects (data) ================')
+              console.log(JSON.stringify(config.config.objects, null, 2))
+            }
           }
+
           if (hasArg('p')) {
             for (const priority of config.config.priorities) {
               console.log(JSON.stringify(priority))
             }
           }
+
           if (hasArg('g')) {
             const easyToRead = _.cloneDeep(config.config.generators)
             for (const semantic of easyToRead) {
@@ -1568,6 +1612,7 @@ const knowledgeModuleImpl = async ({
             }
             console.dir(easyToRead)
           }
+
           if (hasArg('s')) {
             const easyToRead = _.cloneDeep(config.config.semantics)
             for (const semantic of easyToRead) {
@@ -1579,16 +1624,20 @@ const knowledgeModuleImpl = async ({
         }
       }
 
+      // setupConfig(config)
+      processResults = processResults({ config, errorHandler, print: printConfig })
+
+
       checkTemplate(template)
 
       if (template) {
         let needsRebuild
         if (args.rebuildTemplate && !args.rebuildTemplateFull) {
           // get the startOfChanges for the partial rebuild
-          needsRebuild = config.needsRebuild(template.template, template.instance, { ...options, rebuild: false })
+          needsRebuild = config.needsRebuild(template, instance, { ...options, rebuild: false })
         } else {
           // do a check or full rebuild
-          needsRebuild = config.needsRebuild(template.template, template.instance, options)
+          needsRebuild = config.needsRebuild(template, instance, options)
         }
 
         if (needsRebuild.needsRebuild) {
@@ -1604,7 +1653,7 @@ const knowledgeModuleImpl = async ({
           config.config.rebuild = true
         }
         try {
-          await config.load(rebuildTemplate, template.template, template.instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
+          await config.load(rebuildTemplate, template, instance, { rebuild: needsRebuild.needsRebuild || options.rebuild, previousResultss: needsRebuild.previousResultss, startOfChanges: needsRebuild.startOfChanges })
           config.fixtures()
         } catch (e) {
           console.error(`Error loading template for ${config.name}. ${e.error ? e.error : e}${e.stack ? e.stack : ''}`)
@@ -1912,6 +1961,7 @@ const knowledgeModuleImpl = async ({
         try {
           await processResults(_process(config, args.query, { commandLineArgs: args, isProcess, isModule: !isProcess, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
         } catch (error) {
+          printConfig()
           console.log('Error', error)
         }
       }
@@ -1925,8 +1975,8 @@ const knowledgeModuleImpl = async ({
   } else {
     const initConfig = async (config) => {
       if (template) {
-        if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess }).needsRebuild) {
-          config.needsRebuild(template.template, template.instance, { isModule: !isProcess })
+        if (config.needsRebuild(template, instance, { isModule: !isProcess }).needsRebuild) {
+          config.needsRebuild(template, instance, { isModule: !isProcess })
           const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
           throw new Error(error)
         }
@@ -1941,7 +1991,7 @@ const knowledgeModuleImpl = async ({
       }
       if (template) {
         try {
-          await config.load(rebuildTemplate, template.template, template.instance)
+          await config.load(rebuildTemplate, template, instance)
         } catch (e) {
           errorHandler(e)
         }
@@ -1949,33 +1999,17 @@ const knowledgeModuleImpl = async ({
 
       // remove test only stuff
       if (!isProcess && !loadForTesting) {
-        config.removeDevelopmentElements(config.config)
-        /*
-        config.config.operators = config.config.operators.filter((operator) => {
-          if (operator.development) {
-            return false
-          } else {
-            return true
-          }
-        })
-        config.config.bridges = config.config.bridges.filter((bridge) => {
-          if (bridge.development) {
-            return false
-          } else {
-            return true
-          }
-        })
-        */
+        config.removeDevelopmentElements(config.config, config.scope)
       }
     }
 
     // no cache 21 minutes + rebuild fails "node tester_rebuild -m colors"
     // cache okay
-    const createConfigExport = async (useCache = true) => {
+    const createConfigExport = async ({ useCache=true, rootIsProcess, testingModuleName } = {}) => {
       if (useCache && createConfig.cached) {
         return createConfig.cached
       }
-      const config = await createConfig()
+      const config = await createConfig(rootIsProcess, testingModuleName)
       await initConfig(config)
       // config.rebuild({ isModule: true })
       createConfig.cached = config
@@ -2011,9 +2045,15 @@ const ensureTestFile = (module, name, type) => {
 }
 
 const knowledgeModule = async (...args) => {
-  await knowledgeModuleImpl(...args).catch((e) => {
+  await knowledgeModuleImpl(...args).catch(async (e) => {
     console.error(e)
-    process.exit(-1)
+    function sleep(ms) {
+      return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+      });
+    }
+    await sleep(1) // get the stderr to flush
+    await process.exit(-1);   // tiny trick: empty write forces flush of console.error buffer
   })
 }
 
@@ -2038,5 +2078,6 @@ module.exports = {
   gs,
   flattens,
   writeTest,
-  getConfigForTest
+  getConfigForTest,
+  debug: db,
 }

package/lines.js

@@ -5,6 +5,13 @@ class Lines {
     this.rows = []
   }
 
+  static SCREEN_WIDTH = 164
+
+  static addRemainder(widths) {
+    const sum = widths.reduce((a, b) => a + b)
+    return [...widths, Lines.SCREEN_WIDTH - sum]
+  }
+
   addLine () {
     this.lines.push(this.widths.map((width) => ''.padEnd(width)))
   }

package/package.json

@@ -3,14 +3,14 @@
     "@eslint/js": "^9.21.0",
     "@typescript-eslint/eslint-plugin": "^4.28.4",
     "@typescript-eslint/parser": "^4.28.4",
+    "argparse": "^2.0.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^16.0.3",
     "eslint-plugin-import": "^2.23.4",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^5.1.0",
     "globals": "^16.0.0",
-    "jest": "^29.7.0",
-    "argparse": "^2.0.1"
+    "jest": "^30.2.0"
   },
   "scripts": {
     "to:debug": "node inspect node_modules/.bin/jest --runInBand -t NEO23",
@@ -45,6 +45,7 @@
     "runtime.js",
     "src/helpers.js",
     "src/flatten.js",
+    "src/fragments.js",
     "src/unflatten.js",
     "src/config.js",
     "src/configHelpers.js",
@@ -61,6 +62,7 @@
   "dependencies": {
     "base-64": "^1.0.0",
     "deep-equal": "^2.0.4",
+    "flatted": "^3.3.3",
     "fs": "0.0.1-security",
     "json-diff": "^1.0.3",
     "json-stable-stringify": "^1.0.1",
@@ -68,9 +70,8 @@
     "node-fetch": "^2.6.1",
     "readline": "^1.3.0",
     "scriptjs": "^2.5.9",
-    "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.5.1",
+  "version": "9.6.0-beta.1",
   "license": "UNLICENSED"
 }

package/runtime.js

@@ -13,7 +13,6 @@ module.exports = {
   ArgumentParser,
   readline: require('readline'),
   jsonDiff: require('json-diff'),
-  sortJson: require('sort-json'),
   util: require('util'),
   performance: require('perf_hooks')
 }