theprogrammablemind_4wp 9.3.0 → 9.4.1

package/client.js

@@ -1,10 +1,10 @@
 const { Semantics, Semantic } = require('./src/semantics')
 const { Generators, Generator } = require('./src/generators')
 const { Config } = require('./src/config')
-const { loadInstance, ErrorReason, listable, setupArgs, gs, processContext, getObjects, setupProcessB, processContextsB } = require('./src/configHelpers')
-const DigraphInternal = require('./src/digraph_internal')
+const { loadInstance, setupArgs, gs, processContext, getObjects, setupProcessB, processContextsB } = require('./src/configHelpers')
 const Digraph = require('./src/digraph')
 const { project } = require('./src/project')
+const { project:project2 } = require('./src/project2')
 const fetch = require('node-fetch')
 const base64 = require('base-64')
 const deepEqual = require('deep-equal')
@@ -12,7 +12,7 @@ const _ = require('lodash')
 const stringify = require('json-stable-stringify')
 const Lines = require('./lines')
 const flattens = require('./src/flatten')
-const { appendNoDups, InitCalls, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
+const { appendNoDups, updateQueries, safeNoDups, stableId, where, suggestAssociationsFix, suggestAssociationsFixFromSummaries, validProps } = require('./src/helpers')
 const runtime = require('./runtime')
 const sortJson = runtime.sortJson
 const debug = require('./src/debug')
@@ -50,8 +50,8 @@ const getConfig_getContextCheck = (testConfig) => {
   return (testConfig.checks && testConfig.checks.context) || []
 }
 
-const pickContext = (testConfig) => (context) => {
-  return project(context, getConfig_getContextCheck(testConfig))
+const pickContext = (contextChecks) => (context) => {
+  return project2(context, contextChecks)
 }
 
 const pickObjects = (config, testConfig, getObjects) => {
@@ -69,7 +69,11 @@ const pickObjects = (config, testConfig, getObjects) => {
     if (!objects) {
       throw new Error(`In the checks for ${config.name} the KM ${km} does not exist`)
     }
-    projection[km] = project(objects, checks[km])
+    if (checks[km] && checks[km].find((check) => check.match && check.apply)) {
+      projection[km] = project2(objects, checks[km])
+    } else {
+      projection[km] = project(objects, checks[km])
+    }
   }
   return projection
 }
@@ -77,7 +81,7 @@ const pickObjects = (config, testConfig, getObjects) => {
 const sameJSON = (json1, json2) => {
   const sjson1 = sortJson(json1, { depth: 25 })
   const sjson2 = sortJson(json2, { depth: 25 })
-  return JSON.stringify(sjson1) == JSON.stringify(sjson2)
+  return JSON.stringify(sjson1) === JSON.stringify(sjson2)
 }
 
 const vimdiff = (actualJSON, expectedJSON, title) => {
@@ -149,7 +153,7 @@ const convertToStable = (objects) => {
 const writeTestFile = (fn, tests) => {
   const stabilize = (tests) => {
     for (const test of tests) {
-      for (opChoice of test.metadata.opChoices) {
+      for (const opChoice of test.metadata.opChoices) {
         opChoice.ops.sort()
       }
       test.metadata.opChoices.sort()
@@ -185,7 +189,7 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
   }, { depth: 25 })
   let wasSet = false
   tests.forEach((test, index) => {
-    if (test.query == query) {
+    if (test.query === query) {
       tests[index] = results
       wasSet = true
     }
@@ -261,7 +265,7 @@ const throwErrorHandler = (error) => {
   throw error
 }
 
-const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
+const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isProcess, isModule, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = throwErrorHandler } = {}) => {
   if (credentials) {
     config.server(credentials.server, credentials.key)
   }
@@ -316,12 +320,12 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
       if (queries.length === 0) {
         break
       }
-
+      config.updateData(data)
       data.utterance = queries[0]
       data.start_counter = startCounter
       let json = await doWithRetries(retries, url, queryParams, data)
       let resetData = false
-      if (json.code == 'NOT_IN_CACHE') {
+      if (json.code === 'NOT_IN_CACHE') {
         resetData = true
         const setupB = setupProcessB({ config, allowDelta: false })
         data = setupB.data
@@ -346,15 +350,16 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
         throw json
       } else {
         let clientSideTime
+        let start
         if (isTest) {
           start = runtime.performance.performance.now()
         }
         const summary = { summaries: json.summaries, length: json.contexts.length }
         summaries.push(summary)
         const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
-                                  await processContextsB({ isTest, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
+                                  await processContextsB({ isTest, isProcess, isModule, rebuildingTemplate, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
         if (isTest) {
-          end = runtime.performance.performance.now()
+          const end = runtime.performance.performance.now()
           clientSideTime = end - start
         }
         response.associations = json.associations
@@ -386,13 +391,13 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
     }
 
     if (writeTests) {
-      const actual_config = getConfigForTest(config, testConfig)
+      const actualConfig = getConfigForTest(config, testConfig)
       const saveObjects = { ...config.config.objects }
       saveObjects.nameToUUID = {}
       for (const km of config.configs) {
         saveObjects.nameToUUID[km.name] = km.uuid
       }
-      writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized, response.summaries)
+      writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actualConfig, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized, response.summaries)
     }
 
     return response
@@ -485,7 +490,7 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       setupArgs(args, config)
       await testConfig.initializer(args)
     }
-    const result = await _process(config, test, { errorHandler, isTest: true })
+    const result = await _process(config, test, { errorHandler, isTest: true, isProcess: true, isModule: false })
     result.query = test
     if (debug) {
       defaultInnerProcess(config, errorHandler, result)
@@ -499,24 +504,25 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       timings.client += result.clientSideTimes / 1000
       lines.log()
     }
-    const expected_objects = sortJson(convertToStable(expected.objects), { depth: 25 })
-    delete expected_objects.nameToUUID
-    const actual_objects = sortJson(convertToStable(config.config.objects), { depth: 25 })
-    const failed_paraphrases = !matching(result.paraphrases, expected.paraphrases)
-    let failed_paraphrasesParenthesized = !matching(result.paraphrasesParenthesized, expected.paraphrasesParenthesized)
+    const expectedObjects = sortJson(convertToStable(expected.objects), { depth: 25 })
+    delete expectedObjects.nameToUUID
+    const actualObjects = sortJson(convertToStable(config.config.objects), { depth: 25 })
+    const failedParaphrases = !matching(result.paraphrases, expected.paraphrases)
+    let failedParaphrasesParenthesized = !matching(result.paraphrasesParenthesized, expected.paraphrasesParenthesized)
     let failed_generatedParenthesized = !matching(result.generatedParenthesized, expected.generatedParenthesized)
     // TODO fix the naming conventions: camelcase + use actual instead of result
-    const failed_responses = !matching(result.responses, expected.responses)
-    const failed_contexts = !matching(result.contexts, expected.contexts)
-    const failed_objects = !matching(actual_objects, expected_objects)
+    const failedResponses = !matching(result.responses, expected.responses)
+    const failedContexts = !matching(result.contexts, expected.contexts)
+    const failedObjects = !matching(actualObjects, expectedObjects)
 
     if (args.testNoParenthesized) {
-      failed_paraphrasesParenthesized = false
+      failedParaphrasesParenthesized = false
       failed_generatedParenthesized = false
     }
 
-    const pickedResultContexts = result.contexts.map(pickContext(testConfig))
-    const pickedExpectedContexts = expected.contexts.map(pickContext(testConfig))
+    const contextChecks = config.getContextChecks()
+    const pickedResultContexts = result.contexts.map(pickContext(contextChecks))
+    const pickedExpectedContexts = expected.contexts.map(pickContext(contextChecks))
     const failedCheckedContexts = !matching(pickedResultContexts, pickedExpectedContexts)
 
     const expectedGetObjects = (name) => {
@@ -525,23 +531,23 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
       }
       return expected.objects.namespaced[expected.objects.nameToUUID[name]] || {}
     }
-    const expected_checked = sortJson(pickObjects(config, testConfig, expectedGetObjects), { depth: 25 })
+    const expectedChecked = sortJson(pickObjects(config, testConfig, expectedGetObjects), { depth: 25 })
     const actualGetObjects = (name) => {
       if (!name) {
         name = config.name
       }
-      const km = config.configs.find((km) => km.name == name)
+      const km = config.configs.find((km) => km.name === name)
       return config.config.objects.namespaced[km.uuid] || {}
     }
-    const actual_checked = sortJson(pickObjects(config, testConfig, actualGetObjects), { depth: 25 })
-    const failed_checked = !matching(actual_objects, expected_objects)
+    const actualChecked = sortJson(pickObjects(config, testConfig, actualGetObjects), { depth: 25 })
+    const failedChecked = !matching(actualObjects, expectedObjects)
 
-    const failed_checks = !matching(actual_objects, expected_objects)
-    const failed_checked_objects = !matching(actual_checked, expected_checked)
-    const actual_config = sortJson(convertToStable(getConfigForTest(config, testConfig)), { depth: 25 })
-    const expected_config = sortJson(convertToStable(expected.config), { depth: 25 })
-    const failed_config = !matching(actual_config, expected_config)
-    const failed = failed_checked_objects || failed_paraphrases || failed_paraphrasesParenthesized || failed_generatedParenthesized || failed_responses || failed_contexts || failed_objects || failed_config || failed_checked || failedCheckedContexts
+    const failedChecks = !matching(actualObjects, expectedObjects)
+    const failedChecked_objects = !matching(actualChecked, expectedChecked)
+    const actualConfig = sortJson(convertToStable(getConfigForTest(config, testConfig)), { depth: 25 })
+    const expectedConfig = sortJson(convertToStable(expected.config), { depth: 25 })
+    const failedConfig = !matching(actualConfig, expectedConfig)
+    const failed = failedChecked_objects || failedParaphrases || failedParaphrasesParenthesized || failed_generatedParenthesized || failedResponses || failedContexts || failedObjects || failedConfig || failedChecked || failedCheckedContexts
 
     if (expected.metadata && result.metadata && failed) {
       const priorities = analyzeMetaData(expected.metadata, result.metadata)
@@ -559,9 +565,9 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
           paraphrasesParenthesized: expected.paraphrasesParenthesized,
           generatedParenthesized: expected.generatedParenthesized,
           results: expected.contexts,
-          checked: expected_checked,
+          checked: expectedChecked,
           checkedContexts: pickedExpectedContexts,
-          objects: expected_objects,
+          objects: expectedObjects,
           config: expected.config,
           summaries: expected.summaries
         },
@@ -571,10 +577,10 @@ const runTest = async (config, expected, { args, verbose, testConfig, debug, tim
           paraphrasesParenthesized: result.paraphrasesParenthesized,
           generatedParenthesized: result.generatedParenthesized,
           results: result.contexts,
-          checked: actual_checked,
+          checked: actualChecked,
           checkedContexts: pickedResultContexts,
-          objects: actual_objects,
-          config: actual_config,
+          objects: actualObjects,
+          config: actualConfig,
           summaries: result.summaries
         }
       }
@@ -624,7 +630,7 @@ const saveTest = async (testFile, config, test, expected, testConfig, saveDevelo
   await config.rebuild()
   const objects = getObjects(config.config.objects)(config.uuid)
   console.log(test)
-  const result = await _process(config, test, { isTest: true })
+  const result = await _process(config, test, { isTest: true, isProcess: true, isModule: true })
   // const actualObjects = config.config.objects
   const actualConfig = getConfigForTest(config, testConfig)
   const args = {
@@ -667,6 +673,7 @@ const showInfo = (description, section, config) => {
   console.log(JSON.stringify(config.getInfo(), null, 2))
 }
 
+/*
 const submitBugToAPI = async (subscription_id, subscription_password, config) => {
   console.log('********* Submitting bug *********')
   const body = { description: config.config.description, config: config.config }
@@ -689,7 +696,6 @@ const submitBugToAPI = async (subscription_id, subscription_password, config) =>
     })
 }
 
-/*
 const submitBug = async (subscription_id, subscription_password, config, utterance, retries = 2) => {
   // TODO remove these from the config
   const properties = ['expected_contexts', 'expected_generated']
@@ -862,7 +868,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
 
   if (responses.explain_priorities) {
     console.log('Explain Priorities')
-    for ([inputs, output, reason] of responses.explain_priorities) {
+    for (const [inputs, output, reason] of responses.explain_priorities) {
       console.log(`    inputs: ${JSON.stringify(inputs)} output: ${JSON.stringify(output)}  reason: ${reason}`)
     }
   }
@@ -871,7 +877,7 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     if (!name) {
       name = config.name
     }
-    const km = config.configs.find((km) => km.name == name)
+    const km = config.configs.find((km) => km.name === name)
     return config.config.objects.namespaced[km.uuid] || {}
   }
   const picked = sortJson(pickObjects(config, config.testConfig, actualGetObjects), { depth: 25 })
@@ -880,7 +886,8 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
     console.log(JSON.stringify(picked, null, 2))
   }
 
-  const pickedResultContexts = responses.contexts.map(pickContext(config.testConfig))
+  const contextChecks = config.getContextChecks()
+  const pickedResultContexts = responses.contexts.map(pickContext(contextChecks))
   if (pickedResultContexts.some((context) => Object.keys(context).length > 0)) {
     console.log('--- Contexts showing only the checked values ---')
     console.log(JSON.stringify(pickedResultContexts, null, 2))
@@ -967,7 +974,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
       }
       config.config.skipSemantics = skipSemantics && !isFragment
       const transitoryMode = global.transitoryMode
-      if (isFragment || property == 'fragments') {
+      if (isFragment || property === 'fragments') {
         global.transitoryMode = true
       }
       if (hierarchy) {
@@ -991,7 +998,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
             console.log(getSuggestionMessage(suggestion))
           }
         }
-        if (!rebuild && previousResults && previousResults.query == query.query) {
+        if (!rebuild && previousResults && previousResults.query === query.query) {
           results = previousResults
           prMessage = ' Using previous results. use -rtf for a hard rebuild of everything on the server side.'
           await loadInstance(config, { resultss: [results] })
@@ -1139,7 +1146,7 @@ const rebuildTemplate = async ({ config, instance, target, previousResultss, reb
     for (let configsIndex = 0; configsIndex < template.configs.length; ++configsIndex) {
       const query = template.configs[configsIndex]
       // account for things for example associations being added to the config while debugginer
-      const pr = previousResultss && previousResultss.find((pr) => pr.query == query)
+      const pr = previousResultss && previousResultss.find((pr) => pr.query === query)
       todo.push({ property: 'resultss', query, previousResults: pr, skipSemantics: false || query.skipSemantics })
     }
   }
@@ -1309,7 +1316,7 @@ const knowledgeModuleImpl = async ({
       parser.add_argument('--parenthesized', { action: 'store_true', help: 'Show the generated phrases with parenthesis.' })
       parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
       parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
-      parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a == associations j == JSON sent to server. for example --print wb' })
+      parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a === associations j === JSON sent to server. for example --print wb' })
       parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
       parser.add_argument('-fr', '--failRebuild', { action: 'store_true', help: 'If a rebuild is required fail out.' })
       parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
@@ -1361,10 +1368,10 @@ const knowledgeModuleImpl = async ({
             if (what.length !== 2) {
               return false
             }
-            if (!typeof what[0] == 'string') {
+            if (!typeof what[0] === 'string') {
               return false
             }
-            if (!typeof what[1] == 'number') {
+            if (!typeof what[1] === 'number') {
               return false
             }
             return true
@@ -1682,9 +1689,9 @@ const knowledgeModuleImpl = async ({
                       const tidy_summaries = (summaries) => {
                         const summaries_prime = []
                         for (const chunk of summaries) {
-                          chunk_prime = { length: chunk.length, summaries: [] }
+                          const chunk_prime = { length: chunk.length, summaries: [] }
                           for (const summary of chunk.summaries) {
-                            const summary_prime = { counter: summary.counter, operators: [] } 
+                            const summary_prime = { counter: summary.counter, operators: [] }
                             for (const operator of summary.operators) {
                               summary_prime.operators.push(`{ marker: [${operator.marker[0]}, ${operator.marker[1]}] range: { start: ${operator.range.start}, end: ${operator.range.end} } }`)
                             }
@@ -1716,7 +1723,7 @@ const knowledgeModuleImpl = async ({
                               if (lengths > iDiff) {
                                 const suggestion = suggestAssociationsFix(result.expected.summaries[iSummaries].summaries, result.actual.summaries[iSummaries].summaries)
                                 if (suggestion) {
-                                  suggestedFix = suggestion
+                                  const suggestedFix = suggestion
                                   // console.log(`Try adding this to the associations: { context: ${JSON.stringify(getSuggestion(suggestedFix))}, choose: <indexOfMainElement> },`)
                                   console.log(getSuggestionMessage(suggestedFix))
                                   return
@@ -1856,7 +1863,7 @@ const knowledgeModuleImpl = async ({
               f()
             })
             .catch((e) => {
-              if (e.errno == 'ECONNREFUSED') {
+              if (e.errno === 'ECONNREFUSED') {
                 console.log(e)
                 readline.close()
               } else {
@@ -1880,7 +1887,7 @@ const knowledgeModuleImpl = async ({
           global.beforeObjects = _.cloneDeep(objects)
         }
         try {
-          await processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
+          await processResults(_process(config, args.query, { commandLineArgs: args, isProcess, isModule: !isProcess, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
         } catch (error) {
           console.log('Error', error)
         }
@@ -1909,8 +1916,18 @@ const knowledgeModuleImpl = async ({
           loadForTesting = true
         }
       }
+      if (template) {
+        try {
+          await config.load(rebuildTemplate, template.template, template.instance)
+        } catch (e) {
+          errorHandler(e)
+        }
+      }
+
       // remove test only stuff
       if (!isProcess && !loadForTesting) {
+        config.removeDevelopmentElements(config.config)
+        /*
         config.config.operators = config.config.operators.filter((operator) => {
           if (operator.development) {
             return false
@@ -1925,21 +1942,14 @@ const knowledgeModuleImpl = async ({
             return true
           }
         })
-      }
-
-      if (template) {
-        try {
-          await config.load(rebuildTemplate, template.template, template.instance)
-        } catch (e) {
-          errorHandler(e)
-        }
+        */
       }
     }
 
     // no cache 21 minutes + rebuild fails "node tester_rebuild -m colors"
     // cache okay
-    createConfigExport = async () => {
-      if (createConfig.cached) {
+    const createConfigExport = async (useCache = true) => {
+      if (useCache && createConfig.cached) {
         return createConfig.cached
       }
       const config = await createConfig()

package/demo.js

@@ -34,13 +34,13 @@ let config = {
       apply: ({ context }) => context.evalue
     },
     {
-      match: ({ context }) => context.marker == 'unknown',
+      match: ({ context }) => context.marker === 'unknown',
       apply: ({ context }) => context.text
     }
   ],
   semantics: [
     {
-      match: ({ context }) => context.marker == 'went',
+      match: ({ context }) => context.marker === 'went',
       apply: ({ context }) => {
         // same way as kms that I wrote. you dont have to do this though
         context.isResponse = true

package/package.json

@@ -53,6 +53,7 @@
     "src/digraph_internal.js",
     "src/generators.js",
     "src/project.js",
+    "src/project2.js",
     "src/semantics.js"
   ],
   "author": "dev@thinktelligence.com",
@@ -69,6 +70,6 @@
     "sort-json": "^2.0.0",
     "uuid": "^8.3.2"
   },
-  "version": "9.3.0",
+  "version": "9.4.1",
   "license": "UNLICENSED"
 }