theprogrammablemind 7.5.8 → 7.6.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/client.js CHANGED
@@ -2,6 +2,7 @@ const { Semantics, Semantic } = require('./src/semantics')
2
2
  const { Generators, Generator } = require('./src/generators')
3
3
  const DigraphInternal = require('./src/digraph_internal')
4
4
  const Digraph = require('./src/digraph')
5
+ const { project } = require('./src/project')
5
6
  const fetch = require('node-fetch')
6
7
  const base64 = require('base-64')
7
8
  const deepEqual = require('deep-equal')
@@ -9,20 +10,38 @@ const _ = require('lodash')
9
10
  const stringify = require('json-stable-stringify')
10
11
  const Lines = require('./lines')
11
12
  const flattens = require('./src/flatten')
12
- const { appendNoDups, InitCalls, updateQueries } = require('./src/helpers')
13
+ const { appendNoDups, InitCalls, updateQueries, safeNoDups } = require('./src/helpers')
13
14
  const runtime = require('./runtime')
14
15
  const sortJson = runtime.sortJson
15
16
 
17
+ const getConfig_getObjectsCheck = (testConfig) => {
18
+ return (testConfig.checks && testConfig.checks.objects) || []
19
+ }
20
+
21
+ const getConfig_getContextCheck = (testConfig) => {
22
+ return (testConfig.checks && testConfig.checks.context) || []
23
+ }
24
+
25
+ const pickContext = (testConfig) => (context) => {
26
+ return project(context, getConfig_getContextCheck(testConfig))
27
+ }
28
+
29
+ const pickObjects = (testConfig, objects) => {
30
+ return project(objects, getConfig_getObjectsCheck(testConfig))
31
+ }
32
+
16
33
  const getAsk = (config) => (uuid) => (asks) => {
17
34
  for (let ask of asks) {
18
- config.addMotivation({
35
+ config.addSemantic({
19
36
  uuid,
37
+ oneShot: true,
20
38
  match: (args) => ask.matchr(args),
21
39
  apply: (args) => ask.applyr(args)
22
40
  })
23
41
  }
24
- config.addMotivation({
42
+ config.addSemantic({
25
43
  uuid,
44
+ oneShot: true,
26
45
  match: ({context}) => context.marker == 'controlEnd' || context.marker == 'controlBetween',
27
46
  apply: (args) => {
28
47
  for (let ask of asks) {
@@ -37,7 +56,7 @@ const getAsk = (config) => (uuid) => (asks) => {
37
56
  }
38
57
  }
39
58
  if (matchq(args)) {
40
- args.context.motivationKeep = true
59
+ // args.context.motivationKeep = true
41
60
  args.context.verbatim = applyq(args)
42
61
  args.context.isResponse = true;
43
62
  delete args.context.controlRemove;
@@ -49,6 +68,12 @@ const getAsk = (config) => (uuid) => (asks) => {
49
68
  })
50
69
  }
51
70
 
71
+ const sameJSON = (json1, json2) => {
72
+ const sjson1 = sortJson(json1, { depth: 25 })
73
+ const sjson2 = sortJson(json2, { depth: 25 })
74
+ return JSON.stringify(sjson1) == JSON.stringify(sjson2)
75
+ }
76
+
52
77
  const vimdiff = (actualJSON, expectedJSON) => {
53
78
  const path = '.'
54
79
  const actual = sortJson(actualJSON, { depth: 25 })
@@ -58,10 +83,9 @@ const vimdiff = (actualJSON, expectedJSON) => {
58
83
  // console.log(`vimdiff ${path}/actual.json ${path}/expected.json`)
59
84
  {
60
85
  const editor = runtime.process.env.EDITOR || 'vimdiff'
61
- const child = runtime.child_process.spawn(editor, [`${path}/expected.json`, `${path}/actual.json`], { stdio: 'inherit' })
62
- child.on('exit', function (e, code) {
63
- console.log('finished')
64
- })
86
+ // const child = runtime.child_process.spawn(editor, [`${path}/expected.json`, `${path}/actual.json`], { stdio: 'inherit' })
87
+ console.log(`${editor} ${path}/expected.json ${path}/actual.json`)
88
+ runtime.child_process.execSync(`${editor} ${path}/expected.json ${path}/actual.json`, {stdio: 'inherit'})
65
89
  }
66
90
  }
67
91
 
@@ -100,10 +124,24 @@ const asList = (context) => {
100
124
  }
101
125
  }
102
126
 
127
+ class ErrorReason extends Error {
128
+ constructor(context) {
129
+ super(JSON.stringify(context))
130
+ this.reason = context
131
+ }
132
+ }
133
+
103
134
  const setupArgs = (args, config, logs, hierarchy) => {
104
135
  config.setArgs(args)
105
- args.calls = new InitCalls()
136
+ args.calls = new InitCalls(args.isInstance ? `${args.isInstance}#${config.name}` : config.name)
137
+ if (global.theprogrammablemind && global.theprogrammablemind.loadForTesting) {
138
+ args.calls = new InitCalls(Object.keys(global.theprogrammablemind.loadForTesting)[0])
139
+ }
106
140
  args.km = (name) => config.getConfig(name)
141
+ args.api = (name) => config.getConfig(name).api
142
+ args.error = (context) => {
143
+ throw new ErrorReason(context)
144
+ }
107
145
  args.kms = config.getConfigs()
108
146
  args.config = config
109
147
  args.hierarchy = hierarchy
@@ -111,6 +149,7 @@ const setupArgs = (args, config, logs, hierarchy) => {
111
149
  args.listable = listable(hierarchy)
112
150
  args.asList = asList
113
151
  args.retry = () => { throw new RetryError() }
152
+ args.fragments = (query) => config.fragment(query)
114
153
  const scopedAsk = getAsk(config)
115
154
 
116
155
  const getAPI = (uuid) => {
@@ -130,7 +169,6 @@ const setupArgs = (args, config, logs, hierarchy) => {
130
169
  apis: getAPIs(uuid)
131
170
  }
132
171
  }
133
- args.motivation = (m) => config.addMotivation(m)
134
172
  args.breakOnSemantics = false
135
173
  args.theDebugger = {
136
174
  breakOnSemantics: (value) => args.breakOnSemantics = value
@@ -139,12 +177,15 @@ const setupArgs = (args, config, logs, hierarchy) => {
139
177
  args.g = (c) => config.getGenerators(logs).apply(args, c)
140
178
  args.gp = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: true, isResponse: false, response: false})
141
179
  args.gr = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: false, isResponse: true })
142
- args.e = (c) => config.getEvaluator(args.s, logs, c)
180
+ if (!logs) {
181
+ debugger
182
+ }
183
+ args.e = (c) => config.getEvaluator(args.s, args.calls, logs, c)
143
184
  args.log = (message) => logs.push(message)
144
- // config.getAddedArgs(args)
145
185
  args.gs = gs(args.g)
146
186
  args.gsp = gs(args.gp)
147
187
  args.gsr = gs(args.gr)
188
+ config.getAddedArgs(args)
148
189
  }
149
190
 
150
191
  const gs = (g) => (contexts, separator, lastSeparator) => {
@@ -272,7 +313,7 @@ const writeTestFile = (fn, tests) => {
272
313
  runtime.fs.writeFileSync(fn, stringify(tests, { space: 2 }), { encoding: 'utf8', flag: 'w+' })
273
314
  }
274
315
 
275
- const writeTest = (fn, query, objects, generated, paraphrases, responses, contexts, associations, metadata, config, saveDeveloper) => {
316
+ const writeTest = (fn, query, objects, generated, paraphrases, responses, contexts, associations, metadata, config, saveDeveloper, paraphrasesParenthesized, generatedParenthesized) => {
276
317
  let tests = []
277
318
  if (runtime.fs.existsSync(fn)) {
278
319
  tests = JSON.parse(runtime.fs.readFileSync(fn))
@@ -282,7 +323,18 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
282
323
  }
283
324
  associations.sort()
284
325
  // tests[query] = sortJson({ paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
285
- results = sortJson({ query, paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
326
+ results = sortJson({
327
+ query,
328
+ paraphrases,
329
+ responses,
330
+ contexts,
331
+ objects: convertToStable(objects),
332
+ associations,
333
+ metadata,
334
+ config,
335
+ developerTest: saveDeveloper,
336
+ paraphrasesParenthesized,
337
+ generatedParenthesized }, { depth: 25 })
286
338
  let wasSet = false;
287
339
  tests.forEach( (test, index) => {
288
340
  if (test.query == query) {
@@ -339,16 +391,21 @@ const setupContexts = (rawContexts) => {
339
391
  return contexts
340
392
  }
341
393
 
342
- const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, query, data, retries, url, commandLineArgs }) => {
394
+ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, isInstance, instance, query, data, retries, url, commandLineArgs }) => {
343
395
  // TODO fix this name to contextsPrime
344
396
  const contextsPrime = []
345
397
  const generatedPrime = []
346
398
  const paraphrasesPrime = []
399
+ const paraphrasesParenthesizedPrime = []
400
+ const generatedParenthesizedPrime = []
347
401
  const responsesPrime = []
348
402
  const contexts = setupContexts(json.contexts)
349
403
 
350
404
  const objects = config.get('objects')
351
- const args = { objects, isResponse: true, response: json, isTest, getObjects: getObjects(objects) }
405
+ const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance }
406
+ if (!json.logs) {
407
+ json.logs = []
408
+ }
352
409
  setupArgs(args, config, json.logs, hierarchy)
353
410
  const toDo = [...contexts]
354
411
  args.insert = (context) => toDo.unshift(context)
@@ -356,28 +413,6 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
356
413
  config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
357
414
  while (toDo.length > 0) {
358
415
  const context = toDo.shift()
359
- /*
360
- if (false && query) {
361
- if (config.wasChanged()) {
362
- // process contexts that overlap
363
- overlap = lastRange
364
- } else {
365
- config.watch()
366
- }
367
- if (overlap) {
368
- if (overlaps(overlap, context)) {
369
- // okay
370
- query = query.slice(overlap.end+1)
371
- data.utterance = query
372
- const json = await doWithRetries(retries, url, data)
373
- toDo = setupContexts(json.contexts).slice(1) // take off the start context
374
- }
375
- overlap = undefined
376
- }
377
- lastRange = context.range
378
- }
379
- */
380
-
381
416
  args.calls.next()
382
417
  let contextPrime = context
383
418
  context.topLevel = true
@@ -385,25 +420,40 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
385
420
  if (json.has_errors) {
386
421
  throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
387
422
  }
423
+ const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
388
424
  if (!config.get('skipSemantics')) {
389
- if (!config.doMotivations(args, context)) {
390
- const semantics = config.getSemantics(json.logs)
391
- try {
392
- contextPrime = semantics.apply(args, context)
393
- } catch( e ) {
394
- if (e.message == 'Maximum call stack size exceeded') {
395
- const mostCalled = semantics.getMostCalled()
396
- e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
397
- }
398
- throw e;
425
+ const semantics = config.getSemantics(json.logs)
426
+ try {
427
+ contextPrime = semantics.apply(args, context)
428
+ } catch( e ) {
429
+ if (e.message == 'Maximum call stack size exceeded') {
430
+ const mostCalled = semantics.getMostCalled()
431
+ e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
432
+ }
433
+ // contextPrime = semantics.apply(args, { marker: 'error', context, error: e })
434
+ if (isInstance) {
435
+ console.log('error', e.error)
399
436
  }
437
+ contextPrime = semantics.apply(args, {
438
+ marker: 'error',
439
+ context,
440
+ text: e ? e.toString() : 'not available',
441
+ reason: e.reason,
442
+ error: e.stack || e.error
443
+ })
400
444
  }
401
445
  }
402
446
  if (contextPrime.controlRemove) {
403
447
  continue
404
448
  }
405
449
  let assumed = { isResponse: true };
406
- const generated = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
450
+ const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
451
+ let generatedParenthesized = []
452
+ if (generateParenthesized) {
453
+ config.parenthesized = true
454
+ generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
455
+ config.parenthesized = false
456
+ }
407
457
  // assumed = { paraphrase: true, response: false };
408
458
  assumed = { paraphrase: true };
409
459
  args.g = (c) => config.getGenerators(json.logs).apply(args, c, assumed)
@@ -412,7 +462,16 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
412
462
  args.gs = gs(args.g)
413
463
  args.gsp = gs(args.gsp)
414
464
  args.gsr = gs(args.gr)
465
+ if (generateParenthesized) {
466
+ config.parenthesized = false
467
+ }
415
468
  const paraphrases = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
469
+ let paraphrasesParenthesized = []
470
+ if (generateParenthesized) {
471
+ config.parenthesized = true
472
+ paraphrasesParenthesized = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
473
+ config.parenthesized = false
474
+ }
416
475
  args.g = (c) => config.getGenerators(json.logs).apply(args, c)
417
476
  args.gp = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: true, isResponse: false, response: false })
418
477
  args.gr = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: false })
@@ -422,6 +481,10 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
422
481
  contextsPrime.push(contextPrime)
423
482
  generatedPrime.push(generated)
424
483
  paraphrasesPrime.push(paraphrases)
484
+ if (generateParenthesized) {
485
+ paraphrasesParenthesizedPrime.push(paraphrasesParenthesized)
486
+ generatedParenthesizedPrime.push(generatedParenthesized)
487
+ }
425
488
  if (contextPrime.isResponse) {
426
489
  responsesPrime.push(generated)
427
490
  } else {
@@ -431,7 +494,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
431
494
  // add results to processed list
432
495
  config.config.objects.processed = config.config.objects.processed || []
433
496
  config.config.objects.processed = config.config.objects.processed.slice(0, 5)
434
- config.config.objects.processed.unshift({ context: contextPrime, paraphrases: paraphrases, responses: responsesPrime })
497
+ config.config.objects.processed.unshift({ context: contextPrime, paraphrases: paraphrases, paraphrasesParenthesized, generatedParenthesized, responses: responsesPrime })
435
498
  } catch (e) {
436
499
  if (Array.isArray(e)) {
437
500
  e = {
@@ -451,7 +514,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
451
514
  throw e
452
515
  }
453
516
  }
454
- return { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime }
517
+ return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
455
518
  }
456
519
 
457
520
  const doWithRetries = async (n, url, queryParams, data) => {
@@ -473,17 +536,17 @@ const doWithRetries = async (n, url, queryParams, data) => {
473
536
  }
474
537
  if (result.status === 504) {
475
538
  if (n === 0) {
476
- throw `Error ${result.status} - ${result.statusText}`
539
+ throw new Error(`Error ${result.status} - ${result.statusText}`)
477
540
  } else {
478
541
  continue
479
542
  }
480
543
  }
481
544
  if (result.status >= 500 && result.status < 600) {
482
- throw `Error ${result.status} - ${result.statusText}.`
545
+ throw new Error(`Error ${result.status} - ${result.statusText}.`)
483
546
  } if (result.status >= 404) {
484
- throw `Error ${result.status} - ${result.statusText} - Trying it connect to ${url}`
547
+ throw new Error(`Error ${result.status} - ${result.statusText} - Trying it connect to ${url}`)
485
548
  } else {
486
- throw `Error ${result.status} - ${result.statusText}`
549
+ throw new Error(`Error ${result.status} - ${result.statusText}`)
487
550
  }
488
551
  }
489
552
  }
@@ -496,8 +559,11 @@ const setupProcessB = ({ config, initializer, allowDelta=false } = {}) => {
496
559
  // console.log('config', config)
497
560
  data.delta = config.delta()
498
561
  } else {
499
- Object.assign(data, config.config)
562
+ config.toData(data)
563
+ // Object.assign(data, config.config)
500
564
  }
565
+
566
+ // config.toServer(data)
501
567
 
502
568
  if (data.namespaces) {
503
569
  for (const uuid of Object.keys(data.namespaces)) {
@@ -520,10 +586,22 @@ const setupProcessB = ({ config, initializer, allowDelta=false } = {}) => {
520
586
  }
521
587
  }
522
588
 
523
- // instance template
524
- const processInstance = (config, instance) => {
589
+ // instance template loadTemplate
590
+ const loadInstance = (config, instance) => {
525
591
  const transitoryMode = global.transitoryMode
526
592
  global.transitoryMode = false
593
+
594
+ if (instance && (instance.associations || instance.learned_contextual_priorities)) {
595
+ if (!config.config.retrain) {
596
+ if (instance.associations) {
597
+ config.addAssociations(instance.associations)
598
+ }
599
+ if (instance.learned_contextual_priorities && instance.learned_contextual_priorities.length > 0) {
600
+ config.addPriorities(instance.learned_contextual_priorities)
601
+ }
602
+ }
603
+ }
604
+
527
605
  const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
528
606
  // for (const results of (instance.resultss || [])) {
529
607
  for (const i in (instance.resultss || [])) {
@@ -531,15 +609,21 @@ const processInstance = (config, instance) => {
531
609
  if (results.extraConfig) {
532
610
  // config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
533
611
  // config.addInternal(config.template.queries[i], { handleCalculatedProps: true } )
534
- config.addInternal(instance.template.queries[i], { handleCalculatedProps: true } )
612
+ config.addInternal(instance.template.queries[i], { addFirst: true, handleCalculatedProps: true } )
535
613
  } else {
536
- processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {} })
614
+ if (results.skipSemantics) {
615
+ config.config.skipSemantics = results.skipSemantics
616
+ }
617
+ processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}`, instance: instance.queries[i] })
618
+ if (results.skipSemantics) {
619
+ config.config.skipSemantics = null
620
+ }
537
621
  }
538
622
  }
539
623
  global.transitoryMode = transitoryMode
540
624
  }
541
625
 
542
- const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
626
+ const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
543
627
  if (credentials) {
544
628
  config.server(credentials.server, credentials.key)
545
629
  }
@@ -553,7 +637,6 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
553
637
  if (writeTests) {
554
638
  config.rebuild()
555
639
  const objects = getObjects(config.config.objects)(config.uuid)
556
- config.beforeQuery({ query, isModule: false, objects })
557
640
  }
558
641
  } catch(error) {
559
642
  throw error
@@ -561,7 +644,10 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
561
644
 
562
645
  let { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config, initializer, allowDelta: true })
563
646
  if (commandLineArgs && commandLineArgs.checkForLoop) {
564
- data.checkForLoop = true
647
+ data.checkForLoop = commandLineArgs.checkForLoop
648
+ }
649
+ if (rebuildingTemplate) {
650
+ data.errors_ignore_contextual_priorities_non_existant_ops = true
565
651
  }
566
652
  let queries = query.split('\\n')
567
653
 
@@ -578,6 +664,8 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
578
664
  contexts: [],
579
665
  generated: [],
580
666
  paraphrases: [],
667
+ paraphrasesParenthesized: [],
668
+ generatedParenthesized: [],
581
669
  responses: [],
582
670
  associations: [],
583
671
  }
@@ -613,9 +701,10 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
613
701
  if (json.status !== 200) {
614
702
  throw json
615
703
  } else {
616
- const { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime } =
704
+ const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
617
705
  processContextsB({ isTest, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
618
706
  response.associations = json.associations
707
+ response.learned_contextual_priorities = json.learned_contextual_priorities
619
708
  response.hierarchy = json.hierarchy
620
709
  response.load_cache_time += json.load_cache_time
621
710
  appendNoDups(response.logs, json.logs)
@@ -626,10 +715,13 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
626
715
  response.times += json.times
627
716
  response.trace = response.trace.concat(json.trace)
628
717
  response.version = json.version
718
+ response.explain_priorities = json.explain_priorities
629
719
 
630
720
  response.contexts = response.contexts.concat(contextsPrime)
631
721
  response.generated = response.generated.concat(generatedPrime)
632
722
  response.paraphrases = response.paraphrases.concat(paraphrasesPrime)
723
+ response.paraphrasesParenthesized = response.paraphrasesParenthesized.concat(paraphrasesParenthesizedPrime)
724
+ response.generatedParenthesized = response.generatedParenthesized.concat(generatedParenthesizedPrime)
633
725
  response.responses = response.responses.concat(responsesPrime)
634
726
  queries = queries.slice(1)
635
727
  }
@@ -637,13 +729,18 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
637
729
 
638
730
  if (writeTests) {
639
731
  const actual_config = getConfigForTest(config, testConfig)
640
- writeTest(testsFN, query, config.config.objects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper)
732
+ const saveObjects = {...config.config.objects}
733
+ saveObjects.nameToUUID = {}
734
+ for (let km of config.configs) {
735
+ saveObjects.nameToUUID[km.name] = km.uuid
736
+ }
737
+ writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized)
641
738
  }
642
739
 
643
740
  return response
644
741
  } catch(error) {
645
742
  error.query = query
646
- throw error
743
+ errorHandler(error)
647
744
  }
648
745
  }
649
746
 
@@ -677,12 +774,10 @@ const getConfigForTest = (config, testConfig) => {
677
774
  return configForTest
678
775
  }
679
776
 
680
- const runTest = async (config, expected, { verbose, afterTest, testConfig, debug }) => {
777
+ const runTest = async (config, expected, { args, verbose, testConfig, debug }) => {
681
778
  const test = expected.query
682
779
  // initialize in between test so state is not preserved since the test was adding without state
683
780
  config.rebuild()
684
- config.addAssociationsFromTests(config.tests)
685
- // config.addAssocationsFromTests(
686
781
  const errorHandler = (error) => {
687
782
  if (error.metadata) {
688
783
  const priorities = analyzeMetaData(expected.metadata, error.metadata)
@@ -700,8 +795,6 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
700
795
  objects = getObjects(config.config.objects)(config.getConfigs()[testConfig.testModuleName].uuid)
701
796
  testConfigName = testConfig.testModuleName
702
797
  }
703
- config.beforeQuery({ query: test, isModule: false, objects })
704
- // config.resetMotivations()
705
798
  try {
706
799
  const result = await _process(config, test, { errorHandler, isTest: true })
707
800
  result.query = test
@@ -716,66 +809,48 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
716
809
  lines.log()
717
810
  }
718
811
  const expected_objects = sortJson(convertToStable(expected.objects), { depth: 25 })
812
+ delete expected_objects.nameToUUID
719
813
  const actual_objects = sortJson(convertToStable(config.config.objects), { depth: 25 })
720
814
  const failed_paraphrases = !matching(result.paraphrases, expected.paraphrases)
815
+ let failed_paraphrasesParenthesized = !matching(result.paraphrasesParenthesized, expected.paraphrasesParenthesized)
816
+ let failed_generatedParenthesized = !matching(result.generatedParenthesized, expected.generatedParenthesized)
817
+ // TODO fix the naming conventions: camelcase + use actual instead of result
721
818
  const failed_responses = !matching(result.responses, expected.responses)
722
819
  const failed_contexts = !matching(result.contexts, expected.contexts)
723
820
  const failed_objects = !matching(actual_objects, expected_objects)
724
821
 
725
- const pickEm = (getObjects) => {
726
- const picked = {}
727
- for (let prop of (testConfig.check || [])) {
728
- if (prop.km) {
729
- c = config.getConfig(prop.km)
730
- o = getObjects(prop.km)
731
- picked[prop.km] = {}
732
- for (let p of c.testConfig.check) {
733
- if (p.km) {
734
- continue
735
- }
736
- picked[p] = o[p]
737
- }
738
- } else {
739
- picked[prop] = getObjects(testConfigName)[prop]
740
- }
741
- }
742
- return picked
822
+ if (args.testNoParenthesized) {
823
+ failed_paraphrasesParenthesized = false
824
+ failed_generatedParenthesized = false
743
825
  }
826
+
827
+ const pickedResultContexts = result.contexts.map(pickContext(testConfig))
828
+ const pickedExpectedContexts = expected.contexts.map(pickContext(testConfig))
829
+ const failedCheckedContexts = !matching(pickedResultContexts, pickedExpectedContexts)
830
+
744
831
  const expectedGetObjects = (name) => {
745
832
  if (!name) {
746
833
  name = config.name
747
834
  }
748
- return expected.objects.namespaced[expected.objects.nameToUUID[name]]
835
+ return expected.objects.namespaced[expected.objects.nameToUUID[name]] || {}
749
836
  }
750
- const expected_checked = sortJson(pickEm(expectedGetObjects), { depth: 25 })
837
+ const expected_checked = sortJson(pickObjects(testConfig, expectedGetObjects(testConfigName)), { depth: 25 })
751
838
  const actualGetObjects = (name) => {
752
839
  if (!name) {
753
840
  name = config.name
754
841
  }
755
842
  const km = config.configs.find( (km) => km.name == name )
756
- return config.config.objects.namespaced[km.uuid]
843
+ return config.config.objects.namespaced[km.uuid] || {}
757
844
  }
758
- const actual_checked = sortJson(pickEm(actualGetObjects), { depth: 25 })
845
+ const actual_checked = sortJson(pickObjects(testConfig, actualGetObjects(testConfigName)), { depth: 25 })
759
846
  const failed_checked = !matching(actual_objects, expected_objects)
760
847
 
761
848
  const failed_checks = !matching(actual_objects, expected_objects)
849
+ const failed_checked_objects = !matching(actual_checked, expected_checked)
762
850
  const actual_config = sortJson(convertToStable(getConfigForTest(config, testConfig)), { depth: 25 })
763
851
  const expected_config = sortJson(convertToStable(expected.config), { depth: 25 })
764
852
  const failed_config = !matching(actual_config, expected_config)
765
- let failed = failed_paraphrases || failed_responses || failed_contexts || failed_objects || failed_config || failed_checked
766
- if (!failed) {
767
- if (config.afterTest) {
768
- failed = config.afterTest({ query: test, expected, actual: result, config })
769
- if (failed) {
770
- return {
771
- utterance: test,
772
- errorFromAfterTest: failed,
773
- expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, checked: expected_checked, objects: expected_objects, config: expected.config },
774
- actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, checked: actual_checked, objects: actual_objects, config: actual_config }
775
- }
776
- }
777
- }
778
- }
853
+ let failed = failed_checked_objects || failed_paraphrases || failed_paraphrasesParenthesized || failed_generatedParenthesized || failed_responses || failed_contexts || failed_objects || failed_config || failed_checked || failedCheckedContexts
779
854
 
780
855
  if (expected.metadata && result.metadata && failed) {
781
856
  const priorities = analyzeMetaData(expected.metadata, result.metadata)
@@ -787,8 +862,28 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
787
862
  if (failed) {
788
863
  return {
789
864
  utterance: test,
790
- expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, checked: expected_checked, objects: expected_objects, config: expected.config },
791
- actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, checked: actual_checked, objects: actual_objects, config: actual_config }
865
+ expected: {
866
+ responses: expected.responses,
867
+ paraphrases: expected.paraphrases,
868
+ paraphrasesParenthesized: expected.paraphrasesParenthesized,
869
+ generatedParenthesized: expected.generatedParenthesized,
870
+ results: expected.contexts,
871
+ checked: expected_checked,
872
+ checkedContexts: pickedExpectedContexts,
873
+ objects: expected_objects,
874
+ config: expected.config
875
+ },
876
+ actual: {
877
+ responses: result.responses,
878
+ paraphrases: result.paraphrases,
879
+ paraphrasesParenthesized: result.paraphrasesParenthesized,
880
+ generatedParenthesized: result.generatedParenthesized,
881
+ results: result.contexts,
882
+ checked: actual_checked,
883
+ checkedContexts: pickedResultContexts,
884
+ objects: actual_objects,
885
+ config: actual_config
886
+ }
792
887
  }
793
888
  }
794
889
  } catch(error) {
@@ -825,20 +920,16 @@ const runTestsHelper = async (config, tests, failed, juicyBits) => {
825
920
 
826
921
  const runTests = async (config, testFile, juicyBits) => {
827
922
  const tests = JSON.parse(runtime.fs.readFileSync(testFile))
828
- config.beforeTests()
829
923
  if (juicyBits.verbose) {
830
924
  console.log('\n', testFile, '-----------------------------------------------', '\n')
831
925
  }
832
926
  const v = await runTestsHelper(config, [...tests], [], juicyBits)
833
- config.afterTests()
834
927
  return v
835
928
  }
836
929
 
837
930
  const saveTest = async (testFile, config, test, expected, testConfig, saveDeveloper) => {
838
931
  config.rebuild()
839
932
  const objects = getObjects(config.config.objects)(config.uuid)
840
- //config.resetMotivations()
841
- config.beforeQuery({ query: test, isModule: false, objects })
842
933
  console.log(test)
843
934
  const result = await _process(config, test, { isTest: true })
844
935
  // const actualObjects = config.config.objects
@@ -850,7 +941,7 @@ const saveTest = async (testFile, config, test, expected, testConfig, saveDevelo
850
941
  for (let km of config.configs) {
851
942
  saveObjects.nameToUUID[km.name] = km.uuid
852
943
  }
853
- writeTest(testFile, test, saveObjects, result.generated, result.paraphrases, result.responses, result.contexts, result.associations, result.metadata, actualConfig, saveDeveloper)
944
+ writeTest(testFile, test, saveObjects, result.generated, result.paraphrases, result.responses, result.contexts, result.associations, result.metadata, actualConfig, saveDeveloper, result.paraphrasesParenthesized, result.generatedParenthesized)
854
945
  }
855
946
 
856
947
  const saveTestsHelper = async (testFile, config, tests, todo, testConfig, saveDeveloper) => {
@@ -1020,10 +1111,11 @@ const defaultErrorHandler = async (error) => {
1020
1111
  doErrorExit = true
1021
1112
  }
1022
1113
 
1023
- if (doErrorExit) {
1114
+ if (typeof runtime.process.exit == 'function' && doErrorExit) {
1024
1115
  runtime.process.exit(-1)
1025
1116
  }
1026
- // throw error
1117
+
1118
+ throw error
1027
1119
  }
1028
1120
 
1029
1121
  const defaultInnerProcess = (config, errorHandler, responses) => {
@@ -1033,6 +1125,13 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
1033
1125
  }
1034
1126
  console.log("KM's loaded", config.configs.map((c) => c.name))
1035
1127
  console.log('This is the global objects from running semantics:\n', config.objects)
1128
+ if (!_.isEmpty(responses.learned_contextual_priorities)) {
1129
+ console.log('\nThe learned contextual priorties are :\n')
1130
+ for (const lcp of responses.learned_contextual_priorities) {
1131
+ console.log(` ${JSON.stringify(lcp)},\n`)
1132
+ }
1133
+ console.log("\n")
1134
+ }
1036
1135
  if (responses.logs) {
1037
1136
  console.log('Logs')
1038
1137
  responses.logs.forEach((log) => console.log(` ${log}`))
@@ -1044,6 +1143,50 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
1044
1143
  } else {
1045
1144
  console.log('objects', runtime.util.inspect(config.get('objects'), { depth: Infinity, sorted: true }))
1046
1145
  }
1146
+
1147
+ const pickEm = () => {
1148
+ const picked = {}
1149
+ const namespaced = config.get('objects')['namespaced']
1150
+ for (let prop of getConfig_getObjectCheck(config.testConfig)) {
1151
+ if (prop.km) {
1152
+ /*
1153
+ const objects = namespaced[prop.km]]
1154
+ picked[prop.km] = {}
1155
+ for (let p of c.testConfig.check) {
1156
+ if (p.km) {
1157
+ continue
1158
+ }
1159
+ picked[p] = objects[p]
1160
+ }
1161
+ */
1162
+ console.log('TODO implement this if needed')
1163
+ } else {
1164
+ const objects = namespaced[config.uuid]
1165
+ picked[prop] = objects[prop]
1166
+ }
1167
+ }
1168
+ return picked
1169
+ }
1170
+
1171
+ if (responses.explain_priorities) {
1172
+ console.log("Explain Priorities")
1173
+ for ([inputss, outpus, reason] of responses.explain_priorities) {
1174
+ console.log(` ${JSON.stringify(inputss)} reason: ${reason}`)
1175
+ }
1176
+ }
1177
+ const objects = config.get('objects').namespaced[config.uuid]
1178
+ const picked = sortJson(pickObjects(config.testConfig, objects), { depth: 25 })
1179
+ if (!_.isEmpty(picked)) {
1180
+ console.log('--- Object showing only the checked values ---')
1181
+ console.log(JSON.stringify(picked, null, 2))
1182
+ }
1183
+
1184
+ const pickedResultContexts = responses.contexts.map(pickContext(config.testConfig))
1185
+ if (pickedResultContexts.some( (context) => Object.keys(context).length > 0 )) {
1186
+ console.log('--- Contexts showing only the checked values ---')
1187
+ console.log(JSON.stringify(pickedResultContexts, null, 2))
1188
+ }
1189
+
1047
1190
  console.log('--- The contexts are ----------')
1048
1191
  console.log(JSON.stringify(sortJson(responses.contexts, { depth: 25 }), null, 2))
1049
1192
  console.log('')
@@ -1084,37 +1227,14 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
1084
1227
  }
1085
1228
  }
1086
1229
 
1087
- /*
1088
- const kmFileTemplate = (kmBaseName, kmName) =>
1089
- `const entodicton = require('entodicton')
1090
- const base = require('./${kmBaseName}').copy()
1091
- const ${kmName}_tests = require('./${kmName}.test.json')
1092
- const ${kmName}_instance = require('./${kmBaseName}.${kmName}.instance.json')
1093
-
1094
- const config = new entodicton.Config({ name: '${kmName}' })
1095
- config.add(base)
1096
- kirk_instance.base = '${kmBaseName}'
1097
- config.load(${kmName}_instance)
1098
-
1099
- entodicton.knowledgeModule( {
1100
- module,
1101
- description: '${kmName} related concepts',
1102
- section,
1103
- config,
1104
- test: {
1105
- name: './${kmName}.test.json',
1106
- contents: ${kmName}_tests
1107
- },
1108
- })
1109
- `
1110
- */
1111
-
1112
- const build = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
1230
+ // builtTemplate saveInstance
1231
+ const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
1113
1232
  const accumulators = {
1114
1233
  resultss: [],
1115
1234
  fragments: [],
1116
1235
  semantics: [],
1117
1236
  associations: [],
1237
+ learned_contextual_priorities: [],
1118
1238
  }
1119
1239
  const looper = async (queries) => {
1120
1240
  if (queries.length === 0) {
@@ -1133,8 +1253,6 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
1133
1253
  if (property == 'fragments') {
1134
1254
  global.transitoryMode = true
1135
1255
  }
1136
- // greg32
1137
- // config.addInternal( query )
1138
1256
  if (hierarchy) {
1139
1257
  for (let edge of hierarchy) {
1140
1258
  if (Array.isArray(edge)) {
@@ -1144,25 +1262,28 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
1144
1262
  }
1145
1263
  }
1146
1264
  }
1147
-
1148
1265
  try {
1149
- const results = await _process(config, query.query, {initializer})
1266
+ const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
1150
1267
  if (config.config.debug) {
1151
1268
  // TODO pass in the error handler like the other ones
1152
1269
  defaultInnerProcess(config, defaultErrorHandler, results)
1153
1270
  }
1154
1271
  if (results.contexts.length > 1) {
1155
- console.log(`query ${query.query}. There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
1272
+ console.log(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
1273
+ } else if (results.paraphrases[0] != query.query) {
1274
+ console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".`)
1156
1275
  } else {
1157
1276
  console.log(`query ${query.query}`)
1158
1277
  }
1159
1278
  global.transitoryMode = transitoryMode
1160
1279
  config.config.skipSemantics = null
1161
1280
  results.query = query.query
1281
+ results.skipSemantics = skipSemantics
1162
1282
  results.development = query.development
1163
1283
  results.key = { query: query.query, hierarchy }
1164
1284
  accumulators[property].push(results)
1165
1285
  accumulators.associations = accumulators.associations.concat(results.associations)
1286
+ accumulators.learned_contextual_priorities = accumulators.learned_contextual_priorities.concat(results.learned_contextual_priorities)
1166
1287
  await looper(queries)
1167
1288
  } catch(e) {
1168
1289
  const error = { errors: [e], query: query.query };
@@ -1174,6 +1295,12 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
1174
1295
  // it will just get added to the config
1175
1296
  const extraConfig = queryOrExtraConfig
1176
1297
  console.log('config', extraConfig)
1298
+ try {
1299
+ config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
1300
+ } catch ( e ) {
1301
+ const where = extraConfig.where ? ` ${extraConfig.where}` : ''
1302
+ throw new Error(`Error processing extra config${where}: ${e.stack}}`)
1303
+ }
1177
1304
  accumulators[property].push({ extraConfig: true, ...extraConfig })
1178
1305
  await looper(queries)
1179
1306
  }
@@ -1201,6 +1328,7 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
1201
1328
  delete result.version
1202
1329
  result.hierarchy.sort()
1203
1330
  stabilizeAssociations(result.associations)
1331
+ result.learned_contextual_priorities = safeNoDups(result.learned_contextual_priorities)
1204
1332
  }
1205
1333
  }
1206
1334
  }
@@ -1221,14 +1349,14 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
1221
1349
 
1222
1350
  const toProperties = (queryStringOrProperties) => {
1223
1351
  if (typeof queryStringOrProperties == 'string') {
1224
- return { query: queryStringOrProperties}
1352
+ return { query: queryStringOrProperties }
1225
1353
  } else {
1226
1354
  return queryStringOrProperties
1227
1355
  }
1228
1356
  }
1229
1357
  let todo = []
1230
- todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false } }))
1231
- todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false } }))
1358
+ todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
1359
+ todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false || query.skipSemantics} }))
1232
1360
  todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
1233
1361
  todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
1234
1362
  await looper(Object.assign([], todo))
@@ -1238,114 +1366,86 @@ const knowledgeModule = async ({
1238
1366
  module: moduleFromJSFile,
1239
1367
  description,
1240
1368
  section,
1241
- config,
1369
+ // config, createConfig,
1370
+ createConfig,
1371
+ newWay,
1242
1372
  demo,
1243
1373
  test,
1244
1374
  template,
1245
1375
  errorHandler = defaultErrorHandler,
1246
1376
  process: processResults = defaultProcess,
1247
1377
  stopAtFirstFailure = true,
1248
- beforeQuery = () => {},
1249
- beforeTests = () => {},
1250
- afterTests = () => {},
1251
- beforeTest = () => {},
1252
- afterTest = () => {}
1378
+ ...rest
1253
1379
  } = {}) => {
1254
1380
 
1255
- config.beforeQuery = beforeQuery
1256
- config.beforeTests = beforeTests
1257
- config.afterTests = afterTests
1258
- config.beforeTest = beforeTest
1259
- config.afterTest = afterTest
1381
+ const unknownArgs = Object.keys(rest)
1382
+ if (unknownArgs.length > 0) {
1383
+ throw new Error(`Unknown arguments to knowledgeModule: ${unknownArgs.join()}`)
1384
+ }
1260
1385
 
1261
1386
  const testConfig = test
1262
1387
 
1263
1388
  if (!moduleFromJSFile) {
1264
- throw "'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module."
1389
+ throw new Error("'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module.")
1265
1390
  }
1266
- if (!config) {
1267
- throw "'config' is a required parameter. The value should the config that defines the knowledge module."
1268
- }
1269
- if (!config.name) {
1270
- throw "config must have 'name' set to the knowledge module name."
1391
+ // if (!config && !createConfig) {
1392
+ if (!createConfig) {
1393
+ throw new Error("'config' or 'createConfig' is a required parameter. The value should the config that defines the knowledge module.")
1271
1394
  }
1272
1395
  if (!description) {
1273
- throw "'description' is a required parameter. The value should the description of the knowledge module."
1396
+ throw new Error("'description' is a required parameter. The value should the description of the knowledge module.")
1274
1397
  }
1275
1398
  if (!test) {
1276
- throw "'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }."
1399
+ throw new Error("'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }.")
1277
1400
  }
1278
1401
 
1279
1402
  const isProcess = require.main === moduleFromJSFile
1280
- let loadForTesting = false
1281
- if (global.theprogrammablemind) {
1282
- if (global.theprogrammablemind.loadForTesting[config.name]) {
1283
- loadForTesting = true
1403
+
1404
+ const setupConfig = (config) => {
1405
+ if (!config.name) {
1406
+ throw new Error("config must have 'name' set to the knowledge module name.")
1284
1407
  }
1285
- }
1286
1408
 
1287
- // remove test only stuff
1288
- if (!isProcess && !loadForTesting) {
1289
- config.config.operators = config.config.operators.filter( (operator) => {
1290
- if (operator.development) {
1291
- return false
1292
- } else {
1293
- return true
1409
+ config.description = description
1410
+ if (typeof test === 'object') {
1411
+ if (test.contents) {
1412
+ config.tests = test.contents
1413
+ test = test.name
1294
1414
  }
1295
- })
1296
- config.config.bridges = config.config.bridges.filter( (bridge) => {
1297
- if (bridge.development) {
1298
- return false
1299
- } else {
1300
- return true
1301
- }
1302
- })
1303
- }
1304
-
1305
- let module
1306
- if (_.isFunction(moduleFromJSFile)) {
1307
- module = moduleFromJSFile
1308
- } else {
1309
- module = () => {
1310
- config.rebuild({ isModule: true })
1311
- moduleFromJSFile.exports = config
1312
- }
1313
- }
1314
- processResults = processResults({ config, errorHandler })
1315
- config.description = description
1316
- config.demo = demo
1317
- if (typeof test === 'object') {
1318
- if (test.contents) {
1319
- config.tests = test.contents
1320
- test = test.name
1321
- }
1322
- } else {
1323
- if (runtime.fs && runtime.fs.existsSync(test)) {
1324
- config.tests = JSON.parse(runtime.fs.readFileSync(test))
1325
1415
  } else {
1326
- config.tests = {}
1327
- }
1328
- }
1329
- config.setTestConfig(testConfig)
1330
-
1331
- if (!isProcess) {
1332
- if (template) {
1333
- if (config.needsRebuild(template.template, template.instance)) {
1334
- throw `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguements or the -rt argument to rebuild.`
1416
+ if (runtime.fs && runtime.fs.existsSync(test)) {
1417
+ config.tests = JSON.parse(runtime.fs.readFileSync(test))
1418
+ } else {
1419
+ config.tests = []
1335
1420
  }
1336
- config.load(template.template, template.instance)
1337
1421
  }
1422
+ config.setTestConfig(testConfig)
1338
1423
  }
1424
+
1425
+
1339
1426
  if (isProcess) {
1427
+ const config = createConfig()
1428
+ setupConfig(config)
1429
+ processResults = processResults({ config, errorHandler })
1340
1430
  // setup();
1341
1431
  const parser = new runtime.ArgumentParser({
1342
1432
  description: 'Entodicton knowledge module'
1343
1433
  })
1344
1434
 
1435
+ const helpDebugAssociation = 'In order to get a debug break when a specific association is created set the DEBUG_ASSOCIATION environment variable to the JSON of the association to break on. For example DEBUG_ASSOCIATION=\'[["the", 0], ["mammal", 1]]\''
1436
+ const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'[["cat", 1], ["mammel", 1]]\''
1437
+ const helpDebugPriority = 'In order to get a debug break when a specific set of priorities is created set set DEBUG_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_PRIORITY=\'[["verb", 0], ["article", 0]]\''
1438
+ const helpDebugContextualPriority = 'In order to get a debug break when a specific set of contextual priorities is created set set DEBUG_CONTEXTUAL_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_CONTEXTUAL_PRIORITY=\'{ context: [["verb", 0], ["article", 0], select: 1}\''
1439
+ const helpDebugBridge = 'In order to get a debug break when a specific bridge is created set the DEBUG_BRIDGE environment variable to id/level to break on. For example DEBUG_BRIDGE=\'id#level\''
1440
+ const helpDebugOperator = 'In order to get a debug break when a specific hierarcy is created set the DEBUG_OPERATOR environment variable to debug any config loaded. For example DEBUG_OPERATOR=\'([operator] ([arg]))\''
1441
+
1442
+
1345
1443
  parser.add_argument('-tmn', '--testModuleName', { help: 'When running tests instead of using the current modules tests use the specified modules tests' })
1346
1444
  parser.add_argument('-t', '--test', { action: 'store_true', help: 'Run the tests. Create tests by running with the --query + --save flag' })
1347
1445
  parser.add_argument('-tv', '--testVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. Create tests by running with the --query or --loop with the --save flag' })
1446
+ // parser.add_argument('-ttr', '--testToRun', { help: 'Only the specified test will be run' })
1348
1447
  parser.add_argument('-tva', '--testAllVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. All the tests will be run instead of stopping at first failure. Create tests by running with the --query or --loop with the --save flag' })
1448
+ parser.add_argument('-tnp', '--testNoParenthesized', { action: 'store_true', help: 'Don\' check parenthesized differences for the tests' })
1349
1449
  parser.add_argument('-n', '--count', { help: 'Number of times to run the tests. Default is one. Use this to check for flakey test. If possible the system will print out a message with the word "hint" suggesting how to fix the problem' })
1350
1450
  // parser.add_argument('-b', '--build', { help: 'Specify the template file name of the form <kmName>. There should be a file called <baseKmName>.<kmName>.template.json with the queries to run. For example { queries: [...] }. The template file will be run and generate an instantiation called <baseKmName>.<kmName>.instance.json and a file called <kmName>.js that will load the template file (this is file generated only if not already existing) and a test file called <KmName>.tests.json. This can then be loaded into an instance of the current knowledge module to setup initial conditions.' })
1351
1451
  parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild' })
@@ -1353,45 +1453,83 @@ const knowledgeModule = async ({
1353
1453
  parser.add_argument('-i', '--info', { action: 'store_true', help: 'Print meta-data for the module' })
1354
1454
  parser.add_argument('-v', '--vimdiff', { action: 'store_true', help: 'For failures run vimdiff' })
1355
1455
  parser.add_argument('-g', '--greg', { action: 'store_true', help: 'Set the server to be localhost so I can debug stuff' })
1356
- parser.add_argument('-cl', '--checkForLoop', { action: 'store_true', help: 'Check for loops in the priorities' })
1456
+ parser.add_argument('-cl', '--checkForLoop', { nargs: "?", help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
1357
1457
  parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
1358
1458
  parser.add_argument('-q', '--query', { help: 'Run the specified query' })
1359
1459
  parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
1360
1460
  parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
1361
1461
  parser.add_argument('-dt', '--deleteTest', { help: 'Delete the specified query from the tests file.' })
1462
+ parser.add_argument('--parenthesized', { action: 'store_true', help: 'Show the generated phrases with parenthesis.' })
1362
1463
  parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
1363
1464
  parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
1364
- parser.add_argument('-daa', '--dontAddAssociations', { action: 'store_true', help: 'Do not add associations from the tests.' })
1365
1465
  parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a == associations j == JSON sent to server. for example --print wb' })
1366
1466
  parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
1367
1467
  parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
1368
- parser.add_argument('-dic', '--debugIncludeConvolutions', { action: 'store_true', help: 'When running with the --debugIncludeConvolutions flag the logs will include convolutions which are somewhat annoying verbose. Default is false' })
1369
1468
  parser.add_argument('-dl', '--debugLoops', { action: 'store_true', help: 'When running with the --debugLoops flag the logs calls to semantics and generators will be immediately written to the console '})
1370
1469
  parser.add_argument('-d', '--debug', { action: 'store_true', help: 'When running with the --debug flag this set the debug flag in the config' })
1371
- parser.add_argument('-da', '--debugAssociation', { help: 'When running with the --debugAssociation flag the debugging will break when the specified association is added to the config' })
1372
- parser.add_argument('-dh', '--debugHierarchy', { help: 'When running with the --debugHierarchy flag the debugging will break when the specified child-parent pair is added to the config for the main config. Set DEBUG_HIERARCHY to debug any config loaded. For example DEBUG_HIERARCHY=\'["cat", "mammel"]\'' })
1373
- parser.add_argument('-db', '--debugBridge', { help: 'When running with the --debugBridge flag the debugging will break when the specified bridge is added to the config for the main config. Set DEBUG_BRIDGE to debug any config loaded. For example DEBUG_BRIDGE=\'id/level\'' })
1374
- parser.add_argument('-do', '--debugOperator', { help: 'When running with the --debugOperator flag the debugging will break when the specified operator is added to the config for the main config. Set DEBUG_OPERATOR to debug any config loaded. For example DEBUG_OPERATOR=\'([operator] ([arg]))\'' })
1470
+ parser.add_argument('-da', '--debugAssociation', { action: 'store_true', help: helpDebugAssociation })
1471
+ parser.add_argument('-dh', '--debugHierarchy', { action: 'store_true', help: helpDebugHierarchy })
1472
+ parser.add_argument('-dp', '--debugPriority', { action: 'store_true', help: helpDebugPriority })
1473
+ parser.add_argument('-dcp', '--debugContextualPriority', { action: 'store_true', help: helpDebugContextualPriority })
1474
+ parser.add_argument('-db', '--debugBridge', { action: 'store_true', help: helpDebugBridge })
1475
+ parser.add_argument('-do', '--debugOperator', { action: 'store_true', help: helpDebugOperator })
1476
+ parser.add_argument('-ep', '--explainPriorities', { action: 'store_true', help: "The server will return all priorities including the generated one along with an explanation of there they came from"})
1477
+ parser.add_argument('-dic', '--debugIncludeConvolutions', { nargs: "?", help: 'When running with the --debugIncludeConvolutions flag the logs will include convolutions which are somewhat annoyingly verbose. Default is false' })
1375
1478
 
1376
1479
  const args = parser.parse_args()
1377
1480
  args.count = args.count || 1
1378
1481
 
1482
+ if (args.parenthesized) {
1483
+ config.parenthesized = true
1484
+ }
1485
+ if (args.checkForLoop) {
1486
+ try {
1487
+ args.checkForLoop = JSON.parse(args.checkForLoop)
1488
+ const isKey = (what) => {
1489
+ if (!Array.isArray(what)) {
1490
+ return false
1491
+ }
1492
+ if (what.length !== 2) {
1493
+ return false
1494
+ }
1495
+ if (!typeof what[0] == 'string') {
1496
+ return false
1497
+ }
1498
+ if (!typeof what[1] == 'number') {
1499
+ return false
1500
+ }
1501
+ return true
1502
+ }
1503
+ if (!Array.isArray(args.checkForLoop) || args.checkForLoop.some((value) => !isKey(value))) {
1504
+ throw new Error(`Error for the checkForLoop argument. Expected a JSON array of operator keys of the form "[<id>, <level>]"`)
1505
+ }
1506
+ } catch( e ) {
1507
+ throw new Error(`Error parsing JSON of the checkForLoop argument. ${e}`)
1508
+ }
1509
+ } else {
1510
+ if (process.argv.includes('--checkForLoop') || process.argv.includes('-cl')) {
1511
+ args.checkForLoop = true
1512
+ }
1513
+ }
1379
1514
  if (args.debugAssociation) {
1380
- global.entodictonDebugAssociation = JSON.parse(args.debugAssociation)
1515
+ console.log(helpDebugAssociation)
1516
+ runtime.process.exit(-1)
1381
1517
  }
1382
1518
  if (args.debugHierarchy) {
1383
- global.entodictonDebugHierarchy = JSON.parse(args.debugHierarchy)
1519
+ console.log(helpDebugHierarchy)
1520
+ runtime.process.exit(-1)
1521
+ }
1522
+ if (args.debugPriority) {
1523
+ console.log(helpDebugPriority)
1524
+ runtime.process.exit(-1)
1384
1525
  }
1385
1526
  if (args.debugBridge) {
1386
- // id/level
1387
- global.entodictonDebugBridge = args.debugBridge.split('/')
1388
- if (global.entodictonDebugBridge.length !== 2) {
1389
- console.log('Expected DEBUG_BRIDGE to be of the form "id/level"');
1390
- }
1527
+ console.log(helpDebugBridge)
1528
+ runtime.process.exit(-1)
1391
1529
  }
1392
1530
  if (args.debugOperator) {
1393
- // id/level
1394
- global.entodictonDebugOperator = args.debugOperator
1531
+ console.log(helpDebugOperator)
1532
+ runtime.process.exit(-1)
1395
1533
  }
1396
1534
 
1397
1535
  if (args.clean) {
@@ -1430,117 +1568,115 @@ const knowledgeModule = async ({
1430
1568
  if (args.debug) {
1431
1569
  config.config.debug = true
1432
1570
  }
1433
- config.config.debugIncludeConvolutions = args.debugIncludeConvolutions
1434
1571
 
1435
- if (template) {
1436
- const needsRebuild = config.needsRebuild(template.template, template.instance, options)
1437
- if (needsRebuild) {
1438
- console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
1439
-
1440
- }
1441
- config.load(template.template, template.instance, { rebuild: needsRebuild })
1442
- if (needsRebuild) {
1443
- return
1444
- }
1572
+ if (args.explainPriorities) {
1573
+ config.config.explain_priorities = true
1445
1574
  }
1446
1575
 
1447
- if (!args.save && !args.rebuildTemplate && !args.dontAddAssociations) {
1448
- config.addAssociationsFromTests(config.tests);
1449
- //for (let query in config.tests) {
1450
- // config.addAssociations(config.tests[query].associations || []);
1451
- //}
1452
- }
1576
+ config.config.debugIncludeConvolutions = args.debugIncludeConvolutions || process.argv.includes('--debugIncludeConvolutions') || process.argv.includes('-dic')
1453
1577
 
1454
- /*
1455
- if (args.buildTemplate) {
1456
- if (template) {
1457
- config.rebuild()
1458
- config.load(template.template, template.instance, { rebuild: true })
1578
+ let configPrinted = false
1579
+ const printConfig = () => {
1580
+ if (configPrinted) {
1581
+ return
1459
1582
  }
1460
- }
1461
- */
1462
-
1463
- if (args.print) {
1464
- if (args.print.includes('t')) {
1465
- console.log("Test queries")
1466
- let counter = 0
1467
- for (const test of config.tests) {
1468
- console.log(`${counter} - ${test.query}`)
1469
- counter += 1
1583
+ configPrinted = true
1584
+ if (args.print) {
1585
+ if (args.print.includes('t')) {
1586
+ console.log("Test queries")
1587
+ let counter = 0
1588
+ for (const test of config.tests) {
1589
+ console.log(`${counter} - ${test.query}`)
1590
+ counter += 1
1591
+ }
1592
+ }
1593
+ if (args.print.includes('c')) {
1594
+ const { data } = setupProcessB({ config })
1595
+ console.log("Config as sent to server")
1596
+ console.log(JSON.stringify(data, null, 2));
1470
1597
  }
1471
- }
1472
- if (args.print.includes('c')) {
1473
- const { data } = setupProcessB({ config })
1474
- console.log("Config as sent to server")
1475
- console.log(JSON.stringify(data, null, 2));
1476
- }
1477
1598
 
1478
- if (args.print.includes('l')) {
1479
- console.log('Module load ordering')
1480
- for (const km of config.configs) {
1481
- console.log(` ${km.name}`)
1599
+ if (args.print.includes('l')) {
1600
+ console.log('Module load ordering')
1601
+ for (const km of config.configs) {
1602
+ console.log(` ${km.name}`)
1603
+ }
1482
1604
  }
1483
- }
1484
- if (args.print.includes('w')) {
1485
- for (const word in config.config.words) {
1486
- console.log(word.concat(' ', ...config.config.words[word].map((def) => JSON.stringify(def))))
1605
+ if (args.print.includes('w')) {
1606
+ for (const word in config.config.words) {
1607
+ console.log(word.concat(' ', ...config.config.words[word].map((def) => JSON.stringify(def))))
1608
+ }
1487
1609
  }
1488
- }
1489
- if (args.print.includes('b')) {
1490
- for (const bridge of config.config.bridges) {
1491
- console.log(JSON.stringify(bridge))
1610
+ if (args.print.includes('b')) {
1611
+ for (const bridge of config.config.bridges) {
1612
+ console.log(JSON.stringify(bridge))
1613
+ }
1492
1614
  }
1493
- }
1494
- if (args.print.includes('o')) {
1495
- for (const operator of config.config.operators) {
1496
- console.log(JSON.stringify(operator))
1615
+ if (args.print.includes('o')) {
1616
+ for (const operator of config.config.operators) {
1617
+ console.log(JSON.stringify(operator))
1618
+ }
1497
1619
  }
1498
- }
1499
- if (args.print.includes('j')) {
1500
- const { data } = setupProcessB( { config } )
1501
- console.log(JSON.stringify(data, null, 2))
1502
- }
1503
- if (args.print.includes('a')) {
1504
- console.log('associations ================')
1505
- const properties = ['negative', 'positive']
1506
- for (let property of properties) {
1507
- console.log(` ${property} ===============`)
1508
- for (let association of config.config.associations[property]) {
1509
- console.log(` ${JSON.stringify(association)}`)
1620
+ if (args.print.includes('j')) {
1621
+ const { data } = setupProcessB( { config } )
1622
+ console.log(JSON.stringify(data, null, 2))
1623
+ }
1624
+ if (args.print.includes('a')) {
1625
+ console.log('associations ================')
1626
+ const properties = ['negative', 'positive']
1627
+ for (let property of properties) {
1628
+ console.log(` ${property} ===============`)
1629
+ for (let association of config.config.associations[property]) {
1630
+ console.log(` ${JSON.stringify(association)}`)
1631
+ }
1510
1632
  }
1511
1633
  }
1512
- }
1513
- if (args.print.includes('d')) {
1514
- console.log(JSON.stringify(config.config.objects, null, 2))
1515
- }
1516
- if (args.print.includes('p')) {
1517
- for (let priority of config.config.priorities) {
1518
- console.log(JSON.stringify(priority))
1634
+ if (args.print.includes('d')) {
1635
+ console.log(JSON.stringify(config.config.objects, null, 2))
1519
1636
  }
1520
- }
1521
- if (args.print.includes('h')) {
1522
- for (let edge of config.config.hierarchy) {
1523
- console.log(JSON.stringify(edge))
1637
+ if (args.print.includes('p')) {
1638
+ for (let priority of config.config.priorities) {
1639
+ console.log(JSON.stringify(priority))
1640
+ }
1524
1641
  }
1525
- }
1526
- if (args.print.includes('g')) {
1527
- const easyToRead = _.cloneDeep(config.config.generators)
1528
- for (const semantic of easyToRead) {
1529
- semantic.match = semantic.match.toString()
1530
- semantic.apply = semantic.apply.toString()
1531
- if (semantic.applyWrapped) {
1532
- semantic.applyWrapped = semantic.applyWrapped.toString()
1642
+ if (args.print.includes('h')) {
1643
+ for (let edge of config.config.hierarchy) {
1644
+ console.log(JSON.stringify(edge))
1533
1645
  }
1534
1646
  }
1535
- console.dir(easyToRead)
1536
- }
1537
- if (args.print.includes('s')) {
1538
- const easyToRead = _.cloneDeep(config.config.semantics)
1539
- for (const semantic of easyToRead) {
1540
- semantic.match = semantic.match.toString()
1541
- semantic.apply = semantic.apply.toString()
1647
+ if (args.print.includes('g')) {
1648
+ const easyToRead = _.cloneDeep(config.config.generators)
1649
+ for (const semantic of easyToRead) {
1650
+ semantic.match = semantic.match.toString()
1651
+ semantic.apply = semantic.apply.toString()
1652
+ if (semantic.applyWrapped) {
1653
+ semantic.applyWrapped = semantic.applyWrapped.toString()
1654
+ }
1655
+ }
1656
+ console.dir(easyToRead)
1542
1657
  }
1543
- console.dir(easyToRead)
1658
+ if (args.print.includes('s')) {
1659
+ const easyToRead = _.cloneDeep(config.config.semantics)
1660
+ for (const semantic of easyToRead) {
1661
+ semantic.match = semantic.match.toString()
1662
+ semantic.apply = semantic.apply.toString()
1663
+ }
1664
+ console.dir(easyToRead)
1665
+ }
1666
+ }
1667
+ }
1668
+
1669
+ if (template) {
1670
+ const needsRebuild = config.needsRebuild(template.template, template.instance, options)
1671
+ if (needsRebuild) {
1672
+ console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
1673
+ options.rebuild = true
1674
+ config.config.rebuild = true
1675
+ }
1676
+ config.load(template.template, template.instance, { rebuild: needsRebuild })
1677
+ printConfig()
1678
+ if (needsRebuild) {
1679
+ return
1544
1680
  }
1545
1681
  }
1546
1682
 
@@ -1575,12 +1711,7 @@ const knowledgeModule = async ({
1575
1711
  test = useTestConfig.name
1576
1712
 
1577
1713
  }
1578
- runTests(config, test, { debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
1579
- if (results.length > 0 && args.vimdiff) {
1580
- for (const result of results) {
1581
- vimdiff(result.expected, result.actual)
1582
- }
1583
- }
1714
+ runTests(config, test, { args, debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
1584
1715
  let newError = false
1585
1716
  if (results.length > 0) {
1586
1717
  let headerShown = false
@@ -1590,18 +1721,49 @@ const knowledgeModule = async ({
1590
1721
  if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
1591
1722
  hasError = true
1592
1723
  }
1724
+ if (!args.testNoParenthesized) {
1725
+ if (JSON.stringify(result.expected.paraphrasesParenthesized) !== JSON.stringify(result.actual.paraphrasesParenthesized)) {
1726
+ hasError = true
1727
+ }
1728
+ if (JSON.stringify(result.expected.generatedParenthesized) !== JSON.stringify(result.actual.generatedParenthesized)) {
1729
+ hasError = true
1730
+ }
1731
+ }
1593
1732
  if (JSON.stringify(result.expected.responses) !== JSON.stringify(result.actual.responses)) {
1594
1733
  hasError = true
1595
1734
  }
1596
1735
  if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
1597
1736
  hasError = true
1598
1737
  }
1738
+ if (!sameJSON(result.expected.checkedContexts, result.actual.checkedContexts)) {
1739
+ hasError = true
1740
+ }
1599
1741
  }
1600
1742
 
1601
1743
  if (hasError) {
1602
1744
  console.log('**************************** ERRORS ************************')
1603
1745
  for (const result of results) {
1604
1746
  console.log('Utterance: ', result.utterance)
1747
+ const show = (label, expected, actual) => {
1748
+ if (JSON.stringify(expected) !== JSON.stringify(actual)) {
1749
+ if (!headerShown) {
1750
+ console.log(' Failure')
1751
+ }
1752
+ console.log(` expected ${label}`, expected)
1753
+ console.log(` actual ${label} `, actual)
1754
+ newError = true
1755
+ headerShown = true
1756
+ }
1757
+ }
1758
+ show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
1759
+ if (!args.testNoParenthesized) {
1760
+ show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
1761
+ }
1762
+ show('responses', result.expected.responses, result.actual.responses)
1763
+ if (!args.testNoParenthesized) {
1764
+ show('responses parenthesized', result.expected.generatedParenthesized, result.actual.generatedParenthesized)
1765
+ }
1766
+ /*
1605
1767
  if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
1606
1768
  if (!headerShown) {
1607
1769
  console.log(' Failure')
@@ -1620,6 +1782,7 @@ const knowledgeModule = async ({
1620
1782
  newError = true
1621
1783
  headerShown = true
1622
1784
  }
1785
+ */
1623
1786
  if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
1624
1787
  if (!headerShown) {
1625
1788
  console.log(' Failure')
@@ -1632,10 +1795,37 @@ const knowledgeModule = async ({
1632
1795
  lines.setElement(1, 1, 'actual checked')
1633
1796
  lines.setElement(2, 2, JSON.stringify(result.actual.checked, null, 2))
1634
1797
  lines.log()
1798
+ if (args.vimdiff) {
1799
+ vimdiff(result.actual.checked, result.expected.checked)
1800
+ }
1801
+ newError = true
1802
+ headerShown = true
1803
+ }
1804
+ if (!sameJSON(result.expected.checkedContexts, result.actual.checkedContexts)) {
1805
+ if (!headerShown) {
1806
+ console.log(' Failure')
1807
+ }
1808
+ const widths = [4, 18, 72]
1809
+ const lines = new Lines(widths)
1810
+ lines.setElement(1, 1, 'expected checkedContexts', true)
1811
+ lines.setElement(2, 2, JSON.stringify(result.expected.checkedContexts, null, 2))
1812
+ lines.log()
1813
+ lines.setElement(1, 1, 'actual checkedContexts', true)
1814
+ lines.setElement(2, 2, JSON.stringify(result.actual.checkedContexts, null, 2))
1815
+ lines.log()
1816
+ if (args.vimdiff) {
1817
+ vimdiff(result.actual.checkedContexts, result.expected.checkedContexts)
1818
+ }
1635
1819
  newError = true
1636
1820
  headerShown = true
1637
1821
  }
1638
1822
  }
1823
+ } else {
1824
+ if (results.length > 0 && args.vimdiff) {
1825
+ for (const result of results) {
1826
+ vimdiff(result.actual, result.expected)
1827
+ }
1828
+ }
1639
1829
  }
1640
1830
  if (!headerShown) {
1641
1831
  if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
@@ -1667,12 +1857,11 @@ const knowledgeModule = async ({
1667
1857
  if (query.length === 0) {
1668
1858
  return readline.close()
1669
1859
  }
1670
- // const promise = processResults(_process(config, query, { testsFN: test }))
1671
1860
  const promise = _process(config, query, { testsFN: test }).then((results) => {
1672
1861
  console.log(results.responses.join(' '))
1673
1862
  })
1674
1863
  if (!('then' in promise)) {
1675
- throw 'Return a promise from process in the definition of knowledgeModule'
1864
+ throw new Error('Return a promise from process in the definition of knowledgeModule')
1676
1865
  }
1677
1866
  promise
1678
1867
  .then(() => {
@@ -1695,20 +1884,63 @@ const knowledgeModule = async ({
1695
1884
  if (args.objectDiff) {
1696
1885
  global.beforeObjects = _.cloneDeep(objects)
1697
1886
  }
1698
- config.beforeQuery({ query: args.query, isModule: false, objects })
1699
1887
  try {
1700
- processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
1888
+ await processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
1701
1889
  } catch( error ) {
1702
1890
  console.log('Error', error);
1703
1891
  }
1704
1892
  }
1893
+ printConfig()
1705
1894
  } else {
1706
- config.addAssociationsFromTests(config.tests);
1707
- //for (let query in config.tests) {
1708
- // config.addAssociations(config.tests[query].associations || []);
1709
- //}
1710
- module()
1895
+ const initConfig = (config) => {
1896
+ setupConfig(config)
1897
+
1898
+ let loadForTesting = false
1899
+ if (global.theprogrammablemind) {
1900
+ if (global.theprogrammablemind.loadForTesting[config.name]) {
1901
+ loadForTesting = true
1902
+ }
1903
+ }
1904
+ // remove test only stuff
1905
+ if (!isProcess && !loadForTesting) {
1906
+ config.config.operators = config.config.operators.filter( (operator) => {
1907
+ if (operator.development) {
1908
+ return false
1909
+ } else {
1910
+ return true
1911
+ }
1912
+ })
1913
+ config.config.bridges = config.config.bridges.filter( (bridge) => {
1914
+ if (bridge.development) {
1915
+ return false
1916
+ } else {
1917
+ return true
1918
+ }
1919
+ })
1920
+ }
1921
+
1922
+ if (template) {
1923
+ if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess })) {
1924
+ const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
1925
+ throw new Error(error)
1926
+ }
1927
+ try {
1928
+ config.load(template.template, template.instance)
1929
+ } catch( e ) {
1930
+ errorHandler(e)
1931
+ }
1932
+ }
1933
+ }
1934
+
1935
+ createConfigExport = () => {
1936
+ const config = createConfig()
1937
+ initConfig(config)
1938
+ config.rebuild({ isModule: true })
1939
+ return config
1940
+ }
1941
+ moduleFromJSFile.exports = createConfigExport
1711
1942
  }
1943
+
1712
1944
  }
1713
1945
 
1714
1946
  /*
@@ -1762,7 +1994,7 @@ module.exports = {
1762
1994
  w,
1763
1995
  // submitBug,
1764
1996
  ensureTestFile,
1765
- build,
1997
+ rebuildTemplate,
1766
1998
  processContext,
1767
1999
  processContexts,
1768
2000
  runTests,
@@ -1774,8 +2006,9 @@ module.exports = {
1774
2006
  Digraph,
1775
2007
  analyzeMetaData,
1776
2008
  processContextsB,
1777
- processInstance,
2009
+ loadInstance,
1778
2010
  gs,
1779
2011
  flattens,
1780
2012
  writeTest
1781
2013
  }
2014
+