theprogrammablemind 7.5.8 → 7.6.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client.js +580 -347
- package/index.js +1 -0
- package/lines.js +3 -3
- package/package.json +2 -1
- package/runtime.js +1 -1
- package/src/config.js +699 -331
- package/src/flatten.js +9 -1
- package/src/generators.js +2 -2
- package/src/helpers.js +38 -10
- package/src/project.js +81 -0
- package/src/semantics.js +8 -3
package/client.js
CHANGED
@@ -2,6 +2,7 @@ const { Semantics, Semantic } = require('./src/semantics')
|
|
2
2
|
const { Generators, Generator } = require('./src/generators')
|
3
3
|
const DigraphInternal = require('./src/digraph_internal')
|
4
4
|
const Digraph = require('./src/digraph')
|
5
|
+
const { project } = require('./src/project')
|
5
6
|
const fetch = require('node-fetch')
|
6
7
|
const base64 = require('base-64')
|
7
8
|
const deepEqual = require('deep-equal')
|
@@ -9,20 +10,38 @@ const _ = require('lodash')
|
|
9
10
|
const stringify = require('json-stable-stringify')
|
10
11
|
const Lines = require('./lines')
|
11
12
|
const flattens = require('./src/flatten')
|
12
|
-
const { appendNoDups, InitCalls, updateQueries } = require('./src/helpers')
|
13
|
+
const { appendNoDups, InitCalls, updateQueries, safeNoDups } = require('./src/helpers')
|
13
14
|
const runtime = require('./runtime')
|
14
15
|
const sortJson = runtime.sortJson
|
15
16
|
|
17
|
+
const getConfig_getObjectsCheck = (testConfig) => {
|
18
|
+
return (testConfig.checks && testConfig.checks.objects) || []
|
19
|
+
}
|
20
|
+
|
21
|
+
const getConfig_getContextCheck = (testConfig) => {
|
22
|
+
return (testConfig.checks && testConfig.checks.context) || []
|
23
|
+
}
|
24
|
+
|
25
|
+
const pickContext = (testConfig) => (context) => {
|
26
|
+
return project(context, getConfig_getContextCheck(testConfig))
|
27
|
+
}
|
28
|
+
|
29
|
+
const pickObjects = (testConfig, objects) => {
|
30
|
+
return project(objects, getConfig_getObjectsCheck(testConfig))
|
31
|
+
}
|
32
|
+
|
16
33
|
const getAsk = (config) => (uuid) => (asks) => {
|
17
34
|
for (let ask of asks) {
|
18
|
-
config.
|
35
|
+
config.addSemantic({
|
19
36
|
uuid,
|
37
|
+
oneShot: true,
|
20
38
|
match: (args) => ask.matchr(args),
|
21
39
|
apply: (args) => ask.applyr(args)
|
22
40
|
})
|
23
41
|
}
|
24
|
-
config.
|
42
|
+
config.addSemantic({
|
25
43
|
uuid,
|
44
|
+
oneShot: true,
|
26
45
|
match: ({context}) => context.marker == 'controlEnd' || context.marker == 'controlBetween',
|
27
46
|
apply: (args) => {
|
28
47
|
for (let ask of asks) {
|
@@ -37,7 +56,7 @@ const getAsk = (config) => (uuid) => (asks) => {
|
|
37
56
|
}
|
38
57
|
}
|
39
58
|
if (matchq(args)) {
|
40
|
-
args.context.motivationKeep = true
|
59
|
+
// args.context.motivationKeep = true
|
41
60
|
args.context.verbatim = applyq(args)
|
42
61
|
args.context.isResponse = true;
|
43
62
|
delete args.context.controlRemove;
|
@@ -49,6 +68,12 @@ const getAsk = (config) => (uuid) => (asks) => {
|
|
49
68
|
})
|
50
69
|
}
|
51
70
|
|
71
|
+
const sameJSON = (json1, json2) => {
|
72
|
+
const sjson1 = sortJson(json1, { depth: 25 })
|
73
|
+
const sjson2 = sortJson(json2, { depth: 25 })
|
74
|
+
return JSON.stringify(sjson1) == JSON.stringify(sjson2)
|
75
|
+
}
|
76
|
+
|
52
77
|
const vimdiff = (actualJSON, expectedJSON) => {
|
53
78
|
const path = '.'
|
54
79
|
const actual = sortJson(actualJSON, { depth: 25 })
|
@@ -58,10 +83,9 @@ const vimdiff = (actualJSON, expectedJSON) => {
|
|
58
83
|
// console.log(`vimdiff ${path}/actual.json ${path}/expected.json`)
|
59
84
|
{
|
60
85
|
const editor = runtime.process.env.EDITOR || 'vimdiff'
|
61
|
-
const child = runtime.child_process.spawn(editor, [`${path}/expected.json`, `${path}/actual.json`], { stdio: 'inherit' })
|
62
|
-
|
63
|
-
|
64
|
-
})
|
86
|
+
// const child = runtime.child_process.spawn(editor, [`${path}/expected.json`, `${path}/actual.json`], { stdio: 'inherit' })
|
87
|
+
console.log(`${editor} ${path}/expected.json ${path}/actual.json`)
|
88
|
+
runtime.child_process.execSync(`${editor} ${path}/expected.json ${path}/actual.json`, {stdio: 'inherit'})
|
65
89
|
}
|
66
90
|
}
|
67
91
|
|
@@ -100,10 +124,24 @@ const asList = (context) => {
|
|
100
124
|
}
|
101
125
|
}
|
102
126
|
|
127
|
+
class ErrorReason extends Error {
|
128
|
+
constructor(context) {
|
129
|
+
super(JSON.stringify(context))
|
130
|
+
this.reason = context
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
103
134
|
const setupArgs = (args, config, logs, hierarchy) => {
|
104
135
|
config.setArgs(args)
|
105
|
-
args.calls = new InitCalls()
|
136
|
+
args.calls = new InitCalls(args.isInstance ? `${args.isInstance}#${config.name}` : config.name)
|
137
|
+
if (global.theprogrammablemind && global.theprogrammablemind.loadForTesting) {
|
138
|
+
args.calls = new InitCalls(Object.keys(global.theprogrammablemind.loadForTesting)[0])
|
139
|
+
}
|
106
140
|
args.km = (name) => config.getConfig(name)
|
141
|
+
args.api = (name) => config.getConfig(name).api
|
142
|
+
args.error = (context) => {
|
143
|
+
throw new ErrorReason(context)
|
144
|
+
}
|
107
145
|
args.kms = config.getConfigs()
|
108
146
|
args.config = config
|
109
147
|
args.hierarchy = hierarchy
|
@@ -111,6 +149,7 @@ const setupArgs = (args, config, logs, hierarchy) => {
|
|
111
149
|
args.listable = listable(hierarchy)
|
112
150
|
args.asList = asList
|
113
151
|
args.retry = () => { throw new RetryError() }
|
152
|
+
args.fragments = (query) => config.fragment(query)
|
114
153
|
const scopedAsk = getAsk(config)
|
115
154
|
|
116
155
|
const getAPI = (uuid) => {
|
@@ -130,7 +169,6 @@ const setupArgs = (args, config, logs, hierarchy) => {
|
|
130
169
|
apis: getAPIs(uuid)
|
131
170
|
}
|
132
171
|
}
|
133
|
-
args.motivation = (m) => config.addMotivation(m)
|
134
172
|
args.breakOnSemantics = false
|
135
173
|
args.theDebugger = {
|
136
174
|
breakOnSemantics: (value) => args.breakOnSemantics = value
|
@@ -139,12 +177,15 @@ const setupArgs = (args, config, logs, hierarchy) => {
|
|
139
177
|
args.g = (c) => config.getGenerators(logs).apply(args, c)
|
140
178
|
args.gp = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: true, isResponse: false, response: false})
|
141
179
|
args.gr = (c) => config.getGenerators(logs).apply(args, { ...c, paraphrase: false, isResponse: true })
|
142
|
-
|
180
|
+
if (!logs) {
|
181
|
+
debugger
|
182
|
+
}
|
183
|
+
args.e = (c) => config.getEvaluator(args.s, args.calls, logs, c)
|
143
184
|
args.log = (message) => logs.push(message)
|
144
|
-
// config.getAddedArgs(args)
|
145
185
|
args.gs = gs(args.g)
|
146
186
|
args.gsp = gs(args.gp)
|
147
187
|
args.gsr = gs(args.gr)
|
188
|
+
config.getAddedArgs(args)
|
148
189
|
}
|
149
190
|
|
150
191
|
const gs = (g) => (contexts, separator, lastSeparator) => {
|
@@ -272,7 +313,7 @@ const writeTestFile = (fn, tests) => {
|
|
272
313
|
runtime.fs.writeFileSync(fn, stringify(tests, { space: 2 }), { encoding: 'utf8', flag: 'w+' })
|
273
314
|
}
|
274
315
|
|
275
|
-
const writeTest = (fn, query, objects, generated, paraphrases, responses, contexts, associations, metadata, config, saveDeveloper) => {
|
316
|
+
const writeTest = (fn, query, objects, generated, paraphrases, responses, contexts, associations, metadata, config, saveDeveloper, paraphrasesParenthesized, generatedParenthesized) => {
|
276
317
|
let tests = []
|
277
318
|
if (runtime.fs.existsSync(fn)) {
|
278
319
|
tests = JSON.parse(runtime.fs.readFileSync(fn))
|
@@ -282,7 +323,18 @@ const writeTest = (fn, query, objects, generated, paraphrases, responses, contex
|
|
282
323
|
}
|
283
324
|
associations.sort()
|
284
325
|
// tests[query] = sortJson({ paraphrases, responses, contexts, objects: convertToStable(objects), associations, metadata, config, developerTest: saveDeveloper }, { depth: 25 })
|
285
|
-
results = sortJson({
|
326
|
+
results = sortJson({
|
327
|
+
query,
|
328
|
+
paraphrases,
|
329
|
+
responses,
|
330
|
+
contexts,
|
331
|
+
objects: convertToStable(objects),
|
332
|
+
associations,
|
333
|
+
metadata,
|
334
|
+
config,
|
335
|
+
developerTest: saveDeveloper,
|
336
|
+
paraphrasesParenthesized,
|
337
|
+
generatedParenthesized }, { depth: 25 })
|
286
338
|
let wasSet = false;
|
287
339
|
tests.forEach( (test, index) => {
|
288
340
|
if (test.query == query) {
|
@@ -339,16 +391,21 @@ const setupContexts = (rawContexts) => {
|
|
339
391
|
return contexts
|
340
392
|
}
|
341
393
|
|
342
|
-
const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, query, data, retries, url, commandLineArgs }) => {
|
394
|
+
const processContextsB = ({ config, hierarchy, semantics, generators, json, isTest, isInstance, instance, query, data, retries, url, commandLineArgs }) => {
|
343
395
|
// TODO fix this name to contextsPrime
|
344
396
|
const contextsPrime = []
|
345
397
|
const generatedPrime = []
|
346
398
|
const paraphrasesPrime = []
|
399
|
+
const paraphrasesParenthesizedPrime = []
|
400
|
+
const generatedParenthesizedPrime = []
|
347
401
|
const responsesPrime = []
|
348
402
|
const contexts = setupContexts(json.contexts)
|
349
403
|
|
350
404
|
const objects = config.get('objects')
|
351
|
-
const args = { objects, isResponse: true, response: json, isTest, getObjects: getObjects(objects) }
|
405
|
+
const args = { objects, isResponse: true, response: json, isTest, isInstance, getObjects: getObjects(objects), instance }
|
406
|
+
if (!json.logs) {
|
407
|
+
json.logs = []
|
408
|
+
}
|
352
409
|
setupArgs(args, config, json.logs, hierarchy)
|
353
410
|
const toDo = [...contexts]
|
354
411
|
args.insert = (context) => toDo.unshift(context)
|
@@ -356,28 +413,6 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
356
413
|
config.debugLoops = commandLineArgs && commandLineArgs.debugLoops
|
357
414
|
while (toDo.length > 0) {
|
358
415
|
const context = toDo.shift()
|
359
|
-
/*
|
360
|
-
if (false && query) {
|
361
|
-
if (config.wasChanged()) {
|
362
|
-
// process contexts that overlap
|
363
|
-
overlap = lastRange
|
364
|
-
} else {
|
365
|
-
config.watch()
|
366
|
-
}
|
367
|
-
if (overlap) {
|
368
|
-
if (overlaps(overlap, context)) {
|
369
|
-
// okay
|
370
|
-
query = query.slice(overlap.end+1)
|
371
|
-
data.utterance = query
|
372
|
-
const json = await doWithRetries(retries, url, data)
|
373
|
-
toDo = setupContexts(json.contexts).slice(1) // take off the start context
|
374
|
-
}
|
375
|
-
overlap = undefined
|
376
|
-
}
|
377
|
-
lastRange = context.range
|
378
|
-
}
|
379
|
-
*/
|
380
|
-
|
381
416
|
args.calls.next()
|
382
417
|
let contextPrime = context
|
383
418
|
context.topLevel = true
|
@@ -385,25 +420,40 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
385
420
|
if (json.has_errors) {
|
386
421
|
throw new Error('There are errors in the logs. Run with the -d flag and grep for Error')
|
387
422
|
}
|
423
|
+
const generateParenthesized = isTest || (commandLineArgs && commandLineArgs.save)
|
388
424
|
if (!config.get('skipSemantics')) {
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
425
|
+
const semantics = config.getSemantics(json.logs)
|
426
|
+
try {
|
427
|
+
contextPrime = semantics.apply(args, context)
|
428
|
+
} catch( e ) {
|
429
|
+
if (e.message == 'Maximum call stack size exceeded') {
|
430
|
+
const mostCalled = semantics.getMostCalled()
|
431
|
+
e.message += `\nThe most called semantic was:\nnotes: ${mostCalled.notes}\nmatch: ${mostCalled.matcher.toString()}\napply: ${mostCalled._apply.toString()}\n`
|
432
|
+
}
|
433
|
+
// contextPrime = semantics.apply(args, { marker: 'error', context, error: e })
|
434
|
+
if (isInstance) {
|
435
|
+
console.log('error', e.error)
|
399
436
|
}
|
437
|
+
contextPrime = semantics.apply(args, {
|
438
|
+
marker: 'error',
|
439
|
+
context,
|
440
|
+
text: e ? e.toString() : 'not available',
|
441
|
+
reason: e.reason,
|
442
|
+
error: e.stack || e.error
|
443
|
+
})
|
400
444
|
}
|
401
445
|
}
|
402
446
|
if (contextPrime.controlRemove) {
|
403
447
|
continue
|
404
448
|
}
|
405
449
|
let assumed = { isResponse: true };
|
406
|
-
const generated = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
|
450
|
+
const generated = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
|
451
|
+
let generatedParenthesized = []
|
452
|
+
if (generateParenthesized) {
|
453
|
+
config.parenthesized = true
|
454
|
+
generatedParenthesized = contextPrime.isResponse ? config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0] : ''
|
455
|
+
config.parenthesized = false
|
456
|
+
}
|
407
457
|
// assumed = { paraphrase: true, response: false };
|
408
458
|
assumed = { paraphrase: true };
|
409
459
|
args.g = (c) => config.getGenerators(json.logs).apply(args, c, assumed)
|
@@ -412,7 +462,16 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
412
462
|
args.gs = gs(args.g)
|
413
463
|
args.gsp = gs(args.gsp)
|
414
464
|
args.gsr = gs(args.gr)
|
465
|
+
if (generateParenthesized) {
|
466
|
+
config.parenthesized = false
|
467
|
+
}
|
415
468
|
const paraphrases = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
|
469
|
+
let paraphrasesParenthesized = []
|
470
|
+
if (generateParenthesized) {
|
471
|
+
config.parenthesized = true
|
472
|
+
paraphrasesParenthesized = config.getGenerators(json.logs).apply(args, contextPrime, assumed)[0]
|
473
|
+
config.parenthesized = false
|
474
|
+
}
|
416
475
|
args.g = (c) => config.getGenerators(json.logs).apply(args, c)
|
417
476
|
args.gp = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: true, isResponse: false, response: false })
|
418
477
|
args.gr = (c) => config.getGenerators(json.logs).apply(args, {...c, paraphrase: false })
|
@@ -422,6 +481,10 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
422
481
|
contextsPrime.push(contextPrime)
|
423
482
|
generatedPrime.push(generated)
|
424
483
|
paraphrasesPrime.push(paraphrases)
|
484
|
+
if (generateParenthesized) {
|
485
|
+
paraphrasesParenthesizedPrime.push(paraphrasesParenthesized)
|
486
|
+
generatedParenthesizedPrime.push(generatedParenthesized)
|
487
|
+
}
|
425
488
|
if (contextPrime.isResponse) {
|
426
489
|
responsesPrime.push(generated)
|
427
490
|
} else {
|
@@ -431,7 +494,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
431
494
|
// add results to processed list
|
432
495
|
config.config.objects.processed = config.config.objects.processed || []
|
433
496
|
config.config.objects.processed = config.config.objects.processed.slice(0, 5)
|
434
|
-
config.config.objects.processed.unshift({ context: contextPrime, paraphrases: paraphrases, responses: responsesPrime })
|
497
|
+
config.config.objects.processed.unshift({ context: contextPrime, paraphrases: paraphrases, paraphrasesParenthesized, generatedParenthesized, responses: responsesPrime })
|
435
498
|
} catch (e) {
|
436
499
|
if (Array.isArray(e)) {
|
437
500
|
e = {
|
@@ -451,7 +514,7 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
451
514
|
throw e
|
452
515
|
}
|
453
516
|
}
|
454
|
-
return { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime }
|
517
|
+
return { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime }
|
455
518
|
}
|
456
519
|
|
457
520
|
const doWithRetries = async (n, url, queryParams, data) => {
|
@@ -473,17 +536,17 @@ const doWithRetries = async (n, url, queryParams, data) => {
|
|
473
536
|
}
|
474
537
|
if (result.status === 504) {
|
475
538
|
if (n === 0) {
|
476
|
-
throw `Error ${result.status} - ${result.statusText}`
|
539
|
+
throw new Error(`Error ${result.status} - ${result.statusText}`)
|
477
540
|
} else {
|
478
541
|
continue
|
479
542
|
}
|
480
543
|
}
|
481
544
|
if (result.status >= 500 && result.status < 600) {
|
482
|
-
throw `Error ${result.status} - ${result.statusText}.`
|
545
|
+
throw new Error(`Error ${result.status} - ${result.statusText}.`)
|
483
546
|
} if (result.status >= 404) {
|
484
|
-
throw `Error ${result.status} - ${result.statusText} - Trying it connect to ${url}`
|
547
|
+
throw new Error(`Error ${result.status} - ${result.statusText} - Trying it connect to ${url}`)
|
485
548
|
} else {
|
486
|
-
throw `Error ${result.status} - ${result.statusText}`
|
549
|
+
throw new Error(`Error ${result.status} - ${result.statusText}`)
|
487
550
|
}
|
488
551
|
}
|
489
552
|
}
|
@@ -496,8 +559,11 @@ const setupProcessB = ({ config, initializer, allowDelta=false } = {}) => {
|
|
496
559
|
// console.log('config', config)
|
497
560
|
data.delta = config.delta()
|
498
561
|
} else {
|
499
|
-
|
562
|
+
config.toData(data)
|
563
|
+
// Object.assign(data, config.config)
|
500
564
|
}
|
565
|
+
|
566
|
+
// config.toServer(data)
|
501
567
|
|
502
568
|
if (data.namespaces) {
|
503
569
|
for (const uuid of Object.keys(data.namespaces)) {
|
@@ -520,10 +586,22 @@ const setupProcessB = ({ config, initializer, allowDelta=false } = {}) => {
|
|
520
586
|
}
|
521
587
|
}
|
522
588
|
|
523
|
-
// instance template
|
524
|
-
const
|
589
|
+
// instance template loadTemplate
|
590
|
+
const loadInstance = (config, instance) => {
|
525
591
|
const transitoryMode = global.transitoryMode
|
526
592
|
global.transitoryMode = false
|
593
|
+
|
594
|
+
if (instance && (instance.associations || instance.learned_contextual_priorities)) {
|
595
|
+
if (!config.config.retrain) {
|
596
|
+
if (instance.associations) {
|
597
|
+
config.addAssociations(instance.associations)
|
598
|
+
}
|
599
|
+
if (instance.learned_contextual_priorities && instance.learned_contextual_priorities.length > 0) {
|
600
|
+
config.addPriorities(instance.learned_contextual_priorities)
|
601
|
+
}
|
602
|
+
}
|
603
|
+
}
|
604
|
+
|
527
605
|
const { /* data, generators, semantics, */ hierarchy } = setupProcessB({ config })
|
528
606
|
// for (const results of (instance.resultss || [])) {
|
529
607
|
for (const i in (instance.resultss || [])) {
|
@@ -531,15 +609,21 @@ const processInstance = (config, instance) => {
|
|
531
609
|
if (results.extraConfig) {
|
532
610
|
// config.addInternal(results, useOldVersion = true, skipObjects = false, includeNamespaces = true, allowNameToBeNull = false)
|
533
611
|
// config.addInternal(config.template.queries[i], { handleCalculatedProps: true } )
|
534
|
-
config.addInternal(instance.template.queries[i], { handleCalculatedProps: true } )
|
612
|
+
config.addInternal(instance.template.queries[i], { addFirst: true, handleCalculatedProps: true } )
|
535
613
|
} else {
|
536
|
-
|
614
|
+
if (results.skipSemantics) {
|
615
|
+
config.config.skipSemantics = results.skipSemantics
|
616
|
+
}
|
617
|
+
processContextsB({ config, hierarchy, json: results/*, generators, semantics */, commandLineArgs: {}, isInstance: `instance${i}`, instance: instance.queries[i] })
|
618
|
+
if (results.skipSemantics) {
|
619
|
+
config.config.skipSemantics = null
|
620
|
+
}
|
537
621
|
}
|
538
622
|
}
|
539
623
|
global.transitoryMode = transitoryMode
|
540
624
|
}
|
541
625
|
|
542
|
-
const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
|
626
|
+
const _process = async (config, query, { initializer, commandLineArgs, credentials, writeTests, isTest, saveDeveloper, rebuildingTemplate, testConfig, testsFN, errorHandler = defaultErrorHandler } = {}) => {
|
543
627
|
if (credentials) {
|
544
628
|
config.server(credentials.server, credentials.key)
|
545
629
|
}
|
@@ -553,7 +637,6 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
|
|
553
637
|
if (writeTests) {
|
554
638
|
config.rebuild()
|
555
639
|
const objects = getObjects(config.config.objects)(config.uuid)
|
556
|
-
config.beforeQuery({ query, isModule: false, objects })
|
557
640
|
}
|
558
641
|
} catch(error) {
|
559
642
|
throw error
|
@@ -561,7 +644,10 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
|
|
561
644
|
|
562
645
|
let { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config, initializer, allowDelta: true })
|
563
646
|
if (commandLineArgs && commandLineArgs.checkForLoop) {
|
564
|
-
data.checkForLoop =
|
647
|
+
data.checkForLoop = commandLineArgs.checkForLoop
|
648
|
+
}
|
649
|
+
if (rebuildingTemplate) {
|
650
|
+
data.errors_ignore_contextual_priorities_non_existant_ops = true
|
565
651
|
}
|
566
652
|
let queries = query.split('\\n')
|
567
653
|
|
@@ -578,6 +664,8 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
|
|
578
664
|
contexts: [],
|
579
665
|
generated: [],
|
580
666
|
paraphrases: [],
|
667
|
+
paraphrasesParenthesized: [],
|
668
|
+
generatedParenthesized: [],
|
581
669
|
responses: [],
|
582
670
|
associations: [],
|
583
671
|
}
|
@@ -613,9 +701,10 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
|
|
613
701
|
if (json.status !== 200) {
|
614
702
|
throw json
|
615
703
|
} else {
|
616
|
-
const { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime } =
|
704
|
+
const { contextsPrime, generatedPrime, paraphrasesPrime, paraphrasesParenthesizedPrime, generatedParenthesizedPrime, responsesPrime } =
|
617
705
|
processContextsB({ isTest, config, hierarchy, json, commandLineArgs /*, generators, semantics */ })
|
618
706
|
response.associations = json.associations
|
707
|
+
response.learned_contextual_priorities = json.learned_contextual_priorities
|
619
708
|
response.hierarchy = json.hierarchy
|
620
709
|
response.load_cache_time += json.load_cache_time
|
621
710
|
appendNoDups(response.logs, json.logs)
|
@@ -626,10 +715,13 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
|
|
626
715
|
response.times += json.times
|
627
716
|
response.trace = response.trace.concat(json.trace)
|
628
717
|
response.version = json.version
|
718
|
+
response.explain_priorities = json.explain_priorities
|
629
719
|
|
630
720
|
response.contexts = response.contexts.concat(contextsPrime)
|
631
721
|
response.generated = response.generated.concat(generatedPrime)
|
632
722
|
response.paraphrases = response.paraphrases.concat(paraphrasesPrime)
|
723
|
+
response.paraphrasesParenthesized = response.paraphrasesParenthesized.concat(paraphrasesParenthesizedPrime)
|
724
|
+
response.generatedParenthesized = response.generatedParenthesized.concat(generatedParenthesizedPrime)
|
633
725
|
response.responses = response.responses.concat(responsesPrime)
|
634
726
|
queries = queries.slice(1)
|
635
727
|
}
|
@@ -637,13 +729,18 @@ const _process = async (config, query, { initializer, commandLineArgs, credentia
|
|
637
729
|
|
638
730
|
if (writeTests) {
|
639
731
|
const actual_config = getConfigForTest(config, testConfig)
|
640
|
-
|
732
|
+
const saveObjects = {...config.config.objects}
|
733
|
+
saveObjects.nameToUUID = {}
|
734
|
+
for (let km of config.configs) {
|
735
|
+
saveObjects.nameToUUID[km.name] = km.uuid
|
736
|
+
}
|
737
|
+
writeTest(testsFN, query, saveObjects, response.generated, response.paraphrases, response.responses, response.contexts, response.associations, response.metadata, actual_config, saveDeveloper, response.paraphrasesParenthesized, response.generatedParenthesized)
|
641
738
|
}
|
642
739
|
|
643
740
|
return response
|
644
741
|
} catch(error) {
|
645
742
|
error.query = query
|
646
|
-
|
743
|
+
errorHandler(error)
|
647
744
|
}
|
648
745
|
}
|
649
746
|
|
@@ -677,12 +774,10 @@ const getConfigForTest = (config, testConfig) => {
|
|
677
774
|
return configForTest
|
678
775
|
}
|
679
776
|
|
680
|
-
const runTest = async (config, expected, {
|
777
|
+
const runTest = async (config, expected, { args, verbose, testConfig, debug }) => {
|
681
778
|
const test = expected.query
|
682
779
|
// initialize in between test so state is not preserved since the test was adding without state
|
683
780
|
config.rebuild()
|
684
|
-
config.addAssociationsFromTests(config.tests)
|
685
|
-
// config.addAssocationsFromTests(
|
686
781
|
const errorHandler = (error) => {
|
687
782
|
if (error.metadata) {
|
688
783
|
const priorities = analyzeMetaData(expected.metadata, error.metadata)
|
@@ -700,8 +795,6 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
|
|
700
795
|
objects = getObjects(config.config.objects)(config.getConfigs()[testConfig.testModuleName].uuid)
|
701
796
|
testConfigName = testConfig.testModuleName
|
702
797
|
}
|
703
|
-
config.beforeQuery({ query: test, isModule: false, objects })
|
704
|
-
// config.resetMotivations()
|
705
798
|
try {
|
706
799
|
const result = await _process(config, test, { errorHandler, isTest: true })
|
707
800
|
result.query = test
|
@@ -716,66 +809,48 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
|
|
716
809
|
lines.log()
|
717
810
|
}
|
718
811
|
const expected_objects = sortJson(convertToStable(expected.objects), { depth: 25 })
|
812
|
+
delete expected_objects.nameToUUID
|
719
813
|
const actual_objects = sortJson(convertToStable(config.config.objects), { depth: 25 })
|
720
814
|
const failed_paraphrases = !matching(result.paraphrases, expected.paraphrases)
|
815
|
+
let failed_paraphrasesParenthesized = !matching(result.paraphrasesParenthesized, expected.paraphrasesParenthesized)
|
816
|
+
let failed_generatedParenthesized = !matching(result.generatedParenthesized, expected.generatedParenthesized)
|
817
|
+
// TODO fix the naming conventions: camelcase + use actual instead of result
|
721
818
|
const failed_responses = !matching(result.responses, expected.responses)
|
722
819
|
const failed_contexts = !matching(result.contexts, expected.contexts)
|
723
820
|
const failed_objects = !matching(actual_objects, expected_objects)
|
724
821
|
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
if (prop.km) {
|
729
|
-
c = config.getConfig(prop.km)
|
730
|
-
o = getObjects(prop.km)
|
731
|
-
picked[prop.km] = {}
|
732
|
-
for (let p of c.testConfig.check) {
|
733
|
-
if (p.km) {
|
734
|
-
continue
|
735
|
-
}
|
736
|
-
picked[p] = o[p]
|
737
|
-
}
|
738
|
-
} else {
|
739
|
-
picked[prop] = getObjects(testConfigName)[prop]
|
740
|
-
}
|
741
|
-
}
|
742
|
-
return picked
|
822
|
+
if (args.testNoParenthesized) {
|
823
|
+
failed_paraphrasesParenthesized = false
|
824
|
+
failed_generatedParenthesized = false
|
743
825
|
}
|
826
|
+
|
827
|
+
const pickedResultContexts = result.contexts.map(pickContext(testConfig))
|
828
|
+
const pickedExpectedContexts = expected.contexts.map(pickContext(testConfig))
|
829
|
+
const failedCheckedContexts = !matching(pickedResultContexts, pickedExpectedContexts)
|
830
|
+
|
744
831
|
const expectedGetObjects = (name) => {
|
745
832
|
if (!name) {
|
746
833
|
name = config.name
|
747
834
|
}
|
748
|
-
return expected.objects.namespaced[expected.objects.nameToUUID[name]]
|
835
|
+
return expected.objects.namespaced[expected.objects.nameToUUID[name]] || {}
|
749
836
|
}
|
750
|
-
const expected_checked = sortJson(
|
837
|
+
const expected_checked = sortJson(pickObjects(testConfig, expectedGetObjects(testConfigName)), { depth: 25 })
|
751
838
|
const actualGetObjects = (name) => {
|
752
839
|
if (!name) {
|
753
840
|
name = config.name
|
754
841
|
}
|
755
842
|
const km = config.configs.find( (km) => km.name == name )
|
756
|
-
return config.config.objects.namespaced[km.uuid]
|
843
|
+
return config.config.objects.namespaced[km.uuid] || {}
|
757
844
|
}
|
758
|
-
const actual_checked = sortJson(
|
845
|
+
const actual_checked = sortJson(pickObjects(testConfig, actualGetObjects(testConfigName)), { depth: 25 })
|
759
846
|
const failed_checked = !matching(actual_objects, expected_objects)
|
760
847
|
|
761
848
|
const failed_checks = !matching(actual_objects, expected_objects)
|
849
|
+
const failed_checked_objects = !matching(actual_checked, expected_checked)
|
762
850
|
const actual_config = sortJson(convertToStable(getConfigForTest(config, testConfig)), { depth: 25 })
|
763
851
|
const expected_config = sortJson(convertToStable(expected.config), { depth: 25 })
|
764
852
|
const failed_config = !matching(actual_config, expected_config)
|
765
|
-
let failed = failed_paraphrases || failed_responses || failed_contexts || failed_objects || failed_config || failed_checked
|
766
|
-
if (!failed) {
|
767
|
-
if (config.afterTest) {
|
768
|
-
failed = config.afterTest({ query: test, expected, actual: result, config })
|
769
|
-
if (failed) {
|
770
|
-
return {
|
771
|
-
utterance: test,
|
772
|
-
errorFromAfterTest: failed,
|
773
|
-
expected: { responses: expected.responses, paraphrases: expected.paraphrases, results: expected.contexts, checked: expected_checked, objects: expected_objects, config: expected.config },
|
774
|
-
actual: { responses: result.responses, paraphrases: result.paraphrases, results: result.contexts, checked: actual_checked, objects: actual_objects, config: actual_config }
|
775
|
-
}
|
776
|
-
}
|
777
|
-
}
|
778
|
-
}
|
853
|
+
let failed = failed_checked_objects || failed_paraphrases || failed_paraphrasesParenthesized || failed_generatedParenthesized || failed_responses || failed_contexts || failed_objects || failed_config || failed_checked || failedCheckedContexts
|
779
854
|
|
780
855
|
if (expected.metadata && result.metadata && failed) {
|
781
856
|
const priorities = analyzeMetaData(expected.metadata, result.metadata)
|
@@ -787,8 +862,28 @@ const runTest = async (config, expected, { verbose, afterTest, testConfig, debug
|
|
787
862
|
if (failed) {
|
788
863
|
return {
|
789
864
|
utterance: test,
|
790
|
-
expected: {
|
791
|
-
|
865
|
+
expected: {
|
866
|
+
responses: expected.responses,
|
867
|
+
paraphrases: expected.paraphrases,
|
868
|
+
paraphrasesParenthesized: expected.paraphrasesParenthesized,
|
869
|
+
generatedParenthesized: expected.generatedParenthesized,
|
870
|
+
results: expected.contexts,
|
871
|
+
checked: expected_checked,
|
872
|
+
checkedContexts: pickedExpectedContexts,
|
873
|
+
objects: expected_objects,
|
874
|
+
config: expected.config
|
875
|
+
},
|
876
|
+
actual: {
|
877
|
+
responses: result.responses,
|
878
|
+
paraphrases: result.paraphrases,
|
879
|
+
paraphrasesParenthesized: result.paraphrasesParenthesized,
|
880
|
+
generatedParenthesized: result.generatedParenthesized,
|
881
|
+
results: result.contexts,
|
882
|
+
checked: actual_checked,
|
883
|
+
checkedContexts: pickedResultContexts,
|
884
|
+
objects: actual_objects,
|
885
|
+
config: actual_config
|
886
|
+
}
|
792
887
|
}
|
793
888
|
}
|
794
889
|
} catch(error) {
|
@@ -825,20 +920,16 @@ const runTestsHelper = async (config, tests, failed, juicyBits) => {
|
|
825
920
|
|
826
921
|
const runTests = async (config, testFile, juicyBits) => {
|
827
922
|
const tests = JSON.parse(runtime.fs.readFileSync(testFile))
|
828
|
-
config.beforeTests()
|
829
923
|
if (juicyBits.verbose) {
|
830
924
|
console.log('\n', testFile, '-----------------------------------------------', '\n')
|
831
925
|
}
|
832
926
|
const v = await runTestsHelper(config, [...tests], [], juicyBits)
|
833
|
-
config.afterTests()
|
834
927
|
return v
|
835
928
|
}
|
836
929
|
|
837
930
|
const saveTest = async (testFile, config, test, expected, testConfig, saveDeveloper) => {
|
838
931
|
config.rebuild()
|
839
932
|
const objects = getObjects(config.config.objects)(config.uuid)
|
840
|
-
//config.resetMotivations()
|
841
|
-
config.beforeQuery({ query: test, isModule: false, objects })
|
842
933
|
console.log(test)
|
843
934
|
const result = await _process(config, test, { isTest: true })
|
844
935
|
// const actualObjects = config.config.objects
|
@@ -850,7 +941,7 @@ const saveTest = async (testFile, config, test, expected, testConfig, saveDevelo
|
|
850
941
|
for (let km of config.configs) {
|
851
942
|
saveObjects.nameToUUID[km.name] = km.uuid
|
852
943
|
}
|
853
|
-
writeTest(testFile, test, saveObjects, result.generated, result.paraphrases, result.responses, result.contexts, result.associations, result.metadata, actualConfig, saveDeveloper)
|
944
|
+
writeTest(testFile, test, saveObjects, result.generated, result.paraphrases, result.responses, result.contexts, result.associations, result.metadata, actualConfig, saveDeveloper, result.paraphrasesParenthesized, result.generatedParenthesized)
|
854
945
|
}
|
855
946
|
|
856
947
|
const saveTestsHelper = async (testFile, config, tests, todo, testConfig, saveDeveloper) => {
|
@@ -1020,10 +1111,11 @@ const defaultErrorHandler = async (error) => {
|
|
1020
1111
|
doErrorExit = true
|
1021
1112
|
}
|
1022
1113
|
|
1023
|
-
if (doErrorExit) {
|
1114
|
+
if (typeof runtime.process.exit == 'function' && doErrorExit) {
|
1024
1115
|
runtime.process.exit(-1)
|
1025
1116
|
}
|
1026
|
-
|
1117
|
+
|
1118
|
+
throw error
|
1027
1119
|
}
|
1028
1120
|
|
1029
1121
|
const defaultInnerProcess = (config, errorHandler, responses) => {
|
@@ -1033,6 +1125,13 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
|
|
1033
1125
|
}
|
1034
1126
|
console.log("KM's loaded", config.configs.map((c) => c.name))
|
1035
1127
|
console.log('This is the global objects from running semantics:\n', config.objects)
|
1128
|
+
if (!_.isEmpty(responses.learned_contextual_priorities)) {
|
1129
|
+
console.log('\nThe learned contextual priorties are :\n')
|
1130
|
+
for (const lcp of responses.learned_contextual_priorities) {
|
1131
|
+
console.log(` ${JSON.stringify(lcp)},\n`)
|
1132
|
+
}
|
1133
|
+
console.log("\n")
|
1134
|
+
}
|
1036
1135
|
if (responses.logs) {
|
1037
1136
|
console.log('Logs')
|
1038
1137
|
responses.logs.forEach((log) => console.log(` ${log}`))
|
@@ -1044,6 +1143,50 @@ const defaultInnerProcess = (config, errorHandler, responses) => {
|
|
1044
1143
|
} else {
|
1045
1144
|
console.log('objects', runtime.util.inspect(config.get('objects'), { depth: Infinity, sorted: true }))
|
1046
1145
|
}
|
1146
|
+
|
1147
|
+
const pickEm = () => {
|
1148
|
+
const picked = {}
|
1149
|
+
const namespaced = config.get('objects')['namespaced']
|
1150
|
+
for (let prop of getConfig_getObjectCheck(config.testConfig)) {
|
1151
|
+
if (prop.km) {
|
1152
|
+
/*
|
1153
|
+
const objects = namespaced[prop.km]]
|
1154
|
+
picked[prop.km] = {}
|
1155
|
+
for (let p of c.testConfig.check) {
|
1156
|
+
if (p.km) {
|
1157
|
+
continue
|
1158
|
+
}
|
1159
|
+
picked[p] = objects[p]
|
1160
|
+
}
|
1161
|
+
*/
|
1162
|
+
console.log('TODO implement this if needed')
|
1163
|
+
} else {
|
1164
|
+
const objects = namespaced[config.uuid]
|
1165
|
+
picked[prop] = objects[prop]
|
1166
|
+
}
|
1167
|
+
}
|
1168
|
+
return picked
|
1169
|
+
}
|
1170
|
+
|
1171
|
+
if (responses.explain_priorities) {
|
1172
|
+
console.log("Explain Priorities")
|
1173
|
+
for ([inputss, outpus, reason] of responses.explain_priorities) {
|
1174
|
+
console.log(` ${JSON.stringify(inputss)} reason: ${reason}`)
|
1175
|
+
}
|
1176
|
+
}
|
1177
|
+
const objects = config.get('objects').namespaced[config.uuid]
|
1178
|
+
const picked = sortJson(pickObjects(config.testConfig, objects), { depth: 25 })
|
1179
|
+
if (!_.isEmpty(picked)) {
|
1180
|
+
console.log('--- Object showing only the checked values ---')
|
1181
|
+
console.log(JSON.stringify(picked, null, 2))
|
1182
|
+
}
|
1183
|
+
|
1184
|
+
const pickedResultContexts = responses.contexts.map(pickContext(config.testConfig))
|
1185
|
+
if (pickedResultContexts.some( (context) => Object.keys(context).length > 0 )) {
|
1186
|
+
console.log('--- Contexts showing only the checked values ---')
|
1187
|
+
console.log(JSON.stringify(pickedResultContexts, null, 2))
|
1188
|
+
}
|
1189
|
+
|
1047
1190
|
console.log('--- The contexts are ----------')
|
1048
1191
|
console.log(JSON.stringify(sortJson(responses.contexts, { depth: 25 }), null, 2))
|
1049
1192
|
console.log('')
|
@@ -1084,37 +1227,14 @@ const defaultProcess = ({ config, errorHandler }) => async (promise) => {
|
|
1084
1227
|
}
|
1085
1228
|
}
|
1086
1229
|
|
1087
|
-
|
1088
|
-
const
|
1089
|
-
`const entodicton = require('entodicton')
|
1090
|
-
const base = require('./${kmBaseName}').copy()
|
1091
|
-
const ${kmName}_tests = require('./${kmName}.test.json')
|
1092
|
-
const ${kmName}_instance = require('./${kmBaseName}.${kmName}.instance.json')
|
1093
|
-
|
1094
|
-
const config = new entodicton.Config({ name: '${kmName}' })
|
1095
|
-
config.add(base)
|
1096
|
-
kirk_instance.base = '${kmBaseName}'
|
1097
|
-
config.load(${kmName}_instance)
|
1098
|
-
|
1099
|
-
entodicton.knowledgeModule( {
|
1100
|
-
module,
|
1101
|
-
description: '${kmName} related concepts',
|
1102
|
-
section,
|
1103
|
-
config,
|
1104
|
-
test: {
|
1105
|
-
name: './${kmName}.test.json',
|
1106
|
-
contents: ${kmName}_tests
|
1107
|
-
},
|
1108
|
-
})
|
1109
|
-
`
|
1110
|
-
*/
|
1111
|
-
|
1112
|
-
const build = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
|
1230
|
+
// builtTemplate saveInstance
|
1231
|
+
const rebuildTemplate = async ({ config, target, template, errorHandler = defaultErrorHandler }) => {
|
1113
1232
|
const accumulators = {
|
1114
1233
|
resultss: [],
|
1115
1234
|
fragments: [],
|
1116
1235
|
semantics: [],
|
1117
1236
|
associations: [],
|
1237
|
+
learned_contextual_priorities: [],
|
1118
1238
|
}
|
1119
1239
|
const looper = async (queries) => {
|
1120
1240
|
if (queries.length === 0) {
|
@@ -1133,8 +1253,6 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
|
|
1133
1253
|
if (property == 'fragments') {
|
1134
1254
|
global.transitoryMode = true
|
1135
1255
|
}
|
1136
|
-
// greg32
|
1137
|
-
// config.addInternal( query )
|
1138
1256
|
if (hierarchy) {
|
1139
1257
|
for (let edge of hierarchy) {
|
1140
1258
|
if (Array.isArray(edge)) {
|
@@ -1144,25 +1262,28 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
|
|
1144
1262
|
}
|
1145
1263
|
}
|
1146
1264
|
}
|
1147
|
-
|
1148
1265
|
try {
|
1149
|
-
const results = await _process(config, query.query, {initializer})
|
1266
|
+
const results = await _process(config, query.query, {initializer, rebuildingTemplate: true})
|
1150
1267
|
if (config.config.debug) {
|
1151
1268
|
// TODO pass in the error handler like the other ones
|
1152
1269
|
defaultInnerProcess(config, defaultErrorHandler, results)
|
1153
1270
|
}
|
1154
1271
|
if (results.contexts.length > 1) {
|
1155
|
-
console.log(`query ${query.query}. There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
|
1272
|
+
console.log(`query "${query.query}". There is ${results.contexts.length} contexts in the results. Make sure its producing the results that you expect.`)
|
1273
|
+
} else if (results.paraphrases[0] != query.query) {
|
1274
|
+
console.log(`query "${query.query}". The paraphrase is different from the query "${results.paraphrases[0]}".`)
|
1156
1275
|
} else {
|
1157
1276
|
console.log(`query ${query.query}`)
|
1158
1277
|
}
|
1159
1278
|
global.transitoryMode = transitoryMode
|
1160
1279
|
config.config.skipSemantics = null
|
1161
1280
|
results.query = query.query
|
1281
|
+
results.skipSemantics = skipSemantics
|
1162
1282
|
results.development = query.development
|
1163
1283
|
results.key = { query: query.query, hierarchy }
|
1164
1284
|
accumulators[property].push(results)
|
1165
1285
|
accumulators.associations = accumulators.associations.concat(results.associations)
|
1286
|
+
accumulators.learned_contextual_priorities = accumulators.learned_contextual_priorities.concat(results.learned_contextual_priorities)
|
1166
1287
|
await looper(queries)
|
1167
1288
|
} catch(e) {
|
1168
1289
|
const error = { errors: [e], query: query.query };
|
@@ -1174,6 +1295,12 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
|
|
1174
1295
|
// it will just get added to the config
|
1175
1296
|
const extraConfig = queryOrExtraConfig
|
1176
1297
|
console.log('config', extraConfig)
|
1298
|
+
try {
|
1299
|
+
config.addInternal(_.cloneDeep(extraConfig), { handleCalculatedProps: true } )
|
1300
|
+
} catch ( e ) {
|
1301
|
+
const where = extraConfig.where ? ` ${extraConfig.where}` : ''
|
1302
|
+
throw new Error(`Error processing extra config${where}: ${e.stack}}`)
|
1303
|
+
}
|
1177
1304
|
accumulators[property].push({ extraConfig: true, ...extraConfig })
|
1178
1305
|
await looper(queries)
|
1179
1306
|
}
|
@@ -1201,6 +1328,7 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
|
|
1201
1328
|
delete result.version
|
1202
1329
|
result.hierarchy.sort()
|
1203
1330
|
stabilizeAssociations(result.associations)
|
1331
|
+
result.learned_contextual_priorities = safeNoDups(result.learned_contextual_priorities)
|
1204
1332
|
}
|
1205
1333
|
}
|
1206
1334
|
}
|
@@ -1221,14 +1349,14 @@ const build = async ({ config, target, template, errorHandler = defaultErrorHand
|
|
1221
1349
|
|
1222
1350
|
const toProperties = (queryStringOrProperties) => {
|
1223
1351
|
if (typeof queryStringOrProperties == 'string') {
|
1224
|
-
return { query: queryStringOrProperties}
|
1352
|
+
return { query: queryStringOrProperties }
|
1225
1353
|
} else {
|
1226
1354
|
return queryStringOrProperties
|
1227
1355
|
}
|
1228
1356
|
}
|
1229
1357
|
let todo = []
|
1230
|
-
todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false } }))
|
1231
|
-
todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false } }))
|
1358
|
+
todo = todo.concat((template.initializers || []).map((query) => { return { initializer: true, property: 'resultss', query, skipSemantics: false || query.skipSemantics } }))
|
1359
|
+
todo = todo.concat((template.queries || []).map((query) => { return { property: 'resultss', query, skipSemantics: false || query.skipSemantics} }))
|
1232
1360
|
todo = todo.concat((template.fragments || []).map((query) => { return Object.assign({}, toProperties(query), { property: 'fragments', skipSemantics: false }) }))
|
1233
1361
|
todo = todo.concat((template.semantics || []).map((definition) => { return { property: 'semantics', query: `${definition.from}\n${definition.to}`, skipSemantics: true } }))
|
1234
1362
|
await looper(Object.assign([], todo))
|
@@ -1238,114 +1366,86 @@ const knowledgeModule = async ({
|
|
1238
1366
|
module: moduleFromJSFile,
|
1239
1367
|
description,
|
1240
1368
|
section,
|
1241
|
-
config,
|
1369
|
+
// config, createConfig,
|
1370
|
+
createConfig,
|
1371
|
+
newWay,
|
1242
1372
|
demo,
|
1243
1373
|
test,
|
1244
1374
|
template,
|
1245
1375
|
errorHandler = defaultErrorHandler,
|
1246
1376
|
process: processResults = defaultProcess,
|
1247
1377
|
stopAtFirstFailure = true,
|
1248
|
-
|
1249
|
-
beforeTests = () => {},
|
1250
|
-
afterTests = () => {},
|
1251
|
-
beforeTest = () => {},
|
1252
|
-
afterTest = () => {}
|
1378
|
+
...rest
|
1253
1379
|
} = {}) => {
|
1254
1380
|
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
config.afterTest = afterTest
|
1381
|
+
const unknownArgs = Object.keys(rest)
|
1382
|
+
if (unknownArgs.length > 0) {
|
1383
|
+
throw new Error(`Unknown arguments to knowledgeModule: ${unknownArgs.join()}`)
|
1384
|
+
}
|
1260
1385
|
|
1261
1386
|
const testConfig = test
|
1262
1387
|
|
1263
1388
|
if (!moduleFromJSFile) {
|
1264
|
-
throw "'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module."
|
1389
|
+
throw new Error("'module' is a required parameter. The value should be either 'module' or a lambda that will be called when the file is acting as a module.")
|
1265
1390
|
}
|
1266
|
-
if (!config) {
|
1267
|
-
|
1268
|
-
|
1269
|
-
if (!config.name) {
|
1270
|
-
throw "config must have 'name' set to the knowledge module name."
|
1391
|
+
// if (!config && !createConfig) {
|
1392
|
+
if (!createConfig) {
|
1393
|
+
throw new Error("'config' or 'createConfig' is a required parameter. The value should the config that defines the knowledge module.")
|
1271
1394
|
}
|
1272
1395
|
if (!description) {
|
1273
|
-
throw "'description' is a required parameter. The value should the description of the knowledge module."
|
1396
|
+
throw new Error("'description' is a required parameter. The value should the description of the knowledge module.")
|
1274
1397
|
}
|
1275
1398
|
if (!test) {
|
1276
|
-
throw "'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }."
|
1399
|
+
throw new Error("'test' is a required parameter. The value should the path to the file used to store the tests of the knowledge module and the contents of the file in the form { name: <filePath>, contexts: <json> }.")
|
1277
1400
|
}
|
1278
1401
|
|
1279
1402
|
const isProcess = require.main === moduleFromJSFile
|
1280
|
-
|
1281
|
-
|
1282
|
-
if (
|
1283
|
-
|
1403
|
+
|
1404
|
+
const setupConfig = (config) => {
|
1405
|
+
if (!config.name) {
|
1406
|
+
throw new Error("config must have 'name' set to the knowledge module name.")
|
1284
1407
|
}
|
1285
|
-
}
|
1286
1408
|
|
1287
|
-
|
1288
|
-
|
1289
|
-
|
1290
|
-
|
1291
|
-
|
1292
|
-
} else {
|
1293
|
-
return true
|
1409
|
+
config.description = description
|
1410
|
+
if (typeof test === 'object') {
|
1411
|
+
if (test.contents) {
|
1412
|
+
config.tests = test.contents
|
1413
|
+
test = test.name
|
1294
1414
|
}
|
1295
|
-
})
|
1296
|
-
config.config.bridges = config.config.bridges.filter( (bridge) => {
|
1297
|
-
if (bridge.development) {
|
1298
|
-
return false
|
1299
|
-
} else {
|
1300
|
-
return true
|
1301
|
-
}
|
1302
|
-
})
|
1303
|
-
}
|
1304
|
-
|
1305
|
-
let module
|
1306
|
-
if (_.isFunction(moduleFromJSFile)) {
|
1307
|
-
module = moduleFromJSFile
|
1308
|
-
} else {
|
1309
|
-
module = () => {
|
1310
|
-
config.rebuild({ isModule: true })
|
1311
|
-
moduleFromJSFile.exports = config
|
1312
|
-
}
|
1313
|
-
}
|
1314
|
-
processResults = processResults({ config, errorHandler })
|
1315
|
-
config.description = description
|
1316
|
-
config.demo = demo
|
1317
|
-
if (typeof test === 'object') {
|
1318
|
-
if (test.contents) {
|
1319
|
-
config.tests = test.contents
|
1320
|
-
test = test.name
|
1321
|
-
}
|
1322
|
-
} else {
|
1323
|
-
if (runtime.fs && runtime.fs.existsSync(test)) {
|
1324
|
-
config.tests = JSON.parse(runtime.fs.readFileSync(test))
|
1325
1415
|
} else {
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1329
|
-
|
1330
|
-
|
1331
|
-
if (!isProcess) {
|
1332
|
-
if (template) {
|
1333
|
-
if (config.needsRebuild(template.template, template.instance)) {
|
1334
|
-
throw `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguements or the -rt argument to rebuild.`
|
1416
|
+
if (runtime.fs && runtime.fs.existsSync(test)) {
|
1417
|
+
config.tests = JSON.parse(runtime.fs.readFileSync(test))
|
1418
|
+
} else {
|
1419
|
+
config.tests = []
|
1335
1420
|
}
|
1336
|
-
config.load(template.template, template.instance)
|
1337
1421
|
}
|
1422
|
+
config.setTestConfig(testConfig)
|
1338
1423
|
}
|
1424
|
+
|
1425
|
+
|
1339
1426
|
if (isProcess) {
|
1427
|
+
const config = createConfig()
|
1428
|
+
setupConfig(config)
|
1429
|
+
processResults = processResults({ config, errorHandler })
|
1340
1430
|
// setup();
|
1341
1431
|
const parser = new runtime.ArgumentParser({
|
1342
1432
|
description: 'Entodicton knowledge module'
|
1343
1433
|
})
|
1344
1434
|
|
1435
|
+
const helpDebugAssociation = 'In order to get a debug break when a specific association is created set the DEBUG_ASSOCIATION environment variable to the JSON of the association to break on. For example DEBUG_ASSOCIATION=\'[["the", 0], ["mammal", 1]]\''
|
1436
|
+
const helpDebugHierarchy = 'In order to get a debug break when a specific hierarchy is created set the DEBUG_HIERARCHY environment variable to the JSON of the child-parent pair to break on. For example DEBUG_HIERARCHY=\'[["cat", 1], ["mammel", 1]]\''
|
1437
|
+
const helpDebugPriority = 'In order to get a debug break when a specific set of priorities is created set set DEBUG_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_PRIORITY=\'[["verb", 0], ["article", 0]]\''
|
1438
|
+
const helpDebugContextualPriority = 'In order to get a debug break when a specific set of contextual priorities is created set set DEBUG_CONTEXTUAL_PRIORITY environment variable to the JSON of the priorities that you want to break on. For example DEBUG_CONTEXTUAL_PRIORITY=\'{ context: [["verb", 0], ["article", 0], select: 1}\''
|
1439
|
+
const helpDebugBridge = 'In order to get a debug break when a specific bridge is created set the DEBUG_BRIDGE environment variable to id/level to break on. For example DEBUG_BRIDGE=\'id#level\''
|
1440
|
+
const helpDebugOperator = 'In order to get a debug break when a specific hierarcy is created set the DEBUG_OPERATOR environment variable to debug any config loaded. For example DEBUG_OPERATOR=\'([operator] ([arg]))\''
|
1441
|
+
|
1442
|
+
|
1345
1443
|
parser.add_argument('-tmn', '--testModuleName', { help: 'When running tests instead of using the current modules tests use the specified modules tests' })
|
1346
1444
|
parser.add_argument('-t', '--test', { action: 'store_true', help: 'Run the tests. Create tests by running with the --query + --save flag' })
|
1347
1445
|
parser.add_argument('-tv', '--testVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. Create tests by running with the --query or --loop with the --save flag' })
|
1446
|
+
// parser.add_argument('-ttr', '--testToRun', { help: 'Only the specified test will be run' })
|
1348
1447
|
parser.add_argument('-tva', '--testAllVerbose', { action: 'store_true', help: 'Run the tests in verbose mode. All the tests will be run instead of stopping at first failure. Create tests by running with the --query or --loop with the --save flag' })
|
1448
|
+
parser.add_argument('-tnp', '--testNoParenthesized', { action: 'store_true', help: 'Don\' check parenthesized differences for the tests' })
|
1349
1449
|
parser.add_argument('-n', '--count', { help: 'Number of times to run the tests. Default is one. Use this to check for flakey test. If possible the system will print out a message with the word "hint" suggesting how to fix the problem' })
|
1350
1450
|
// parser.add_argument('-b', '--build', { help: 'Specify the template file name of the form <kmName>. There should be a file called <baseKmName>.<kmName>.template.json with the queries to run. For example { queries: [...] }. The template file will be run and generate an instantiation called <baseKmName>.<kmName>.instance.json and a file called <kmName>.js that will load the template file (this is file generated only if not already existing) and a test file called <KmName>.tests.json. This can then be loaded into an instance of the current knowledge module to setup initial conditions.' })
|
1351
1451
|
parser.add_argument('-rt', '--rebuildTemplate', { action: 'store_true', help: 'Force a template rebuild' })
|
@@ -1353,45 +1453,83 @@ const knowledgeModule = async ({
|
|
1353
1453
|
parser.add_argument('-i', '--info', { action: 'store_true', help: 'Print meta-data for the module' })
|
1354
1454
|
parser.add_argument('-v', '--vimdiff', { action: 'store_true', help: 'For failures run vimdiff' })
|
1355
1455
|
parser.add_argument('-g', '--greg', { action: 'store_true', help: 'Set the server to be localhost so I can debug stuff' })
|
1356
|
-
parser.add_argument('-cl', '--checkForLoop', {
|
1456
|
+
parser.add_argument('-cl', '--checkForLoop', { nargs: "?", help: 'Check for loops in the priorities, Optional argument is list of operator keys to consider. For example [["banana", 0], ["food", 1]]' })
|
1357
1457
|
parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
|
1358
1458
|
parser.add_argument('-q', '--query', { help: 'Run the specified query' })
|
1359
1459
|
parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
|
1360
1460
|
parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
|
1361
1461
|
parser.add_argument('-dt', '--deleteTest', { help: 'Delete the specified query from the tests file.' })
|
1462
|
+
parser.add_argument('--parenthesized', { action: 'store_true', help: 'Show the generated phrases with parenthesis.' })
|
1362
1463
|
parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
|
1363
1464
|
parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
|
1364
|
-
parser.add_argument('-daa', '--dontAddAssociations', { action: 'store_true', help: 'Do not add associations from the tests.' })
|
1365
1465
|
parser.add_argument('-p', '--print', { help: 'Print the specified elements c === config, w === words, b === bridges, o === operators d === objects (d for data), h === hierarchy, g === generators, s === semantics, l === load t=tests ordering p === priorities a == associations j == JSON sent to server. for example --print wb' })
|
1366
1466
|
parser.add_argument('-s', '--save', { action: 'store_true', help: 'When running with the --query flag this will save the current run to the test file. When running without the --query flag all tests will be run and resaved.' })
|
1367
1467
|
parser.add_argument('-sd', '--saveDeveloper', { action: 'store_true', help: 'Same as -s but the query will not show up in the info command.' })
|
1368
|
-
parser.add_argument('-dic', '--debugIncludeConvolutions', { action: 'store_true', help: 'When running with the --debugIncludeConvolutions flag the logs will include convolutions which are somewhat annoying verbose. Default is false' })
|
1369
1468
|
parser.add_argument('-dl', '--debugLoops', { action: 'store_true', help: 'When running with the --debugLoops flag the logs calls to semantics and generators will be immediately written to the console '})
|
1370
1469
|
parser.add_argument('-d', '--debug', { action: 'store_true', help: 'When running with the --debug flag this set the debug flag in the config' })
|
1371
|
-
parser.add_argument('-da', '--debugAssociation', {
|
1372
|
-
parser.add_argument('-dh', '--debugHierarchy', {
|
1373
|
-
parser.add_argument('-
|
1374
|
-
parser.add_argument('-
|
1470
|
+
parser.add_argument('-da', '--debugAssociation', { action: 'store_true', help: helpDebugAssociation })
|
1471
|
+
parser.add_argument('-dh', '--debugHierarchy', { action: 'store_true', help: helpDebugHierarchy })
|
1472
|
+
parser.add_argument('-dp', '--debugPriority', { action: 'store_true', help: helpDebugPriority })
|
1473
|
+
parser.add_argument('-dcp', '--debugContextualPriority', { action: 'store_true', help: helpDebugContextualPriority })
|
1474
|
+
parser.add_argument('-db', '--debugBridge', { action: 'store_true', help: helpDebugBridge })
|
1475
|
+
parser.add_argument('-do', '--debugOperator', { action: 'store_true', help: helpDebugOperator })
|
1476
|
+
parser.add_argument('-ep', '--explainPriorities', { action: 'store_true', help: "The server will return all priorities including the generated one along with an explanation of there they came from"})
|
1477
|
+
parser.add_argument('-dic', '--debugIncludeConvolutions', { nargs: "?", help: 'When running with the --debugIncludeConvolutions flag the logs will include convolutions which are somewhat annoyingly verbose. Default is false' })
|
1375
1478
|
|
1376
1479
|
const args = parser.parse_args()
|
1377
1480
|
args.count = args.count || 1
|
1378
1481
|
|
1482
|
+
if (args.parenthesized) {
|
1483
|
+
config.parenthesized = true
|
1484
|
+
}
|
1485
|
+
if (args.checkForLoop) {
|
1486
|
+
try {
|
1487
|
+
args.checkForLoop = JSON.parse(args.checkForLoop)
|
1488
|
+
const isKey = (what) => {
|
1489
|
+
if (!Array.isArray(what)) {
|
1490
|
+
return false
|
1491
|
+
}
|
1492
|
+
if (what.length !== 2) {
|
1493
|
+
return false
|
1494
|
+
}
|
1495
|
+
if (!typeof what[0] == 'string') {
|
1496
|
+
return false
|
1497
|
+
}
|
1498
|
+
if (!typeof what[1] == 'number') {
|
1499
|
+
return false
|
1500
|
+
}
|
1501
|
+
return true
|
1502
|
+
}
|
1503
|
+
if (!Array.isArray(args.checkForLoop) || args.checkForLoop.some((value) => !isKey(value))) {
|
1504
|
+
throw new Error(`Error for the checkForLoop argument. Expected a JSON array of operator keys of the form "[<id>, <level>]"`)
|
1505
|
+
}
|
1506
|
+
} catch( e ) {
|
1507
|
+
throw new Error(`Error parsing JSON of the checkForLoop argument. ${e}`)
|
1508
|
+
}
|
1509
|
+
} else {
|
1510
|
+
if (process.argv.includes('--checkForLoop') || process.argv.includes('-cl')) {
|
1511
|
+
args.checkForLoop = true
|
1512
|
+
}
|
1513
|
+
}
|
1379
1514
|
if (args.debugAssociation) {
|
1380
|
-
|
1515
|
+
console.log(helpDebugAssociation)
|
1516
|
+
runtime.process.exit(-1)
|
1381
1517
|
}
|
1382
1518
|
if (args.debugHierarchy) {
|
1383
|
-
|
1519
|
+
console.log(helpDebugHierarchy)
|
1520
|
+
runtime.process.exit(-1)
|
1521
|
+
}
|
1522
|
+
if (args.debugPriority) {
|
1523
|
+
console.log(helpDebugPriority)
|
1524
|
+
runtime.process.exit(-1)
|
1384
1525
|
}
|
1385
1526
|
if (args.debugBridge) {
|
1386
|
-
|
1387
|
-
|
1388
|
-
if (global.entodictonDebugBridge.length !== 2) {
|
1389
|
-
console.log('Expected DEBUG_BRIDGE to be of the form "id/level"');
|
1390
|
-
}
|
1527
|
+
console.log(helpDebugBridge)
|
1528
|
+
runtime.process.exit(-1)
|
1391
1529
|
}
|
1392
1530
|
if (args.debugOperator) {
|
1393
|
-
|
1394
|
-
|
1531
|
+
console.log(helpDebugOperator)
|
1532
|
+
runtime.process.exit(-1)
|
1395
1533
|
}
|
1396
1534
|
|
1397
1535
|
if (args.clean) {
|
@@ -1430,117 +1568,115 @@ const knowledgeModule = async ({
|
|
1430
1568
|
if (args.debug) {
|
1431
1569
|
config.config.debug = true
|
1432
1570
|
}
|
1433
|
-
config.config.debugIncludeConvolutions = args.debugIncludeConvolutions
|
1434
1571
|
|
1435
|
-
if (
|
1436
|
-
|
1437
|
-
if (needsRebuild) {
|
1438
|
-
console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
|
1439
|
-
|
1440
|
-
}
|
1441
|
-
config.load(template.template, template.instance, { rebuild: needsRebuild })
|
1442
|
-
if (needsRebuild) {
|
1443
|
-
return
|
1444
|
-
}
|
1572
|
+
if (args.explainPriorities) {
|
1573
|
+
config.config.explain_priorities = true
|
1445
1574
|
}
|
1446
1575
|
|
1447
|
-
|
1448
|
-
config.addAssociationsFromTests(config.tests);
|
1449
|
-
//for (let query in config.tests) {
|
1450
|
-
// config.addAssociations(config.tests[query].associations || []);
|
1451
|
-
//}
|
1452
|
-
}
|
1576
|
+
config.config.debugIncludeConvolutions = args.debugIncludeConvolutions || process.argv.includes('--debugIncludeConvolutions') || process.argv.includes('-dic')
|
1453
1577
|
|
1454
|
-
|
1455
|
-
|
1456
|
-
if (
|
1457
|
-
|
1458
|
-
config.load(template.template, template.instance, { rebuild: true })
|
1578
|
+
let configPrinted = false
|
1579
|
+
const printConfig = () => {
|
1580
|
+
if (configPrinted) {
|
1581
|
+
return
|
1459
1582
|
}
|
1460
|
-
|
1461
|
-
|
1462
|
-
|
1463
|
-
|
1464
|
-
|
1465
|
-
|
1466
|
-
|
1467
|
-
|
1468
|
-
|
1469
|
-
|
1583
|
+
configPrinted = true
|
1584
|
+
if (args.print) {
|
1585
|
+
if (args.print.includes('t')) {
|
1586
|
+
console.log("Test queries")
|
1587
|
+
let counter = 0
|
1588
|
+
for (const test of config.tests) {
|
1589
|
+
console.log(`${counter} - ${test.query}`)
|
1590
|
+
counter += 1
|
1591
|
+
}
|
1592
|
+
}
|
1593
|
+
if (args.print.includes('c')) {
|
1594
|
+
const { data } = setupProcessB({ config })
|
1595
|
+
console.log("Config as sent to server")
|
1596
|
+
console.log(JSON.stringify(data, null, 2));
|
1470
1597
|
}
|
1471
|
-
}
|
1472
|
-
if (args.print.includes('c')) {
|
1473
|
-
const { data } = setupProcessB({ config })
|
1474
|
-
console.log("Config as sent to server")
|
1475
|
-
console.log(JSON.stringify(data, null, 2));
|
1476
|
-
}
|
1477
1598
|
|
1478
|
-
|
1479
|
-
|
1480
|
-
|
1481
|
-
|
1599
|
+
if (args.print.includes('l')) {
|
1600
|
+
console.log('Module load ordering')
|
1601
|
+
for (const km of config.configs) {
|
1602
|
+
console.log(` ${km.name}`)
|
1603
|
+
}
|
1482
1604
|
}
|
1483
|
-
|
1484
|
-
|
1485
|
-
|
1486
|
-
|
1605
|
+
if (args.print.includes('w')) {
|
1606
|
+
for (const word in config.config.words) {
|
1607
|
+
console.log(word.concat(' ', ...config.config.words[word].map((def) => JSON.stringify(def))))
|
1608
|
+
}
|
1487
1609
|
}
|
1488
|
-
|
1489
|
-
|
1490
|
-
|
1491
|
-
|
1610
|
+
if (args.print.includes('b')) {
|
1611
|
+
for (const bridge of config.config.bridges) {
|
1612
|
+
console.log(JSON.stringify(bridge))
|
1613
|
+
}
|
1492
1614
|
}
|
1493
|
-
|
1494
|
-
|
1495
|
-
|
1496
|
-
|
1615
|
+
if (args.print.includes('o')) {
|
1616
|
+
for (const operator of config.config.operators) {
|
1617
|
+
console.log(JSON.stringify(operator))
|
1618
|
+
}
|
1497
1619
|
}
|
1498
|
-
|
1499
|
-
|
1500
|
-
|
1501
|
-
|
1502
|
-
|
1503
|
-
|
1504
|
-
|
1505
|
-
|
1506
|
-
|
1507
|
-
|
1508
|
-
|
1509
|
-
|
1620
|
+
if (args.print.includes('j')) {
|
1621
|
+
const { data } = setupProcessB( { config } )
|
1622
|
+
console.log(JSON.stringify(data, null, 2))
|
1623
|
+
}
|
1624
|
+
if (args.print.includes('a')) {
|
1625
|
+
console.log('associations ================')
|
1626
|
+
const properties = ['negative', 'positive']
|
1627
|
+
for (let property of properties) {
|
1628
|
+
console.log(` ${property} ===============`)
|
1629
|
+
for (let association of config.config.associations[property]) {
|
1630
|
+
console.log(` ${JSON.stringify(association)}`)
|
1631
|
+
}
|
1510
1632
|
}
|
1511
1633
|
}
|
1512
|
-
|
1513
|
-
|
1514
|
-
console.log(JSON.stringify(config.config.objects, null, 2))
|
1515
|
-
}
|
1516
|
-
if (args.print.includes('p')) {
|
1517
|
-
for (let priority of config.config.priorities) {
|
1518
|
-
console.log(JSON.stringify(priority))
|
1634
|
+
if (args.print.includes('d')) {
|
1635
|
+
console.log(JSON.stringify(config.config.objects, null, 2))
|
1519
1636
|
}
|
1520
|
-
|
1521
|
-
|
1522
|
-
|
1523
|
-
|
1637
|
+
if (args.print.includes('p')) {
|
1638
|
+
for (let priority of config.config.priorities) {
|
1639
|
+
console.log(JSON.stringify(priority))
|
1640
|
+
}
|
1524
1641
|
}
|
1525
|
-
|
1526
|
-
|
1527
|
-
|
1528
|
-
for (const semantic of easyToRead) {
|
1529
|
-
semantic.match = semantic.match.toString()
|
1530
|
-
semantic.apply = semantic.apply.toString()
|
1531
|
-
if (semantic.applyWrapped) {
|
1532
|
-
semantic.applyWrapped = semantic.applyWrapped.toString()
|
1642
|
+
if (args.print.includes('h')) {
|
1643
|
+
for (let edge of config.config.hierarchy) {
|
1644
|
+
console.log(JSON.stringify(edge))
|
1533
1645
|
}
|
1534
1646
|
}
|
1535
|
-
|
1536
|
-
|
1537
|
-
|
1538
|
-
|
1539
|
-
|
1540
|
-
|
1541
|
-
|
1647
|
+
if (args.print.includes('g')) {
|
1648
|
+
const easyToRead = _.cloneDeep(config.config.generators)
|
1649
|
+
for (const semantic of easyToRead) {
|
1650
|
+
semantic.match = semantic.match.toString()
|
1651
|
+
semantic.apply = semantic.apply.toString()
|
1652
|
+
if (semantic.applyWrapped) {
|
1653
|
+
semantic.applyWrapped = semantic.applyWrapped.toString()
|
1654
|
+
}
|
1655
|
+
}
|
1656
|
+
console.dir(easyToRead)
|
1542
1657
|
}
|
1543
|
-
|
1658
|
+
if (args.print.includes('s')) {
|
1659
|
+
const easyToRead = _.cloneDeep(config.config.semantics)
|
1660
|
+
for (const semantic of easyToRead) {
|
1661
|
+
semantic.match = semantic.match.toString()
|
1662
|
+
semantic.apply = semantic.apply.toString()
|
1663
|
+
}
|
1664
|
+
console.dir(easyToRead)
|
1665
|
+
}
|
1666
|
+
}
|
1667
|
+
}
|
1668
|
+
|
1669
|
+
if (template) {
|
1670
|
+
const needsRebuild = config.needsRebuild(template.template, template.instance, options)
|
1671
|
+
if (needsRebuild) {
|
1672
|
+
console.log(`This module "${config.name}" needs rebuilding all other arguments will be ignored. Try again after the template is rebuilt.`)
|
1673
|
+
options.rebuild = true
|
1674
|
+
config.config.rebuild = true
|
1675
|
+
}
|
1676
|
+
config.load(template.template, template.instance, { rebuild: needsRebuild })
|
1677
|
+
printConfig()
|
1678
|
+
if (needsRebuild) {
|
1679
|
+
return
|
1544
1680
|
}
|
1545
1681
|
}
|
1546
1682
|
|
@@ -1575,12 +1711,7 @@ const knowledgeModule = async ({
|
|
1575
1711
|
test = useTestConfig.name
|
1576
1712
|
|
1577
1713
|
}
|
1578
|
-
runTests(config, test, { debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
|
1579
|
-
if (results.length > 0 && args.vimdiff) {
|
1580
|
-
for (const result of results) {
|
1581
|
-
vimdiff(result.expected, result.actual)
|
1582
|
-
}
|
1583
|
-
}
|
1714
|
+
runTests(config, test, { args, debug: args.debug, testConfig: useTestConfig, verbose: args.testVerbose || args.testAllVerbose, stopAtFirstError: !args.testAllVerbose }).then((results) => {
|
1584
1715
|
let newError = false
|
1585
1716
|
if (results.length > 0) {
|
1586
1717
|
let headerShown = false
|
@@ -1590,18 +1721,49 @@ const knowledgeModule = async ({
|
|
1590
1721
|
if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
|
1591
1722
|
hasError = true
|
1592
1723
|
}
|
1724
|
+
if (!args.testNoParenthesized) {
|
1725
|
+
if (JSON.stringify(result.expected.paraphrasesParenthesized) !== JSON.stringify(result.actual.paraphrasesParenthesized)) {
|
1726
|
+
hasError = true
|
1727
|
+
}
|
1728
|
+
if (JSON.stringify(result.expected.generatedParenthesized) !== JSON.stringify(result.actual.generatedParenthesized)) {
|
1729
|
+
hasError = true
|
1730
|
+
}
|
1731
|
+
}
|
1593
1732
|
if (JSON.stringify(result.expected.responses) !== JSON.stringify(result.actual.responses)) {
|
1594
1733
|
hasError = true
|
1595
1734
|
}
|
1596
1735
|
if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
|
1597
1736
|
hasError = true
|
1598
1737
|
}
|
1738
|
+
if (!sameJSON(result.expected.checkedContexts, result.actual.checkedContexts)) {
|
1739
|
+
hasError = true
|
1740
|
+
}
|
1599
1741
|
}
|
1600
1742
|
|
1601
1743
|
if (hasError) {
|
1602
1744
|
console.log('**************************** ERRORS ************************')
|
1603
1745
|
for (const result of results) {
|
1604
1746
|
console.log('Utterance: ', result.utterance)
|
1747
|
+
const show = (label, expected, actual) => {
|
1748
|
+
if (JSON.stringify(expected) !== JSON.stringify(actual)) {
|
1749
|
+
if (!headerShown) {
|
1750
|
+
console.log(' Failure')
|
1751
|
+
}
|
1752
|
+
console.log(` expected ${label}`, expected)
|
1753
|
+
console.log(` actual ${label} `, actual)
|
1754
|
+
newError = true
|
1755
|
+
headerShown = true
|
1756
|
+
}
|
1757
|
+
}
|
1758
|
+
show('paraphrases', result.expected.paraphrases, result.actual.paraphrases)
|
1759
|
+
if (!args.testNoParenthesized) {
|
1760
|
+
show('paraphrases parenthesized', result.expected.paraphrasesParenthesized, result.actual.paraphrasesParenthesized)
|
1761
|
+
}
|
1762
|
+
show('responses', result.expected.responses, result.actual.responses)
|
1763
|
+
if (!args.testNoParenthesized) {
|
1764
|
+
show('responses parenthesized', result.expected.generatedParenthesized, result.actual.generatedParenthesized)
|
1765
|
+
}
|
1766
|
+
/*
|
1605
1767
|
if (JSON.stringify(result.expected.paraphrases) !== JSON.stringify(result.actual.paraphrases)) {
|
1606
1768
|
if (!headerShown) {
|
1607
1769
|
console.log(' Failure')
|
@@ -1620,6 +1782,7 @@ const knowledgeModule = async ({
|
|
1620
1782
|
newError = true
|
1621
1783
|
headerShown = true
|
1622
1784
|
}
|
1785
|
+
*/
|
1623
1786
|
if (JSON.stringify(result.expected.checked) !== JSON.stringify(result.actual.checked)) {
|
1624
1787
|
if (!headerShown) {
|
1625
1788
|
console.log(' Failure')
|
@@ -1632,10 +1795,37 @@ const knowledgeModule = async ({
|
|
1632
1795
|
lines.setElement(1, 1, 'actual checked')
|
1633
1796
|
lines.setElement(2, 2, JSON.stringify(result.actual.checked, null, 2))
|
1634
1797
|
lines.log()
|
1798
|
+
if (args.vimdiff) {
|
1799
|
+
vimdiff(result.actual.checked, result.expected.checked)
|
1800
|
+
}
|
1801
|
+
newError = true
|
1802
|
+
headerShown = true
|
1803
|
+
}
|
1804
|
+
if (!sameJSON(result.expected.checkedContexts, result.actual.checkedContexts)) {
|
1805
|
+
if (!headerShown) {
|
1806
|
+
console.log(' Failure')
|
1807
|
+
}
|
1808
|
+
const widths = [4, 18, 72]
|
1809
|
+
const lines = new Lines(widths)
|
1810
|
+
lines.setElement(1, 1, 'expected checkedContexts', true)
|
1811
|
+
lines.setElement(2, 2, JSON.stringify(result.expected.checkedContexts, null, 2))
|
1812
|
+
lines.log()
|
1813
|
+
lines.setElement(1, 1, 'actual checkedContexts', true)
|
1814
|
+
lines.setElement(2, 2, JSON.stringify(result.actual.checkedContexts, null, 2))
|
1815
|
+
lines.log()
|
1816
|
+
if (args.vimdiff) {
|
1817
|
+
vimdiff(result.actual.checkedContexts, result.expected.checkedContexts)
|
1818
|
+
}
|
1635
1819
|
newError = true
|
1636
1820
|
headerShown = true
|
1637
1821
|
}
|
1638
1822
|
}
|
1823
|
+
} else {
|
1824
|
+
if (results.length > 0 && args.vimdiff) {
|
1825
|
+
for (const result of results) {
|
1826
|
+
vimdiff(result.actual, result.expected)
|
1827
|
+
}
|
1828
|
+
}
|
1639
1829
|
}
|
1640
1830
|
if (!headerShown) {
|
1641
1831
|
if (!(useTestConfig.check && useTestConfig.check.length > 0)) {
|
@@ -1667,12 +1857,11 @@ const knowledgeModule = async ({
|
|
1667
1857
|
if (query.length === 0) {
|
1668
1858
|
return readline.close()
|
1669
1859
|
}
|
1670
|
-
// const promise = processResults(_process(config, query, { testsFN: test }))
|
1671
1860
|
const promise = _process(config, query, { testsFN: test }).then((results) => {
|
1672
1861
|
console.log(results.responses.join(' '))
|
1673
1862
|
})
|
1674
1863
|
if (!('then' in promise)) {
|
1675
|
-
throw 'Return a promise from process in the definition of knowledgeModule'
|
1864
|
+
throw new Error('Return a promise from process in the definition of knowledgeModule')
|
1676
1865
|
}
|
1677
1866
|
promise
|
1678
1867
|
.then(() => {
|
@@ -1695,20 +1884,63 @@ const knowledgeModule = async ({
|
|
1695
1884
|
if (args.objectDiff) {
|
1696
1885
|
global.beforeObjects = _.cloneDeep(objects)
|
1697
1886
|
}
|
1698
|
-
config.beforeQuery({ query: args.query, isModule: false, objects })
|
1699
1887
|
try {
|
1700
|
-
processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
|
1888
|
+
await processResults(_process(config, args.query, { commandLineArgs: args, dontAddAssociations: args.dontAddAssociations, writeTests: args.save || args.saveDeveloper, saveDeveloper: args.saveDeveloper, testConfig, testsFN: test }))
|
1701
1889
|
} catch( error ) {
|
1702
1890
|
console.log('Error', error);
|
1703
1891
|
}
|
1704
1892
|
}
|
1893
|
+
printConfig()
|
1705
1894
|
} else {
|
1706
|
-
|
1707
|
-
|
1708
|
-
|
1709
|
-
|
1710
|
-
|
1895
|
+
const initConfig = (config) => {
|
1896
|
+
setupConfig(config)
|
1897
|
+
|
1898
|
+
let loadForTesting = false
|
1899
|
+
if (global.theprogrammablemind) {
|
1900
|
+
if (global.theprogrammablemind.loadForTesting[config.name]) {
|
1901
|
+
loadForTesting = true
|
1902
|
+
}
|
1903
|
+
}
|
1904
|
+
// remove test only stuff
|
1905
|
+
if (!isProcess && !loadForTesting) {
|
1906
|
+
config.config.operators = config.config.operators.filter( (operator) => {
|
1907
|
+
if (operator.development) {
|
1908
|
+
return false
|
1909
|
+
} else {
|
1910
|
+
return true
|
1911
|
+
}
|
1912
|
+
})
|
1913
|
+
config.config.bridges = config.config.bridges.filter( (bridge) => {
|
1914
|
+
if (bridge.development) {
|
1915
|
+
return false
|
1916
|
+
} else {
|
1917
|
+
return true
|
1918
|
+
}
|
1919
|
+
})
|
1920
|
+
}
|
1921
|
+
|
1922
|
+
if (template) {
|
1923
|
+
if (config.needsRebuild(template.template, template.instance, { isModule: !isProcess })) {
|
1924
|
+
const error = `This module "${config.name}" cannot be used because the instance file needs rebuilding. Run on the command line with no arguments or the -rt argument to rebuild.`
|
1925
|
+
throw new Error(error)
|
1926
|
+
}
|
1927
|
+
try {
|
1928
|
+
config.load(template.template, template.instance)
|
1929
|
+
} catch( e ) {
|
1930
|
+
errorHandler(e)
|
1931
|
+
}
|
1932
|
+
}
|
1933
|
+
}
|
1934
|
+
|
1935
|
+
createConfigExport = () => {
|
1936
|
+
const config = createConfig()
|
1937
|
+
initConfig(config)
|
1938
|
+
config.rebuild({ isModule: true })
|
1939
|
+
return config
|
1940
|
+
}
|
1941
|
+
moduleFromJSFile.exports = createConfigExport
|
1711
1942
|
}
|
1943
|
+
|
1712
1944
|
}
|
1713
1945
|
|
1714
1946
|
/*
|
@@ -1762,7 +1994,7 @@ module.exports = {
|
|
1762
1994
|
w,
|
1763
1995
|
// submitBug,
|
1764
1996
|
ensureTestFile,
|
1765
|
-
|
1997
|
+
rebuildTemplate,
|
1766
1998
|
processContext,
|
1767
1999
|
processContexts,
|
1768
2000
|
runTests,
|
@@ -1774,8 +2006,9 @@ module.exports = {
|
|
1774
2006
|
Digraph,
|
1775
2007
|
analyzeMetaData,
|
1776
2008
|
processContextsB,
|
1777
|
-
|
2009
|
+
loadInstance,
|
1778
2010
|
gs,
|
1779
2011
|
flattens,
|
1780
2012
|
writeTest
|
1781
2013
|
}
|
2014
|
+
|