theprogrammablemind_4wp 7.4.1 → 7.4.3-beta.0
Sign up to get free protection for your applications and to get access to all the features.
- package/client.js +61 -11
- package/package.json +1 -1
- package/src/config.js +66 -9
package/client.js
CHANGED
@@ -436,9 +436,12 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
|
|
436
436
|
return { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime }
|
437
437
|
}
|
438
438
|
|
439
|
-
const doWithRetries = async (n, url, data) => {
|
439
|
+
const doWithRetries = async (n, url, queryParams, data) => {
|
440
|
+
if (!queryParams) {
|
441
|
+
queryParams = ''
|
442
|
+
}
|
440
443
|
for (let i = 0; i < n; ++i) {
|
441
|
-
const result = await fetch(`${url}/process`, {
|
444
|
+
const result = await fetch(`${url}/process${queryParams}`, {
|
442
445
|
method: 'POST',
|
443
446
|
body: JSON.stringify(data),
|
444
447
|
timeout: 1000 * 60 * 5, // it does not respect this timeout so that's why I have the retries
|
@@ -467,13 +470,22 @@ const doWithRetries = async (n, url, data) => {
|
|
467
470
|
}
|
468
471
|
}
|
469
472
|
|
470
|
-
const setupProcessB = ({ config }) => {
|
473
|
+
const setupProcessB = ({ config, allowDelta=false } = {}) => {
|
471
474
|
const key = config._key
|
472
475
|
|
473
|
-
const data = Object.assign({ key, version: '3' }, config.
|
474
|
-
|
475
|
-
|
476
|
-
data.
|
476
|
+
const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
|
477
|
+
if (allowDelta && config.allowDelta && config.hasDelta()) {
|
478
|
+
// console.log('config', config)
|
479
|
+
data.delta = config.delta()
|
480
|
+
} else {
|
481
|
+
Object.assign(data, config.config)
|
482
|
+
}
|
483
|
+
|
484
|
+
if (data.namespaces) {
|
485
|
+
for (const uuid of Object.keys(data.namespaces)) {
|
486
|
+
const km = config.configs.find((km) => km.uuid === uuid)
|
487
|
+
data.namespaces[uuid].name = km.name
|
488
|
+
}
|
477
489
|
}
|
478
490
|
|
479
491
|
// const generators = new Generators((data.generators || []).map((g) => new Generator(normalizeGenerator(g))))
|
@@ -511,6 +523,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
|
|
511
523
|
config.server(credentials.server, credentials.key)
|
512
524
|
}
|
513
525
|
const url = config._server
|
526
|
+
const queryParams = config._queryParams || ''
|
514
527
|
const retries = 2
|
515
528
|
writeTests = !!writeTests
|
516
529
|
|
@@ -525,7 +538,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
|
|
525
538
|
throw error
|
526
539
|
}
|
527
540
|
|
528
|
-
|
541
|
+
let { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config, allowDelta: true })
|
529
542
|
let queries = query.split('\\n')
|
530
543
|
|
531
544
|
try {
|
@@ -549,8 +562,28 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
|
|
549
562
|
if (queries.length === 0) {
|
550
563
|
break;
|
551
564
|
}
|
565
|
+
|
552
566
|
data.utterance = queries[0]
|
553
|
-
|
567
|
+
let json = await doWithRetries(retries, url, queryParams, data)
|
568
|
+
let resetData = false
|
569
|
+
if (json.code == 'NOT_IN_CACHE') {
|
570
|
+
resetData = true
|
571
|
+
const setupB = setupProcessB({ config, allowDelta: false })
|
572
|
+
data = setupB.data
|
573
|
+
hierarchy = setupB.hierarchy
|
574
|
+
data.utterance = queries[0]
|
575
|
+
json = await doWithRetries(retries, url, queryParams, data)
|
576
|
+
}
|
577
|
+
if (json.cacheKey) {
|
578
|
+
config.cacheKey = json.cacheKey
|
579
|
+
if (resetData) {
|
580
|
+
if (queries.length > 1) {
|
581
|
+
const setupB = setupProcessB({ config, allowDelta: true })
|
582
|
+
data = setupB.data
|
583
|
+
hierarchy = setupB.hierarchy
|
584
|
+
}
|
585
|
+
}
|
586
|
+
}
|
554
587
|
json.contexts = json.results
|
555
588
|
delete json.results
|
556
589
|
if (json.status !== 200) {
|
@@ -1220,6 +1253,7 @@ const knowledgeModule = async ({
|
|
1220
1253
|
parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
|
1221
1254
|
parser.add_argument('-q', '--query', { help: 'Run the specified query' })
|
1222
1255
|
parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
|
1256
|
+
parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
|
1223
1257
|
parser.add_argument('-qd', '--queryDelete', { help: 'Delete the specified query from the tests file' })
|
1224
1258
|
parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
|
1225
1259
|
parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
|
@@ -1278,6 +1312,10 @@ const knowledgeModule = async ({
|
|
1278
1312
|
config.server(args.server)
|
1279
1313
|
}
|
1280
1314
|
|
1315
|
+
if (args.queryParams) {
|
1316
|
+
config.setQueryParams(args.queryParams)
|
1317
|
+
}
|
1318
|
+
|
1281
1319
|
if (args.debug) {
|
1282
1320
|
config.config.debug = true
|
1283
1321
|
}
|
@@ -1320,7 +1358,7 @@ const knowledgeModule = async ({
|
|
1320
1358
|
}
|
1321
1359
|
}
|
1322
1360
|
if (args.print.includes('c')) {
|
1323
|
-
const { data
|
1361
|
+
const { data } = setupProcessB({ config })
|
1324
1362
|
console.log("Config as sent to server")
|
1325
1363
|
console.log(JSON.stringify(data, null, 2));
|
1326
1364
|
}
|
@@ -1479,7 +1517,19 @@ const knowledgeModule = async ({
|
|
1479
1517
|
if (!('then' in promise)) {
|
1480
1518
|
throw 'Return a promise from process in the definition of knowledgeModule'
|
1481
1519
|
}
|
1482
|
-
promise
|
1520
|
+
promise
|
1521
|
+
.then(() => {
|
1522
|
+
f()
|
1523
|
+
})
|
1524
|
+
.catch( (e) => {
|
1525
|
+
if (e.errno == 'ECONNREFUSED') {
|
1526
|
+
console.log(e)
|
1527
|
+
readline.close()
|
1528
|
+
} else {
|
1529
|
+
console.log(e)
|
1530
|
+
f()
|
1531
|
+
}
|
1532
|
+
})
|
1483
1533
|
})
|
1484
1534
|
f()
|
1485
1535
|
} else if (args.query) {
|
package/package.json
CHANGED
package/src/config.js
CHANGED
@@ -572,6 +572,7 @@ class Config {
|
|
572
572
|
}
|
573
573
|
}
|
574
574
|
this.config.associations.positive.push(association)
|
575
|
+
this._delta.json.associations.push({ action: 'add', association })
|
575
576
|
}
|
576
577
|
|
577
578
|
// TODO add more error checking to these like addHierarchy has
|
@@ -580,6 +581,7 @@ class Config {
|
|
580
581
|
this.config.priorities = []
|
581
582
|
}
|
582
583
|
this.config.priorities.push(priorities)
|
584
|
+
this._delta.json.priorities.push({ action: 'add', priorities })
|
583
585
|
}
|
584
586
|
|
585
587
|
addHierarchy (child, parent) {
|
@@ -603,6 +605,7 @@ class Config {
|
|
603
605
|
}
|
604
606
|
// this.config.hierarchy.push([properties.child, properties.parent])
|
605
607
|
this.config.hierarchy.push(properties)
|
608
|
+
this._delta.json.hierarchy.push([child, parent])
|
606
609
|
}
|
607
610
|
|
608
611
|
addHierarchyChildParent (child, parent) {
|
@@ -630,6 +633,7 @@ class Config {
|
|
630
633
|
}
|
631
634
|
|
632
635
|
this.config.hierarchy.push([child, parent])
|
636
|
+
this._delta.json.hierarchy.push([child, parent])
|
633
637
|
}
|
634
638
|
|
635
639
|
getBridge (id, level) {
|
@@ -659,6 +663,7 @@ class Config {
|
|
659
663
|
}
|
660
664
|
bridges.push(def)
|
661
665
|
this.checkBridges();
|
666
|
+
this._delta.json.bridges.push({ action: 'add', bridge: def })
|
662
667
|
}
|
663
668
|
|
664
669
|
addGenerator (match, apply) {
|
@@ -735,17 +740,19 @@ class Config {
|
|
735
740
|
|
736
741
|
operators.unshift(operator)
|
737
742
|
this.checkOperators()
|
743
|
+
|
744
|
+
this._delta.json.operators.push({ action: 'add', operator })
|
738
745
|
}
|
739
746
|
|
740
747
|
addWord (word, def) {
|
741
|
-
this.addWordInternal(
|
748
|
+
this.addWordInternal(word, def)
|
742
749
|
}
|
743
750
|
|
744
|
-
addWordInternal (
|
745
|
-
if (!config.words) {
|
746
|
-
config.words = {}
|
751
|
+
addWordInternal (word, def) {
|
752
|
+
if (!this.config.words) {
|
753
|
+
this.config.words = {}
|
747
754
|
}
|
748
|
-
const words = config.words
|
755
|
+
const words = this.config.words
|
749
756
|
def = Object.assign({}, def, { uuid: this._uuid })
|
750
757
|
if (words[word]) {
|
751
758
|
if (!words[word].some((e) => helpers.safeEquals(e, def))) {
|
@@ -754,6 +761,8 @@ class Config {
|
|
754
761
|
} else {
|
755
762
|
words[word] = [def]
|
756
763
|
}
|
764
|
+
|
765
|
+
this._delta.json.words.push({ action: 'add', word, def })
|
757
766
|
}
|
758
767
|
|
759
768
|
getAPI (uuid) {
|
@@ -792,13 +801,22 @@ class Config {
|
|
792
801
|
return this._key
|
793
802
|
}
|
794
803
|
|
795
|
-
|
804
|
+
getQueryParams() {
|
805
|
+
return this._queryParams
|
806
|
+
}
|
807
|
+
|
808
|
+
setQueryParams(queryParams) {
|
809
|
+
this._queryParams = queryParams
|
810
|
+
}
|
811
|
+
|
812
|
+
server (server, apiKey, queryParams) {
|
796
813
|
if (server) {
|
797
814
|
this._server = server
|
798
815
|
}
|
799
816
|
if (apiKey) {
|
800
817
|
this._key = apiKey
|
801
818
|
}
|
819
|
+
this._queryParams = queryParams
|
802
820
|
|
803
821
|
if (this._api && this._api.multiApi) {
|
804
822
|
for (const key of Object.keys(this._api.apis)) {
|
@@ -961,6 +979,9 @@ class Config {
|
|
961
979
|
throw 'Excepted the config argument to be a hash not a Config object'
|
962
980
|
}
|
963
981
|
|
982
|
+
this.allowDelta = false
|
983
|
+
this.resetDelta()
|
984
|
+
|
964
985
|
this.addedArgss = []
|
965
986
|
let isProcess = require.main === module
|
966
987
|
if (global.theprogrammablemind && config) {
|
@@ -1042,6 +1063,15 @@ class Config {
|
|
1042
1063
|
// this.config.words = {}
|
1043
1064
|
}
|
1044
1065
|
for (let bridge of (this.config.bridges || [])) {
|
1066
|
+
const valid = [ 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generators', 'hierarchy', 'id', 'inverted', 'isA',
|
1067
|
+
'level', 'optional', 'selector', 'semantic', 'words' ]
|
1068
|
+
for (let prop of Object.keys(bridge)) {
|
1069
|
+
if (!valid.includes(prop)) {
|
1070
|
+
if (!prop.endsWith("Bridge")) {
|
1071
|
+
throw `Unknown property "${prop}" in the bridge. Valid properties are ${valid}. The bridge is ${JSON.stringify(bridge)}`
|
1072
|
+
}
|
1073
|
+
}
|
1074
|
+
}
|
1045
1075
|
/*
|
1046
1076
|
if (bridge.generator) {
|
1047
1077
|
this.config.generators.push({
|
@@ -1063,11 +1093,11 @@ class Config {
|
|
1063
1093
|
if (bridge.words) {
|
1064
1094
|
for (let def of bridge.words) {
|
1065
1095
|
if (typeof def == 'string') {
|
1066
|
-
this.addWordInternal(
|
1096
|
+
this.addWordInternal(def, {"id": bridge.id, "initial": `{ value: "${def}"}` })
|
1067
1097
|
} else {
|
1068
1098
|
const word = def.word
|
1069
1099
|
def = { initial: JSON.stringify(def), id: bridge.id, word: undefined }
|
1070
|
-
this.addWordInternal(
|
1100
|
+
this.addWordInternal(word, def)
|
1071
1101
|
}
|
1072
1102
|
}
|
1073
1103
|
}
|
@@ -1165,6 +1195,32 @@ class Config {
|
|
1165
1195
|
}
|
1166
1196
|
}
|
1167
1197
|
|
1198
|
+
hasDelta () {
|
1199
|
+
return this._delta.json
|
1200
|
+
}
|
1201
|
+
|
1202
|
+
delta () {
|
1203
|
+
return { cacheKey: this._delta.cacheKey, json: this._delta.json }
|
1204
|
+
}
|
1205
|
+
|
1206
|
+
resetDelta (cacheKey) {
|
1207
|
+
this._delta = {
|
1208
|
+
cacheKey,
|
1209
|
+
json: {
|
1210
|
+
words: [],
|
1211
|
+
operators: [],
|
1212
|
+
bridges: [],
|
1213
|
+
associations: [],
|
1214
|
+
priorities: [],
|
1215
|
+
hierarchy: [],
|
1216
|
+
}
|
1217
|
+
}
|
1218
|
+
}
|
1219
|
+
|
1220
|
+
set cacheKey (cacheKey) {
|
1221
|
+
this.resetDelta(cacheKey)
|
1222
|
+
}
|
1223
|
+
|
1168
1224
|
get api () {
|
1169
1225
|
if (this._api && this._api.multiApi) {
|
1170
1226
|
return this._api.api(this._api)
|
@@ -1637,6 +1693,7 @@ class Config {
|
|
1637
1693
|
|
1638
1694
|
// rebuild ({ isModule: mainIsModule = false } = {}) {
|
1639
1695
|
rebuild ({ isModule: mainIsModule } = {}) {
|
1696
|
+
this.resetDelta()
|
1640
1697
|
const debug = this.config.debug;
|
1641
1698
|
this.config = _.cloneDeep(this.initConfig)
|
1642
1699
|
if (debug) {
|
@@ -2061,7 +2118,7 @@ class Config {
|
|
2061
2118
|
more.valid()
|
2062
2119
|
// copy so i don't have to copy later
|
2063
2120
|
more = more.copy()
|
2064
|
-
more.server(this._server, this._key)
|
2121
|
+
more.server(this._server, this._key, this._queryParams)
|
2065
2122
|
|
2066
2123
|
this.loadOrder.addList(more.configs.map((km) => km.name || km.uuid))
|
2067
2124
|
|