theprogrammablemind_4wp 7.4.1 → 7.4.3-beta.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (3) hide show
  1. package/client.js +61 -11
  2. package/package.json +1 -1
  3. package/src/config.js +66 -9
package/client.js CHANGED
@@ -436,9 +436,12 @@ const processContextsB = ({ config, hierarchy, semantics, generators, json, isTe
436
436
  return { contextsPrime, generatedPrime, paraphrasesPrime, responsesPrime }
437
437
  }
438
438
 
439
- const doWithRetries = async (n, url, data) => {
439
+ const doWithRetries = async (n, url, queryParams, data) => {
440
+ if (!queryParams) {
441
+ queryParams = ''
442
+ }
440
443
  for (let i = 0; i < n; ++i) {
441
- const result = await fetch(`${url}/process`, {
444
+ const result = await fetch(`${url}/process${queryParams}`, {
442
445
  method: 'POST',
443
446
  body: JSON.stringify(data),
444
447
  timeout: 1000 * 60 * 5, // it does not respect this timeout so that's why I have the retries
@@ -467,13 +470,22 @@ const doWithRetries = async (n, url, data) => {
467
470
  }
468
471
  }
469
472
 
470
- const setupProcessB = ({ config }) => {
473
+ const setupProcessB = ({ config, allowDelta=false } = {}) => {
471
474
  const key = config._key
472
475
 
473
- const data = Object.assign({ key, version: '3' }, config.config)
474
- for (const uuid of Object.keys(data.namespaces)) {
475
- const km = config.configs.find((km) => km.uuid === uuid)
476
- data.namespaces[uuid].name = km.name
476
+ const data = Object.assign({ key, version: '3' }, { uuid: config._uuid })
477
+ if (allowDelta && config.allowDelta && config.hasDelta()) {
478
+ // console.log('config', config)
479
+ data.delta = config.delta()
480
+ } else {
481
+ Object.assign(data, config.config)
482
+ }
483
+
484
+ if (data.namespaces) {
485
+ for (const uuid of Object.keys(data.namespaces)) {
486
+ const km = config.configs.find((km) => km.uuid === uuid)
487
+ data.namespaces[uuid].name = km.name
488
+ }
477
489
  }
478
490
 
479
491
  // const generators = new Generators((data.generators || []).map((g) => new Generator(normalizeGenerator(g))))
@@ -511,6 +523,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
511
523
  config.server(credentials.server, credentials.key)
512
524
  }
513
525
  const url = config._server
526
+ const queryParams = config._queryParams || ''
514
527
  const retries = 2
515
528
  writeTests = !!writeTests
516
529
 
@@ -525,7 +538,7 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
525
538
  throw error
526
539
  }
527
540
 
528
- const { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config })
541
+ let { data, /* generators, semantics, */ hierarchy } = setupProcessB({ config, allowDelta: true })
529
542
  let queries = query.split('\\n')
530
543
 
531
544
  try {
@@ -549,8 +562,28 @@ const _process = async (config, query, { credentials, writeTests, isTest, saveDe
549
562
  if (queries.length === 0) {
550
563
  break;
551
564
  }
565
+
552
566
  data.utterance = queries[0]
553
- const json = await doWithRetries(retries, url, data)
567
+ let json = await doWithRetries(retries, url, queryParams, data)
568
+ let resetData = false
569
+ if (json.code == 'NOT_IN_CACHE') {
570
+ resetData = true
571
+ const setupB = setupProcessB({ config, allowDelta: false })
572
+ data = setupB.data
573
+ hierarchy = setupB.hierarchy
574
+ data.utterance = queries[0]
575
+ json = await doWithRetries(retries, url, queryParams, data)
576
+ }
577
+ if (json.cacheKey) {
578
+ config.cacheKey = json.cacheKey
579
+ if (resetData) {
580
+ if (queries.length > 1) {
581
+ const setupB = setupProcessB({ config, allowDelta: true })
582
+ data = setupB.data
583
+ hierarchy = setupB.hierarchy
584
+ }
585
+ }
586
+ }
554
587
  json.contexts = json.results
555
588
  delete json.results
556
589
  if (json.status !== 200) {
@@ -1220,6 +1253,7 @@ const knowledgeModule = async ({
1220
1253
  parser.add_argument('-r', '--retrain', { action: 'store_true', help: 'Get the server to retrain the neural nets' })
1221
1254
  parser.add_argument('-q', '--query', { help: 'Run the specified query' })
1222
1255
  parser.add_argument('-ip ', '--server', { help: 'Server to run against' })
1256
+ parser.add_argument('-qp ', '--queryParams', { help: 'Query params for the server call' })
1223
1257
  parser.add_argument('-qd', '--queryDelete', { help: 'Delete the specified query from the tests file' })
1224
1258
  parser.add_argument('-c', '--clean', { help: 'Remove data from the test files. a === association' })
1225
1259
  parser.add_argument('-od', '--objectDiff', { action: 'store_true', help: 'When showing the objects use a colour diff' })
@@ -1278,6 +1312,10 @@ const knowledgeModule = async ({
1278
1312
  config.server(args.server)
1279
1313
  }
1280
1314
 
1315
+ if (args.queryParams) {
1316
+ config.setQueryParams(args.queryParams)
1317
+ }
1318
+
1281
1319
  if (args.debug) {
1282
1320
  config.config.debug = true
1283
1321
  }
@@ -1320,7 +1358,7 @@ const knowledgeModule = async ({
1320
1358
  }
1321
1359
  }
1322
1360
  if (args.print.includes('c')) {
1323
- const { data } = setupProcessB({ config })
1361
+ const { data } = setupProcessB({ config })
1324
1362
  console.log("Config as sent to server")
1325
1363
  console.log(JSON.stringify(data, null, 2));
1326
1364
  }
@@ -1479,7 +1517,19 @@ const knowledgeModule = async ({
1479
1517
  if (!('then' in promise)) {
1480
1518
  throw 'Return a promise from process in the definition of knowledgeModule'
1481
1519
  }
1482
- promise.then(() => f()).catch( (e) => f() )
1520
+ promise
1521
+ .then(() => {
1522
+ f()
1523
+ })
1524
+ .catch( (e) => {
1525
+ if (e.errno == 'ECONNREFUSED') {
1526
+ console.log(e)
1527
+ readline.close()
1528
+ } else {
1529
+ console.log(e)
1530
+ f()
1531
+ }
1532
+ })
1483
1533
  })
1484
1534
  f()
1485
1535
  } else if (args.query) {
package/package.json CHANGED
@@ -61,6 +61,6 @@
61
61
  "json-stable-stringify": "^1.0.1",
62
62
  "node-fetch": "^2.6.1"
63
63
  },
64
- "version": "7.4.1",
64
+ "version": "7.4.3-beta.0",
65
65
  "license": "ISC"
66
66
  }
package/src/config.js CHANGED
@@ -572,6 +572,7 @@ class Config {
572
572
  }
573
573
  }
574
574
  this.config.associations.positive.push(association)
575
+ this._delta.json.associations.push({ action: 'add', association })
575
576
  }
576
577
 
577
578
  // TODO add more error checking to these like addHierarchy has
@@ -580,6 +581,7 @@ class Config {
580
581
  this.config.priorities = []
581
582
  }
582
583
  this.config.priorities.push(priorities)
584
+ this._delta.json.priorities.push({ action: 'add', priorities })
583
585
  }
584
586
 
585
587
  addHierarchy (child, parent) {
@@ -603,6 +605,7 @@ class Config {
603
605
  }
604
606
  // this.config.hierarchy.push([properties.child, properties.parent])
605
607
  this.config.hierarchy.push(properties)
608
+ this._delta.json.hierarchy.push([child, parent])
606
609
  }
607
610
 
608
611
  addHierarchyChildParent (child, parent) {
@@ -630,6 +633,7 @@ class Config {
630
633
  }
631
634
 
632
635
  this.config.hierarchy.push([child, parent])
636
+ this._delta.json.hierarchy.push([child, parent])
633
637
  }
634
638
 
635
639
  getBridge (id, level) {
@@ -659,6 +663,7 @@ class Config {
659
663
  }
660
664
  bridges.push(def)
661
665
  this.checkBridges();
666
+ this._delta.json.bridges.push({ action: 'add', bridge: def })
662
667
  }
663
668
 
664
669
  addGenerator (match, apply) {
@@ -735,17 +740,19 @@ class Config {
735
740
 
736
741
  operators.unshift(operator)
737
742
  this.checkOperators()
743
+
744
+ this._delta.json.operators.push({ action: 'add', operator })
738
745
  }
739
746
 
740
747
  addWord (word, def) {
741
- this.addWordInternal(this.config, word, def)
748
+ this.addWordInternal(word, def)
742
749
  }
743
750
 
744
- addWordInternal (config, word, def) {
745
- if (!config.words) {
746
- config.words = {}
751
+ addWordInternal (word, def) {
752
+ if (!this.config.words) {
753
+ this.config.words = {}
747
754
  }
748
- const words = config.words
755
+ const words = this.config.words
749
756
  def = Object.assign({}, def, { uuid: this._uuid })
750
757
  if (words[word]) {
751
758
  if (!words[word].some((e) => helpers.safeEquals(e, def))) {
@@ -754,6 +761,8 @@ class Config {
754
761
  } else {
755
762
  words[word] = [def]
756
763
  }
764
+
765
+ this._delta.json.words.push({ action: 'add', word, def })
757
766
  }
758
767
 
759
768
  getAPI (uuid) {
@@ -792,13 +801,22 @@ class Config {
792
801
  return this._key
793
802
  }
794
803
 
795
- server (server, apiKey) {
804
+ getQueryParams() {
805
+ return this._queryParams
806
+ }
807
+
808
+ setQueryParams(queryParams) {
809
+ this._queryParams = queryParams
810
+ }
811
+
812
+ server (server, apiKey, queryParams) {
796
813
  if (server) {
797
814
  this._server = server
798
815
  }
799
816
  if (apiKey) {
800
817
  this._key = apiKey
801
818
  }
819
+ this._queryParams = queryParams
802
820
 
803
821
  if (this._api && this._api.multiApi) {
804
822
  for (const key of Object.keys(this._api.apis)) {
@@ -961,6 +979,9 @@ class Config {
961
979
  throw 'Excepted the config argument to be a hash not a Config object'
962
980
  }
963
981
 
982
+ this.allowDelta = false
983
+ this.resetDelta()
984
+
964
985
  this.addedArgss = []
965
986
  let isProcess = require.main === module
966
987
  if (global.theprogrammablemind && config) {
@@ -1042,6 +1063,15 @@ class Config {
1042
1063
  // this.config.words = {}
1043
1064
  }
1044
1065
  for (let bridge of (this.config.bridges || [])) {
1066
+ const valid = [ 'before', 'bridge', 'development', 'evaluator', 'generatorp', 'generatorr', 'generators', 'hierarchy', 'id', 'inverted', 'isA',
1067
+ 'level', 'optional', 'selector', 'semantic', 'words' ]
1068
+ for (let prop of Object.keys(bridge)) {
1069
+ if (!valid.includes(prop)) {
1070
+ if (!prop.endsWith("Bridge")) {
1071
+ throw `Unknown property "${prop}" in the bridge. Valid properties are ${valid}. The bridge is ${JSON.stringify(bridge)}`
1072
+ }
1073
+ }
1074
+ }
1045
1075
  /*
1046
1076
  if (bridge.generator) {
1047
1077
  this.config.generators.push({
@@ -1063,11 +1093,11 @@ class Config {
1063
1093
  if (bridge.words) {
1064
1094
  for (let def of bridge.words) {
1065
1095
  if (typeof def == 'string') {
1066
- this.addWordInternal(this.config, def, {"id": bridge.id, "initial": "{}" })
1096
+ this.addWordInternal(def, {"id": bridge.id, "initial": `{ value: "${def}"}` })
1067
1097
  } else {
1068
1098
  const word = def.word
1069
1099
  def = { initial: JSON.stringify(def), id: bridge.id, word: undefined }
1070
- this.addWordInternal(this.config, word, def)
1100
+ this.addWordInternal(word, def)
1071
1101
  }
1072
1102
  }
1073
1103
  }
@@ -1165,6 +1195,32 @@ class Config {
1165
1195
  }
1166
1196
  }
1167
1197
 
1198
+ hasDelta () {
1199
+ return this._delta.json
1200
+ }
1201
+
1202
+ delta () {
1203
+ return { cacheKey: this._delta.cacheKey, json: this._delta.json }
1204
+ }
1205
+
1206
+ resetDelta (cacheKey) {
1207
+ this._delta = {
1208
+ cacheKey,
1209
+ json: {
1210
+ words: [],
1211
+ operators: [],
1212
+ bridges: [],
1213
+ associations: [],
1214
+ priorities: [],
1215
+ hierarchy: [],
1216
+ }
1217
+ }
1218
+ }
1219
+
1220
+ set cacheKey (cacheKey) {
1221
+ this.resetDelta(cacheKey)
1222
+ }
1223
+
1168
1224
  get api () {
1169
1225
  if (this._api && this._api.multiApi) {
1170
1226
  return this._api.api(this._api)
@@ -1637,6 +1693,7 @@ class Config {
1637
1693
 
1638
1694
  // rebuild ({ isModule: mainIsModule = false } = {}) {
1639
1695
  rebuild ({ isModule: mainIsModule } = {}) {
1696
+ this.resetDelta()
1640
1697
  const debug = this.config.debug;
1641
1698
  this.config = _.cloneDeep(this.initConfig)
1642
1699
  if (debug) {
@@ -2061,7 +2118,7 @@ class Config {
2061
2118
  more.valid()
2062
2119
  // copy so i don't have to copy later
2063
2120
  more = more.copy()
2064
- more.server(this._server, this._key)
2121
+ more.server(this._server, this._key, this._queryParams)
2065
2122
 
2066
2123
  this.loadOrder.addList(more.configs.map((km) => km.name || km.uuid))
2067
2124