retold-facto 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/.claude/launch.json +11 -0
  2. package/.dockerignore +8 -0
  3. package/.quackage.json +19 -0
  4. package/Dockerfile +26 -0
  5. package/bin/retold-facto.js +909 -0
  6. package/examples/facto-government-data.sqlite +0 -0
  7. package/examples/government-data-catalog.json +137 -0
  8. package/examples/government-data-loader.js +1432 -0
  9. package/package.json +91 -0
  10. package/scripts/facto-download.js +425 -0
  11. package/source/Retold-Facto.js +1042 -0
  12. package/source/services/Retold-Facto-BeaconProvider.js +511 -0
  13. package/source/services/Retold-Facto-CatalogManager.js +1252 -0
  14. package/source/services/Retold-Facto-DataLakeService.js +1642 -0
  15. package/source/services/Retold-Facto-DatasetManager.js +417 -0
  16. package/source/services/Retold-Facto-IngestEngine.js +1315 -0
  17. package/source/services/Retold-Facto-ProjectionEngine.js +3960 -0
  18. package/source/services/Retold-Facto-RecordManager.js +360 -0
  19. package/source/services/Retold-Facto-SchemaManager.js +1110 -0
  20. package/source/services/Retold-Facto-SourceFolderScanner.js +2243 -0
  21. package/source/services/Retold-Facto-SourceManager.js +730 -0
  22. package/source/services/Retold-Facto-StoreConnectionManager.js +441 -0
  23. package/source/services/Retold-Facto-ThroughputMonitor.js +478 -0
  24. package/source/services/web-app/codemirror-entry.js +7 -0
  25. package/source/services/web-app/pict-app/Pict-Application-Facto-Configuration.json +9 -0
  26. package/source/services/web-app/pict-app/Pict-Application-Facto.js +70 -0
  27. package/source/services/web-app/pict-app/Pict-Facto-Bundle.js +11 -0
  28. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto-UI.js +66 -0
  29. package/source/services/web-app/pict-app/providers/Pict-Provider-Facto.js +69 -0
  30. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Catalog.js +93 -0
  31. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Connections.js +42 -0
  32. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Datasets.js +605 -0
  33. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Projections.js +188 -0
  34. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Scanner.js +80 -0
  35. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Schema.js +116 -0
  36. package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Sources.js +104 -0
  37. package/source/services/web-app/pict-app/views/PictView-Facto-Catalog.js +526 -0
  38. package/source/services/web-app/pict-app/views/PictView-Facto-Datasets.js +173 -0
  39. package/source/services/web-app/pict-app/views/PictView-Facto-Ingest.js +259 -0
  40. package/source/services/web-app/pict-app/views/PictView-Facto-Layout.js +191 -0
  41. package/source/services/web-app/pict-app/views/PictView-Facto-Projections.js +231 -0
  42. package/source/services/web-app/pict-app/views/PictView-Facto-Records.js +326 -0
  43. package/source/services/web-app/pict-app/views/PictView-Facto-Scanner.js +624 -0
  44. package/source/services/web-app/pict-app/views/PictView-Facto-Sources.js +201 -0
  45. package/source/services/web-app/pict-app/views/PictView-Facto-Throughput.js +456 -0
  46. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full-Configuration.json +14 -0
  47. package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full.js +391 -0
  48. package/source/services/web-app/pict-app-full/providers/PictRouter-Facto-Configuration.json +56 -0
  49. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-BottomBar.js +68 -0
  50. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Connections.js +340 -0
  51. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboard.js +149 -0
  52. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboards.js +819 -0
  53. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Datasets.js +178 -0
  54. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-IngestJobs.js +99 -0
  55. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Layout.js +62 -0
  56. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-MappingEditor.js +158 -0
  57. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-ProjectionDetail.js +1120 -0
  58. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Projections.js +172 -0
  59. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-QueryPanel.js +119 -0
  60. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-RecordViewer.js +663 -0
  61. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Records.js +648 -0
  62. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Scanner.js +1017 -0
  63. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDetail.js +1404 -0
  64. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDocEditor.js +1036 -0
  65. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaEditor.js +636 -0
  66. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaResearch.js +357 -0
  67. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceDetail.js +822 -0
  68. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceEditor.js +1036 -0
  69. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceResearch.js +487 -0
  70. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Sources.js +165 -0
  71. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Throughput.js +439 -0
  72. package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-TopBar.js +335 -0
  73. package/source/services/web-app/pict-app-full/views/projections/Facto-Projections-Constants.js +71 -0
  74. package/source/services/web-app/web/chart.min.js +20 -0
  75. package/source/services/web-app/web/codemirror-bundle.js +30099 -0
  76. package/source/services/web-app/web/css/facto-themes.css +467 -0
  77. package/source/services/web-app/web/css/facto.css +502 -0
  78. package/source/services/web-app/web/index.html +28 -0
  79. package/source/services/web-app/web/retold-facto.js +12138 -0
  80. package/source/services/web-app/web/retold-facto.js.map +1 -0
  81. package/source/services/web-app/web/retold-facto.min.js +2 -0
  82. package/source/services/web-app/web/retold-facto.min.js.map +1 -0
  83. package/source/services/web-app/web/simple/index.html +17 -0
  84. package/test/Facto_Browser_Integration_tests.js +798 -0
  85. package/test/RetoldFacto_tests.js +4117 -0
  86. package/test/fixtures/weather-readings.csv +17 -0
  87. package/test/fixtures/weather-stations.csv +9 -0
  88. package/test/model/MeadowModel-Extended.json +8497 -0
  89. package/test/model/MeadowModel-PICT.json +1 -0
  90. package/test/model/MeadowModel.json +1355 -0
  91. package/test/model/ddl/Facto.ddl +225 -0
  92. package/test/model/fable-configuration.json +14 -0
@@ -0,0 +1,4117 @@
1
+ /**
2
+ * Unit tests for Retold Facto
3
+ *
4
+ * @license MIT
5
+ *
6
+ * @author Steven Velozo <steven@velozo.com>
7
+ */
8
+
9
+ var Chai = require("chai");
10
+ var Expect = Chai.expect;
11
+
12
+ const libFable = require('pict');
13
+ const libSuperTest = require('supertest');
14
+ const libMeadowConnectionManager = require('meadow-connection-manager');
15
+ const libRetoldFacto = require('../source/Retold-Facto.js');
16
+ const libFs = require('fs');
17
+ const libPath = require('path');
18
+
19
+ const _APIServerPort = 9340;
20
+ const _BaseURL = `http://localhost:${_APIServerPort}/`;
21
+
22
+ let _Fable;
23
+ let _RetoldFacto;
24
+ let _SuperTest;
25
+
26
+ suite
27
+ (
28
+ 'Retold Facto',
29
+ function()
30
+ {
31
+ suiteSetup
32
+ (
33
+ function(fDone)
34
+ {
35
+ this.timeout(10000);
36
+
37
+ let tmpSettings = {
38
+ Product: 'RetoldFactoTest',
39
+ ProductVersion: '0.0.1',
40
+ APIServerPort: _APIServerPort,
41
+ SQLite:
42
+ {
43
+ SQLiteFilePath: ':memory:'
44
+ },
45
+ LogStreams:
46
+ [
47
+ {
48
+ streamtype: 'console',
49
+ level: 'fatal'
50
+ }
51
+ ]
52
+ };
53
+
54
+ _Fable = new libFable(tmpSettings);
55
+
56
+ // Register the connection manager and connect the catalog database
57
+ _Fable.serviceManager.addServiceType('MeadowConnectionManager', libMeadowConnectionManager);
58
+ _Fable.serviceManager.instantiateServiceProvider('MeadowConnectionManager');
59
+
60
+ _Fable.MeadowConnectionManager.connect('facto',
61
+ {
62
+ Type: 'SQLite',
63
+ SQLiteFilePath: ':memory:'
64
+ },
65
+ (pError, pConnection) =>
66
+ {
67
+ if (pError)
68
+ {
69
+ return fDone(pError);
70
+ }
71
+
72
+ // Bridge: Meadow DAL providers look up fable.MeadowSQLiteProvider
73
+ _Fable.MeadowSQLiteProvider = pConnection.instance;
74
+
75
+ let tmpDB = _Fable.MeadowSQLiteProvider.db;
76
+
77
+ // Create all tables using the canonical schema from the module
78
+ tmpDB.exec(libRetoldFacto.FACTO_SCHEMA_SQL);
79
+
80
+ _Fable.settings.MeadowProvider = 'SQLite';
81
+
82
+ _Fable.serviceManager.addServiceType('RetoldFacto', libRetoldFacto);
83
+ _RetoldFacto = _Fable.serviceManager.instantiateServiceProvider('RetoldFacto',
84
+ {
85
+ StorageProvider: 'SQLite',
86
+
87
+ FullMeadowSchemaPath: `${__dirname}/model/`,
88
+ FullMeadowSchemaFilename: 'MeadowModel-Extended.json',
89
+
90
+ AutoStartOrator: true,
91
+
92
+ Endpoints:
93
+ {
94
+ MeadowEndpoints: true,
95
+ SourceManager: true,
96
+ RecordManager: true,
97
+ DatasetManager: true,
98
+ IngestEngine: true,
99
+ ProjectionEngine: true,
100
+ CatalogManager: true,
101
+ StoreConnectionManager: true,
102
+ SchemaManager: true,
103
+ WebUI: false
104
+ }
105
+ });
106
+
107
+ // Enable JSON body parsing
108
+ _RetoldFacto.onBeforeInitialize = (fCallback) =>
109
+ {
110
+ _Fable.OratorServiceServer.server.use(_Fable.OratorServiceServer.bodyParser());
111
+ _Fable.OratorServiceServer.server.use(require('restify').plugins.queryParser());
112
+ return fCallback();
113
+ };
114
+
115
+ _RetoldFacto.initializeService(
116
+ (pInitError) =>
117
+ {
118
+ if (pInitError)
119
+ {
120
+ return fDone(pInitError);
121
+ }
122
+ _SuperTest = libSuperTest(`http://localhost:${_APIServerPort}`);
123
+ return fDone();
124
+ });
125
+ });
126
+ }
127
+ );
128
+
129
+ suiteTeardown
130
+ (
131
+ function(fDone)
132
+ {
133
+ this.timeout(5000);
134
+ if (_RetoldFacto && _RetoldFacto.serviceInitialized)
135
+ {
136
+ _RetoldFacto.stopService(fDone);
137
+ }
138
+ else
139
+ {
140
+ return fDone();
141
+ }
142
+ }
143
+ );
144
+
145
+ suite
146
+ (
147
+ 'Object Sanity',
148
+ function()
149
+ {
150
+ test
151
+ (
152
+ 'The RetoldFacto class should exist',
153
+ function()
154
+ {
155
+ const libRetoldFacto = require('../source/Retold-Facto.js');
156
+ Expect(libRetoldFacto).to.be.a('function');
157
+ }
158
+ );
159
+ test
160
+ (
161
+ 'The service should be initialized',
162
+ function()
163
+ {
164
+ Expect(_RetoldFacto).to.be.an('object');
165
+ Expect(_RetoldFacto.serviceInitialized).to.equal(true);
166
+ }
167
+ );
168
+ test
169
+ (
170
+ 'The entity list should contain all 15 entities',
171
+ function()
172
+ {
173
+ Expect(_RetoldFacto.entityList).to.be.an('array');
174
+ Expect(_RetoldFacto.entityList).to.include('Source');
175
+ Expect(_RetoldFacto.entityList).to.include('SourceDocumentation');
176
+ Expect(_RetoldFacto.entityList).to.include('Dataset');
177
+ Expect(_RetoldFacto.entityList).to.include('DatasetSource');
178
+ Expect(_RetoldFacto.entityList).to.include('Record');
179
+ Expect(_RetoldFacto.entityList).to.include('RecordBinary');
180
+ Expect(_RetoldFacto.entityList).to.include('CertaintyIndex');
181
+ Expect(_RetoldFacto.entityList).to.include('IngestJob');
182
+ Expect(_RetoldFacto.entityList).to.include('SourceCatalogEntry');
183
+ Expect(_RetoldFacto.entityList).to.include('CatalogDatasetDefinition');
184
+ Expect(_RetoldFacto.entityList).to.include('StoreConnection');
185
+ Expect(_RetoldFacto.entityList).to.include('ProjectionStore');
186
+ Expect(_RetoldFacto.entityList).to.include('ProjectionMapping');
187
+ Expect(_RetoldFacto.entityList).to.include('MultiSetProjection');
188
+ Expect(_RetoldFacto.entityList).to.include('ProjectionCertaintyLog');
189
+ Expect(_RetoldFacto.entityList.length).to.equal(18);
190
+ }
191
+ );
192
+ test
193
+ (
194
+ 'DAL objects should exist for all entities',
195
+ function()
196
+ {
197
+ Expect(_RetoldFacto._DAL).to.be.an('object');
198
+ Expect(_RetoldFacto._DAL.Source).to.be.an('object');
199
+ Expect(_RetoldFacto._DAL.Record).to.be.an('object');
200
+ Expect(_RetoldFacto._DAL.CertaintyIndex).to.be.an('object');
201
+ }
202
+ );
203
+ test
204
+ (
205
+ 'MeadowEndpoints should exist for all entities',
206
+ function()
207
+ {
208
+ Expect(_RetoldFacto._MeadowEndpoints).to.be.an('object');
209
+ Expect(_RetoldFacto._MeadowEndpoints.Source).to.be.an('object');
210
+ Expect(_RetoldFacto._MeadowEndpoints.Record).to.be.an('object');
211
+ Expect(_RetoldFacto._MeadowEndpoints.Dataset).to.be.an('object');
212
+ }
213
+ );
214
+ }
215
+ );
216
+
217
+ suite
218
+ (
219
+ 'Service Lifecycle',
220
+ function()
221
+ {
222
+ test
223
+ (
224
+ 'Should not allow double initialization',
225
+ function(fDone)
226
+ {
227
+ _RetoldFacto.initializeService(
228
+ (pError) =>
229
+ {
230
+ Expect(pError).to.be.an.instanceOf(Error);
231
+ Expect(pError.message).to.contain('already been initialized');
232
+ return fDone();
233
+ });
234
+ }
235
+ );
236
+ test
237
+ (
238
+ 'Lifecycle hooks should exist',
239
+ function()
240
+ {
241
+ Expect(_RetoldFacto.onBeforeInitialize).to.be.a('function');
242
+ Expect(_RetoldFacto.onInitialize).to.be.a('function');
243
+ Expect(_RetoldFacto.onAfterInitialize).to.be.a('function');
244
+ }
245
+ );
246
+ test
247
+ (
248
+ 'Endpoint group check should work',
249
+ function()
250
+ {
251
+ Expect(_RetoldFacto.isEndpointGroupEnabled('MeadowEndpoints')).to.equal(true);
252
+ Expect(_RetoldFacto.isEndpointGroupEnabled('SourceManager')).to.equal(true);
253
+ Expect(_RetoldFacto.isEndpointGroupEnabled('IngestEngine')).to.equal(true);
254
+ Expect(_RetoldFacto.isEndpointGroupEnabled('ProjectionEngine')).to.equal(true);
255
+ Expect(_RetoldFacto.isEndpointGroupEnabled('NonExistent')).to.equal(false);
256
+ }
257
+ );
258
+ }
259
+ );
260
+
261
+ suite
262
+ (
263
+ 'Source CRUD Endpoints',
264
+ function()
265
+ {
266
+ test
267
+ (
268
+ 'Should create a Source',
269
+ function(fDone)
270
+ {
271
+ _SuperTest
272
+ .post('/1.0/Source')
273
+ .send({ Name: 'US Census API', Type: 'API', URL: 'https://api.census.gov', Protocol: 'HTTPS', Active: 1 })
274
+ .expect(200)
275
+ .end(
276
+ (pError, pResponse) =>
277
+ {
278
+ if (pError) return fDone(pError);
279
+ Expect(pResponse.body.Name).to.equal('US Census API');
280
+ Expect(pResponse.body.IDSource).to.be.greaterThan(0);
281
+ return fDone();
282
+ });
283
+ }
284
+ );
285
+ test
286
+ (
287
+ 'Should create a second Source',
288
+ function(fDone)
289
+ {
290
+ _SuperTest
291
+ .post('/1.0/Source')
292
+ .send({ Name: 'Dept of Labor CSV', Type: 'File', URL: 'https://www.bls.gov/data/', Protocol: 'HTTPS' })
293
+ .expect(200)
294
+ .end(
295
+ (pError, pResponse) =>
296
+ {
297
+ if (pError) return fDone(pError);
298
+ Expect(pResponse.body.Name).to.equal('Dept of Labor CSV');
299
+ return fDone();
300
+ });
301
+ }
302
+ );
303
+ test
304
+ (
305
+ 'Should read a Source by ID',
306
+ function(fDone)
307
+ {
308
+ _SuperTest
309
+ .get('/1.0/Source/1')
310
+ .expect(200)
311
+ .end(
312
+ (pError, pResponse) =>
313
+ {
314
+ if (pError) return fDone(pError);
315
+ Expect(pResponse.body.Name).to.equal('US Census API');
316
+ Expect(pResponse.body.Type).to.equal('API');
317
+ return fDone();
318
+ });
319
+ }
320
+ );
321
+ test
322
+ (
323
+ 'Should list Sources',
324
+ function(fDone)
325
+ {
326
+ _SuperTest
327
+ .get('/1.0/Sources/0/10')
328
+ .expect(200)
329
+ .end(
330
+ (pError, pResponse) =>
331
+ {
332
+ if (pError) return fDone(pError);
333
+ Expect(pResponse.body).to.be.an('array');
334
+ Expect(pResponse.body.length).to.equal(2);
335
+ return fDone();
336
+ });
337
+ }
338
+ );
339
+ }
340
+ );
341
+
342
+ suite
343
+ (
344
+ 'Source Manager Domain Endpoints',
345
+ function()
346
+ {
347
+ test
348
+ (
349
+ 'Should return active sources (only source 1 is active)',
350
+ function(fDone)
351
+ {
352
+ _SuperTest
353
+ .get('/facto/sources/active')
354
+ .expect(200)
355
+ .end(
356
+ (pError, pResponse) =>
357
+ {
358
+ if (pError) return fDone(pError);
359
+ Expect(pResponse.body.Active).to.equal(true);
360
+ Expect(pResponse.body.Sources).to.be.an('array');
361
+ Expect(pResponse.body.Sources.length).to.equal(1);
362
+ Expect(pResponse.body.Sources[0].Name).to.equal('US Census API');
363
+ return fDone();
364
+ });
365
+ }
366
+ );
367
+ test
368
+ (
369
+ 'Should activate source 2',
370
+ function(fDone)
371
+ {
372
+ _SuperTest
373
+ .put('/facto/source/2/activate')
374
+ .expect(200)
375
+ .end(
376
+ (pError, pResponse) =>
377
+ {
378
+ if (pError) return fDone(pError);
379
+ Expect(pResponse.body.Success).to.equal(true);
380
+ return fDone();
381
+ });
382
+ }
383
+ );
384
+ test
385
+ (
386
+ 'Should now return 2 active sources',
387
+ function(fDone)
388
+ {
389
+ _SuperTest
390
+ .get('/facto/sources/active')
391
+ .expect(200)
392
+ .end(
393
+ (pError, pResponse) =>
394
+ {
395
+ if (pError) return fDone(pError);
396
+ Expect(pResponse.body.Sources.length).to.equal(2);
397
+ return fDone();
398
+ });
399
+ }
400
+ );
401
+ test
402
+ (
403
+ 'Should deactivate source 2',
404
+ function(fDone)
405
+ {
406
+ _SuperTest
407
+ .put('/facto/source/2/deactivate')
408
+ .expect(200)
409
+ .end(
410
+ (pError, pResponse) =>
411
+ {
412
+ if (pError) return fDone(pError);
413
+ Expect(pResponse.body.Success).to.equal(true);
414
+ return fDone();
415
+ });
416
+ }
417
+ );
418
+ test
419
+ (
420
+ 'Should return 1 active source after deactivation',
421
+ function(fDone)
422
+ {
423
+ _SuperTest
424
+ .get('/facto/sources/active')
425
+ .expect(200)
426
+ .end(
427
+ (pError, pResponse) =>
428
+ {
429
+ if (pError) return fDone(pError);
430
+ Expect(pResponse.body.Sources.length).to.equal(1);
431
+ return fDone();
432
+ });
433
+ }
434
+ );
435
+ test
436
+ (
437
+ 'Should create source documentation',
438
+ function(fDone)
439
+ {
440
+ _SuperTest
441
+ .post('/1.0/SourceDocumentation')
442
+ .send({ IDSource: 1, Name: 'Census API Docs', DocumentType: 'markdown', MimeType: 'text/markdown', Description: 'API documentation' })
443
+ .expect(200)
444
+ .end(
445
+ (pError, pResponse) =>
446
+ {
447
+ if (pError) return fDone(pError);
448
+ Expect(pResponse.body.IDSourceDocumentation).to.be.greaterThan(0);
449
+ return fDone();
450
+ });
451
+ }
452
+ );
453
+ test
454
+ (
455
+ 'Should list documentation for source 1',
456
+ function(fDone)
457
+ {
458
+ _SuperTest
459
+ .get('/facto/source/1/documentation')
460
+ .expect(200)
461
+ .end(
462
+ (pError, pResponse) =>
463
+ {
464
+ if (pError) return fDone(pError);
465
+ Expect(pResponse.body.Documentation).to.be.an('array');
466
+ Expect(pResponse.body.Documentation.length).to.equal(1);
467
+ Expect(pResponse.body.Documentation[0].Name).to.equal('Census API Docs');
468
+ return fDone();
469
+ });
470
+ }
471
+ );
472
+ test
473
+ (
474
+ 'Should get source summary with counts',
475
+ function(fDone)
476
+ {
477
+ _SuperTest
478
+ .get('/facto/source/1/summary')
479
+ .expect(200)
480
+ .end(
481
+ (pError, pResponse) =>
482
+ {
483
+ if (pError) return fDone(pError);
484
+ Expect(pResponse.body.Source).to.be.an('object');
485
+ Expect(pResponse.body.Source.Name).to.equal('US Census API');
486
+ Expect(pResponse.body.DocumentationCount).to.equal(1);
487
+ return fDone();
488
+ });
489
+ }
490
+ );
491
+ }
492
+ );
493
+
494
+ suite
495
+ (
496
+ 'Dataset CRUD Endpoints',
497
+ function()
498
+ {
499
+ test
500
+ (
501
+ 'Should create a Raw Dataset',
502
+ function(fDone)
503
+ {
504
+ _SuperTest
505
+ .post('/1.0/Dataset')
506
+ .send({ Name: 'Census Population 2020', Type: 'Raw', Description: 'US Census population counts by county' })
507
+ .expect(200)
508
+ .end(
509
+ (pError, pResponse) =>
510
+ {
511
+ if (pError) return fDone(pError);
512
+ Expect(pResponse.body.Name).to.equal('Census Population 2020');
513
+ Expect(pResponse.body.Type).to.equal('Raw');
514
+ return fDone();
515
+ });
516
+ }
517
+ );
518
+ test
519
+ (
520
+ 'Should create a Projection Dataset',
521
+ function(fDone)
522
+ {
523
+ _SuperTest
524
+ .post('/1.0/Dataset')
525
+ .send({ Name: 'Population Summary View', Type: 'Projection', Description: 'Flattened population data for charting' })
526
+ .expect(200)
527
+ .end(
528
+ (pError, pResponse) =>
529
+ {
530
+ if (pError) return fDone(pError);
531
+ Expect(pResponse.body.Name).to.equal('Population Summary View');
532
+ Expect(pResponse.body.Type).to.equal('Projection');
533
+ return fDone();
534
+ });
535
+ }
536
+ );
537
+ test
538
+ (
539
+ 'Should create a DatasetSource link',
540
+ function(fDone)
541
+ {
542
+ _SuperTest
543
+ .post('/1.0/DatasetSource')
544
+ .send({ IDDataset: 1, IDSource: 1, ReliabilityWeight: 0.85 })
545
+ .expect(200)
546
+ .end(
547
+ (pError, pResponse) =>
548
+ {
549
+ if (pError) return fDone(pError);
550
+ Expect(pResponse.body.IDDataset).to.equal(1);
551
+ Expect(pResponse.body.IDSource).to.equal(1);
552
+ return fDone();
553
+ });
554
+ }
555
+ );
556
+ test
557
+ (
558
+ 'Should list dataset types via domain endpoint',
559
+ function(fDone)
560
+ {
561
+ _SuperTest
562
+ .get('/facto/datasets/types')
563
+ .expect(200)
564
+ .end(
565
+ (pError, pResponse) =>
566
+ {
567
+ if (pError) return fDone(pError);
568
+ Expect(pResponse.body.Types).to.be.an('array');
569
+ Expect(pResponse.body.Types).to.include('Raw');
570
+ Expect(pResponse.body.Types).to.include('Compositional');
571
+ Expect(pResponse.body.Types).to.include('Projection');
572
+ Expect(pResponse.body.Types).to.include('Derived');
573
+ return fDone();
574
+ });
575
+ }
576
+ );
577
+ }
578
+ );
579
+
580
+ suite
581
+ (
582
+ 'Dataset Manager Domain Endpoints',
583
+ function()
584
+ {
585
+ test
586
+ (
587
+ 'Should link a source to a dataset via domain endpoint',
588
+ function(fDone)
589
+ {
590
+ _SuperTest
591
+ .post('/facto/dataset/1/source')
592
+ .send({ IDSource: 2, ReliabilityWeight: 0.6 })
593
+ .expect(200)
594
+ .end(
595
+ (pError, pResponse) =>
596
+ {
597
+ if (pError) return fDone(pError);
598
+ Expect(pResponse.body.Success).to.equal(true);
599
+ Expect(pResponse.body.DatasetSource).to.be.an('object');
600
+ return fDone();
601
+ });
602
+ }
603
+ );
604
+ test
605
+ (
606
+ 'Should list sources linked to dataset 1',
607
+ function(fDone)
608
+ {
609
+ _SuperTest
610
+ .get('/facto/dataset/1/sources')
611
+ .expect(200)
612
+ .end(
613
+ (pError, pResponse) =>
614
+ {
615
+ if (pError) return fDone(pError);
616
+ Expect(pResponse.body.Sources).to.be.an('array');
617
+ Expect(pResponse.body.Sources.length).to.equal(2);
618
+ return fDone();
619
+ });
620
+ }
621
+ );
622
+ test
623
+ (
624
+ 'Should get dataset stats',
625
+ function(fDone)
626
+ {
627
+ _SuperTest
628
+ .get('/facto/dataset/1/stats')
629
+ .expect(200)
630
+ .end(
631
+ (pError, pResponse) =>
632
+ {
633
+ if (pError) return fDone(pError);
634
+ Expect(pResponse.body.Dataset).to.be.an('object');
635
+ Expect(pResponse.body.Dataset.Name).to.equal('Census Population 2020');
636
+ Expect(pResponse.body.SourceCount).to.equal(2);
637
+ Expect(pResponse.body.RecordCount).to.equal(0);
638
+ return fDone();
639
+ });
640
+ }
641
+ );
642
+ }
643
+ );
644
+
645
+ suite
646
+ (
647
+ 'Record CRUD Endpoints',
648
+ function()
649
+ {
650
+ test
651
+ (
652
+ 'Should create a Record',
653
+ function(fDone)
654
+ {
655
+ _SuperTest
656
+ .post('/1.0/Record')
657
+ .send(
658
+ {
659
+ IDDataset: 1,
660
+ IDSource: 1,
661
+ Type: 'census-population',
662
+ Version: 1,
663
+ RepresentedTimeStampStart: 1577836800,
664
+ RepresentedTimeStampStop: 1609459199,
665
+ Content: JSON.stringify({ county: 'Los Angeles', state: 'CA', population: 10014009 })
666
+ })
667
+ .expect(200)
668
+ .end(
669
+ (pError, pResponse) =>
670
+ {
671
+ if (pError) return fDone(pError);
672
+ Expect(pResponse.body.IDRecord).to.be.greaterThan(0);
673
+ Expect(pResponse.body.Type).to.equal('census-population');
674
+ return fDone();
675
+ });
676
+ }
677
+ );
678
+ test
679
+ (
680
+ 'Should create a second Record',
681
+ function(fDone)
682
+ {
683
+ _SuperTest
684
+ .post('/1.0/Record')
685
+ .send(
686
+ {
687
+ IDDataset: 1,
688
+ IDSource: 1,
689
+ Type: 'census-population',
690
+ Version: 1,
691
+ RepresentedTimeStampStart: 1577836800,
692
+ RepresentedTimeStampStop: 1609459199,
693
+ Content: JSON.stringify({ county: 'Cook', state: 'IL', population: 5275541 })
694
+ })
695
+ .expect(200)
696
+ .end(
697
+ (pError, pResponse) =>
698
+ {
699
+ if (pError) return fDone(pError);
700
+ Expect(pResponse.body.IDRecord).to.be.greaterThan(0);
701
+ return fDone();
702
+ });
703
+ }
704
+ );
705
+ test
706
+ (
707
+ 'Should read a Record by ID',
708
+ function(fDone)
709
+ {
710
+ _SuperTest
711
+ .get('/1.0/Record/1')
712
+ .expect(200)
713
+ .end(
714
+ (pError, pResponse) =>
715
+ {
716
+ if (pError) return fDone(pError);
717
+ Expect(pResponse.body.Type).to.equal('census-population');
718
+ let tmpContent = JSON.parse(pResponse.body.Content);
719
+ Expect(tmpContent.county).to.equal('Los Angeles');
720
+ return fDone();
721
+ });
722
+ }
723
+ );
724
+ test
725
+ (
726
+ 'Should list Records with pagination',
727
+ function(fDone)
728
+ {
729
+ _SuperTest
730
+ .get('/1.0/Records/0/50')
731
+ .expect(200)
732
+ .end(
733
+ (pError, pResponse) =>
734
+ {
735
+ if (pError) return fDone(pError);
736
+ Expect(pResponse.body).to.be.an('array');
737
+ Expect(pResponse.body.length).to.equal(2);
738
+ return fDone();
739
+ });
740
+ }
741
+ );
742
+ test
743
+ (
744
+ 'Should count Records',
745
+ function(fDone)
746
+ {
747
+ _SuperTest
748
+ .get('/1.0/Records/Count')
749
+ .expect(200)
750
+ .end(
751
+ (pError, pResponse) =>
752
+ {
753
+ if (pError) return fDone(pError);
754
+ Expect(pResponse.body.Count).to.equal(2);
755
+ return fDone();
756
+ });
757
+ }
758
+ );
759
+ }
760
+ );
761
+
762
+ suite
763
+ (
764
+ 'Record Manager Domain Endpoints',
765
+ function()
766
+ {
767
+ test
768
+ (
769
+ 'Should batch ingest records with auto-certainty',
770
+ function(fDone)
771
+ {
772
+ this.timeout(10000);
773
+ _SuperTest
774
+ .post('/facto/record/ingest')
775
+ .send(
776
+ {
777
+ IDDataset: 1,
778
+ IDSource: 1,
779
+ Records:
780
+ [
781
+ {
782
+ Type: 'census-population',
783
+ Content: JSON.stringify({ county: 'Harris', state: 'TX', population: 4713325 }),
784
+ RepresentedTimeStampStart: 1577836800,
785
+ RepresentedTimeStampStop: 1609459199
786
+ },
787
+ {
788
+ Type: 'census-population',
789
+ Content: JSON.stringify({ county: 'Maricopa', state: 'AZ', population: 4485414 }),
790
+ RepresentedTimeStampStart: 1577836800,
791
+ RepresentedTimeStampStop: 1609459199
792
+ }
793
+ ]
794
+ })
795
+ .expect(200)
796
+ .end(
797
+ (pError, pResponse) =>
798
+ {
799
+ if (pError) return fDone(pError);
800
+ Expect(pResponse.body.Ingested).to.equal(2);
801
+ Expect(pResponse.body.Errors).to.equal(0);
802
+ Expect(pResponse.body.Total).to.equal(2);
803
+ Expect(pResponse.body.DefaultCertainty).to.equal(0.5);
804
+ Expect(pResponse.body.Records).to.be.an('array');
805
+ Expect(pResponse.body.Records.length).to.equal(2);
806
+ return fDone();
807
+ });
808
+ }
809
+ );
810
+ test
811
+ (
812
+ 'Should return error for empty ingest request',
813
+ function(fDone)
814
+ {
815
+ _SuperTest
816
+ .post('/facto/record/ingest')
817
+ .send({ IDDataset: 1, IDSource: 1 })
818
+ .expect(200)
819
+ .end(
820
+ (pError, pResponse) =>
821
+ {
822
+ if (pError) return fDone(pError);
823
+ Expect(pResponse.body.Error).to.be.a('string');
824
+ Expect(pResponse.body.Ingested).to.equal(0);
825
+ return fDone();
826
+ });
827
+ }
828
+ );
829
+ test
830
+ (
831
+ 'Should get certainty indices for an ingested record',
832
+ function(fDone)
833
+ {
834
+ // Record 3 was created by batch ingest and should have auto-certainty
835
+ _SuperTest
836
+ .get('/facto/record/3/certainty')
837
+ .expect(200)
838
+ .end(
839
+ (pError, pResponse) =>
840
+ {
841
+ if (pError) return fDone(pError);
842
+ Expect(pResponse.body.CertaintyIndices).to.be.an('array');
843
+ Expect(pResponse.body.CertaintyIndices.length).to.be.greaterThan(0);
844
+ Expect(pResponse.body.CertaintyIndices[0].Dimension).to.equal('overall');
845
+ return fDone();
846
+ });
847
+ }
848
+ );
849
+ test
850
+ (
851
+ 'Should add a new certainty index entry',
852
+ function(fDone)
853
+ {
854
+ _SuperTest
855
+ .post('/facto/record/1/certainty')
856
+ .send({ CertaintyValue: 0.9, Dimension: 'accuracy', Justification: 'Verified against official source' })
857
+ .expect(200)
858
+ .end(
859
+ (pError, pResponse) =>
860
+ {
861
+ if (pError) return fDone(pError);
862
+ Expect(pResponse.body.Success).to.equal(true);
863
+ Expect(pResponse.body.CertaintyIndex).to.be.an('object');
864
+ Expect(pResponse.body.CertaintyIndex.Dimension).to.equal('accuracy');
865
+ return fDone();
866
+ });
867
+ }
868
+ );
869
+ test
870
+ (
871
+ 'Should list binary attachments for a record (empty)',
872
+ function(fDone)
873
+ {
874
+ _SuperTest
875
+ .get('/facto/record/1/binary')
876
+ .expect(200)
877
+ .end(
878
+ (pError, pResponse) =>
879
+ {
880
+ if (pError) return fDone(pError);
881
+ Expect(pResponse.body.Binaries).to.be.an('array');
882
+ Expect(pResponse.body.Binaries.length).to.equal(0);
883
+ return fDone();
884
+ });
885
+ }
886
+ );
887
+ test
888
+ (
889
+ 'Should get record versions by GUIDRecord',
890
+ function(fDone)
891
+ {
892
+ _SuperTest
893
+ .get('/facto/record/1/versions')
894
+ .expect(200)
895
+ .end(
896
+ (pError, pResponse) =>
897
+ {
898
+ if (pError) return fDone(pError);
899
+ Expect(pResponse.body.Versions).to.be.an('array');
900
+ Expect(pResponse.body.Versions.length).to.be.greaterThan(0);
901
+ return fDone();
902
+ });
903
+ }
904
+ );
905
+ }
906
+ );
907
+
908
+ suite
909
+ (
910
+ 'CertaintyIndex CRUD Endpoints',
911
+ function()
912
+ {
913
+ test
914
+ (
915
+ 'Should create a CertaintyIndex entry',
916
+ function(fDone)
917
+ {
918
+ _SuperTest
919
+ .post('/1.0/CertaintyIndex')
920
+ .send({ IDRecord: 1, CertaintyValue: 0.5, Dimension: 'completeness', Justification: 'Default initial certainty' })
921
+ .expect(200)
922
+ .end(
923
+ (pError, pResponse) =>
924
+ {
925
+ if (pError) return fDone(pError);
926
+ Expect(pResponse.body.IDCertaintyIndex).to.be.greaterThan(0);
927
+ Expect(pResponse.body.Dimension).to.equal('completeness');
928
+ return fDone();
929
+ });
930
+ }
931
+ );
932
+ test
933
+ (
934
+ 'Should read CertaintyIndex by ID',
935
+ function(fDone)
936
+ {
937
+ _SuperTest
938
+ .get('/1.0/CertaintyIndex/1')
939
+ .expect(200)
940
+ .end(
941
+ (pError, pResponse) =>
942
+ {
943
+ if (pError) return fDone(pError);
944
+ Expect(pResponse.body.IDRecord).to.be.greaterThan(0);
945
+ return fDone();
946
+ });
947
+ }
948
+ );
949
+ }
950
+ );
951
+
952
+ suite
953
+ (
954
+ 'Dataset Manager Advanced',
955
+ function()
956
+ {
957
+ test
958
+ (
959
+ 'Should get updated dataset stats with records',
960
+ function(fDone)
961
+ {
962
+ _SuperTest
963
+ .get('/facto/dataset/1/stats')
964
+ .expect(200)
965
+ .end(
966
+ (pError, pResponse) =>
967
+ {
968
+ if (pError) return fDone(pError);
969
+ Expect(pResponse.body.RecordCount).to.equal(4);
970
+ Expect(pResponse.body.SourceCount).to.equal(2);
971
+ return fDone();
972
+ });
973
+ }
974
+ );
975
+ test
976
+ (
977
+ 'Should list records for a dataset with pagination',
978
+ function(fDone)
979
+ {
980
+ _SuperTest
981
+ .get('/facto/dataset/1/records/0/10')
982
+ .expect(200)
983
+ .end(
984
+ (pError, pResponse) =>
985
+ {
986
+ if (pError) return fDone(pError);
987
+ Expect(pResponse.body.Records).to.be.an('array');
988
+ Expect(pResponse.body.Records.length).to.equal(4);
989
+ Expect(pResponse.body.IDDataset).to.equal(1);
990
+ return fDone();
991
+ });
992
+ }
993
+ );
994
+ }
995
+ );
996
+
997
+ suite
998
+ (
999
+ 'Ingest Engine Domain Endpoints',
1000
+ function()
1001
+ {
1002
+ test
1003
+ (
1004
+ 'Should create an ingest job',
1005
+ function(fDone)
1006
+ {
1007
+ _SuperTest
1008
+ .post('/facto/ingest/job')
1009
+ .send({ IDSource: 1, IDDataset: 1, Configuration: { format: 'csv', delimiter: ',' } })
1010
+ .expect(200)
1011
+ .end(
1012
+ (pError, pResponse) =>
1013
+ {
1014
+ if (pError) return fDone(pError);
1015
+ Expect(pResponse.body.Success).to.equal(true);
1016
+ Expect(pResponse.body.Job).to.be.an('object');
1017
+ Expect(pResponse.body.Job.Status).to.equal('Pending');
1018
+ Expect(pResponse.body.Job.IDSource).to.equal(1);
1019
+ return fDone();
1020
+ });
1021
+ }
1022
+ );
1023
+ test
1024
+ (
1025
+ 'Should list ingest jobs',
1026
+ function(fDone)
1027
+ {
1028
+ _SuperTest
1029
+ .get('/facto/ingest/jobs')
1030
+ .expect(200)
1031
+ .end(
1032
+ (pError, pResponse) =>
1033
+ {
1034
+ if (pError) return fDone(pError);
1035
+ Expect(pResponse.body.Jobs).to.be.an('array');
1036
+ Expect(pResponse.body.Jobs.length).to.equal(1);
1037
+ return fDone();
1038
+ });
1039
+ }
1040
+ );
1041
+ test
1042
+ (
1043
+ 'Should get ingest job details',
1044
+ function(fDone)
1045
+ {
1046
+ _SuperTest
1047
+ .get('/facto/ingest/job/1')
1048
+ .expect(200)
1049
+ .end(
1050
+ (pError, pResponse) =>
1051
+ {
1052
+ if (pError) return fDone(pError);
1053
+ Expect(pResponse.body.Job).to.be.an('object');
1054
+ Expect(pResponse.body.Job.Status).to.equal('Pending');
1055
+ Expect(pResponse.body.Job.Log).to.contain('Job created');
1056
+ return fDone();
1057
+ });
1058
+ }
1059
+ );
1060
+ test
1061
+ (
1062
+ 'Should start an ingest job',
1063
+ function(fDone)
1064
+ {
1065
+ _SuperTest
1066
+ .put('/facto/ingest/job/1/start')
1067
+ .expect(200)
1068
+ .end(
1069
+ (pError, pResponse) =>
1070
+ {
1071
+ if (pError) return fDone(pError);
1072
+ Expect(pResponse.body.Success).to.equal(true);
1073
+ return fDone();
1074
+ });
1075
+ }
1076
+ );
1077
+ test
1078
+ (
1079
+ 'Should complete an ingest job with counters',
1080
+ function(fDone)
1081
+ {
1082
+ _SuperTest
1083
+ .put('/facto/ingest/job/1/complete')
1084
+ .send({ RecordsProcessed: 100, RecordsCreated: 95, RecordsErrored: 5 })
1085
+ .expect(200)
1086
+ .end(
1087
+ (pError, pResponse) =>
1088
+ {
1089
+ if (pError) return fDone(pError);
1090
+ Expect(pResponse.body.Success).to.equal(true);
1091
+ return fDone();
1092
+ });
1093
+ }
1094
+ );
1095
+ test
1096
+ (
1097
+ 'Should verify completed job has log entries',
1098
+ function(fDone)
1099
+ {
1100
+ _SuperTest
1101
+ .get('/facto/ingest/job/1')
1102
+ .expect(200)
1103
+ .end(
1104
+ (pError, pResponse) =>
1105
+ {
1106
+ if (pError) return fDone(pError);
1107
+ Expect(pResponse.body.Job.Log).to.contain('Job created');
1108
+ Expect(pResponse.body.Job.Log).to.contain('Job started');
1109
+ Expect(pResponse.body.Job.Log).to.contain('Job completed');
1110
+ return fDone();
1111
+ });
1112
+ }
1113
+ );
1114
+ test
1115
+ (
1116
+ 'Should list valid job statuses',
1117
+ function(fDone)
1118
+ {
1119
+ _SuperTest
1120
+ .get('/facto/ingest/statuses')
1121
+ .expect(200)
1122
+ .end(
1123
+ (pError, pResponse) =>
1124
+ {
1125
+ if (pError) return fDone(pError);
1126
+ Expect(pResponse.body.Statuses).to.be.an('array');
1127
+ Expect(pResponse.body.Statuses).to.include('Pending');
1128
+ Expect(pResponse.body.Statuses).to.include('Running');
1129
+ Expect(pResponse.body.Statuses).to.include('Completed');
1130
+ Expect(pResponse.body.Statuses).to.include('Failed');
1131
+ return fDone();
1132
+ });
1133
+ }
1134
+ );
1135
+ }
1136
+ );
1137
+
1138
+ suite
1139
+ (
1140
+ 'Projection Engine Domain Endpoints',
1141
+ function()
1142
+ {
1143
+ test
1144
+ (
1145
+ 'Should list projection datasets',
1146
+ function(fDone)
1147
+ {
1148
+ _SuperTest
1149
+ .get('/facto/projections')
1150
+ .expect(200)
1151
+ .end(
1152
+ (pError, pResponse) =>
1153
+ {
1154
+ if (pError) return fDone(pError);
1155
+ Expect(pResponse.body.Projections).to.be.an('array');
1156
+ Expect(pResponse.body.Projections.length).to.equal(1);
1157
+ Expect(pResponse.body.Projections[0].Name).to.equal('Population Summary View');
1158
+ return fDone();
1159
+ });
1160
+ }
1161
+ );
1162
+ test
1163
+ (
1164
+ 'Should list datasets by type',
1165
+ function(fDone)
1166
+ {
1167
+ _SuperTest
1168
+ .get('/facto/datasets/by-type/Raw')
1169
+ .expect(200)
1170
+ .end(
1171
+ (pError, pResponse) =>
1172
+ {
1173
+ if (pError) return fDone(pError);
1174
+ Expect(pResponse.body.Datasets).to.be.an('array');
1175
+ Expect(pResponse.body.Datasets.length).to.equal(1);
1176
+ Expect(pResponse.body.Type).to.equal('Raw');
1177
+ return fDone();
1178
+ });
1179
+ }
1180
+ );
1181
+ }
1182
+ );
1183
+
1184
+ suite
1185
+ (
1186
+ 'Schema Auto-Creation',
1187
+ function()
1188
+ {
1189
+ test
1190
+ (
1191
+ 'Should expose FACTO_SCHEMA_SQL on module exports',
1192
+ function()
1193
+ {
1194
+ const libRetoldFacto = require('../source/Retold-Facto.js');
1195
+ Expect(libRetoldFacto.FACTO_SCHEMA_SQL).to.be.a('string');
1196
+ Expect(libRetoldFacto.FACTO_SCHEMA_SQL).to.contain('CREATE TABLE IF NOT EXISTS Source');
1197
+ Expect(libRetoldFacto.FACTO_SCHEMA_SQL).to.contain('CREATE TABLE IF NOT EXISTS IngestJob');
1198
+ }
1199
+ );
1200
+ test
1201
+ (
1202
+ 'Should have createSchema method on the service instance',
1203
+ function()
1204
+ {
1205
+ Expect(_RetoldFacto.createSchema).to.be.a('function');
1206
+ }
1207
+ );
1208
+ test
1209
+ (
1210
+ 'Should be able to run createSchema without error (tables already exist)',
1211
+ function(fDone)
1212
+ {
1213
+ _RetoldFacto.createSchema(
1214
+ (pError) =>
1215
+ {
1216
+ Expect(pError).to.not.exist;
1217
+ return fDone();
1218
+ });
1219
+ }
1220
+ );
1221
+ }
1222
+ );
1223
+
1224
+ suite
1225
+ (
1226
+ 'CSV/JSON File Ingest via API',
1227
+ function()
1228
+ {
1229
+ test
1230
+ (
1231
+ 'Should ingest CSV content via POST /facto/ingest/file',
1232
+ function(fDone)
1233
+ {
1234
+ this.timeout(10000);
1235
+ let tmpCSVContent = 'name,state,population\nAlaska,AK,733391\nDelaware,DE,989948\nVermont,VT,643077';
1236
+
1237
+ _SuperTest
1238
+ .post('/facto/ingest/file')
1239
+ .send(
1240
+ {
1241
+ IDDataset: 1,
1242
+ IDSource: 1,
1243
+ Format: 'csv',
1244
+ Type: 'state-population',
1245
+ Content: tmpCSVContent
1246
+ })
1247
+ .expect(200)
1248
+ .end(
1249
+ (pError, pResponse) =>
1250
+ {
1251
+ if (pError) return fDone(pError);
1252
+ Expect(pResponse.body.Ingested).to.equal(3);
1253
+ Expect(pResponse.body.Errors).to.equal(0);
1254
+ Expect(pResponse.body.Total).to.equal(3);
1255
+ Expect(pResponse.body.Format).to.equal('csv');
1256
+ Expect(pResponse.body.Records).to.be.an('array');
1257
+ Expect(pResponse.body.Records.length).to.equal(3);
1258
+ // Verify content was stored as JSON
1259
+ let tmpContent = JSON.parse(pResponse.body.Records[0].Content);
1260
+ Expect(tmpContent.name).to.equal('Alaska');
1261
+ Expect(tmpContent.state).to.equal('AK');
1262
+ return fDone();
1263
+ });
1264
+ }
1265
+ );
1266
+ test
1267
+ (
1268
+ 'Should ingest JSON array content via POST /facto/ingest/file',
1269
+ function(fDone)
1270
+ {
1271
+ this.timeout(10000);
1272
+ let tmpJSONContent = JSON.stringify([
1273
+ { county: 'San Diego', state: 'CA', population: 3338330 },
1274
+ { county: 'Orange', state: 'CA', population: 3186989 }
1275
+ ]);
1276
+
1277
+ _SuperTest
1278
+ .post('/facto/ingest/file')
1279
+ .send(
1280
+ {
1281
+ IDDataset: 1,
1282
+ IDSource: 1,
1283
+ Format: 'json',
1284
+ Type: 'county-population',
1285
+ Content: tmpJSONContent
1286
+ })
1287
+ .expect(200)
1288
+ .end(
1289
+ (pError, pResponse) =>
1290
+ {
1291
+ if (pError) return fDone(pError);
1292
+ Expect(pResponse.body.Ingested).to.equal(2);
1293
+ Expect(pResponse.body.Errors).to.equal(0);
1294
+ Expect(pResponse.body.Format).to.equal('json');
1295
+ return fDone();
1296
+ });
1297
+ }
1298
+ );
1299
+ test
1300
+ (
1301
+ 'Should auto-detect JSON format',
1302
+ function(fDone)
1303
+ {
1304
+ this.timeout(10000);
1305
+ let tmpContent = JSON.stringify({ data: [{ metric: 'gdp', value: 21433 }] });
1306
+
1307
+ _SuperTest
1308
+ .post('/facto/ingest/file')
1309
+ .send(
1310
+ {
1311
+ IDDataset: 1,
1312
+ IDSource: 1,
1313
+ Type: 'economic',
1314
+ Content: tmpContent
1315
+ })
1316
+ .expect(200)
1317
+ .end(
1318
+ (pError, pResponse) =>
1319
+ {
1320
+ if (pError) return fDone(pError);
1321
+ Expect(pResponse.body.Ingested).to.equal(1);
1322
+ Expect(pResponse.body.Format).to.equal('json');
1323
+ return fDone();
1324
+ });
1325
+ }
1326
+ );
1327
+ test
1328
+ (
1329
+ 'Should auto-detect CSV format',
1330
+ function(fDone)
1331
+ {
1332
+ this.timeout(10000);
1333
+ let tmpContent = 'key,value\nalpha,100\nbeta,200';
1334
+
1335
+ _SuperTest
1336
+ .post('/facto/ingest/file')
1337
+ .send(
1338
+ {
1339
+ IDDataset: 1,
1340
+ IDSource: 1,
1341
+ Content: tmpContent
1342
+ })
1343
+ .expect(200)
1344
+ .end(
1345
+ (pError, pResponse) =>
1346
+ {
1347
+ if (pError) return fDone(pError);
1348
+ Expect(pResponse.body.Ingested).to.equal(2);
1349
+ Expect(pResponse.body.Format).to.equal('csv');
1350
+ return fDone();
1351
+ });
1352
+ }
1353
+ );
1354
+ test
1355
+ (
1356
+ 'Should return error when Content is missing',
1357
+ function(fDone)
1358
+ {
1359
+ _SuperTest
1360
+ .post('/facto/ingest/file')
1361
+ .send({ IDDataset: 1, IDSource: 1 })
1362
+ .expect(200)
1363
+ .end(
1364
+ (pError, pResponse) =>
1365
+ {
1366
+ if (pError) return fDone(pError);
1367
+ Expect(pResponse.body.Error).to.be.a('string');
1368
+ Expect(pResponse.body.Ingested).to.equal(0);
1369
+ return fDone();
1370
+ });
1371
+ }
1372
+ );
1373
+ }
1374
+ );
1375
+
1376
+ suite
1377
+ (
1378
+ 'Programmatic File Ingest',
1379
+ function()
1380
+ {
1381
+ test
1382
+ (
1383
+ 'Should ingest a CSV file from disk',
1384
+ function(fDone)
1385
+ {
1386
+ this.timeout(10000);
1387
+
1388
+ // Write a temp CSV file
1389
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.csv');
1390
+ libFs.writeFileSync(tmpFilePath, 'city,state,zip\nPortland,OR,97201\nSeattle,WA,98101\nBoise,ID,83702\n');
1391
+
1392
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
1393
+ { type: 'city-data' },
1394
+ (pError, pResult) =>
1395
+ {
1396
+ // Clean up temp file
1397
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
1398
+
1399
+ if (pError) return fDone(pError);
1400
+ Expect(pResult.Ingested).to.equal(3);
1401
+ Expect(pResult.Errors).to.equal(0);
1402
+ Expect(pResult.Format).to.equal('csv');
1403
+ Expect(pResult.Records).to.be.an('array');
1404
+ return fDone();
1405
+ });
1406
+ }
1407
+ );
1408
+ test
1409
+ (
1410
+ 'Should ingest a JSON file from disk',
1411
+ function(fDone)
1412
+ {
1413
+ this.timeout(10000);
1414
+
1415
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.json');
1416
+ libFs.writeFileSync(tmpFilePath, JSON.stringify([
1417
+ { region: 'West', count: 50 },
1418
+ { region: 'East', count: 45 }
1419
+ ]));
1420
+
1421
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
1422
+ { type: 'region-data' },
1423
+ (pError, pResult) =>
1424
+ {
1425
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
1426
+
1427
+ if (pError) return fDone(pError);
1428
+ Expect(pResult.Ingested).to.equal(2);
1429
+ Expect(pResult.Format).to.equal('json');
1430
+ return fDone();
1431
+ });
1432
+ }
1433
+ );
1434
+ test
1435
+ (
1436
+ 'Should return error for non-existent file',
1437
+ function(fDone)
1438
+ {
1439
+ _Fable.RetoldFactoIngestEngine.ingestFile('/tmp/non-existent-file-12345.csv', 1, 1,
1440
+ (pError, pResult) =>
1441
+ {
1442
+ Expect(pError).to.be.an.instanceOf(Error);
1443
+ return fDone();
1444
+ });
1445
+ }
1446
+ );
1447
+ test
1448
+ (
1449
+ 'Should return error for unknown file extension',
1450
+ function(fDone)
1451
+ {
1452
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.xyz');
1453
+ libFs.writeFileSync(tmpFilePath, 'some unknown format data');
1454
+
1455
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
1456
+ (pError, pResult) =>
1457
+ {
1458
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
1459
+
1460
+ Expect(pError).to.be.an.instanceOf(Error);
1461
+ Expect(pError.message).to.contain('Cannot determine format');
1462
+ return fDone();
1463
+ });
1464
+ }
1465
+ );
1466
+ test
1467
+ (
1468
+ 'Should handle TSV files with auto-detected tab delimiter',
1469
+ function(fDone)
1470
+ {
1471
+ this.timeout(10000);
1472
+
1473
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.tsv');
1474
+ libFs.writeFileSync(tmpFilePath, 'name\tscore\nAlice\t95\nBob\t87\n');
1475
+
1476
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
1477
+ { type: 'scores' },
1478
+ (pError, pResult) =>
1479
+ {
1480
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
1481
+
1482
+ if (pError) return fDone(pError);
1483
+ Expect(pResult.Ingested).to.equal(2);
1484
+ Expect(pResult.Format).to.equal('csv');
1485
+ return fDone();
1486
+ });
1487
+ }
1488
+ );
1489
+ }
1490
+ );
1491
+
1492
+ suite
1493
+ (
1494
+ 'Projection Engine Advanced',
1495
+ function()
1496
+ {
1497
+ test
1498
+ (
1499
+ 'Should query records across datasets',
1500
+ function(fDone)
1501
+ {
1502
+ this.timeout(10000);
1503
+ _SuperTest
1504
+ .post('/facto/projections/query')
1505
+ .send({ DatasetIDs: [1], Begin: 0, Cap: 100 })
1506
+ .expect(200)
1507
+ .end(
1508
+ (pError, pResponse) =>
1509
+ {
1510
+ if (pError) return fDone(pError);
1511
+ Expect(pResponse.body.Query).to.be.an('object');
1512
+ Expect(pResponse.body.Records).to.be.an('array');
1513
+ Expect(pResponse.body.Records.length).to.be.greaterThan(0);
1514
+ Expect(pResponse.body.Count).to.be.greaterThan(0);
1515
+ return fDone();
1516
+ });
1517
+ }
1518
+ );
1519
+ test
1520
+ (
1521
+ 'Should query with type filter',
1522
+ function(fDone)
1523
+ {
1524
+ this.timeout(10000);
1525
+ _SuperTest
1526
+ .post('/facto/projections/query')
1527
+ .send({ DatasetIDs: [1], Type: 'census-population', Begin: 0, Cap: 100 })
1528
+ .expect(200)
1529
+ .end(
1530
+ (pError, pResponse) =>
1531
+ {
1532
+ if (pError) return fDone(pError);
1533
+ Expect(pResponse.body.Records).to.be.an('array');
1534
+ // All returned records should be of the filtered type
1535
+ for (let i = 0; i < pResponse.body.Records.length; i++)
1536
+ {
1537
+ Expect(pResponse.body.Records[i].Type).to.equal('census-population');
1538
+ }
1539
+ return fDone();
1540
+ });
1541
+ }
1542
+ );
1543
+ test
1544
+ (
1545
+ 'Should aggregate records by dataset',
1546
+ function(fDone)
1547
+ {
1548
+ this.timeout(10000);
1549
+ _SuperTest
1550
+ .post('/facto/projections/aggregate')
1551
+ .send({ DatasetIDs: [1], GroupBy: 'IDDataset' })
1552
+ .expect(200)
1553
+ .end(
1554
+ (pError, pResponse) =>
1555
+ {
1556
+ if (pError) return fDone(pError);
1557
+ Expect(pResponse.body.Aggregation).to.be.an('array');
1558
+ Expect(pResponse.body.Aggregation.length).to.be.greaterThan(0);
1559
+ Expect(pResponse.body.Total).to.be.greaterThan(0);
1560
+ return fDone();
1561
+ });
1562
+ }
1563
+ );
1564
+ test
1565
+ (
1566
+ 'Should query certainty-weighted records',
1567
+ function(fDone)
1568
+ {
1569
+ this.timeout(10000);
1570
+ _SuperTest
1571
+ .post('/facto/projections/certainty')
1572
+ .send({ DatasetIDs: [1], MinCertainty: 0.0, MaxCertainty: 1.0, Begin: 0, Cap: 100 })
1573
+ .expect(200)
1574
+ .end(
1575
+ (pError, pResponse) =>
1576
+ {
1577
+ if (pError) return fDone(pError);
1578
+ Expect(pResponse.body.Records).to.be.an('array');
1579
+ Expect(pResponse.body.Count).to.be.a('number');
1580
+ return fDone();
1581
+ });
1582
+ }
1583
+ );
1584
+ test
1585
+ (
1586
+ 'Should compare datasets',
1587
+ function(fDone)
1588
+ {
1589
+ this.timeout(10000);
1590
+ _SuperTest
1591
+ .post('/facto/projections/compare')
1592
+ .send({ DatasetIDs: [1, 2] })
1593
+ .expect(200)
1594
+ .end(
1595
+ (pError, pResponse) =>
1596
+ {
1597
+ if (pError) return fDone(pError);
1598
+ Expect(pResponse.body.Datasets).to.be.an('array');
1599
+ Expect(pResponse.body.Datasets.length).to.equal(2);
1600
+ // First dataset should have records, second should have none
1601
+ Expect(pResponse.body.Datasets[0].IDDataset).to.equal(1);
1602
+ Expect(pResponse.body.Datasets[0].RecordCount).to.be.greaterThan(0);
1603
+ Expect(pResponse.body.Datasets[1].IDDataset).to.equal(2);
1604
+ return fDone();
1605
+ });
1606
+ }
1607
+ );
1608
+ test
1609
+ (
1610
+ 'Should get warehouse summary statistics',
1611
+ function(fDone)
1612
+ {
1613
+ this.timeout(10000);
1614
+ _SuperTest
1615
+ .get('/facto/projections/summary')
1616
+ .expect(200)
1617
+ .end(
1618
+ (pError, pResponse) =>
1619
+ {
1620
+ if (pError) return fDone(pError);
1621
+ Expect(pResponse.body.Sources).to.be.a('number');
1622
+ Expect(pResponse.body.Sources).to.be.greaterThan(0);
1623
+ Expect(pResponse.body.Datasets).to.be.a('number');
1624
+ Expect(pResponse.body.Datasets).to.be.greaterThan(0);
1625
+ Expect(pResponse.body.Records).to.be.a('number');
1626
+ Expect(pResponse.body.Records).to.be.greaterThan(0);
1627
+ Expect(pResponse.body.DatasetsByType).to.be.an('object');
1628
+ Expect(pResponse.body.DatasetsByType.Raw).to.be.a('number');
1629
+ Expect(pResponse.body.DatasetsByType.Projection).to.be.a('number');
1630
+ return fDone();
1631
+ });
1632
+ }
1633
+ );
1634
+ test
1635
+ (
1636
+ 'Should return empty results for non-existent dataset query',
1637
+ function(fDone)
1638
+ {
1639
+ this.timeout(10000);
1640
+ _SuperTest
1641
+ .post('/facto/projections/query')
1642
+ .send({ DatasetIDs: [999], Begin: 0, Cap: 100 })
1643
+ .expect(200)
1644
+ .end(
1645
+ (pError, pResponse) =>
1646
+ {
1647
+ if (pError) return fDone(pError);
1648
+ Expect(pResponse.body.Records).to.be.an('array');
1649
+ Expect(pResponse.body.Records.length).to.equal(0);
1650
+ return fDone();
1651
+ });
1652
+ }
1653
+ );
1654
+ }
1655
+ );
1656
+
1657
+ suite
1658
+ (
1659
+ 'CSV/JSON Parsing Methods',
1660
+ function()
1661
+ {
1662
+ test
1663
+ (
1664
+ 'parseCSV should parse a basic CSV string',
1665
+ function(fDone)
1666
+ {
1667
+ _Fable.RetoldFactoIngestEngine.parseCSV('a,b\n1,2\n3,4',
1668
+ (pError, pRecords) =>
1669
+ {
1670
+ Expect(pError).to.not.exist;
1671
+ Expect(pRecords).to.be.an('array');
1672
+ Expect(pRecords.length).to.equal(2);
1673
+ Expect(pRecords[0].a).to.equal('1');
1674
+ Expect(pRecords[0].b).to.equal('2');
1675
+ return fDone();
1676
+ });
1677
+ }
1678
+ );
1679
+ test
1680
+ (
1681
+ 'parseJSON should parse an array',
1682
+ function(fDone)
1683
+ {
1684
+ _Fable.RetoldFactoIngestEngine.parseJSON('[{"x":1},{"x":2}]',
1685
+ (pError, pRecords) =>
1686
+ {
1687
+ Expect(pError).to.not.exist;
1688
+ Expect(pRecords).to.be.an('array');
1689
+ Expect(pRecords.length).to.equal(2);
1690
+ return fDone();
1691
+ });
1692
+ }
1693
+ );
1694
+ test
1695
+ (
1696
+ 'parseJSON should extract data key from object',
1697
+ function(fDone)
1698
+ {
1699
+ _Fable.RetoldFactoIngestEngine.parseJSON('{"data":[{"y":10}]}',
1700
+ (pError, pRecords) =>
1701
+ {
1702
+ Expect(pError).to.not.exist;
1703
+ Expect(pRecords).to.be.an('array');
1704
+ Expect(pRecords.length).to.equal(1);
1705
+ Expect(pRecords[0].y).to.equal(10);
1706
+ return fDone();
1707
+ });
1708
+ }
1709
+ );
1710
+ test
1711
+ (
1712
+ 'parseJSON should wrap a single object in an array',
1713
+ function(fDone)
1714
+ {
1715
+ _Fable.RetoldFactoIngestEngine.parseJSON('{"solo":true}',
1716
+ (pError, pRecords) =>
1717
+ {
1718
+ Expect(pError).to.not.exist;
1719
+ Expect(pRecords).to.be.an('array');
1720
+ Expect(pRecords.length).to.equal(1);
1721
+ Expect(pRecords[0].solo).to.equal(true);
1722
+ return fDone();
1723
+ });
1724
+ }
1725
+ );
1726
+ test
1727
+ (
1728
+ 'parseJSON should return error for invalid JSON',
1729
+ function(fDone)
1730
+ {
1731
+ _Fable.RetoldFactoIngestEngine.parseJSON('not valid json {{{',
1732
+ (pError, pRecords) =>
1733
+ {
1734
+ Expect(pError).to.be.an.instanceOf(Error);
1735
+ return fDone();
1736
+ });
1737
+ }
1738
+ );
1739
+ }
1740
+ );
1741
+ suite
1742
+ (
1743
+ 'Multi-Format Ingest',
1744
+ function()
1745
+ {
1746
+ // ========================================================
1747
+ // XML Parsing
1748
+ // ========================================================
1749
+ test
1750
+ (
1751
+ 'parseXML should parse a basic XML array',
1752
+ function(fDone)
1753
+ {
1754
+ let tmpXML = '<root><item><name>Alice</name><score>95</score></item><item><name>Bob</name><score>87</score></item></root>';
1755
+ _Fable.RetoldFactoIngestEngine.parseXML(tmpXML,
1756
+ (pError, pRecords) =>
1757
+ {
1758
+ Expect(pError).to.not.exist;
1759
+ Expect(pRecords).to.be.an('array');
1760
+ Expect(pRecords.length).to.equal(2);
1761
+ Expect(pRecords[0].name).to.equal('Alice');
1762
+ Expect(pRecords[1].score).to.equal(87);
1763
+ return fDone();
1764
+ });
1765
+ }
1766
+ );
1767
+ test
1768
+ (
1769
+ 'parseXML should auto-detect nested record array',
1770
+ function(fDone)
1771
+ {
1772
+ let tmpXML = '<?xml version="1.0"?><response><meta><total>2</total></meta><data><record><id>1</id><val>A</val></record><record><id>2</id><val>B</val></record></data></response>';
1773
+ _Fable.RetoldFactoIngestEngine.parseXML(tmpXML,
1774
+ (pError, pRecords) =>
1775
+ {
1776
+ Expect(pError).to.not.exist;
1777
+ Expect(pRecords).to.be.an('array');
1778
+ Expect(pRecords.length).to.equal(2);
1779
+ Expect(pRecords[0].id).to.equal(1);
1780
+ Expect(pRecords[1].val).to.equal('B');
1781
+ return fDone();
1782
+ });
1783
+ }
1784
+ );
1785
+ test
1786
+ (
1787
+ 'parseXML should navigate with recordPath option',
1788
+ function(fDone)
1789
+ {
1790
+ let tmpXML = '<api><results><items><item><x>10</x></item><item><x>20</x></item></items></results></api>';
1791
+ _Fable.RetoldFactoIngestEngine.parseXML(tmpXML, { recordPath: 'api.results.items.item' },
1792
+ (pError, pRecords) =>
1793
+ {
1794
+ Expect(pError).to.not.exist;
1795
+ Expect(pRecords).to.be.an('array');
1796
+ Expect(pRecords.length).to.equal(2);
1797
+ Expect(pRecords[0].x).to.equal(10);
1798
+ return fDone();
1799
+ });
1800
+ }
1801
+ );
1802
+ test
1803
+ (
1804
+ 'parseXML should return error for malformed XML',
1805
+ function(fDone)
1806
+ {
1807
+ _Fable.RetoldFactoIngestEngine.parseXML('<<<not xml at all>>>',
1808
+ (pError, pRecords) =>
1809
+ {
1810
+ // fast-xml-parser may not error on malformed XML but should at least return something
1811
+ // The key test is that it doesn't crash
1812
+ Expect(pRecords).to.exist;
1813
+ return fDone();
1814
+ });
1815
+ }
1816
+ );
1817
+ test
1818
+ (
1819
+ 'parseXML should return error for invalid recordPath',
1820
+ function(fDone)
1821
+ {
1822
+ let tmpXML = '<root><item>test</item></root>';
1823
+ _Fable.RetoldFactoIngestEngine.parseXML(tmpXML, { recordPath: 'root.nonexistent.path' },
1824
+ (pError, pRecords) =>
1825
+ {
1826
+ Expect(pError).to.be.an.instanceOf(Error);
1827
+ Expect(pError.message).to.contain('not found in XML');
1828
+ return fDone();
1829
+ });
1830
+ }
1831
+ );
1832
+
1833
+ // ========================================================
1834
+ // Excel Parsing
1835
+ // ========================================================
1836
+ test
1837
+ (
1838
+ 'parseExcel should parse a basic workbook',
1839
+ function(fDone)
1840
+ {
1841
+ let libXLSX = require('xlsx');
1842
+ let tmpWorksheet = libXLSX.utils.aoa_to_sheet([
1843
+ ['name', 'score'],
1844
+ ['Alice', 95],
1845
+ ['Bob', 87]
1846
+ ]);
1847
+ let tmpWorkbook = libXLSX.utils.book_new();
1848
+ libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Sheet1');
1849
+ let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
1850
+
1851
+ _Fable.RetoldFactoIngestEngine.parseExcel(tmpBuffer,
1852
+ (pError, pRecords) =>
1853
+ {
1854
+ Expect(pError).to.not.exist;
1855
+ Expect(pRecords).to.be.an('array');
1856
+ Expect(pRecords.length).to.equal(2);
1857
+ Expect(pRecords[0].name).to.equal('Alice');
1858
+ Expect(pRecords[0].score).to.equal(95);
1859
+ return fDone();
1860
+ });
1861
+ }
1862
+ );
1863
+ test
1864
+ (
1865
+ 'parseExcel should select sheet by name',
1866
+ function(fDone)
1867
+ {
1868
+ let libXLSX = require('xlsx');
1869
+ let tmpSheet1 = libXLSX.utils.aoa_to_sheet([['a'], [1]]);
1870
+ let tmpSheet2 = libXLSX.utils.aoa_to_sheet([['b'], [2], [3]]);
1871
+ let tmpWorkbook = libXLSX.utils.book_new();
1872
+ libXLSX.utils.book_append_sheet(tmpWorkbook, tmpSheet1, 'First');
1873
+ libXLSX.utils.book_append_sheet(tmpWorkbook, tmpSheet2, 'Second');
1874
+ let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
1875
+
1876
+ _Fable.RetoldFactoIngestEngine.parseExcel(tmpBuffer, { sheet: 'Second' },
1877
+ (pError, pRecords) =>
1878
+ {
1879
+ Expect(pError).to.not.exist;
1880
+ Expect(pRecords).to.be.an('array');
1881
+ Expect(pRecords.length).to.equal(2);
1882
+ Expect(pRecords[0].b).to.equal(2);
1883
+ return fDone();
1884
+ });
1885
+ }
1886
+ );
1887
+ test
1888
+ (
1889
+ 'parseExcel should return empty array for empty sheet',
1890
+ function(fDone)
1891
+ {
1892
+ let libXLSX = require('xlsx');
1893
+ let tmpWorksheet = libXLSX.utils.aoa_to_sheet([]);
1894
+ let tmpWorkbook = libXLSX.utils.book_new();
1895
+ libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Empty');
1896
+ let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
1897
+
1898
+ _Fable.RetoldFactoIngestEngine.parseExcel(tmpBuffer,
1899
+ (pError, pRecords) =>
1900
+ {
1901
+ Expect(pError).to.not.exist;
1902
+ Expect(pRecords).to.be.an('array');
1903
+ Expect(pRecords.length).to.equal(0);
1904
+ return fDone();
1905
+ });
1906
+ }
1907
+ );
1908
+
1909
+ // ========================================================
1910
+ // Fixed-Width Parsing
1911
+ // ========================================================
1912
+ test
1913
+ (
1914
+ 'parseFixedWidth should extract columns by position',
1915
+ function(fDone)
1916
+ {
1917
+ let tmpContent = 'ACM12345678 37.7749 -122.4194\nBCN98765432 41.3851 2.1734 \n';
1918
+ _Fable.RetoldFactoIngestEngine.parseFixedWidth(tmpContent,
1919
+ {
1920
+ columns: [
1921
+ { name: 'ID', start: 1, width: 11 },
1922
+ { name: 'Lat', start: 13, width: 8 },
1923
+ { name: 'Lon', start: 22, width: 9 }
1924
+ ]
1925
+ },
1926
+ (pError, pRecords) =>
1927
+ {
1928
+ Expect(pError).to.not.exist;
1929
+ Expect(pRecords).to.be.an('array');
1930
+ Expect(pRecords.length).to.equal(2);
1931
+ Expect(pRecords[0].ID).to.equal('ACM12345678');
1932
+ Expect(pRecords[0].Lat).to.equal('37.7749');
1933
+ Expect(pRecords[1].ID).to.equal('BCN98765432');
1934
+ return fDone();
1935
+ });
1936
+ }
1937
+ );
1938
+ test
1939
+ (
1940
+ 'parseFixedWidth should skip lines with skipLines option',
1941
+ function(fDone)
1942
+ {
1943
+ let tmpContent = 'HEADER LINE 1\nHEADER LINE 2\nDAT 100\nDAT 200\n';
1944
+ _Fable.RetoldFactoIngestEngine.parseFixedWidth(tmpContent,
1945
+ {
1946
+ columns: [
1947
+ { name: 'Code', start: 1, width: 3 },
1948
+ { name: 'Value', start: 5, width: 3 }
1949
+ ],
1950
+ skipLines: 2
1951
+ },
1952
+ (pError, pRecords) =>
1953
+ {
1954
+ Expect(pError).to.not.exist;
1955
+ Expect(pRecords).to.be.an('array');
1956
+ Expect(pRecords.length).to.equal(2);
1957
+ Expect(pRecords[0].Code).to.equal('DAT');
1958
+ Expect(pRecords[0].Value).to.equal('100');
1959
+ return fDone();
1960
+ });
1961
+ }
1962
+ );
1963
+ test
1964
+ (
1965
+ 'parseFixedWidth should return error without columns option',
1966
+ function(fDone)
1967
+ {
1968
+ _Fable.RetoldFactoIngestEngine.parseFixedWidth('some data',
1969
+ (pError, pRecords) =>
1970
+ {
1971
+ Expect(pError).to.be.an.instanceOf(Error);
1972
+ Expect(pError.message).to.contain('columns');
1973
+ return fDone();
1974
+ });
1975
+ }
1976
+ );
1977
+
1978
+ // ========================================================
1979
+ // CSV Comment Stripping
1980
+ // ========================================================
1981
+ test
1982
+ (
1983
+ 'parseCSV should strip comment lines when stripCommentLines is true',
1984
+ function(fDone)
1985
+ {
1986
+ let tmpContent = '# This is a comment\n# Another comment\nname,score\nAlice,95\nBob,87\n';
1987
+ _Fable.RetoldFactoIngestEngine.parseCSV(tmpContent, { stripCommentLines: true },
1988
+ (pError, pRecords) =>
1989
+ {
1990
+ Expect(pError).to.not.exist;
1991
+ Expect(pRecords).to.be.an('array');
1992
+ Expect(pRecords.length).to.equal(2);
1993
+ Expect(pRecords[0].name).to.equal('Alice');
1994
+ return fDone();
1995
+ });
1996
+ }
1997
+ );
1998
+
1999
+ // ========================================================
2000
+ // ingestFile with new extensions
2001
+ // ========================================================
2002
+ test
2003
+ (
2004
+ 'ingestFile should handle .xml files',
2005
+ function(fDone)
2006
+ {
2007
+ this.timeout(10000);
2008
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.xml');
2009
+ libFs.writeFileSync(tmpFilePath, '<root><item><name>XMLAlice</name><value>100</value></item><item><name>XMLBob</name><value>200</value></item></root>');
2010
+
2011
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
2012
+ (pError, pResult) =>
2013
+ {
2014
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
2015
+
2016
+ Expect(pError).to.not.exist;
2017
+ Expect(pResult.Format).to.equal('xml');
2018
+ Expect(pResult.Ingested).to.equal(2);
2019
+ return fDone();
2020
+ });
2021
+ }
2022
+ );
2023
+ test
2024
+ (
2025
+ 'ingestFile should handle .xlsx files',
2026
+ function(fDone)
2027
+ {
2028
+ this.timeout(10000);
2029
+ let libXLSX = require('xlsx');
2030
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.xlsx');
2031
+ let tmpWorksheet = libXLSX.utils.aoa_to_sheet([
2032
+ ['city', 'pop'],
2033
+ ['NYC', 8336817],
2034
+ ['LA', 3979576]
2035
+ ]);
2036
+ let tmpWorkbook = libXLSX.utils.book_new();
2037
+ libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Cities');
2038
+ libXLSX.writeFile(tmpWorkbook, tmpFilePath);
2039
+
2040
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
2041
+ (pError, pResult) =>
2042
+ {
2043
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
2044
+
2045
+ Expect(pError).to.not.exist;
2046
+ Expect(pResult.Format).to.equal('excel');
2047
+ Expect(pResult.Ingested).to.equal(2);
2048
+ return fDone();
2049
+ });
2050
+ }
2051
+ );
2052
+ test
2053
+ (
2054
+ 'ingestFile should handle .fw (fixed-width) files',
2055
+ function(fDone)
2056
+ {
2057
+ this.timeout(10000);
2058
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.fw');
2059
+ libFs.writeFileSync(tmpFilePath, 'AAA 100\nBBB 200\nCCC 300\n');
2060
+
2061
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
2062
+ {
2063
+ columns: [
2064
+ { name: 'Code', start: 1, width: 3 },
2065
+ { name: 'Val', start: 5, width: 3 }
2066
+ ]
2067
+ },
2068
+ (pError, pResult) =>
2069
+ {
2070
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
2071
+
2072
+ Expect(pError).to.not.exist;
2073
+ Expect(pResult.Format).to.equal('fixed-width');
2074
+ Expect(pResult.Ingested).to.equal(3);
2075
+ return fDone();
2076
+ });
2077
+ }
2078
+ );
2079
+ test
2080
+ (
2081
+ 'ingestFile should handle .rdb files as TSV with comment stripping',
2082
+ function(fDone)
2083
+ {
2084
+ this.timeout(10000);
2085
+ let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.rdb');
2086
+ let tmpContent = '# USGS comment line\n# Another comment\nagency_cd\tsite_no\tvalue\n5s\t15s\t14n\nUSGS\t01646500\t1234\nUSGS\t01646500\t5678\n';
2087
+ libFs.writeFileSync(tmpFilePath, tmpContent);
2088
+
2089
+ _Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
2090
+ (pError, pResult) =>
2091
+ {
2092
+ try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
2093
+
2094
+ Expect(pError).to.not.exist;
2095
+ Expect(pResult.Format).to.equal('csv');
2096
+ Expect(pResult.Ingested).to.be.greaterThan(0);
2097
+ return fDone();
2098
+ });
2099
+ }
2100
+ );
2101
+
2102
+ // ========================================================
2103
+ // POST /facto/ingest/file with new formats
2104
+ // ========================================================
2105
+ test
2106
+ (
2107
+ 'POST /facto/ingest/file should auto-detect XML content',
2108
+ function(fDone)
2109
+ {
2110
+ _SuperTest.post('/facto/ingest/file')
2111
+ .send(
2112
+ {
2113
+ Content: '<items><item><name>PostXML1</name></item><item><name>PostXML2</name></item></items>',
2114
+ IDDataset: 1,
2115
+ IDSource: 1,
2116
+ Type: 'xml-test'
2117
+ })
2118
+ .expect(200)
2119
+ .end(
2120
+ (pError, pResponse) =>
2121
+ {
2122
+ Expect(pError).to.not.exist;
2123
+ Expect(pResponse.body.Format).to.equal('xml');
2124
+ Expect(pResponse.body.Ingested).to.equal(2);
2125
+ return fDone();
2126
+ });
2127
+ }
2128
+ );
2129
+ test
2130
+ (
2131
+ 'POST /facto/ingest/file should accept explicit Format=xml',
2132
+ function(fDone)
2133
+ {
2134
+ _SuperTest.post('/facto/ingest/file')
2135
+ .send(
2136
+ {
2137
+ Content: '<data><row><a>1</a></row><row><a>2</a></row><row><a>3</a></row></data>',
2138
+ Format: 'xml',
2139
+ IDDataset: 1,
2140
+ IDSource: 1
2141
+ })
2142
+ .expect(200)
2143
+ .end(
2144
+ (pError, pResponse) =>
2145
+ {
2146
+ Expect(pError).to.not.exist;
2147
+ Expect(pResponse.body.Format).to.equal('xml');
2148
+ Expect(pResponse.body.Ingested).to.equal(3);
2149
+ return fDone();
2150
+ });
2151
+ }
2152
+ );
2153
+ test
2154
+ (
2155
+ 'POST /facto/ingest/file should accept Format=fixed-width with Columns',
2156
+ function(fDone)
2157
+ {
2158
+ _SuperTest.post('/facto/ingest/file')
2159
+ .send(
2160
+ {
2161
+ Content: 'AAA 100\nBBB 200\n',
2162
+ Format: 'fixed-width',
2163
+ Columns: [
2164
+ { name: 'Code', start: 1, width: 3 },
2165
+ { name: 'Val', start: 5, width: 3 }
2166
+ ],
2167
+ IDDataset: 1,
2168
+ IDSource: 1
2169
+ })
2170
+ .expect(200)
2171
+ .end(
2172
+ (pError, pResponse) =>
2173
+ {
2174
+ Expect(pError).to.not.exist;
2175
+ Expect(pResponse.body.Format).to.equal('fixed-width');
2176
+ Expect(pResponse.body.Ingested).to.equal(2);
2177
+ return fDone();
2178
+ });
2179
+ }
2180
+ );
2181
+ test
2182
+ (
2183
+ 'POST /facto/ingest/file should reject fixed-width without Columns',
2184
+ function(fDone)
2185
+ {
2186
+ _SuperTest.post('/facto/ingest/file')
2187
+ .send(
2188
+ {
2189
+ Content: 'AAA 100\nBBB 200\n',
2190
+ Format: 'fixed-width',
2191
+ IDDataset: 1,
2192
+ IDSource: 1
2193
+ })
2194
+ .expect(200)
2195
+ .end(
2196
+ (pError, pResponse) =>
2197
+ {
2198
+ Expect(pError).to.not.exist;
2199
+ Expect(pResponse.body.Error).to.contain('Columns');
2200
+ return fDone();
2201
+ });
2202
+ }
2203
+ );
2204
+ test
2205
+ (
2206
+ 'POST /facto/ingest/file should accept Format=excel with base64 content',
2207
+ function(fDone)
2208
+ {
2209
+ let libXLSX = require('xlsx');
2210
+ let tmpWorksheet = libXLSX.utils.aoa_to_sheet([
2211
+ ['x', 'y'],
2212
+ [10, 20],
2213
+ [30, 40]
2214
+ ]);
2215
+ let tmpWorkbook = libXLSX.utils.book_new();
2216
+ libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Data');
2217
+ let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
2218
+ let tmpBase64 = tmpBuffer.toString('base64');
2219
+
2220
+ _SuperTest.post('/facto/ingest/file')
2221
+ .send(
2222
+ {
2223
+ Content: tmpBase64,
2224
+ Format: 'excel',
2225
+ IDDataset: 1,
2226
+ IDSource: 1,
2227
+ Type: 'excel-test'
2228
+ })
2229
+ .expect(200)
2230
+ .end(
2231
+ (pError, pResponse) =>
2232
+ {
2233
+ Expect(pError).to.not.exist;
2234
+ Expect(pResponse.body.Format).to.equal('excel');
2235
+ Expect(pResponse.body.Ingested).to.equal(2);
2236
+ return fDone();
2237
+ });
2238
+ }
2239
+ );
2240
+ }
2241
+ );
2242
+ suite
2243
+ (
2244
+ 'Dataset Versioning',
2245
+ function()
2246
+ {
2247
+ test
2248
+ (
2249
+ 'ingest should auto-create IngestJob with DatasetVersion=1',
2250
+ function(fDone)
2251
+ {
2252
+ this.timeout(10000);
2253
+
2254
+ // Create a dedicated dataset for version testing
2255
+ _SuperTest.post('/1.0/Dataset')
2256
+ .send({ Name: 'Version Test Dataset', Type: 'Raw', Description: 'Testing versioning' })
2257
+ .expect(200)
2258
+ .end(
2259
+ (pError, pResponse) =>
2260
+ {
2261
+ Expect(pError).to.not.exist;
2262
+ let tmpIDDataset = pResponse.body.IDDataset;
2263
+ Expect(tmpIDDataset).to.be.greaterThan(0);
2264
+
2265
+ // Ingest CSV data
2266
+ _SuperTest.post('/facto/ingest/file')
2267
+ .send(
2268
+ {
2269
+ Content: 'name,value\nAlpha,100\nBeta,200\n',
2270
+ Format: 'csv',
2271
+ IDDataset: tmpIDDataset,
2272
+ IDSource: 1
2273
+ })
2274
+ .expect(200)
2275
+ .end(
2276
+ (pIngestError, pIngestResponse) =>
2277
+ {
2278
+ Expect(pIngestError).to.not.exist;
2279
+ Expect(pIngestResponse.body.Ingested).to.equal(2);
2280
+ Expect(pIngestResponse.body.DatasetVersion).to.equal(1);
2281
+ Expect(pIngestResponse.body.ContentSignature).to.be.a('string');
2282
+ Expect(pIngestResponse.body.ContentSignature.length).to.equal(64);
2283
+ Expect(pIngestResponse.body.IngestJob).to.exist;
2284
+ Expect(pIngestResponse.body.IngestJob.DatasetVersion).to.equal(1);
2285
+ return fDone();
2286
+ });
2287
+ });
2288
+ }
2289
+ );
2290
+ test
2291
+ (
2292
+ 'second ingest should auto-increment to DatasetVersion=2',
2293
+ function(fDone)
2294
+ {
2295
+ this.timeout(10000);
2296
+
2297
+ _SuperTest.post('/1.0/Dataset')
2298
+ .send({ Name: 'Version Increment Dataset', Type: 'Raw', Description: 'Testing version increment' })
2299
+ .expect(200)
2300
+ .end(
2301
+ (pError, pResponse) =>
2302
+ {
2303
+ let tmpIDDataset = pResponse.body.IDDataset;
2304
+
2305
+ // First ingest
2306
+ _SuperTest.post('/facto/ingest/file')
2307
+ .send(
2308
+ {
2309
+ Content: 'a,b\n1,2\n',
2310
+ Format: 'csv',
2311
+ IDDataset: tmpIDDataset,
2312
+ IDSource: 1
2313
+ })
2314
+ .expect(200)
2315
+ .end(
2316
+ (pErr1, pRes1) =>
2317
+ {
2318
+ Expect(pRes1.body.DatasetVersion).to.equal(1);
2319
+
2320
+ // Second ingest (different content)
2321
+ _SuperTest.post('/facto/ingest/file')
2322
+ .send(
2323
+ {
2324
+ Content: 'a,b\n3,4\n5,6\n',
2325
+ Format: 'csv',
2326
+ IDDataset: tmpIDDataset,
2327
+ IDSource: 1
2328
+ })
2329
+ .expect(200)
2330
+ .end(
2331
+ (pErr2, pRes2) =>
2332
+ {
2333
+ Expect(pRes2.body.DatasetVersion).to.equal(2);
2334
+ Expect(pRes2.body.Ingested).to.equal(2);
2335
+ return fDone();
2336
+ });
2337
+ });
2338
+ });
2339
+ }
2340
+ );
2341
+ test
2342
+ (
2343
+ 'records should have correct IDIngestJob FK',
2344
+ function(fDone)
2345
+ {
2346
+ this.timeout(10000);
2347
+
2348
+ _SuperTest.post('/1.0/Dataset')
2349
+ .send({ Name: 'IngestJob FK Dataset', Type: 'Raw', Description: 'Testing IDIngestJob on records' })
2350
+ .expect(200)
2351
+ .end(
2352
+ (pError, pResponse) =>
2353
+ {
2354
+ let tmpIDDataset = pResponse.body.IDDataset;
2355
+
2356
+ _SuperTest.post('/facto/ingest/file')
2357
+ .send(
2358
+ {
2359
+ Content: 'x,y\n10,20\n',
2360
+ Format: 'csv',
2361
+ IDDataset: tmpIDDataset,
2362
+ IDSource: 1
2363
+ })
2364
+ .expect(200)
2365
+ .end(
2366
+ (pErr, pRes) =>
2367
+ {
2368
+ Expect(pRes.body.IngestJob).to.exist;
2369
+ let tmpIDIngestJob = pRes.body.IngestJob.IDIngestJob;
2370
+ Expect(tmpIDIngestJob).to.be.greaterThan(0);
2371
+
2372
+ // Check the records have the correct IDIngestJob
2373
+ Expect(pRes.body.Records).to.be.an('array');
2374
+ Expect(pRes.body.Records.length).to.equal(1);
2375
+ Expect(parseInt(pRes.body.Records[0].IDIngestJob, 10)).to.equal(tmpIDIngestJob);
2376
+ return fDone();
2377
+ });
2378
+ });
2379
+ }
2380
+ );
2381
+ }
2382
+ );
2383
+ suite
2384
+ (
2385
+ 'Content Signatures',
2386
+ function()
2387
+ {
2388
+ test
2389
+ (
2390
+ 'identical content should produce isDuplicate=true on second ingest',
2391
+ function(fDone)
2392
+ {
2393
+ this.timeout(10000);
2394
+
2395
+ let tmpContent = 'col1,col2\nfoo,bar\nbaz,qux\n';
2396
+
2397
+ _SuperTest.post('/1.0/Dataset')
2398
+ .send({ Name: 'Duplicate Sig Dataset', Type: 'Raw', Description: 'Testing duplicate signatures' })
2399
+ .expect(200)
2400
+ .end(
2401
+ (pError, pResponse) =>
2402
+ {
2403
+ let tmpIDDataset = pResponse.body.IDDataset;
2404
+
2405
+ // First ingest
2406
+ _SuperTest.post('/facto/ingest/file')
2407
+ .send({ Content: tmpContent, Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2408
+ .expect(200)
2409
+ .end(
2410
+ (pErr1, pRes1) =>
2411
+ {
2412
+ Expect(pRes1.body.IsDuplicate).to.equal(false);
2413
+ let tmpSig1 = pRes1.body.ContentSignature;
2414
+
2415
+ // Second ingest with identical content
2416
+ _SuperTest.post('/facto/ingest/file')
2417
+ .send({ Content: tmpContent, Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2418
+ .expect(200)
2419
+ .end(
2420
+ (pErr2, pRes2) =>
2421
+ {
2422
+ Expect(pRes2.body.IsDuplicate).to.equal(true);
2423
+ Expect(pRes2.body.ContentSignature).to.equal(tmpSig1);
2424
+ Expect(pRes2.body.DatasetVersion).to.equal(2);
2425
+ return fDone();
2426
+ });
2427
+ });
2428
+ });
2429
+ }
2430
+ );
2431
+ test
2432
+ (
2433
+ 'different content should produce isDuplicate=false',
2434
+ function(fDone)
2435
+ {
2436
+ this.timeout(10000);
2437
+
2438
+ _SuperTest.post('/1.0/Dataset')
2439
+ .send({ Name: 'Different Sig Dataset', Type: 'Raw', Description: 'Testing different signatures' })
2440
+ .expect(200)
2441
+ .end(
2442
+ (pError, pResponse) =>
2443
+ {
2444
+ let tmpIDDataset = pResponse.body.IDDataset;
2445
+
2446
+ _SuperTest.post('/facto/ingest/file')
2447
+ .send({ Content: 'a,b\n1,2\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2448
+ .expect(200)
2449
+ .end(
2450
+ (pErr1, pRes1) =>
2451
+ {
2452
+ let tmpSig1 = pRes1.body.ContentSignature;
2453
+
2454
+ _SuperTest.post('/facto/ingest/file')
2455
+ .send({ Content: 'a,b\n3,4\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2456
+ .expect(200)
2457
+ .end(
2458
+ (pErr2, pRes2) =>
2459
+ {
2460
+ Expect(pRes2.body.IsDuplicate).to.equal(false);
2461
+ Expect(pRes2.body.ContentSignature).to.not.equal(tmpSig1);
2462
+ return fDone();
2463
+ });
2464
+ });
2465
+ });
2466
+ }
2467
+ );
2468
+ }
2469
+ );
2470
+ suite
2471
+ (
2472
+ 'Version Policy',
2473
+ function()
2474
+ {
2475
+ test
2476
+ (
2477
+ 'PUT /facto/dataset/:id/version-policy should set policy',
2478
+ function(fDone)
2479
+ {
2480
+ _SuperTest.post('/1.0/Dataset')
2481
+ .send({ Name: 'Policy Test Dataset', Type: 'Raw', Description: 'Testing VersionPolicy' })
2482
+ .expect(200)
2483
+ .end(
2484
+ (pError, pResponse) =>
2485
+ {
2486
+ let tmpIDDataset = pResponse.body.IDDataset;
2487
+
2488
+ _SuperTest.put(`/facto/dataset/${tmpIDDataset}/version-policy`)
2489
+ .send({ VersionPolicy: 'Replace' })
2490
+ .expect(200)
2491
+ .end(
2492
+ (pErr, pRes) =>
2493
+ {
2494
+ Expect(pRes.body.Success).to.equal(true);
2495
+ Expect(pRes.body.Dataset.VersionPolicy).to.equal('Replace');
2496
+ return fDone();
2497
+ });
2498
+ });
2499
+ }
2500
+ );
2501
+ test
2502
+ (
2503
+ 'PUT /facto/dataset/:id/version-policy should reject invalid policy',
2504
+ function(fDone)
2505
+ {
2506
+ _SuperTest.post('/1.0/Dataset')
2507
+ .send({ Name: 'Invalid Policy Dataset', Type: 'Raw', Description: 'Testing invalid policy' })
2508
+ .expect(200)
2509
+ .end(
2510
+ (pError, pResponse) =>
2511
+ {
2512
+ let tmpIDDataset = pResponse.body.IDDataset;
2513
+
2514
+ _SuperTest.put(`/facto/dataset/${tmpIDDataset}/version-policy`)
2515
+ .send({ VersionPolicy: 'InvalidValue' })
2516
+ .expect(200)
2517
+ .end(
2518
+ (pErr, pRes) =>
2519
+ {
2520
+ Expect(pRes.body.Error).to.exist;
2521
+ return fDone();
2522
+ });
2523
+ });
2524
+ }
2525
+ );
2526
+ test
2527
+ (
2528
+ 'Replace policy should soft-delete old records on re-import',
2529
+ function(fDone)
2530
+ {
2531
+ this.timeout(10000);
2532
+
2533
+ _SuperTest.post('/1.0/Dataset')
2534
+ .send({ Name: 'Replace Policy Dataset', Type: 'Raw', Description: 'Testing Replace policy' })
2535
+ .expect(200)
2536
+ .end(
2537
+ (pError, pResponse) =>
2538
+ {
2539
+ let tmpIDDataset = pResponse.body.IDDataset;
2540
+
2541
+ // Set policy to Replace
2542
+ _SuperTest.put(`/facto/dataset/${tmpIDDataset}/version-policy`)
2543
+ .send({ VersionPolicy: 'Replace' })
2544
+ .expect(200)
2545
+ .end(
2546
+ () =>
2547
+ {
2548
+ // Ingest v1 (3 records)
2549
+ _SuperTest.post('/facto/ingest/file')
2550
+ .send({ Content: 'a,b\n1,2\n3,4\n5,6\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2551
+ .expect(200)
2552
+ .end(
2553
+ (pErr1, pRes1) =>
2554
+ {
2555
+ Expect(pRes1.body.Ingested).to.equal(3);
2556
+ Expect(pRes1.body.DatasetVersion).to.equal(1);
2557
+
2558
+ // Ingest v2 (2 records) — v1 records should be soft-deleted
2559
+ _SuperTest.post('/facto/ingest/file')
2560
+ .send({ Content: 'a,b\n7,8\n9,10\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2561
+ .expect(200)
2562
+ .end(
2563
+ (pErr2, pRes2) =>
2564
+ {
2565
+ Expect(pRes2.body.Ingested).to.equal(2);
2566
+ Expect(pRes2.body.DatasetVersion).to.equal(2);
2567
+
2568
+ // Count active records — should be 2 (only v2)
2569
+ _SuperTest.get(`/facto/dataset/${tmpIDDataset}/stats`)
2570
+ .expect(200)
2571
+ .end(
2572
+ (pErr3, pRes3) =>
2573
+ {
2574
+ Expect(pRes3.body.RecordCount).to.equal(2);
2575
+ return fDone();
2576
+ });
2577
+ });
2578
+ });
2579
+ });
2580
+ });
2581
+ }
2582
+ );
2583
+ }
2584
+ );
2585
+ suite
2586
+ (
2587
+ 'Dataset Version History',
2588
+ function()
2589
+ {
2590
+ test
2591
+ (
2592
+ 'GET /facto/dataset/:id/versions should return version history',
2593
+ function(fDone)
2594
+ {
2595
+ this.timeout(10000);
2596
+
2597
+ _SuperTest.post('/1.0/Dataset')
2598
+ .send({ Name: 'Version History Dataset', Type: 'Raw', Description: 'Testing version history' })
2599
+ .expect(200)
2600
+ .end(
2601
+ (pError, pResponse) =>
2602
+ {
2603
+ let tmpIDDataset = pResponse.body.IDDataset;
2604
+
2605
+ // Ingest v1
2606
+ _SuperTest.post('/facto/ingest/file')
2607
+ .send({ Content: 'k,v\na,1\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2608
+ .expect(200)
2609
+ .end(
2610
+ () =>
2611
+ {
2612
+ // Ingest v2
2613
+ _SuperTest.post('/facto/ingest/file')
2614
+ .send({ Content: 'k,v\nb,2\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2615
+ .expect(200)
2616
+ .end(
2617
+ () =>
2618
+ {
2619
+ // Ingest v3
2620
+ _SuperTest.post('/facto/ingest/file')
2621
+ .send({ Content: 'k,v\nc,3\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
2622
+ .expect(200)
2623
+ .end(
2624
+ () =>
2625
+ {
2626
+ // Get version history
2627
+ _SuperTest.get(`/facto/dataset/${tmpIDDataset}/versions`)
2628
+ .expect(200)
2629
+ .end(
2630
+ (pErr, pRes) =>
2631
+ {
2632
+ Expect(pRes.body.Count).to.equal(3);
2633
+ Expect(pRes.body.Versions).to.be.an('array');
2634
+ Expect(pRes.body.Versions.length).to.equal(3);
2635
+
2636
+ // Should be sorted DESC
2637
+ Expect(parseInt(pRes.body.Versions[0].DatasetVersion, 10)).to.equal(3);
2638
+ Expect(parseInt(pRes.body.Versions[1].DatasetVersion, 10)).to.equal(2);
2639
+ Expect(parseInt(pRes.body.Versions[2].DatasetVersion, 10)).to.equal(1);
2640
+
2641
+ // Each should have a ContentSignature
2642
+ Expect(pRes.body.Versions[0].ContentSignature).to.be.a('string');
2643
+ Expect(pRes.body.Versions[0].ContentSignature.length).to.equal(64);
2644
+
2645
+ return fDone();
2646
+ });
2647
+ });
2648
+ });
2649
+ });
2650
+ });
2651
+ }
2652
+ );
2653
+ }
2654
+ );
2655
+
2656
+ // ================================================================
2657
+ // Source Catalog - CRUD
2658
+ // ================================================================
2659
+ suite
2660
+ (
2661
+ 'Source Catalog CRUD',
2662
+ function()
2663
+ {
2664
+ let _CatalogEntryID = 0;
2665
+
2666
+ test
2667
+ (
2668
+ 'Create a catalog entry',
2669
+ function(fDone)
2670
+ {
2671
+ _SuperTest
2672
+ .post('/facto/catalog/entry')
2673
+ .send(
2674
+ {
2675
+ Agency: 'US Geological Survey (USGS)',
2676
+ Name: 'USGS Data Portal',
2677
+ Type: 'API',
2678
+ URL: 'https://waterservices.usgs.gov',
2679
+ Protocol: 'HTTPS',
2680
+ Category: 'Science',
2681
+ Region: 'US',
2682
+ UpdateFrequency: 'Continuous',
2683
+ Description: 'Geological and hydrological data',
2684
+ Notes: 'Free, no auth required',
2685
+ Verified: true
2686
+ })
2687
+ .end(
2688
+ (pError, pRes) =>
2689
+ {
2690
+ Expect(pRes.body.Success).to.equal(true);
2691
+ Expect(pRes.body.Entry).to.be.an('object');
2692
+ Expect(pRes.body.Entry.IDSourceCatalogEntry).to.be.greaterThan(0);
2693
+ Expect(pRes.body.Entry.Agency).to.equal('US Geological Survey (USGS)');
2694
+ Expect(pRes.body.Entry.Verified).to.equal(1);
2695
+ _CatalogEntryID = pRes.body.Entry.IDSourceCatalogEntry;
2696
+ return fDone();
2697
+ });
2698
+ }
2699
+ );
2700
+
2701
+ test
2702
+ (
2703
+ 'Read catalog entry back',
2704
+ function(fDone)
2705
+ {
2706
+ _SuperTest
2707
+ .get(`/facto/catalog/entry/${_CatalogEntryID}`)
2708
+ .end(
2709
+ (pError, pRes) =>
2710
+ {
2711
+ Expect(pRes.body.Entry).to.be.an('object');
2712
+ Expect(pRes.body.Entry.Agency).to.equal('US Geological Survey (USGS)');
2713
+ Expect(pRes.body.Entry.Category).to.equal('Science');
2714
+ Expect(pRes.body.Datasets).to.be.an('array');
2715
+ return fDone();
2716
+ });
2717
+ }
2718
+ );
2719
+
2720
+ test
2721
+ (
2722
+ 'Update catalog entry',
2723
+ function(fDone)
2724
+ {
2725
+ _SuperTest
2726
+ .put(`/facto/catalog/entry/${_CatalogEntryID}`)
2727
+ .send({ Category: 'Earth Science', Notes: 'Updated notes' })
2728
+ .end(
2729
+ (pError, pRes) =>
2730
+ {
2731
+ Expect(pRes.body.Success).to.equal(true);
2732
+ return fDone();
2733
+ });
2734
+ }
2735
+ );
2736
+
2737
+ test
2738
+ (
2739
+ 'List catalog entries',
2740
+ function(fDone)
2741
+ {
2742
+ _SuperTest
2743
+ .get('/facto/catalog/entries')
2744
+ .end(
2745
+ (pError, pRes) =>
2746
+ {
2747
+ Expect(pRes.body.Count).to.be.greaterThan(0);
2748
+ Expect(pRes.body.Entries).to.be.an('array');
2749
+ return fDone();
2750
+ });
2751
+ }
2752
+ );
2753
+
2754
+ test
2755
+ (
2756
+ 'Soft-delete catalog entry',
2757
+ function(fDone)
2758
+ {
2759
+ _SuperTest
2760
+ .delete(`/facto/catalog/entry/${_CatalogEntryID}`)
2761
+ .end(
2762
+ (pError, pRes) =>
2763
+ {
2764
+ Expect(pRes.body.Success).to.equal(true);
2765
+ Expect(pRes.body.Deleted).to.equal(_CatalogEntryID);
2766
+
2767
+ // Verify it no longer appears in listing
2768
+ _SuperTest
2769
+ .get('/facto/catalog/entries')
2770
+ .end(
2771
+ (pError2, pRes2) =>
2772
+ {
2773
+ let tmpFound = pRes2.body.Entries.filter((e) => e.IDSourceCatalogEntry === _CatalogEntryID);
2774
+ Expect(tmpFound.length).to.equal(0);
2775
+ return fDone();
2776
+ });
2777
+ });
2778
+ }
2779
+ );
2780
+ }
2781
+ );
2782
+
2783
+ // ================================================================
2784
+ // Source Catalog - Dataset Definitions
2785
+ // ================================================================
2786
+ suite
2787
+ (
2788
+ 'Catalog Dataset Definitions',
2789
+ function()
2790
+ {
2791
+ let _CatalogEntryID = 0;
2792
+ let _CatalogDatasetID = 0;
2793
+
2794
+ test
2795
+ (
2796
+ 'Create catalog entry for dataset tests',
2797
+ function(fDone)
2798
+ {
2799
+ _SuperTest
2800
+ .post('/facto/catalog/entry')
2801
+ .send(
2802
+ {
2803
+ Agency: 'NOAA',
2804
+ Name: 'National Weather Service',
2805
+ Type: 'API',
2806
+ URL: 'https://www.weather.gov',
2807
+ Category: 'Weather'
2808
+ })
2809
+ .end(
2810
+ (pError, pRes) =>
2811
+ {
2812
+ Expect(pRes.body.Success).to.equal(true);
2813
+ _CatalogEntryID = pRes.body.Entry.IDSourceCatalogEntry;
2814
+ return fDone();
2815
+ });
2816
+ }
2817
+ );
2818
+
2819
+ test
2820
+ (
2821
+ 'Add dataset definition to catalog entry',
2822
+ function(fDone)
2823
+ {
2824
+ _SuperTest
2825
+ .post(`/facto/catalog/entry/${_CatalogEntryID}/dataset`)
2826
+ .send(
2827
+ {
2828
+ Name: 'Weather Stations',
2829
+ Format: 'fixed-width',
2830
+ MimeType: 'text/plain',
2831
+ EndpointURL: 'https://www.weather.gov/stations.dat',
2832
+ Description: 'NOAA weather station master list',
2833
+ ParseOptions: JSON.stringify({ columns: [{ name: 'StationID', start: 1, width: 11 }] }),
2834
+ AuthRequirements: 'None',
2835
+ VersionPolicy: 'Append'
2836
+ })
2837
+ .end(
2838
+ (pError, pRes) =>
2839
+ {
2840
+ Expect(pRes.body.Success).to.equal(true);
2841
+ Expect(pRes.body.DatasetDefinition).to.be.an('object');
2842
+ Expect(pRes.body.DatasetDefinition.IDCatalogDatasetDefinition).to.be.greaterThan(0);
2843
+ Expect(pRes.body.DatasetDefinition.Format).to.equal('fixed-width');
2844
+ Expect(pRes.body.DatasetDefinition.Provisioned).to.equal(0);
2845
+ _CatalogDatasetID = pRes.body.DatasetDefinition.IDCatalogDatasetDefinition;
2846
+ return fDone();
2847
+ });
2848
+ }
2849
+ );
2850
+
2851
+ test
2852
+ (
2853
+ 'List dataset definitions for entry',
2854
+ function(fDone)
2855
+ {
2856
+ _SuperTest
2857
+ .get(`/facto/catalog/entry/${_CatalogEntryID}/datasets`)
2858
+ .end(
2859
+ (pError, pRes) =>
2860
+ {
2861
+ Expect(pRes.body.Count).to.equal(1);
2862
+ Expect(pRes.body.Datasets).to.be.an('array');
2863
+ Expect(pRes.body.Datasets[0].Name).to.equal('Weather Stations');
2864
+ return fDone();
2865
+ });
2866
+ }
2867
+ );
2868
+
2869
+ test
2870
+ (
2871
+ 'Update dataset definition',
2872
+ function(fDone)
2873
+ {
2874
+ _SuperTest
2875
+ .put(`/facto/catalog/dataset/${_CatalogDatasetID}`)
2876
+ .send({ Description: 'Updated description', VersionPolicy: 'Replace' })
2877
+ .end(
2878
+ (pError, pRes) =>
2879
+ {
2880
+ Expect(pRes.body.Success).to.equal(true);
2881
+ return fDone();
2882
+ });
2883
+ }
2884
+ );
2885
+
2886
+ test
2887
+ (
2888
+ 'Soft-delete dataset definition',
2889
+ function(fDone)
2890
+ {
2891
+ _SuperTest
2892
+ .delete(`/facto/catalog/dataset/${_CatalogDatasetID}`)
2893
+ .end(
2894
+ (pError, pRes) =>
2895
+ {
2896
+ Expect(pRes.body.Success).to.equal(true);
2897
+
2898
+ // Verify it no longer appears
2899
+ _SuperTest
2900
+ .get(`/facto/catalog/entry/${_CatalogEntryID}/datasets`)
2901
+ .end(
2902
+ (pError2, pRes2) =>
2903
+ {
2904
+ Expect(pRes2.body.Count).to.equal(0);
2905
+ return fDone();
2906
+ });
2907
+ });
2908
+ }
2909
+ );
2910
+ }
2911
+ );
2912
+
2913
+ // ================================================================
2914
+ // Source Catalog - Search
2915
+ // ================================================================
2916
+ suite
2917
+ (
2918
+ 'Catalog Search',
2919
+ function()
2920
+ {
2921
+ test
2922
+ (
2923
+ 'Create entries for search tests',
2924
+ function(fDone)
2925
+ {
2926
+ _SuperTest
2927
+ .post('/facto/catalog/entry')
2928
+ .send({ Agency: 'EPA', Name: 'Environmental Data', Category: 'Environment', Region: 'US' })
2929
+ .end(
2930
+ (pError, pRes) =>
2931
+ {
2932
+ Expect(pRes.body.Success).to.equal(true);
2933
+ _SuperTest
2934
+ .post('/facto/catalog/entry')
2935
+ .send({ Agency: 'NASA', Name: 'Space Science Data', Category: 'Space', Region: 'Global' })
2936
+ .end(
2937
+ (pError2, pRes2) =>
2938
+ {
2939
+ Expect(pRes2.body.Success).to.equal(true);
2940
+ _SuperTest
2941
+ .post('/facto/catalog/entry')
2942
+ .send({ Agency: 'CDC', Name: 'Health Statistics', Category: 'Health', Region: 'US' })
2943
+ .end(
2944
+ (pError3, pRes3) =>
2945
+ {
2946
+ Expect(pRes3.body.Success).to.equal(true);
2947
+ return fDone();
2948
+ });
2949
+ });
2950
+ });
2951
+ }
2952
+ );
2953
+
2954
+ test
2955
+ (
2956
+ 'Search by name',
2957
+ function(fDone)
2958
+ {
2959
+ _SuperTest
2960
+ .get('/facto/catalog/search?q=Space')
2961
+ .end(
2962
+ (pError, pRes) =>
2963
+ {
2964
+ Expect(pRes.body.Query).to.equal('space');
2965
+ Expect(pRes.body.Count).to.be.greaterThan(0);
2966
+ let tmpNames = pRes.body.Entries.map((e) => e.Name);
2967
+ Expect(tmpNames).to.include('Space Science Data');
2968
+ return fDone();
2969
+ });
2970
+ }
2971
+ );
2972
+
2973
+ test
2974
+ (
2975
+ 'Search by category',
2976
+ function(fDone)
2977
+ {
2978
+ _SuperTest
2979
+ .get('/facto/catalog/search?q=health')
2980
+ .end(
2981
+ (pError, pRes) =>
2982
+ {
2983
+ Expect(pRes.body.Count).to.be.greaterThan(0);
2984
+ let tmpCategories = pRes.body.Entries.map((e) => e.Category);
2985
+ Expect(tmpCategories).to.include('Health');
2986
+ return fDone();
2987
+ });
2988
+ }
2989
+ );
2990
+
2991
+ test
2992
+ (
2993
+ 'Empty search returns all non-deleted entries',
2994
+ function(fDone)
2995
+ {
2996
+ _SuperTest
2997
+ .get('/facto/catalog/search')
2998
+ .end(
2999
+ (pError, pRes) =>
3000
+ {
3001
+ Expect(pRes.body.Query).to.equal('');
3002
+ // Should include NOAA, EPA, NASA, CDC (not the deleted USGS)
3003
+ Expect(pRes.body.Count).to.be.greaterThan(2);
3004
+ return fDone();
3005
+ });
3006
+ }
3007
+ );
3008
+ }
3009
+ );
3010
+
3011
+ // ================================================================
3012
+ // Source Catalog - Provision
3013
+ // ================================================================
3014
+ suite
3015
+ (
3016
+ 'Catalog Provision',
3017
+ function()
3018
+ {
3019
+ let _ProvisionEntryID = 0;
3020
+ let _ProvisionDatasetID = 0;
3021
+
3022
+ test
3023
+ (
3024
+ 'Create catalog entry and dataset for provisioning',
3025
+ function(fDone)
3026
+ {
3027
+ _SuperTest
3028
+ .post('/facto/catalog/entry')
3029
+ .send(
3030
+ {
3031
+ Agency: 'Bureau of Labor Statistics',
3032
+ Name: 'BLS Data Portal',
3033
+ Type: 'API',
3034
+ URL: 'https://www.bls.gov',
3035
+ Protocol: 'HTTPS',
3036
+ Category: 'Economics'
3037
+ })
3038
+ .end(
3039
+ (pError, pRes) =>
3040
+ {
3041
+ Expect(pRes.body.Success).to.equal(true);
3042
+ _ProvisionEntryID = pRes.body.Entry.IDSourceCatalogEntry;
3043
+
3044
+ _SuperTest
3045
+ .post(`/facto/catalog/entry/${_ProvisionEntryID}/dataset`)
3046
+ .send(
3047
+ {
3048
+ Name: 'Consumer Price Index',
3049
+ Format: 'csv',
3050
+ MimeType: 'text/csv',
3051
+ EndpointURL: 'https://www.bls.gov/cpi/data.csv',
3052
+ Description: 'Monthly CPI data',
3053
+ ParseOptions: '{}',
3054
+ AuthRequirements: 'None',
3055
+ VersionPolicy: 'Append'
3056
+ })
3057
+ .end(
3058
+ (pError2, pRes2) =>
3059
+ {
3060
+ Expect(pRes2.body.Success).to.equal(true);
3061
+ _ProvisionDatasetID = pRes2.body.DatasetDefinition.IDCatalogDatasetDefinition;
3062
+ return fDone();
3063
+ });
3064
+ });
3065
+ }
3066
+ );
3067
+
3068
+ test
3069
+ (
3070
+ 'Provision creates runtime Source, Dataset, and DatasetSource',
3071
+ function(fDone)
3072
+ {
3073
+ _SuperTest
3074
+ .post(`/facto/catalog/dataset/${_ProvisionDatasetID}/provision`)
3075
+ .end(
3076
+ (pError, pRes) =>
3077
+ {
3078
+ Expect(pRes.body.Success).to.equal(true);
3079
+ Expect(pRes.body.Source).to.be.an('object');
3080
+ Expect(pRes.body.Source.Name).to.equal('Bureau of Labor Statistics');
3081
+ Expect(pRes.body.Source.IDSource).to.be.greaterThan(0);
3082
+ Expect(pRes.body.Source.Hash).to.equal('Bureau-of-Labor-Statistics');
3083
+ Expect(pRes.body.Dataset).to.be.an('object');
3084
+ Expect(pRes.body.Dataset.Name).to.equal('Consumer Price Index');
3085
+ Expect(pRes.body.Dataset.IDDataset).to.be.greaterThan(0);
3086
+ Expect(pRes.body.Dataset.Hash).to.equal('Consumer-Price-Index');
3087
+ Expect(pRes.body.DatasetSource).to.be.an('object');
3088
+ return fDone();
3089
+ });
3090
+ }
3091
+ );
3092
+
3093
+ test
3094
+ (
3095
+ 'CatalogDatasetDefinition is marked as provisioned',
3096
+ function(fDone)
3097
+ {
3098
+ _SuperTest
3099
+ .get(`/facto/catalog/entry/${_ProvisionEntryID}/datasets`)
3100
+ .end(
3101
+ (pError, pRes) =>
3102
+ {
3103
+ Expect(pRes.body.Count).to.equal(1);
3104
+ let tmpDef = pRes.body.Datasets[0];
3105
+ Expect(tmpDef.Provisioned).to.equal(1);
3106
+ Expect(tmpDef.IDSource).to.be.greaterThan(0);
3107
+ Expect(tmpDef.IDDataset).to.be.greaterThan(0);
3108
+ return fDone();
3109
+ });
3110
+ }
3111
+ );
3112
+
3113
+ test
3114
+ (
3115
+ 'Provisioning again is idempotent (reuses existing Source and Dataset)',
3116
+ function(fDone)
3117
+ {
3118
+ _SuperTest
3119
+ .post(`/facto/catalog/dataset/${_ProvisionDatasetID}/provision`)
3120
+ .end(
3121
+ (pError, pRes) =>
3122
+ {
3123
+ Expect(pRes.body.Success).to.equal(true);
3124
+ // Source and Dataset should have the same IDs
3125
+ Expect(pRes.body.Source.Name).to.equal('Bureau of Labor Statistics');
3126
+ Expect(pRes.body.Dataset.Name).to.equal('Consumer Price Index');
3127
+ return fDone();
3128
+ });
3129
+ }
3130
+ );
3131
+ }
3132
+ );
3133
+
3134
+ // ================================================================
3135
+ // Source Catalog - Import / Export
3136
+ // ================================================================
3137
+ suite
3138
+ (
3139
+ 'Catalog Import/Export',
3140
+ function()
3141
+ {
3142
+ test
3143
+ (
3144
+ 'Import catalog entries with datasets',
3145
+ function(fDone)
3146
+ {
3147
+ let tmpImportData = [
3148
+ {
3149
+ Agency: 'Import Test Agency A',
3150
+ Name: 'Agency A Portal',
3151
+ Type: 'API',
3152
+ Category: 'Test',
3153
+ Datasets: [
3154
+ {
3155
+ Name: 'Dataset Alpha',
3156
+ Format: 'csv',
3157
+ EndpointURL: 'https://example.com/alpha.csv',
3158
+ VersionPolicy: 'Append'
3159
+ },
3160
+ {
3161
+ Name: 'Dataset Beta',
3162
+ Format: 'json',
3163
+ EndpointURL: 'https://example.com/beta.json',
3164
+ VersionPolicy: 'Replace'
3165
+ }
3166
+ ]
3167
+ },
3168
+ {
3169
+ Agency: 'Import Test Agency B',
3170
+ Name: 'Agency B Portal',
3171
+ Type: 'File',
3172
+ Category: 'Test',
3173
+ Datasets: [
3174
+ {
3175
+ Name: 'Dataset Gamma',
3176
+ Format: 'xml',
3177
+ EndpointURL: 'https://example.com/gamma.xml'
3178
+ }
3179
+ ]
3180
+ }
3181
+ ];
3182
+
3183
+ _SuperTest
3184
+ .post('/facto/catalog/import')
3185
+ .send(tmpImportData)
3186
+ .end(
3187
+ (pError, pRes) =>
3188
+ {
3189
+ Expect(pRes.body.Success).to.equal(true);
3190
+ Expect(pRes.body.EntriesCreated).to.equal(2);
3191
+ Expect(pRes.body.DatasetsCreated).to.equal(3);
3192
+ Expect(pRes.body.Errors).to.equal(0);
3193
+ return fDone();
3194
+ });
3195
+ }
3196
+ );
3197
+
3198
+ test
3199
+ (
3200
+ 'Export catalog contains imported entries',
3201
+ function(fDone)
3202
+ {
3203
+ _SuperTest
3204
+ .get('/facto/catalog/export')
3205
+ .end(
3206
+ (pError, pRes) =>
3207
+ {
3208
+ Expect(pRes.body.Count).to.be.greaterThan(0);
3209
+ Expect(pRes.body.Entries).to.be.an('array');
3210
+
3211
+ // Find our imported entries
3212
+ let tmpAgencyA = pRes.body.Entries.find((e) => e.Agency === 'Import Test Agency A');
3213
+ Expect(tmpAgencyA).to.be.an('object');
3214
+ Expect(tmpAgencyA.Datasets).to.be.an('array');
3215
+ Expect(tmpAgencyA.Datasets.length).to.equal(2);
3216
+
3217
+ let tmpAgencyB = pRes.body.Entries.find((e) => e.Agency === 'Import Test Agency B');
3218
+ Expect(tmpAgencyB).to.be.an('object');
3219
+ Expect(tmpAgencyB.Datasets.length).to.equal(1);
3220
+ Expect(tmpAgencyB.Datasets[0].Name).to.equal('Dataset Gamma');
3221
+
3222
+ return fDone();
3223
+ });
3224
+ }
3225
+ );
3226
+ }
3227
+ );
3228
+
3229
+ suite
3230
+ (
3231
+ 'parseJSON dataPath',
3232
+ () =>
3233
+ {
3234
+ test
3235
+ (
3236
+ 'parseJSON with simple dataPath extracts nested array',
3237
+ function(fDone)
3238
+ {
3239
+ let tmpJSON = JSON.stringify({
3240
+ metadata: { page: 1 },
3241
+ data: [
3242
+ { id: 1, name: 'Alpha' },
3243
+ { id: 2, name: 'Beta' }
3244
+ ]
3245
+ });
3246
+
3247
+ _Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON, { dataPath: 'data' },
3248
+ (pError, pRecords) =>
3249
+ {
3250
+ Expect(pError).to.be.null;
3251
+ Expect(pRecords).to.be.an('array');
3252
+ Expect(pRecords.length).to.equal(2);
3253
+ Expect(pRecords[0].name).to.equal('Alpha');
3254
+ return fDone();
3255
+ });
3256
+ }
3257
+ );
3258
+
3259
+ test
3260
+ (
3261
+ 'parseJSON with deep dataPath including array index',
3262
+ function(fDone)
3263
+ {
3264
+ let tmpJSON = JSON.stringify({
3265
+ Results: {
3266
+ series: [
3267
+ {
3268
+ seriesID: 'CUUR0000SA0',
3269
+ data: [
3270
+ { year: '2024', value: '310.1' },
3271
+ { year: '2023', value: '305.2' }
3272
+ ]
3273
+ }
3274
+ ]
3275
+ }
3276
+ });
3277
+
3278
+ _Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON, { dataPath: 'Results.series[0].data' },
3279
+ (pError, pRecords) =>
3280
+ {
3281
+ Expect(pError).to.be.null;
3282
+ Expect(pRecords).to.be.an('array');
3283
+ Expect(pRecords.length).to.equal(2);
3284
+ Expect(pRecords[0].year).to.equal('2024');
3285
+ Expect(pRecords[1].value).to.equal('305.2');
3286
+ return fDone();
3287
+ });
3288
+ }
3289
+ );
3290
+
3291
+ test
3292
+ (
3293
+ 'parseJSON backward compatibility (no options)',
3294
+ function(fDone)
3295
+ {
3296
+ let tmpJSON = JSON.stringify({
3297
+ data: [{ x: 1 }, { x: 2 }]
3298
+ });
3299
+
3300
+ _Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON,
3301
+ (pError, pRecords) =>
3302
+ {
3303
+ Expect(pError).to.be.null;
3304
+ Expect(pRecords).to.be.an('array');
3305
+ Expect(pRecords.length).to.equal(2);
3306
+ return fDone();
3307
+ });
3308
+ }
3309
+ );
3310
+
3311
+ test
3312
+ (
3313
+ 'parseJSON with invalid dataPath returns error',
3314
+ function(fDone)
3315
+ {
3316
+ let tmpJSON = JSON.stringify({ foo: { bar: 1 } });
3317
+
3318
+ _Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON, { dataPath: 'missing.path' },
3319
+ (pError, pRecords) =>
3320
+ {
3321
+ Expect(pError).to.be.an('error');
3322
+ Expect(pError.message).to.contain('not found');
3323
+ return fDone();
3324
+ });
3325
+ }
3326
+ );
3327
+
3328
+ test
3329
+ (
3330
+ '_resolveDataPath handles empty path',
3331
+ function(fDone)
3332
+ {
3333
+ let tmpObj = { a: 1 };
3334
+ let tmpResult = _Fable.RetoldFactoIngestEngine._resolveDataPath(tmpObj, '');
3335
+ Expect(tmpResult).to.deep.equal(tmpObj);
3336
+ return fDone();
3337
+ }
3338
+ );
3339
+ }
3340
+ );
3341
+
3342
+ suite
3343
+ (
3344
+ 'Catalog Fetch',
3345
+ () =>
3346
+ {
3347
+ let _FetchCatalogEntryID;
3348
+ let _FetchCatalogDatasetID;
3349
+
3350
+ test
3351
+ (
3352
+ 'Create catalog entry and dataset for fetch testing',
3353
+ function(fDone)
3354
+ {
3355
+ _SuperTest
3356
+ .post('/facto/catalog/entry')
3357
+ .send({
3358
+ Agency: 'Fetch Test Agency',
3359
+ Name: 'Fetch Test Portal',
3360
+ Type: 'API',
3361
+ URL: 'https://example.invalid',
3362
+ Protocol: 'HTTPS',
3363
+ Category: 'Testing'
3364
+ })
3365
+ .end(
3366
+ (pError, pRes) =>
3367
+ {
3368
+ Expect(pRes.body.Success).to.equal(true);
3369
+ _FetchCatalogEntryID = pRes.body.Entry.IDSourceCatalogEntry;
3370
+
3371
+ _SuperTest
3372
+ .post('/facto/catalog/entry/' + _FetchCatalogEntryID + '/dataset')
3373
+ .send({
3374
+ Name: 'Fetch Test Dataset',
3375
+ Format: 'csv',
3376
+ EndpointURL: 'https://example.invalid/data.csv',
3377
+ Description: 'Test dataset for fetch tests',
3378
+ VersionPolicy: 'Append'
3379
+ })
3380
+ .end(
3381
+ (pError2, pRes2) =>
3382
+ {
3383
+ Expect(pRes2.body.Success).to.equal(true);
3384
+ _FetchCatalogDatasetID = pRes2.body.DatasetDefinition.IDCatalogDatasetDefinition;
3385
+ return fDone();
3386
+ });
3387
+ });
3388
+ }
3389
+ );
3390
+
3391
+ test
3392
+ (
3393
+ 'Fetch rejects non-provisioned dataset definition',
3394
+ function(fDone)
3395
+ {
3396
+ _SuperTest
3397
+ .post('/facto/catalog/dataset/' + _FetchCatalogDatasetID + '/fetch')
3398
+ .end(
3399
+ (pError, pRes) =>
3400
+ {
3401
+ Expect(pRes.body.Error).to.contain('provisioned');
3402
+ return fDone();
3403
+ });
3404
+ }
3405
+ );
3406
+
3407
+ test
3408
+ (
3409
+ 'Provision dataset for fetch testing',
3410
+ function(fDone)
3411
+ {
3412
+ _SuperTest
3413
+ .post('/facto/catalog/dataset/' + _FetchCatalogDatasetID + '/provision')
3414
+ .end(
3415
+ (pError, pRes) =>
3416
+ {
3417
+ Expect(pRes.body.Success).to.equal(true);
3418
+ return fDone();
3419
+ });
3420
+ }
3421
+ );
3422
+
3423
+ test
3424
+ (
3425
+ 'Fetch with unreachable URL returns download error',
3426
+ function(fDone)
3427
+ {
3428
+ this.timeout(35000);
3429
+ _SuperTest
3430
+ .post('/facto/catalog/dataset/' + _FetchCatalogDatasetID + '/fetch')
3431
+ .end(
3432
+ (pError, pRes) =>
3433
+ {
3434
+ Expect(pRes.body.Error).to.be.a('string');
3435
+ Expect(pRes.body.Error).to.contain('Download failed');
3436
+ return fDone();
3437
+ });
3438
+ }
3439
+ );
3440
+
3441
+ test
3442
+ (
3443
+ 'Fetch rejects invalid dataset definition ID',
3444
+ function(fDone)
3445
+ {
3446
+ _SuperTest
3447
+ .post('/facto/catalog/dataset/99999/fetch')
3448
+ .end(
3449
+ (pError, pRes) =>
3450
+ {
3451
+ Expect(pRes.body.Error).to.be.a('string');
3452
+ return fDone();
3453
+ });
3454
+ }
3455
+ );
3456
+
3457
+ test
3458
+ (
3459
+ 'ingestContent works with CSV string directly',
3460
+ function(fDone)
3461
+ {
3462
+ let tmpCSV = 'name,value\nAlpha,100\nBeta,200\nGamma,300';
3463
+
3464
+ _Fable.RetoldFactoIngestEngine.ingestContent(tmpCSV, 1, 1,
3465
+ { format: 'csv', type: 'test-ingest' },
3466
+ (pError, pResult) =>
3467
+ {
3468
+ Expect(pError).to.be.null;
3469
+ Expect(pResult.Ingested).to.equal(3);
3470
+ Expect(pResult.Format).to.equal('csv');
3471
+ Expect(pResult.DatasetVersion).to.be.a('number');
3472
+ Expect(pResult.ContentSignature).to.be.a('string');
3473
+ Expect(pResult.IngestJob).to.be.an('object');
3474
+ return fDone();
3475
+ });
3476
+ }
3477
+ );
3478
+
3479
+ test
3480
+ (
3481
+ 'ingestContent works with JSON string and dataPath',
3482
+ function(fDone)
3483
+ {
3484
+ let tmpJSON = JSON.stringify({
3485
+ metadata: { count: 2 },
3486
+ data: [
3487
+ { id: 1, val: 'x' },
3488
+ { id: 2, val: 'y' }
3489
+ ]
3490
+ });
3491
+
3492
+ _Fable.RetoldFactoIngestEngine.ingestContent(tmpJSON, 1, 1,
3493
+ { format: 'json', dataPath: 'data' },
3494
+ (pError, pResult) =>
3495
+ {
3496
+ Expect(pError).to.be.null;
3497
+ Expect(pResult.Ingested).to.equal(2);
3498
+ Expect(pResult.Format).to.equal('json');
3499
+ return fDone();
3500
+ });
3501
+ }
3502
+ );
3503
+ }
3504
+ );
3505
+
3506
+ suite
3507
+ (
3508
+ 'Multi-Set Projection Pipeline',
3509
+ function()
3510
+ {
3511
+ // Track IDs for test fixtures
3512
+ let _MultiSetSourceA_ID = 0;
3513
+ let _MultiSetSourceB_ID = 0;
3514
+ let _MultiSetDatasetA_ID = 0;
3515
+ let _MultiSetDatasetB_ID = 0;
3516
+ let _MultiSetProjectionDataset_ID = 0;
3517
+ let _MultiSetMappingA_ID = 0;
3518
+ let _MultiSetMappingB_ID = 0;
3519
+ let _MultiSetProjection_ID = 0;
3520
+
3521
+ test
3522
+ (
3523
+ 'Set up multi-set test fixtures: sources, datasets, records',
3524
+ function(fDone)
3525
+ {
3526
+ this.timeout(10000);
3527
+
3528
+ let tmpAnticipate = _Fable.newAnticipate();
3529
+
3530
+ // Create Source A (high reliability)
3531
+ tmpAnticipate.anticipate(
3532
+ (fStepCallback) =>
3533
+ {
3534
+ _SuperTest.post('/1.0/Source')
3535
+ .send({ Name: 'Source A - Census', Type: 'API', Active: 1 })
3536
+ .end((pError, pResponse) =>
3537
+ {
3538
+ _MultiSetSourceA_ID = pResponse.body.IDSource;
3539
+ return fStepCallback();
3540
+ });
3541
+ });
3542
+
3543
+ // Create Source B (lower reliability)
3544
+ tmpAnticipate.anticipate(
3545
+ (fStepCallback) =>
3546
+ {
3547
+ _SuperTest.post('/1.0/Source')
3548
+ .send({ Name: 'Source B - Survey', Type: 'API', Active: 1 })
3549
+ .end((pError, pResponse) =>
3550
+ {
3551
+ _MultiSetSourceB_ID = pResponse.body.IDSource;
3552
+ return fStepCallback();
3553
+ });
3554
+ });
3555
+
3556
+ tmpAnticipate.wait(
3557
+ () =>
3558
+ {
3559
+ let tmpAnticipate2 = _Fable.newAnticipate();
3560
+
3561
+ // Create Dataset A
3562
+ tmpAnticipate2.anticipate(
3563
+ (fStepCallback) =>
3564
+ {
3565
+ _SuperTest.post('/1.0/Dataset')
3566
+ .send({ Name: 'Census Data', Type: 'Raw' })
3567
+ .end((pError, pResponse) =>
3568
+ {
3569
+ _MultiSetDatasetA_ID = pResponse.body.IDDataset;
3570
+ return fStepCallback();
3571
+ });
3572
+ });
3573
+
3574
+ // Create Dataset B
3575
+ tmpAnticipate2.anticipate(
3576
+ (fStepCallback) =>
3577
+ {
3578
+ _SuperTest.post('/1.0/Dataset')
3579
+ .send({ Name: 'Survey Data', Type: 'Raw' })
3580
+ .end((pError, pResponse) =>
3581
+ {
3582
+ _MultiSetDatasetB_ID = pResponse.body.IDDataset;
3583
+ return fStepCallback();
3584
+ });
3585
+ });
3586
+
3587
+ // Create Projection Dataset
3588
+ tmpAnticipate2.anticipate(
3589
+ (fStepCallback) =>
3590
+ {
3591
+ _SuperTest.post('/1.0/Dataset')
3592
+ .send({ Name: 'Multi-Set Projection Target', Type: 'Projection' })
3593
+ .end((pError, pResponse) =>
3594
+ {
3595
+ _MultiSetProjectionDataset_ID = pResponse.body.IDDataset;
3596
+ return fStepCallback();
3597
+ });
3598
+ });
3599
+
3600
+ tmpAnticipate2.wait(
3601
+ () =>
3602
+ {
3603
+ let tmpAnticipate3 = _Fable.newAnticipate();
3604
+
3605
+ // Link DatasetSource A with high weight
3606
+ tmpAnticipate3.anticipate(
3607
+ (fStepCallback) =>
3608
+ {
3609
+ _SuperTest.post('/1.0/DatasetSource')
3610
+ .send({ IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, ReliabilityWeight: 0.9 })
3611
+ .end(() => { return fStepCallback(); });
3612
+ });
3613
+
3614
+ // Link DatasetSource B with lower weight
3615
+ tmpAnticipate3.anticipate(
3616
+ (fStepCallback) =>
3617
+ {
3618
+ _SuperTest.post('/1.0/DatasetSource')
3619
+ .send({ IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, ReliabilityWeight: 0.4 })
3620
+ .end(() => { return fStepCallback(); });
3621
+ });
3622
+
3623
+ // Ingest records for Source A
3624
+ tmpAnticipate3.anticipate(
3625
+ (fStepCallback) =>
3626
+ {
3627
+ let tmpRecords = [
3628
+ { GUIDRecord: 'PERSON-001', IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, Type: 'person', Content: JSON.stringify({ Name: 'Alice', City: 'Portland', Age: 30 }) },
3629
+ { GUIDRecord: 'PERSON-002', IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, Type: 'person', Content: JSON.stringify({ Name: 'Bob', City: 'Seattle', Age: 25 }) },
3630
+ { GUIDRecord: 'PERSON-003', IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, Type: 'person', Content: JSON.stringify({ Name: 'Charlie', City: 'Denver' }) }
3631
+ ];
3632
+ let tmpInner = _Fable.newAnticipate();
3633
+ for (let i = 0; i < tmpRecords.length; i++)
3634
+ {
3635
+ let tmpRec = tmpRecords[i];
3636
+ tmpInner.anticipate(
3637
+ (fInner) =>
3638
+ {
3639
+ _SuperTest.post('/1.0/Record')
3640
+ .send(tmpRec)
3641
+ .end(() => { return fInner(); });
3642
+ });
3643
+ }
3644
+ tmpInner.wait(() => { return fStepCallback(); });
3645
+ });
3646
+
3647
+ // Ingest records for Source B (overlapping GUIDs)
3648
+ tmpAnticipate3.anticipate(
3649
+ (fStepCallback) =>
3650
+ {
3651
+ let tmpRecords = [
3652
+ { GUIDRecord: 'PERSON-002', IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, Type: 'person', Content: JSON.stringify({ Name: 'Robert', City: 'Tacoma', Age: 26, Phone: '555-0102' }) },
3653
+ { GUIDRecord: 'PERSON-003', IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, Type: 'person', Content: JSON.stringify({ Name: 'Chuck', City: 'Boulder', Age: 35, Phone: '555-0103' }) },
3654
+ { GUIDRecord: 'PERSON-004', IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, Type: 'person', Content: JSON.stringify({ Name: 'Diana', City: 'Austin', Age: 28, Phone: '555-0104' }) }
3655
+ ];
3656
+ let tmpInner = _Fable.newAnticipate();
3657
+ for (let i = 0; i < tmpRecords.length; i++)
3658
+ {
3659
+ let tmpRec = tmpRecords[i];
3660
+ tmpInner.anticipate(
3661
+ (fInner) =>
3662
+ {
3663
+ _SuperTest.post('/1.0/Record')
3664
+ .send(tmpRec)
3665
+ .end(() => { return fInner(); });
3666
+ });
3667
+ }
3668
+ tmpInner.wait(() => { return fStepCallback(); });
3669
+ });
3670
+
3671
+ tmpAnticipate3.wait(
3672
+ () =>
3673
+ {
3674
+ return fDone();
3675
+ });
3676
+ });
3677
+ });
3678
+ }
3679
+ );
3680
+
3681
+ test
3682
+ (
3683
+ 'Create projection mappings for both sources',
3684
+ function(fDone)
3685
+ {
3686
+ this.timeout(5000);
3687
+
3688
+ let tmpMappingConfigA = JSON.stringify(
3689
+ {
3690
+ Entity: 'Person',
3691
+ GUIDTemplate: '{~D:Record.GUIDRecord~}',
3692
+ Mappings:
3693
+ {
3694
+ Name: '{~D:Record.Name~}',
3695
+ City: '{~D:Record.City~}',
3696
+ Age: '{~D:Record.Age~}',
3697
+ Phone: '{~D:Record.Phone~}'
3698
+ }
3699
+ });
3700
+
3701
+ let tmpMappingConfigB = JSON.stringify(
3702
+ {
3703
+ Entity: 'Person',
3704
+ GUIDTemplate: '{~D:Record.GUIDRecord~}',
3705
+ Mappings:
3706
+ {
3707
+ Name: '{~D:Record.Name~}',
3708
+ City: '{~D:Record.City~}',
3709
+ Age: '{~D:Record.Age~}',
3710
+ Phone: '{~D:Record.Phone~}'
3711
+ }
3712
+ });
3713
+
3714
+ let tmpAnticipate = _Fable.newAnticipate();
3715
+
3716
+ tmpAnticipate.anticipate(
3717
+ (fStepCallback) =>
3718
+ {
3719
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/mapping`)
3720
+ .send({
3721
+ IDSource: _MultiSetSourceA_ID,
3722
+ Name: 'Census Mapping',
3723
+ MappingConfiguration: tmpMappingConfigA
3724
+ })
3725
+ .end((pError, pResponse) =>
3726
+ {
3727
+ Expect(pResponse.body.Success).to.equal(true);
3728
+ _MultiSetMappingA_ID = pResponse.body.Mapping.IDProjectionMapping;
3729
+ return fStepCallback();
3730
+ });
3731
+ });
3732
+
3733
+ tmpAnticipate.anticipate(
3734
+ (fStepCallback) =>
3735
+ {
3736
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/mapping`)
3737
+ .send({
3738
+ IDSource: _MultiSetSourceB_ID,
3739
+ Name: 'Survey Mapping',
3740
+ MappingConfiguration: tmpMappingConfigB
3741
+ })
3742
+ .end((pError, pResponse) =>
3743
+ {
3744
+ Expect(pResponse.body.Success).to.equal(true);
3745
+ _MultiSetMappingB_ID = pResponse.body.Mapping.IDProjectionMapping;
3746
+ return fStepCallback();
3747
+ });
3748
+ });
3749
+
3750
+ tmpAnticipate.wait(
3751
+ () =>
3752
+ {
3753
+ return fDone();
3754
+ });
3755
+ }
3756
+ );
3757
+
3758
+ test
3759
+ (
3760
+ 'Create a MultiSetProjection with WriteAll strategy',
3761
+ function(fDone)
3762
+ {
3763
+ this.timeout(5000);
3764
+
3765
+ let tmpPipelineConfig =
3766
+ {
3767
+ Steps:
3768
+ [
3769
+ {
3770
+ IDProjectionMapping: 0, // will be set below
3771
+ Ordinal: 0,
3772
+ MergeStrategy: 'WriteAll',
3773
+ Label: 'Census Source',
3774
+ InputType: 'Records'
3775
+ },
3776
+ {
3777
+ IDProjectionMapping: 0,
3778
+ Ordinal: 1,
3779
+ MergeStrategy: 'WriteAll',
3780
+ Label: 'Survey Source',
3781
+ InputType: 'Records'
3782
+ }
3783
+ ],
3784
+ ConfidenceReinforcement:
3785
+ {
3786
+ Enabled: false
3787
+ }
3788
+ };
3789
+
3790
+ // Patch in the mapping IDs
3791
+ tmpPipelineConfig.Steps[0].IDProjectionMapping = _MultiSetMappingA_ID;
3792
+ tmpPipelineConfig.Steps[1].IDProjectionMapping = _MultiSetMappingB_ID;
3793
+
3794
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
3795
+ .send({
3796
+ Name: 'WriteAll Multi-Set Test',
3797
+ Description: 'Tests WriteAll merge with two sources',
3798
+ PipelineConfiguration: tmpPipelineConfig
3799
+ })
3800
+ .end((pError, pResponse) =>
3801
+ {
3802
+ Expect(pResponse.body.Success).to.equal(true);
3803
+ Expect(pResponse.body.MultiSetProjection).to.be.an('object');
3804
+ _MultiSetProjection_ID = pResponse.body.MultiSetProjection.IDMultiSetProjection;
3805
+ return fDone();
3806
+ });
3807
+ }
3808
+ );
3809
+
3810
+ test
3811
+ (
3812
+ 'List MultiSetProjections for dataset',
3813
+ function(fDone)
3814
+ {
3815
+ this.timeout(5000);
3816
+
3817
+ _SuperTest.get(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projections`)
3818
+ .end((pError, pResponse) =>
3819
+ {
3820
+ Expect(pResponse.body.Count).to.be.greaterThan(0);
3821
+ Expect(pResponse.body.MultiSetProjections).to.be.an('array');
3822
+ return fDone();
3823
+ });
3824
+ }
3825
+ );
3826
+
3827
+ test
3828
+ (
3829
+ 'Get a single MultiSetProjection',
3830
+ function(fDone)
3831
+ {
3832
+ this.timeout(5000);
3833
+
3834
+ _SuperTest.get(`/facto/projection/multi-set-projection/${_MultiSetProjection_ID}`)
3835
+ .end((pError, pResponse) =>
3836
+ {
3837
+ Expect(pResponse.body.MultiSetProjection).to.be.an('object');
3838
+ Expect(pResponse.body.MultiSetProjection.Name).to.equal('WriteAll Multi-Set Test');
3839
+ return fDone();
3840
+ });
3841
+ }
3842
+ );
3843
+
3844
+ test
3845
+ (
3846
+ 'Update a MultiSetProjection',
3847
+ function(fDone)
3848
+ {
3849
+ this.timeout(5000);
3850
+
3851
+ _SuperTest.post(`/facto/projection/multi-set-projection/${_MultiSetProjection_ID}/update`)
3852
+ .send({ Description: 'Updated description for WriteAll test' })
3853
+ .end((pError, pResponse) =>
3854
+ {
3855
+ Expect(pResponse.body.Success).to.equal(true);
3856
+ Expect(pResponse.body.MultiSetProjection.Description).to.equal('Updated description for WriteAll test');
3857
+ return fDone();
3858
+ });
3859
+ }
3860
+ );
3861
+
3862
+ test
3863
+ (
3864
+ 'Create and execute FirstWriteWins pipeline',
3865
+ function(fDone)
3866
+ {
3867
+ this.timeout(15000);
3868
+
3869
+ let tmpPipelineConfig =
3870
+ {
3871
+ Steps:
3872
+ [
3873
+ {
3874
+ IDProjectionMapping: _MultiSetMappingA_ID,
3875
+ Ordinal: 0,
3876
+ MergeStrategy: 'WriteAll',
3877
+ Label: 'Census First',
3878
+ InputType: 'Records'
3879
+ },
3880
+ {
3881
+ IDProjectionMapping: _MultiSetMappingB_ID,
3882
+ Ordinal: 1,
3883
+ MergeStrategy: 'FirstWriteWins',
3884
+ Label: 'Survey Second - FirstWriteWins',
3885
+ InputType: 'Records'
3886
+ }
3887
+ ],
3888
+ ConfidenceReinforcement: { Enabled: false }
3889
+ };
3890
+
3891
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
3892
+ .send({
3893
+ Name: 'FirstWriteWins Test',
3894
+ PipelineConfiguration: tmpPipelineConfig
3895
+ })
3896
+ .end((pError, pResponse) =>
3897
+ {
3898
+ Expect(pResponse.body.Success).to.equal(true);
3899
+ let tmpFWW_ID = pResponse.body.MultiSetProjection.IDMultiSetProjection;
3900
+
3901
+ // We cannot fully execute the multi-import without a deployed
3902
+ // projection store, but we can verify the pipeline loads
3903
+ // and the merge logic is applied by checking the response.
3904
+ // For now, test the CRUD path succeeded.
3905
+ Expect(tmpFWW_ID).to.be.greaterThan(0);
3906
+
3907
+ // Verify the pipeline config roundtrips correctly
3908
+ _SuperTest.get(`/facto/projection/multi-set-projection/${tmpFWW_ID}`)
3909
+ .end((pError2, pResponse2) =>
3910
+ {
3911
+ let tmpStored = pResponse2.body.MultiSetProjection;
3912
+ let tmpConfig = JSON.parse(tmpStored.PipelineConfiguration);
3913
+ Expect(tmpConfig.Steps).to.have.length(2);
3914
+ Expect(tmpConfig.Steps[1].MergeStrategy).to.equal('FirstWriteWins');
3915
+ return fDone();
3916
+ });
3917
+ });
3918
+ }
3919
+ );
3920
+
3921
+ test
3922
+ (
3923
+ 'Create MergeAndReinforce pipeline with confidence tracking',
3924
+ function(fDone)
3925
+ {
3926
+ this.timeout(5000);
3927
+
3928
+ let tmpPipelineConfig =
3929
+ {
3930
+ Steps:
3931
+ [
3932
+ {
3933
+ IDProjectionMapping: _MultiSetMappingA_ID,
3934
+ Ordinal: 0,
3935
+ MergeStrategy: 'WriteAll',
3936
+ Label: 'Census Base',
3937
+ InputType: 'Records'
3938
+ },
3939
+ {
3940
+ IDProjectionMapping: _MultiSetMappingB_ID,
3941
+ Ordinal: 1,
3942
+ MergeStrategy: 'MergeAndReinforce',
3943
+ Label: 'Survey Reinforcement',
3944
+ InputType: 'Records'
3945
+ }
3946
+ ],
3947
+ ConfidenceReinforcement:
3948
+ {
3949
+ Enabled: true,
3950
+ Dimension: 'multi-source',
3951
+ BaseValue: 0.5,
3952
+ IncrementPerConfirmation: 0.15,
3953
+ MaxValue: 1.0
3954
+ }
3955
+ };
3956
+
3957
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
3958
+ .send({
3959
+ Name: 'MergeAndReinforce Test',
3960
+ PipelineConfiguration: tmpPipelineConfig
3961
+ })
3962
+ .end((pError, pResponse) =>
3963
+ {
3964
+ Expect(pResponse.body.Success).to.equal(true);
3965
+ let tmpConfig = JSON.parse(pResponse.body.MultiSetProjection.PipelineConfiguration);
3966
+ Expect(tmpConfig.ConfidenceReinforcement.Enabled).to.equal(true);
3967
+ Expect(tmpConfig.ConfidenceReinforcement.IncrementPerConfirmation).to.equal(0.15);
3968
+ return fDone();
3969
+ });
3970
+ }
3971
+ );
3972
+
3973
+ test
3974
+ (
3975
+ 'Create ReliabilityOverwrite pipeline',
3976
+ function(fDone)
3977
+ {
3978
+ this.timeout(5000);
3979
+
3980
+ let tmpPipelineConfig =
3981
+ {
3982
+ Steps:
3983
+ [
3984
+ {
3985
+ IDProjectionMapping: _MultiSetMappingB_ID,
3986
+ Ordinal: 0,
3987
+ MergeStrategy: 'WriteAll',
3988
+ Label: 'Survey First (low reliability)',
3989
+ InputType: 'Records'
3990
+ },
3991
+ {
3992
+ IDProjectionMapping: _MultiSetMappingA_ID,
3993
+ Ordinal: 1,
3994
+ MergeStrategy: 'ReliabilityOverwrite',
3995
+ Label: 'Census Override (high reliability)',
3996
+ InputType: 'Records'
3997
+ }
3998
+ ],
3999
+ ConfidenceReinforcement: { Enabled: false }
4000
+ };
4001
+
4002
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
4003
+ .send({
4004
+ Name: 'ReliabilityOverwrite Test',
4005
+ PipelineConfiguration: tmpPipelineConfig
4006
+ })
4007
+ .end((pError, pResponse) =>
4008
+ {
4009
+ Expect(pResponse.body.Success).to.equal(true);
4010
+ let tmpConfig = JSON.parse(pResponse.body.MultiSetProjection.PipelineConfiguration);
4011
+ Expect(tmpConfig.Steps[1].MergeStrategy).to.equal('ReliabilityOverwrite');
4012
+ return fDone();
4013
+ });
4014
+ }
4015
+ );
4016
+
4017
+ test
4018
+ (
4019
+ 'Create FieldFillOnly pipeline',
4020
+ function(fDone)
4021
+ {
4022
+ this.timeout(5000);
4023
+
4024
+ let tmpPipelineConfig =
4025
+ {
4026
+ Steps:
4027
+ [
4028
+ {
4029
+ IDProjectionMapping: _MultiSetMappingA_ID,
4030
+ Ordinal: 0,
4031
+ MergeStrategy: 'WriteAll',
4032
+ Label: 'Census Base (some fields missing)',
4033
+ InputType: 'Records'
4034
+ },
4035
+ {
4036
+ IDProjectionMapping: _MultiSetMappingB_ID,
4037
+ Ordinal: 1,
4038
+ MergeStrategy: 'FieldFillOnly',
4039
+ Label: 'Survey Fill (adds Phone field)',
4040
+ InputType: 'Records'
4041
+ }
4042
+ ],
4043
+ ConfidenceReinforcement: { Enabled: false }
4044
+ };
4045
+
4046
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
4047
+ .send({
4048
+ Name: 'FieldFillOnly Test',
4049
+ PipelineConfiguration: tmpPipelineConfig
4050
+ })
4051
+ .end((pError, pResponse) =>
4052
+ {
4053
+ Expect(pResponse.body.Success).to.equal(true);
4054
+ let tmpConfig = JSON.parse(pResponse.body.MultiSetProjection.PipelineConfiguration);
4055
+ Expect(tmpConfig.Steps[1].MergeStrategy).to.equal('FieldFillOnly');
4056
+ return fDone();
4057
+ });
4058
+ }
4059
+ );
4060
+
4061
+ test
4062
+ (
4063
+ 'Soft-delete a MultiSetProjection',
4064
+ function(fDone)
4065
+ {
4066
+ this.timeout(5000);
4067
+
4068
+ // Create one to delete
4069
+ _SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
4070
+ .send({
4071
+ Name: 'To Be Deleted',
4072
+ PipelineConfiguration: { Steps: [] }
4073
+ })
4074
+ .end((pError, pResponse) =>
4075
+ {
4076
+ Expect(pResponse.body.Success).to.equal(true);
4077
+ let tmpDeleteID = pResponse.body.MultiSetProjection.IDMultiSetProjection;
4078
+
4079
+ _SuperTest.del(`/facto/projection/multi-set-projection/${tmpDeleteID}`)
4080
+ .end((pError2, pResponse2) =>
4081
+ {
4082
+ Expect(pResponse2.body.Success).to.equal(true);
4083
+
4084
+ // Verify it's soft-deleted (doesn't appear in list)
4085
+ _SuperTest.get(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projections`)
4086
+ .end((pError3, pResponse3) =>
4087
+ {
4088
+ let tmpFound = pResponse3.body.MultiSetProjections.filter(
4089
+ (pRec) => { return pRec.IDMultiSetProjection === tmpDeleteID; });
4090
+ Expect(tmpFound.length).to.equal(0);
4091
+ return fDone();
4092
+ });
4093
+ });
4094
+ });
4095
+ }
4096
+ );
4097
+
4098
+ test
4099
+ (
4100
+ 'Query certainty log (empty for unexecuted pipeline)',
4101
+ function(fDone)
4102
+ {
4103
+ this.timeout(5000);
4104
+
4105
+ _SuperTest.get(`/facto/projection/multi-set-projection/${_MultiSetProjection_ID}/certainty-log`)
4106
+ .end((pError, pResponse) =>
4107
+ {
4108
+ Expect(pResponse.body.CertaintyLog).to.be.an('array');
4109
+ Expect(pResponse.body.Count).to.equal(0);
4110
+ return fDone();
4111
+ });
4112
+ }
4113
+ );
4114
+ }
4115
+ );
4116
+ }
4117
+ );