retold-facto 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/launch.json +11 -0
- package/.dockerignore +8 -0
- package/.quackage.json +19 -0
- package/Dockerfile +26 -0
- package/bin/retold-facto.js +909 -0
- package/examples/facto-government-data.sqlite +0 -0
- package/examples/government-data-catalog.json +137 -0
- package/examples/government-data-loader.js +1432 -0
- package/package.json +91 -0
- package/scripts/facto-download.js +425 -0
- package/source/Retold-Facto.js +1042 -0
- package/source/services/Retold-Facto-BeaconProvider.js +511 -0
- package/source/services/Retold-Facto-CatalogManager.js +1252 -0
- package/source/services/Retold-Facto-DataLakeService.js +1642 -0
- package/source/services/Retold-Facto-DatasetManager.js +417 -0
- package/source/services/Retold-Facto-IngestEngine.js +1315 -0
- package/source/services/Retold-Facto-ProjectionEngine.js +3960 -0
- package/source/services/Retold-Facto-RecordManager.js +360 -0
- package/source/services/Retold-Facto-SchemaManager.js +1110 -0
- package/source/services/Retold-Facto-SourceFolderScanner.js +2243 -0
- package/source/services/Retold-Facto-SourceManager.js +730 -0
- package/source/services/Retold-Facto-StoreConnectionManager.js +441 -0
- package/source/services/Retold-Facto-ThroughputMonitor.js +478 -0
- package/source/services/web-app/codemirror-entry.js +7 -0
- package/source/services/web-app/pict-app/Pict-Application-Facto-Configuration.json +9 -0
- package/source/services/web-app/pict-app/Pict-Application-Facto.js +70 -0
- package/source/services/web-app/pict-app/Pict-Facto-Bundle.js +11 -0
- package/source/services/web-app/pict-app/providers/Pict-Provider-Facto-UI.js +66 -0
- package/source/services/web-app/pict-app/providers/Pict-Provider-Facto.js +69 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Catalog.js +93 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Connections.js +42 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Datasets.js +605 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Projections.js +188 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Scanner.js +80 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Schema.js +116 -0
- package/source/services/web-app/pict-app/providers/facto-api/Facto-API-Sources.js +104 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Catalog.js +526 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Datasets.js +173 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Ingest.js +259 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Layout.js +191 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Projections.js +231 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Records.js +326 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Scanner.js +624 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Sources.js +201 -0
- package/source/services/web-app/pict-app/views/PictView-Facto-Throughput.js +456 -0
- package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full-Configuration.json +14 -0
- package/source/services/web-app/pict-app-full/Pict-Application-Facto-Full.js +391 -0
- package/source/services/web-app/pict-app-full/providers/PictRouter-Facto-Configuration.json +56 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-BottomBar.js +68 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Connections.js +340 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboard.js +149 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Dashboards.js +819 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Datasets.js +178 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-IngestJobs.js +99 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Layout.js +62 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-MappingEditor.js +158 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-ProjectionDetail.js +1120 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Projections.js +172 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-QueryPanel.js +119 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-RecordViewer.js +663 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Records.js +648 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Scanner.js +1017 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDetail.js +1404 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaDocEditor.js +1036 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaEditor.js +636 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SchemaResearch.js +357 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceDetail.js +822 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceEditor.js +1036 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-SourceResearch.js +487 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Sources.js +165 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-Throughput.js +439 -0
- package/source/services/web-app/pict-app-full/views/PictView-Facto-Full-TopBar.js +335 -0
- package/source/services/web-app/pict-app-full/views/projections/Facto-Projections-Constants.js +71 -0
- package/source/services/web-app/web/chart.min.js +20 -0
- package/source/services/web-app/web/codemirror-bundle.js +30099 -0
- package/source/services/web-app/web/css/facto-themes.css +467 -0
- package/source/services/web-app/web/css/facto.css +502 -0
- package/source/services/web-app/web/index.html +28 -0
- package/source/services/web-app/web/retold-facto.js +12138 -0
- package/source/services/web-app/web/retold-facto.js.map +1 -0
- package/source/services/web-app/web/retold-facto.min.js +2 -0
- package/source/services/web-app/web/retold-facto.min.js.map +1 -0
- package/source/services/web-app/web/simple/index.html +17 -0
- package/test/Facto_Browser_Integration_tests.js +798 -0
- package/test/RetoldFacto_tests.js +4117 -0
- package/test/fixtures/weather-readings.csv +17 -0
- package/test/fixtures/weather-stations.csv +9 -0
- package/test/model/MeadowModel-Extended.json +8497 -0
- package/test/model/MeadowModel-PICT.json +1 -0
- package/test/model/MeadowModel.json +1355 -0
- package/test/model/ddl/Facto.ddl +225 -0
- package/test/model/fable-configuration.json +14 -0
|
@@ -0,0 +1,4117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for Retold Facto
|
|
3
|
+
*
|
|
4
|
+
* @license MIT
|
|
5
|
+
*
|
|
6
|
+
* @author Steven Velozo <steven@velozo.com>
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
var Chai = require("chai");
|
|
10
|
+
var Expect = Chai.expect;
|
|
11
|
+
|
|
12
|
+
const libFable = require('pict');
|
|
13
|
+
const libSuperTest = require('supertest');
|
|
14
|
+
const libMeadowConnectionManager = require('meadow-connection-manager');
|
|
15
|
+
const libRetoldFacto = require('../source/Retold-Facto.js');
|
|
16
|
+
const libFs = require('fs');
|
|
17
|
+
const libPath = require('path');
|
|
18
|
+
|
|
19
|
+
const _APIServerPort = 9340;
|
|
20
|
+
const _BaseURL = `http://localhost:${_APIServerPort}/`;
|
|
21
|
+
|
|
22
|
+
let _Fable;
|
|
23
|
+
let _RetoldFacto;
|
|
24
|
+
let _SuperTest;
|
|
25
|
+
|
|
26
|
+
suite
|
|
27
|
+
(
|
|
28
|
+
'Retold Facto',
|
|
29
|
+
function()
|
|
30
|
+
{
|
|
31
|
+
suiteSetup
|
|
32
|
+
(
|
|
33
|
+
function(fDone)
|
|
34
|
+
{
|
|
35
|
+
this.timeout(10000);
|
|
36
|
+
|
|
37
|
+
let tmpSettings = {
|
|
38
|
+
Product: 'RetoldFactoTest',
|
|
39
|
+
ProductVersion: '0.0.1',
|
|
40
|
+
APIServerPort: _APIServerPort,
|
|
41
|
+
SQLite:
|
|
42
|
+
{
|
|
43
|
+
SQLiteFilePath: ':memory:'
|
|
44
|
+
},
|
|
45
|
+
LogStreams:
|
|
46
|
+
[
|
|
47
|
+
{
|
|
48
|
+
streamtype: 'console',
|
|
49
|
+
level: 'fatal'
|
|
50
|
+
}
|
|
51
|
+
]
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
_Fable = new libFable(tmpSettings);
|
|
55
|
+
|
|
56
|
+
// Register the connection manager and connect the catalog database
|
|
57
|
+
_Fable.serviceManager.addServiceType('MeadowConnectionManager', libMeadowConnectionManager);
|
|
58
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowConnectionManager');
|
|
59
|
+
|
|
60
|
+
_Fable.MeadowConnectionManager.connect('facto',
|
|
61
|
+
{
|
|
62
|
+
Type: 'SQLite',
|
|
63
|
+
SQLiteFilePath: ':memory:'
|
|
64
|
+
},
|
|
65
|
+
(pError, pConnection) =>
|
|
66
|
+
{
|
|
67
|
+
if (pError)
|
|
68
|
+
{
|
|
69
|
+
return fDone(pError);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Bridge: Meadow DAL providers look up fable.MeadowSQLiteProvider
|
|
73
|
+
_Fable.MeadowSQLiteProvider = pConnection.instance;
|
|
74
|
+
|
|
75
|
+
let tmpDB = _Fable.MeadowSQLiteProvider.db;
|
|
76
|
+
|
|
77
|
+
// Create all tables using the canonical schema from the module
|
|
78
|
+
tmpDB.exec(libRetoldFacto.FACTO_SCHEMA_SQL);
|
|
79
|
+
|
|
80
|
+
_Fable.settings.MeadowProvider = 'SQLite';
|
|
81
|
+
|
|
82
|
+
_Fable.serviceManager.addServiceType('RetoldFacto', libRetoldFacto);
|
|
83
|
+
_RetoldFacto = _Fable.serviceManager.instantiateServiceProvider('RetoldFacto',
|
|
84
|
+
{
|
|
85
|
+
StorageProvider: 'SQLite',
|
|
86
|
+
|
|
87
|
+
FullMeadowSchemaPath: `${__dirname}/model/`,
|
|
88
|
+
FullMeadowSchemaFilename: 'MeadowModel-Extended.json',
|
|
89
|
+
|
|
90
|
+
AutoStartOrator: true,
|
|
91
|
+
|
|
92
|
+
Endpoints:
|
|
93
|
+
{
|
|
94
|
+
MeadowEndpoints: true,
|
|
95
|
+
SourceManager: true,
|
|
96
|
+
RecordManager: true,
|
|
97
|
+
DatasetManager: true,
|
|
98
|
+
IngestEngine: true,
|
|
99
|
+
ProjectionEngine: true,
|
|
100
|
+
CatalogManager: true,
|
|
101
|
+
StoreConnectionManager: true,
|
|
102
|
+
SchemaManager: true,
|
|
103
|
+
WebUI: false
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
// Enable JSON body parsing
|
|
108
|
+
_RetoldFacto.onBeforeInitialize = (fCallback) =>
|
|
109
|
+
{
|
|
110
|
+
_Fable.OratorServiceServer.server.use(_Fable.OratorServiceServer.bodyParser());
|
|
111
|
+
_Fable.OratorServiceServer.server.use(require('restify').plugins.queryParser());
|
|
112
|
+
return fCallback();
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
_RetoldFacto.initializeService(
|
|
116
|
+
(pInitError) =>
|
|
117
|
+
{
|
|
118
|
+
if (pInitError)
|
|
119
|
+
{
|
|
120
|
+
return fDone(pInitError);
|
|
121
|
+
}
|
|
122
|
+
_SuperTest = libSuperTest(`http://localhost:${_APIServerPort}`);
|
|
123
|
+
return fDone();
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
suiteTeardown
|
|
130
|
+
(
|
|
131
|
+
function(fDone)
|
|
132
|
+
{
|
|
133
|
+
this.timeout(5000);
|
|
134
|
+
if (_RetoldFacto && _RetoldFacto.serviceInitialized)
|
|
135
|
+
{
|
|
136
|
+
_RetoldFacto.stopService(fDone);
|
|
137
|
+
}
|
|
138
|
+
else
|
|
139
|
+
{
|
|
140
|
+
return fDone();
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
);
|
|
144
|
+
|
|
145
|
+
suite
|
|
146
|
+
(
|
|
147
|
+
'Object Sanity',
|
|
148
|
+
function()
|
|
149
|
+
{
|
|
150
|
+
test
|
|
151
|
+
(
|
|
152
|
+
'The RetoldFacto class should exist',
|
|
153
|
+
function()
|
|
154
|
+
{
|
|
155
|
+
const libRetoldFacto = require('../source/Retold-Facto.js');
|
|
156
|
+
Expect(libRetoldFacto).to.be.a('function');
|
|
157
|
+
}
|
|
158
|
+
);
|
|
159
|
+
test
|
|
160
|
+
(
|
|
161
|
+
'The service should be initialized',
|
|
162
|
+
function()
|
|
163
|
+
{
|
|
164
|
+
Expect(_RetoldFacto).to.be.an('object');
|
|
165
|
+
Expect(_RetoldFacto.serviceInitialized).to.equal(true);
|
|
166
|
+
}
|
|
167
|
+
);
|
|
168
|
+
test
|
|
169
|
+
(
|
|
170
|
+
'The entity list should contain all 15 entities',
|
|
171
|
+
function()
|
|
172
|
+
{
|
|
173
|
+
Expect(_RetoldFacto.entityList).to.be.an('array');
|
|
174
|
+
Expect(_RetoldFacto.entityList).to.include('Source');
|
|
175
|
+
Expect(_RetoldFacto.entityList).to.include('SourceDocumentation');
|
|
176
|
+
Expect(_RetoldFacto.entityList).to.include('Dataset');
|
|
177
|
+
Expect(_RetoldFacto.entityList).to.include('DatasetSource');
|
|
178
|
+
Expect(_RetoldFacto.entityList).to.include('Record');
|
|
179
|
+
Expect(_RetoldFacto.entityList).to.include('RecordBinary');
|
|
180
|
+
Expect(_RetoldFacto.entityList).to.include('CertaintyIndex');
|
|
181
|
+
Expect(_RetoldFacto.entityList).to.include('IngestJob');
|
|
182
|
+
Expect(_RetoldFacto.entityList).to.include('SourceCatalogEntry');
|
|
183
|
+
Expect(_RetoldFacto.entityList).to.include('CatalogDatasetDefinition');
|
|
184
|
+
Expect(_RetoldFacto.entityList).to.include('StoreConnection');
|
|
185
|
+
Expect(_RetoldFacto.entityList).to.include('ProjectionStore');
|
|
186
|
+
Expect(_RetoldFacto.entityList).to.include('ProjectionMapping');
|
|
187
|
+
Expect(_RetoldFacto.entityList).to.include('MultiSetProjection');
|
|
188
|
+
Expect(_RetoldFacto.entityList).to.include('ProjectionCertaintyLog');
|
|
189
|
+
Expect(_RetoldFacto.entityList.length).to.equal(18);
|
|
190
|
+
}
|
|
191
|
+
);
|
|
192
|
+
test
|
|
193
|
+
(
|
|
194
|
+
'DAL objects should exist for all entities',
|
|
195
|
+
function()
|
|
196
|
+
{
|
|
197
|
+
Expect(_RetoldFacto._DAL).to.be.an('object');
|
|
198
|
+
Expect(_RetoldFacto._DAL.Source).to.be.an('object');
|
|
199
|
+
Expect(_RetoldFacto._DAL.Record).to.be.an('object');
|
|
200
|
+
Expect(_RetoldFacto._DAL.CertaintyIndex).to.be.an('object');
|
|
201
|
+
}
|
|
202
|
+
);
|
|
203
|
+
test
|
|
204
|
+
(
|
|
205
|
+
'MeadowEndpoints should exist for all entities',
|
|
206
|
+
function()
|
|
207
|
+
{
|
|
208
|
+
Expect(_RetoldFacto._MeadowEndpoints).to.be.an('object');
|
|
209
|
+
Expect(_RetoldFacto._MeadowEndpoints.Source).to.be.an('object');
|
|
210
|
+
Expect(_RetoldFacto._MeadowEndpoints.Record).to.be.an('object');
|
|
211
|
+
Expect(_RetoldFacto._MeadowEndpoints.Dataset).to.be.an('object');
|
|
212
|
+
}
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
suite
|
|
218
|
+
(
|
|
219
|
+
'Service Lifecycle',
|
|
220
|
+
function()
|
|
221
|
+
{
|
|
222
|
+
test
|
|
223
|
+
(
|
|
224
|
+
'Should not allow double initialization',
|
|
225
|
+
function(fDone)
|
|
226
|
+
{
|
|
227
|
+
_RetoldFacto.initializeService(
|
|
228
|
+
(pError) =>
|
|
229
|
+
{
|
|
230
|
+
Expect(pError).to.be.an.instanceOf(Error);
|
|
231
|
+
Expect(pError.message).to.contain('already been initialized');
|
|
232
|
+
return fDone();
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
);
|
|
236
|
+
test
|
|
237
|
+
(
|
|
238
|
+
'Lifecycle hooks should exist',
|
|
239
|
+
function()
|
|
240
|
+
{
|
|
241
|
+
Expect(_RetoldFacto.onBeforeInitialize).to.be.a('function');
|
|
242
|
+
Expect(_RetoldFacto.onInitialize).to.be.a('function');
|
|
243
|
+
Expect(_RetoldFacto.onAfterInitialize).to.be.a('function');
|
|
244
|
+
}
|
|
245
|
+
);
|
|
246
|
+
test
|
|
247
|
+
(
|
|
248
|
+
'Endpoint group check should work',
|
|
249
|
+
function()
|
|
250
|
+
{
|
|
251
|
+
Expect(_RetoldFacto.isEndpointGroupEnabled('MeadowEndpoints')).to.equal(true);
|
|
252
|
+
Expect(_RetoldFacto.isEndpointGroupEnabled('SourceManager')).to.equal(true);
|
|
253
|
+
Expect(_RetoldFacto.isEndpointGroupEnabled('IngestEngine')).to.equal(true);
|
|
254
|
+
Expect(_RetoldFacto.isEndpointGroupEnabled('ProjectionEngine')).to.equal(true);
|
|
255
|
+
Expect(_RetoldFacto.isEndpointGroupEnabled('NonExistent')).to.equal(false);
|
|
256
|
+
}
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
);
|
|
260
|
+
|
|
261
|
+
suite
|
|
262
|
+
(
|
|
263
|
+
'Source CRUD Endpoints',
|
|
264
|
+
function()
|
|
265
|
+
{
|
|
266
|
+
test
|
|
267
|
+
(
|
|
268
|
+
'Should create a Source',
|
|
269
|
+
function(fDone)
|
|
270
|
+
{
|
|
271
|
+
_SuperTest
|
|
272
|
+
.post('/1.0/Source')
|
|
273
|
+
.send({ Name: 'US Census API', Type: 'API', URL: 'https://api.census.gov', Protocol: 'HTTPS', Active: 1 })
|
|
274
|
+
.expect(200)
|
|
275
|
+
.end(
|
|
276
|
+
(pError, pResponse) =>
|
|
277
|
+
{
|
|
278
|
+
if (pError) return fDone(pError);
|
|
279
|
+
Expect(pResponse.body.Name).to.equal('US Census API');
|
|
280
|
+
Expect(pResponse.body.IDSource).to.be.greaterThan(0);
|
|
281
|
+
return fDone();
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
);
|
|
285
|
+
test
|
|
286
|
+
(
|
|
287
|
+
'Should create a second Source',
|
|
288
|
+
function(fDone)
|
|
289
|
+
{
|
|
290
|
+
_SuperTest
|
|
291
|
+
.post('/1.0/Source')
|
|
292
|
+
.send({ Name: 'Dept of Labor CSV', Type: 'File', URL: 'https://www.bls.gov/data/', Protocol: 'HTTPS' })
|
|
293
|
+
.expect(200)
|
|
294
|
+
.end(
|
|
295
|
+
(pError, pResponse) =>
|
|
296
|
+
{
|
|
297
|
+
if (pError) return fDone(pError);
|
|
298
|
+
Expect(pResponse.body.Name).to.equal('Dept of Labor CSV');
|
|
299
|
+
return fDone();
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
);
|
|
303
|
+
test
|
|
304
|
+
(
|
|
305
|
+
'Should read a Source by ID',
|
|
306
|
+
function(fDone)
|
|
307
|
+
{
|
|
308
|
+
_SuperTest
|
|
309
|
+
.get('/1.0/Source/1')
|
|
310
|
+
.expect(200)
|
|
311
|
+
.end(
|
|
312
|
+
(pError, pResponse) =>
|
|
313
|
+
{
|
|
314
|
+
if (pError) return fDone(pError);
|
|
315
|
+
Expect(pResponse.body.Name).to.equal('US Census API');
|
|
316
|
+
Expect(pResponse.body.Type).to.equal('API');
|
|
317
|
+
return fDone();
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
);
|
|
321
|
+
test
|
|
322
|
+
(
|
|
323
|
+
'Should list Sources',
|
|
324
|
+
function(fDone)
|
|
325
|
+
{
|
|
326
|
+
_SuperTest
|
|
327
|
+
.get('/1.0/Sources/0/10')
|
|
328
|
+
.expect(200)
|
|
329
|
+
.end(
|
|
330
|
+
(pError, pResponse) =>
|
|
331
|
+
{
|
|
332
|
+
if (pError) return fDone(pError);
|
|
333
|
+
Expect(pResponse.body).to.be.an('array');
|
|
334
|
+
Expect(pResponse.body.length).to.equal(2);
|
|
335
|
+
return fDone();
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
);
|
|
341
|
+
|
|
342
|
+
suite
|
|
343
|
+
(
|
|
344
|
+
'Source Manager Domain Endpoints',
|
|
345
|
+
function()
|
|
346
|
+
{
|
|
347
|
+
test
|
|
348
|
+
(
|
|
349
|
+
'Should return active sources (only source 1 is active)',
|
|
350
|
+
function(fDone)
|
|
351
|
+
{
|
|
352
|
+
_SuperTest
|
|
353
|
+
.get('/facto/sources/active')
|
|
354
|
+
.expect(200)
|
|
355
|
+
.end(
|
|
356
|
+
(pError, pResponse) =>
|
|
357
|
+
{
|
|
358
|
+
if (pError) return fDone(pError);
|
|
359
|
+
Expect(pResponse.body.Active).to.equal(true);
|
|
360
|
+
Expect(pResponse.body.Sources).to.be.an('array');
|
|
361
|
+
Expect(pResponse.body.Sources.length).to.equal(1);
|
|
362
|
+
Expect(pResponse.body.Sources[0].Name).to.equal('US Census API');
|
|
363
|
+
return fDone();
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
);
|
|
367
|
+
test
|
|
368
|
+
(
|
|
369
|
+
'Should activate source 2',
|
|
370
|
+
function(fDone)
|
|
371
|
+
{
|
|
372
|
+
_SuperTest
|
|
373
|
+
.put('/facto/source/2/activate')
|
|
374
|
+
.expect(200)
|
|
375
|
+
.end(
|
|
376
|
+
(pError, pResponse) =>
|
|
377
|
+
{
|
|
378
|
+
if (pError) return fDone(pError);
|
|
379
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
380
|
+
return fDone();
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
);
|
|
384
|
+
test
|
|
385
|
+
(
|
|
386
|
+
'Should now return 2 active sources',
|
|
387
|
+
function(fDone)
|
|
388
|
+
{
|
|
389
|
+
_SuperTest
|
|
390
|
+
.get('/facto/sources/active')
|
|
391
|
+
.expect(200)
|
|
392
|
+
.end(
|
|
393
|
+
(pError, pResponse) =>
|
|
394
|
+
{
|
|
395
|
+
if (pError) return fDone(pError);
|
|
396
|
+
Expect(pResponse.body.Sources.length).to.equal(2);
|
|
397
|
+
return fDone();
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
);
|
|
401
|
+
test
|
|
402
|
+
(
|
|
403
|
+
'Should deactivate source 2',
|
|
404
|
+
function(fDone)
|
|
405
|
+
{
|
|
406
|
+
_SuperTest
|
|
407
|
+
.put('/facto/source/2/deactivate')
|
|
408
|
+
.expect(200)
|
|
409
|
+
.end(
|
|
410
|
+
(pError, pResponse) =>
|
|
411
|
+
{
|
|
412
|
+
if (pError) return fDone(pError);
|
|
413
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
414
|
+
return fDone();
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
);
|
|
418
|
+
test
|
|
419
|
+
(
|
|
420
|
+
'Should return 1 active source after deactivation',
|
|
421
|
+
function(fDone)
|
|
422
|
+
{
|
|
423
|
+
_SuperTest
|
|
424
|
+
.get('/facto/sources/active')
|
|
425
|
+
.expect(200)
|
|
426
|
+
.end(
|
|
427
|
+
(pError, pResponse) =>
|
|
428
|
+
{
|
|
429
|
+
if (pError) return fDone(pError);
|
|
430
|
+
Expect(pResponse.body.Sources.length).to.equal(1);
|
|
431
|
+
return fDone();
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
);
|
|
435
|
+
test
|
|
436
|
+
(
|
|
437
|
+
'Should create source documentation',
|
|
438
|
+
function(fDone)
|
|
439
|
+
{
|
|
440
|
+
_SuperTest
|
|
441
|
+
.post('/1.0/SourceDocumentation')
|
|
442
|
+
.send({ IDSource: 1, Name: 'Census API Docs', DocumentType: 'markdown', MimeType: 'text/markdown', Description: 'API documentation' })
|
|
443
|
+
.expect(200)
|
|
444
|
+
.end(
|
|
445
|
+
(pError, pResponse) =>
|
|
446
|
+
{
|
|
447
|
+
if (pError) return fDone(pError);
|
|
448
|
+
Expect(pResponse.body.IDSourceDocumentation).to.be.greaterThan(0);
|
|
449
|
+
return fDone();
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
);
|
|
453
|
+
test
|
|
454
|
+
(
|
|
455
|
+
'Should list documentation for source 1',
|
|
456
|
+
function(fDone)
|
|
457
|
+
{
|
|
458
|
+
_SuperTest
|
|
459
|
+
.get('/facto/source/1/documentation')
|
|
460
|
+
.expect(200)
|
|
461
|
+
.end(
|
|
462
|
+
(pError, pResponse) =>
|
|
463
|
+
{
|
|
464
|
+
if (pError) return fDone(pError);
|
|
465
|
+
Expect(pResponse.body.Documentation).to.be.an('array');
|
|
466
|
+
Expect(pResponse.body.Documentation.length).to.equal(1);
|
|
467
|
+
Expect(pResponse.body.Documentation[0].Name).to.equal('Census API Docs');
|
|
468
|
+
return fDone();
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
);
|
|
472
|
+
test
|
|
473
|
+
(
|
|
474
|
+
'Should get source summary with counts',
|
|
475
|
+
function(fDone)
|
|
476
|
+
{
|
|
477
|
+
_SuperTest
|
|
478
|
+
.get('/facto/source/1/summary')
|
|
479
|
+
.expect(200)
|
|
480
|
+
.end(
|
|
481
|
+
(pError, pResponse) =>
|
|
482
|
+
{
|
|
483
|
+
if (pError) return fDone(pError);
|
|
484
|
+
Expect(pResponse.body.Source).to.be.an('object');
|
|
485
|
+
Expect(pResponse.body.Source.Name).to.equal('US Census API');
|
|
486
|
+
Expect(pResponse.body.DocumentationCount).to.equal(1);
|
|
487
|
+
return fDone();
|
|
488
|
+
});
|
|
489
|
+
}
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
);
|
|
493
|
+
|
|
494
|
+
suite
|
|
495
|
+
(
|
|
496
|
+
'Dataset CRUD Endpoints',
|
|
497
|
+
function()
|
|
498
|
+
{
|
|
499
|
+
test
|
|
500
|
+
(
|
|
501
|
+
'Should create a Raw Dataset',
|
|
502
|
+
function(fDone)
|
|
503
|
+
{
|
|
504
|
+
_SuperTest
|
|
505
|
+
.post('/1.0/Dataset')
|
|
506
|
+
.send({ Name: 'Census Population 2020', Type: 'Raw', Description: 'US Census population counts by county' })
|
|
507
|
+
.expect(200)
|
|
508
|
+
.end(
|
|
509
|
+
(pError, pResponse) =>
|
|
510
|
+
{
|
|
511
|
+
if (pError) return fDone(pError);
|
|
512
|
+
Expect(pResponse.body.Name).to.equal('Census Population 2020');
|
|
513
|
+
Expect(pResponse.body.Type).to.equal('Raw');
|
|
514
|
+
return fDone();
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
);
|
|
518
|
+
test
|
|
519
|
+
(
|
|
520
|
+
'Should create a Projection Dataset',
|
|
521
|
+
function(fDone)
|
|
522
|
+
{
|
|
523
|
+
_SuperTest
|
|
524
|
+
.post('/1.0/Dataset')
|
|
525
|
+
.send({ Name: 'Population Summary View', Type: 'Projection', Description: 'Flattened population data for charting' })
|
|
526
|
+
.expect(200)
|
|
527
|
+
.end(
|
|
528
|
+
(pError, pResponse) =>
|
|
529
|
+
{
|
|
530
|
+
if (pError) return fDone(pError);
|
|
531
|
+
Expect(pResponse.body.Name).to.equal('Population Summary View');
|
|
532
|
+
Expect(pResponse.body.Type).to.equal('Projection');
|
|
533
|
+
return fDone();
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
);
|
|
537
|
+
test
|
|
538
|
+
(
|
|
539
|
+
'Should create a DatasetSource link',
|
|
540
|
+
function(fDone)
|
|
541
|
+
{
|
|
542
|
+
_SuperTest
|
|
543
|
+
.post('/1.0/DatasetSource')
|
|
544
|
+
.send({ IDDataset: 1, IDSource: 1, ReliabilityWeight: 0.85 })
|
|
545
|
+
.expect(200)
|
|
546
|
+
.end(
|
|
547
|
+
(pError, pResponse) =>
|
|
548
|
+
{
|
|
549
|
+
if (pError) return fDone(pError);
|
|
550
|
+
Expect(pResponse.body.IDDataset).to.equal(1);
|
|
551
|
+
Expect(pResponse.body.IDSource).to.equal(1);
|
|
552
|
+
return fDone();
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
);
|
|
556
|
+
test
|
|
557
|
+
(
|
|
558
|
+
'Should list dataset types via domain endpoint',
|
|
559
|
+
function(fDone)
|
|
560
|
+
{
|
|
561
|
+
_SuperTest
|
|
562
|
+
.get('/facto/datasets/types')
|
|
563
|
+
.expect(200)
|
|
564
|
+
.end(
|
|
565
|
+
(pError, pResponse) =>
|
|
566
|
+
{
|
|
567
|
+
if (pError) return fDone(pError);
|
|
568
|
+
Expect(pResponse.body.Types).to.be.an('array');
|
|
569
|
+
Expect(pResponse.body.Types).to.include('Raw');
|
|
570
|
+
Expect(pResponse.body.Types).to.include('Compositional');
|
|
571
|
+
Expect(pResponse.body.Types).to.include('Projection');
|
|
572
|
+
Expect(pResponse.body.Types).to.include('Derived');
|
|
573
|
+
return fDone();
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
);
|
|
577
|
+
}
|
|
578
|
+
);
|
|
579
|
+
|
|
580
|
+
suite
|
|
581
|
+
(
|
|
582
|
+
'Dataset Manager Domain Endpoints',
|
|
583
|
+
function()
|
|
584
|
+
{
|
|
585
|
+
test
|
|
586
|
+
(
|
|
587
|
+
'Should link a source to a dataset via domain endpoint',
|
|
588
|
+
function(fDone)
|
|
589
|
+
{
|
|
590
|
+
_SuperTest
|
|
591
|
+
.post('/facto/dataset/1/source')
|
|
592
|
+
.send({ IDSource: 2, ReliabilityWeight: 0.6 })
|
|
593
|
+
.expect(200)
|
|
594
|
+
.end(
|
|
595
|
+
(pError, pResponse) =>
|
|
596
|
+
{
|
|
597
|
+
if (pError) return fDone(pError);
|
|
598
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
599
|
+
Expect(pResponse.body.DatasetSource).to.be.an('object');
|
|
600
|
+
return fDone();
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
);
|
|
604
|
+
test
|
|
605
|
+
(
|
|
606
|
+
'Should list sources linked to dataset 1',
|
|
607
|
+
function(fDone)
|
|
608
|
+
{
|
|
609
|
+
_SuperTest
|
|
610
|
+
.get('/facto/dataset/1/sources')
|
|
611
|
+
.expect(200)
|
|
612
|
+
.end(
|
|
613
|
+
(pError, pResponse) =>
|
|
614
|
+
{
|
|
615
|
+
if (pError) return fDone(pError);
|
|
616
|
+
Expect(pResponse.body.Sources).to.be.an('array');
|
|
617
|
+
Expect(pResponse.body.Sources.length).to.equal(2);
|
|
618
|
+
return fDone();
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
);
|
|
622
|
+
test
|
|
623
|
+
(
|
|
624
|
+
'Should get dataset stats',
|
|
625
|
+
function(fDone)
|
|
626
|
+
{
|
|
627
|
+
_SuperTest
|
|
628
|
+
.get('/facto/dataset/1/stats')
|
|
629
|
+
.expect(200)
|
|
630
|
+
.end(
|
|
631
|
+
(pError, pResponse) =>
|
|
632
|
+
{
|
|
633
|
+
if (pError) return fDone(pError);
|
|
634
|
+
Expect(pResponse.body.Dataset).to.be.an('object');
|
|
635
|
+
Expect(pResponse.body.Dataset.Name).to.equal('Census Population 2020');
|
|
636
|
+
Expect(pResponse.body.SourceCount).to.equal(2);
|
|
637
|
+
Expect(pResponse.body.RecordCount).to.equal(0);
|
|
638
|
+
return fDone();
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
);
|
|
642
|
+
}
|
|
643
|
+
);
|
|
644
|
+
|
|
645
|
+
suite
|
|
646
|
+
(
|
|
647
|
+
'Record CRUD Endpoints',
|
|
648
|
+
function()
|
|
649
|
+
{
|
|
650
|
+
test
|
|
651
|
+
(
|
|
652
|
+
'Should create a Record',
|
|
653
|
+
function(fDone)
|
|
654
|
+
{
|
|
655
|
+
_SuperTest
|
|
656
|
+
.post('/1.0/Record')
|
|
657
|
+
.send(
|
|
658
|
+
{
|
|
659
|
+
IDDataset: 1,
|
|
660
|
+
IDSource: 1,
|
|
661
|
+
Type: 'census-population',
|
|
662
|
+
Version: 1,
|
|
663
|
+
RepresentedTimeStampStart: 1577836800,
|
|
664
|
+
RepresentedTimeStampStop: 1609459199,
|
|
665
|
+
Content: JSON.stringify({ county: 'Los Angeles', state: 'CA', population: 10014009 })
|
|
666
|
+
})
|
|
667
|
+
.expect(200)
|
|
668
|
+
.end(
|
|
669
|
+
(pError, pResponse) =>
|
|
670
|
+
{
|
|
671
|
+
if (pError) return fDone(pError);
|
|
672
|
+
Expect(pResponse.body.IDRecord).to.be.greaterThan(0);
|
|
673
|
+
Expect(pResponse.body.Type).to.equal('census-population');
|
|
674
|
+
return fDone();
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
);
|
|
678
|
+
test
|
|
679
|
+
(
|
|
680
|
+
'Should create a second Record',
|
|
681
|
+
function(fDone)
|
|
682
|
+
{
|
|
683
|
+
_SuperTest
|
|
684
|
+
.post('/1.0/Record')
|
|
685
|
+
.send(
|
|
686
|
+
{
|
|
687
|
+
IDDataset: 1,
|
|
688
|
+
IDSource: 1,
|
|
689
|
+
Type: 'census-population',
|
|
690
|
+
Version: 1,
|
|
691
|
+
RepresentedTimeStampStart: 1577836800,
|
|
692
|
+
RepresentedTimeStampStop: 1609459199,
|
|
693
|
+
Content: JSON.stringify({ county: 'Cook', state: 'IL', population: 5275541 })
|
|
694
|
+
})
|
|
695
|
+
.expect(200)
|
|
696
|
+
.end(
|
|
697
|
+
(pError, pResponse) =>
|
|
698
|
+
{
|
|
699
|
+
if (pError) return fDone(pError);
|
|
700
|
+
Expect(pResponse.body.IDRecord).to.be.greaterThan(0);
|
|
701
|
+
return fDone();
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
);
|
|
705
|
+
test
|
|
706
|
+
(
|
|
707
|
+
'Should read a Record by ID',
|
|
708
|
+
function(fDone)
|
|
709
|
+
{
|
|
710
|
+
_SuperTest
|
|
711
|
+
.get('/1.0/Record/1')
|
|
712
|
+
.expect(200)
|
|
713
|
+
.end(
|
|
714
|
+
(pError, pResponse) =>
|
|
715
|
+
{
|
|
716
|
+
if (pError) return fDone(pError);
|
|
717
|
+
Expect(pResponse.body.Type).to.equal('census-population');
|
|
718
|
+
let tmpContent = JSON.parse(pResponse.body.Content);
|
|
719
|
+
Expect(tmpContent.county).to.equal('Los Angeles');
|
|
720
|
+
return fDone();
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
);
|
|
724
|
+
test
|
|
725
|
+
(
|
|
726
|
+
'Should list Records with pagination',
|
|
727
|
+
function(fDone)
|
|
728
|
+
{
|
|
729
|
+
_SuperTest
|
|
730
|
+
.get('/1.0/Records/0/50')
|
|
731
|
+
.expect(200)
|
|
732
|
+
.end(
|
|
733
|
+
(pError, pResponse) =>
|
|
734
|
+
{
|
|
735
|
+
if (pError) return fDone(pError);
|
|
736
|
+
Expect(pResponse.body).to.be.an('array');
|
|
737
|
+
Expect(pResponse.body.length).to.equal(2);
|
|
738
|
+
return fDone();
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
);
|
|
742
|
+
test
|
|
743
|
+
(
|
|
744
|
+
'Should count Records',
|
|
745
|
+
function(fDone)
|
|
746
|
+
{
|
|
747
|
+
_SuperTest
|
|
748
|
+
.get('/1.0/Records/Count')
|
|
749
|
+
.expect(200)
|
|
750
|
+
.end(
|
|
751
|
+
(pError, pResponse) =>
|
|
752
|
+
{
|
|
753
|
+
if (pError) return fDone(pError);
|
|
754
|
+
Expect(pResponse.body.Count).to.equal(2);
|
|
755
|
+
return fDone();
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
);
|
|
759
|
+
}
|
|
760
|
+
);
|
|
761
|
+
|
|
762
|
+
suite
|
|
763
|
+
(
|
|
764
|
+
'Record Manager Domain Endpoints',
|
|
765
|
+
function()
|
|
766
|
+
{
|
|
767
|
+
test
|
|
768
|
+
(
|
|
769
|
+
'Should batch ingest records with auto-certainty',
|
|
770
|
+
function(fDone)
|
|
771
|
+
{
|
|
772
|
+
this.timeout(10000);
|
|
773
|
+
_SuperTest
|
|
774
|
+
.post('/facto/record/ingest')
|
|
775
|
+
.send(
|
|
776
|
+
{
|
|
777
|
+
IDDataset: 1,
|
|
778
|
+
IDSource: 1,
|
|
779
|
+
Records:
|
|
780
|
+
[
|
|
781
|
+
{
|
|
782
|
+
Type: 'census-population',
|
|
783
|
+
Content: JSON.stringify({ county: 'Harris', state: 'TX', population: 4713325 }),
|
|
784
|
+
RepresentedTimeStampStart: 1577836800,
|
|
785
|
+
RepresentedTimeStampStop: 1609459199
|
|
786
|
+
},
|
|
787
|
+
{
|
|
788
|
+
Type: 'census-population',
|
|
789
|
+
Content: JSON.stringify({ county: 'Maricopa', state: 'AZ', population: 4485414 }),
|
|
790
|
+
RepresentedTimeStampStart: 1577836800,
|
|
791
|
+
RepresentedTimeStampStop: 1609459199
|
|
792
|
+
}
|
|
793
|
+
]
|
|
794
|
+
})
|
|
795
|
+
.expect(200)
|
|
796
|
+
.end(
|
|
797
|
+
(pError, pResponse) =>
|
|
798
|
+
{
|
|
799
|
+
if (pError) return fDone(pError);
|
|
800
|
+
Expect(pResponse.body.Ingested).to.equal(2);
|
|
801
|
+
Expect(pResponse.body.Errors).to.equal(0);
|
|
802
|
+
Expect(pResponse.body.Total).to.equal(2);
|
|
803
|
+
Expect(pResponse.body.DefaultCertainty).to.equal(0.5);
|
|
804
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
805
|
+
Expect(pResponse.body.Records.length).to.equal(2);
|
|
806
|
+
return fDone();
|
|
807
|
+
});
|
|
808
|
+
}
|
|
809
|
+
);
|
|
810
|
+
test
|
|
811
|
+
(
|
|
812
|
+
'Should return error for empty ingest request',
|
|
813
|
+
function(fDone)
|
|
814
|
+
{
|
|
815
|
+
_SuperTest
|
|
816
|
+
.post('/facto/record/ingest')
|
|
817
|
+
.send({ IDDataset: 1, IDSource: 1 })
|
|
818
|
+
.expect(200)
|
|
819
|
+
.end(
|
|
820
|
+
(pError, pResponse) =>
|
|
821
|
+
{
|
|
822
|
+
if (pError) return fDone(pError);
|
|
823
|
+
Expect(pResponse.body.Error).to.be.a('string');
|
|
824
|
+
Expect(pResponse.body.Ingested).to.equal(0);
|
|
825
|
+
return fDone();
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
);
|
|
829
|
+
test
|
|
830
|
+
(
|
|
831
|
+
'Should get certainty indices for an ingested record',
|
|
832
|
+
function(fDone)
|
|
833
|
+
{
|
|
834
|
+
// Record 3 was created by batch ingest and should have auto-certainty
|
|
835
|
+
_SuperTest
|
|
836
|
+
.get('/facto/record/3/certainty')
|
|
837
|
+
.expect(200)
|
|
838
|
+
.end(
|
|
839
|
+
(pError, pResponse) =>
|
|
840
|
+
{
|
|
841
|
+
if (pError) return fDone(pError);
|
|
842
|
+
Expect(pResponse.body.CertaintyIndices).to.be.an('array');
|
|
843
|
+
Expect(pResponse.body.CertaintyIndices.length).to.be.greaterThan(0);
|
|
844
|
+
Expect(pResponse.body.CertaintyIndices[0].Dimension).to.equal('overall');
|
|
845
|
+
return fDone();
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
);
|
|
849
|
+
test
|
|
850
|
+
(
|
|
851
|
+
'Should add a new certainty index entry',
|
|
852
|
+
function(fDone)
|
|
853
|
+
{
|
|
854
|
+
_SuperTest
|
|
855
|
+
.post('/facto/record/1/certainty')
|
|
856
|
+
.send({ CertaintyValue: 0.9, Dimension: 'accuracy', Justification: 'Verified against official source' })
|
|
857
|
+
.expect(200)
|
|
858
|
+
.end(
|
|
859
|
+
(pError, pResponse) =>
|
|
860
|
+
{
|
|
861
|
+
if (pError) return fDone(pError);
|
|
862
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
863
|
+
Expect(pResponse.body.CertaintyIndex).to.be.an('object');
|
|
864
|
+
Expect(pResponse.body.CertaintyIndex.Dimension).to.equal('accuracy');
|
|
865
|
+
return fDone();
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
);
|
|
869
|
+
test
|
|
870
|
+
(
|
|
871
|
+
'Should list binary attachments for a record (empty)',
|
|
872
|
+
function(fDone)
|
|
873
|
+
{
|
|
874
|
+
_SuperTest
|
|
875
|
+
.get('/facto/record/1/binary')
|
|
876
|
+
.expect(200)
|
|
877
|
+
.end(
|
|
878
|
+
(pError, pResponse) =>
|
|
879
|
+
{
|
|
880
|
+
if (pError) return fDone(pError);
|
|
881
|
+
Expect(pResponse.body.Binaries).to.be.an('array');
|
|
882
|
+
Expect(pResponse.body.Binaries.length).to.equal(0);
|
|
883
|
+
return fDone();
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
);
|
|
887
|
+
test
|
|
888
|
+
(
|
|
889
|
+
'Should get record versions by GUIDRecord',
|
|
890
|
+
function(fDone)
|
|
891
|
+
{
|
|
892
|
+
_SuperTest
|
|
893
|
+
.get('/facto/record/1/versions')
|
|
894
|
+
.expect(200)
|
|
895
|
+
.end(
|
|
896
|
+
(pError, pResponse) =>
|
|
897
|
+
{
|
|
898
|
+
if (pError) return fDone(pError);
|
|
899
|
+
Expect(pResponse.body.Versions).to.be.an('array');
|
|
900
|
+
Expect(pResponse.body.Versions.length).to.be.greaterThan(0);
|
|
901
|
+
return fDone();
|
|
902
|
+
});
|
|
903
|
+
}
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
);
|
|
907
|
+
|
|
908
|
+
suite
|
|
909
|
+
(
|
|
910
|
+
'CertaintyIndex CRUD Endpoints',
|
|
911
|
+
function()
|
|
912
|
+
{
|
|
913
|
+
test
|
|
914
|
+
(
|
|
915
|
+
'Should create a CertaintyIndex entry',
|
|
916
|
+
function(fDone)
|
|
917
|
+
{
|
|
918
|
+
_SuperTest
|
|
919
|
+
.post('/1.0/CertaintyIndex')
|
|
920
|
+
.send({ IDRecord: 1, CertaintyValue: 0.5, Dimension: 'completeness', Justification: 'Default initial certainty' })
|
|
921
|
+
.expect(200)
|
|
922
|
+
.end(
|
|
923
|
+
(pError, pResponse) =>
|
|
924
|
+
{
|
|
925
|
+
if (pError) return fDone(pError);
|
|
926
|
+
Expect(pResponse.body.IDCertaintyIndex).to.be.greaterThan(0);
|
|
927
|
+
Expect(pResponse.body.Dimension).to.equal('completeness');
|
|
928
|
+
return fDone();
|
|
929
|
+
});
|
|
930
|
+
}
|
|
931
|
+
);
|
|
932
|
+
test
|
|
933
|
+
(
|
|
934
|
+
'Should read CertaintyIndex by ID',
|
|
935
|
+
function(fDone)
|
|
936
|
+
{
|
|
937
|
+
_SuperTest
|
|
938
|
+
.get('/1.0/CertaintyIndex/1')
|
|
939
|
+
.expect(200)
|
|
940
|
+
.end(
|
|
941
|
+
(pError, pResponse) =>
|
|
942
|
+
{
|
|
943
|
+
if (pError) return fDone(pError);
|
|
944
|
+
Expect(pResponse.body.IDRecord).to.be.greaterThan(0);
|
|
945
|
+
return fDone();
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
);
|
|
951
|
+
|
|
952
|
+
suite
|
|
953
|
+
(
|
|
954
|
+
'Dataset Manager Advanced',
|
|
955
|
+
function()
|
|
956
|
+
{
|
|
957
|
+
test
|
|
958
|
+
(
|
|
959
|
+
'Should get updated dataset stats with records',
|
|
960
|
+
function(fDone)
|
|
961
|
+
{
|
|
962
|
+
_SuperTest
|
|
963
|
+
.get('/facto/dataset/1/stats')
|
|
964
|
+
.expect(200)
|
|
965
|
+
.end(
|
|
966
|
+
(pError, pResponse) =>
|
|
967
|
+
{
|
|
968
|
+
if (pError) return fDone(pError);
|
|
969
|
+
Expect(pResponse.body.RecordCount).to.equal(4);
|
|
970
|
+
Expect(pResponse.body.SourceCount).to.equal(2);
|
|
971
|
+
return fDone();
|
|
972
|
+
});
|
|
973
|
+
}
|
|
974
|
+
);
|
|
975
|
+
test
|
|
976
|
+
(
|
|
977
|
+
'Should list records for a dataset with pagination',
|
|
978
|
+
function(fDone)
|
|
979
|
+
{
|
|
980
|
+
_SuperTest
|
|
981
|
+
.get('/facto/dataset/1/records/0/10')
|
|
982
|
+
.expect(200)
|
|
983
|
+
.end(
|
|
984
|
+
(pError, pResponse) =>
|
|
985
|
+
{
|
|
986
|
+
if (pError) return fDone(pError);
|
|
987
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
988
|
+
Expect(pResponse.body.Records.length).to.equal(4);
|
|
989
|
+
Expect(pResponse.body.IDDataset).to.equal(1);
|
|
990
|
+
return fDone();
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
);
|
|
996
|
+
|
|
997
|
+
suite
|
|
998
|
+
(
|
|
999
|
+
'Ingest Engine Domain Endpoints',
|
|
1000
|
+
function()
|
|
1001
|
+
{
|
|
1002
|
+
test
|
|
1003
|
+
(
|
|
1004
|
+
'Should create an ingest job',
|
|
1005
|
+
function(fDone)
|
|
1006
|
+
{
|
|
1007
|
+
_SuperTest
|
|
1008
|
+
.post('/facto/ingest/job')
|
|
1009
|
+
.send({ IDSource: 1, IDDataset: 1, Configuration: { format: 'csv', delimiter: ',' } })
|
|
1010
|
+
.expect(200)
|
|
1011
|
+
.end(
|
|
1012
|
+
(pError, pResponse) =>
|
|
1013
|
+
{
|
|
1014
|
+
if (pError) return fDone(pError);
|
|
1015
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
1016
|
+
Expect(pResponse.body.Job).to.be.an('object');
|
|
1017
|
+
Expect(pResponse.body.Job.Status).to.equal('Pending');
|
|
1018
|
+
Expect(pResponse.body.Job.IDSource).to.equal(1);
|
|
1019
|
+
return fDone();
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
);
|
|
1023
|
+
test
|
|
1024
|
+
(
|
|
1025
|
+
'Should list ingest jobs',
|
|
1026
|
+
function(fDone)
|
|
1027
|
+
{
|
|
1028
|
+
_SuperTest
|
|
1029
|
+
.get('/facto/ingest/jobs')
|
|
1030
|
+
.expect(200)
|
|
1031
|
+
.end(
|
|
1032
|
+
(pError, pResponse) =>
|
|
1033
|
+
{
|
|
1034
|
+
if (pError) return fDone(pError);
|
|
1035
|
+
Expect(pResponse.body.Jobs).to.be.an('array');
|
|
1036
|
+
Expect(pResponse.body.Jobs.length).to.equal(1);
|
|
1037
|
+
return fDone();
|
|
1038
|
+
});
|
|
1039
|
+
}
|
|
1040
|
+
);
|
|
1041
|
+
test
|
|
1042
|
+
(
|
|
1043
|
+
'Should get ingest job details',
|
|
1044
|
+
function(fDone)
|
|
1045
|
+
{
|
|
1046
|
+
_SuperTest
|
|
1047
|
+
.get('/facto/ingest/job/1')
|
|
1048
|
+
.expect(200)
|
|
1049
|
+
.end(
|
|
1050
|
+
(pError, pResponse) =>
|
|
1051
|
+
{
|
|
1052
|
+
if (pError) return fDone(pError);
|
|
1053
|
+
Expect(pResponse.body.Job).to.be.an('object');
|
|
1054
|
+
Expect(pResponse.body.Job.Status).to.equal('Pending');
|
|
1055
|
+
Expect(pResponse.body.Job.Log).to.contain('Job created');
|
|
1056
|
+
return fDone();
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
);
|
|
1060
|
+
test
|
|
1061
|
+
(
|
|
1062
|
+
'Should start an ingest job',
|
|
1063
|
+
function(fDone)
|
|
1064
|
+
{
|
|
1065
|
+
_SuperTest
|
|
1066
|
+
.put('/facto/ingest/job/1/start')
|
|
1067
|
+
.expect(200)
|
|
1068
|
+
.end(
|
|
1069
|
+
(pError, pResponse) =>
|
|
1070
|
+
{
|
|
1071
|
+
if (pError) return fDone(pError);
|
|
1072
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
1073
|
+
return fDone();
|
|
1074
|
+
});
|
|
1075
|
+
}
|
|
1076
|
+
);
|
|
1077
|
+
test
|
|
1078
|
+
(
|
|
1079
|
+
'Should complete an ingest job with counters',
|
|
1080
|
+
function(fDone)
|
|
1081
|
+
{
|
|
1082
|
+
_SuperTest
|
|
1083
|
+
.put('/facto/ingest/job/1/complete')
|
|
1084
|
+
.send({ RecordsProcessed: 100, RecordsCreated: 95, RecordsErrored: 5 })
|
|
1085
|
+
.expect(200)
|
|
1086
|
+
.end(
|
|
1087
|
+
(pError, pResponse) =>
|
|
1088
|
+
{
|
|
1089
|
+
if (pError) return fDone(pError);
|
|
1090
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
1091
|
+
return fDone();
|
|
1092
|
+
});
|
|
1093
|
+
}
|
|
1094
|
+
);
|
|
1095
|
+
test
|
|
1096
|
+
(
|
|
1097
|
+
'Should verify completed job has log entries',
|
|
1098
|
+
function(fDone)
|
|
1099
|
+
{
|
|
1100
|
+
_SuperTest
|
|
1101
|
+
.get('/facto/ingest/job/1')
|
|
1102
|
+
.expect(200)
|
|
1103
|
+
.end(
|
|
1104
|
+
(pError, pResponse) =>
|
|
1105
|
+
{
|
|
1106
|
+
if (pError) return fDone(pError);
|
|
1107
|
+
Expect(pResponse.body.Job.Log).to.contain('Job created');
|
|
1108
|
+
Expect(pResponse.body.Job.Log).to.contain('Job started');
|
|
1109
|
+
Expect(pResponse.body.Job.Log).to.contain('Job completed');
|
|
1110
|
+
return fDone();
|
|
1111
|
+
});
|
|
1112
|
+
}
|
|
1113
|
+
);
|
|
1114
|
+
test
|
|
1115
|
+
(
|
|
1116
|
+
'Should list valid job statuses',
|
|
1117
|
+
function(fDone)
|
|
1118
|
+
{
|
|
1119
|
+
_SuperTest
|
|
1120
|
+
.get('/facto/ingest/statuses')
|
|
1121
|
+
.expect(200)
|
|
1122
|
+
.end(
|
|
1123
|
+
(pError, pResponse) =>
|
|
1124
|
+
{
|
|
1125
|
+
if (pError) return fDone(pError);
|
|
1126
|
+
Expect(pResponse.body.Statuses).to.be.an('array');
|
|
1127
|
+
Expect(pResponse.body.Statuses).to.include('Pending');
|
|
1128
|
+
Expect(pResponse.body.Statuses).to.include('Running');
|
|
1129
|
+
Expect(pResponse.body.Statuses).to.include('Completed');
|
|
1130
|
+
Expect(pResponse.body.Statuses).to.include('Failed');
|
|
1131
|
+
return fDone();
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
);
|
|
1135
|
+
}
|
|
1136
|
+
);
|
|
1137
|
+
|
|
1138
|
+
suite
|
|
1139
|
+
(
|
|
1140
|
+
'Projection Engine Domain Endpoints',
|
|
1141
|
+
function()
|
|
1142
|
+
{
|
|
1143
|
+
test
|
|
1144
|
+
(
|
|
1145
|
+
'Should list projection datasets',
|
|
1146
|
+
function(fDone)
|
|
1147
|
+
{
|
|
1148
|
+
_SuperTest
|
|
1149
|
+
.get('/facto/projections')
|
|
1150
|
+
.expect(200)
|
|
1151
|
+
.end(
|
|
1152
|
+
(pError, pResponse) =>
|
|
1153
|
+
{
|
|
1154
|
+
if (pError) return fDone(pError);
|
|
1155
|
+
Expect(pResponse.body.Projections).to.be.an('array');
|
|
1156
|
+
Expect(pResponse.body.Projections.length).to.equal(1);
|
|
1157
|
+
Expect(pResponse.body.Projections[0].Name).to.equal('Population Summary View');
|
|
1158
|
+
return fDone();
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
);
|
|
1162
|
+
test
|
|
1163
|
+
(
|
|
1164
|
+
'Should list datasets by type',
|
|
1165
|
+
function(fDone)
|
|
1166
|
+
{
|
|
1167
|
+
_SuperTest
|
|
1168
|
+
.get('/facto/datasets/by-type/Raw')
|
|
1169
|
+
.expect(200)
|
|
1170
|
+
.end(
|
|
1171
|
+
(pError, pResponse) =>
|
|
1172
|
+
{
|
|
1173
|
+
if (pError) return fDone(pError);
|
|
1174
|
+
Expect(pResponse.body.Datasets).to.be.an('array');
|
|
1175
|
+
Expect(pResponse.body.Datasets.length).to.equal(1);
|
|
1176
|
+
Expect(pResponse.body.Type).to.equal('Raw');
|
|
1177
|
+
return fDone();
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
);
|
|
1181
|
+
}
|
|
1182
|
+
);
|
|
1183
|
+
|
|
1184
|
+
suite
|
|
1185
|
+
(
|
|
1186
|
+
'Schema Auto-Creation',
|
|
1187
|
+
function()
|
|
1188
|
+
{
|
|
1189
|
+
test
|
|
1190
|
+
(
|
|
1191
|
+
'Should expose FACTO_SCHEMA_SQL on module exports',
|
|
1192
|
+
function()
|
|
1193
|
+
{
|
|
1194
|
+
const libRetoldFacto = require('../source/Retold-Facto.js');
|
|
1195
|
+
Expect(libRetoldFacto.FACTO_SCHEMA_SQL).to.be.a('string');
|
|
1196
|
+
Expect(libRetoldFacto.FACTO_SCHEMA_SQL).to.contain('CREATE TABLE IF NOT EXISTS Source');
|
|
1197
|
+
Expect(libRetoldFacto.FACTO_SCHEMA_SQL).to.contain('CREATE TABLE IF NOT EXISTS IngestJob');
|
|
1198
|
+
}
|
|
1199
|
+
);
|
|
1200
|
+
test
|
|
1201
|
+
(
|
|
1202
|
+
'Should have createSchema method on the service instance',
|
|
1203
|
+
function()
|
|
1204
|
+
{
|
|
1205
|
+
Expect(_RetoldFacto.createSchema).to.be.a('function');
|
|
1206
|
+
}
|
|
1207
|
+
);
|
|
1208
|
+
test
|
|
1209
|
+
(
|
|
1210
|
+
'Should be able to run createSchema without error (tables already exist)',
|
|
1211
|
+
function(fDone)
|
|
1212
|
+
{
|
|
1213
|
+
_RetoldFacto.createSchema(
|
|
1214
|
+
(pError) =>
|
|
1215
|
+
{
|
|
1216
|
+
Expect(pError).to.not.exist;
|
|
1217
|
+
return fDone();
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
);
|
|
1223
|
+
|
|
1224
|
+
suite
|
|
1225
|
+
(
|
|
1226
|
+
'CSV/JSON File Ingest via API',
|
|
1227
|
+
function()
|
|
1228
|
+
{
|
|
1229
|
+
test
|
|
1230
|
+
(
|
|
1231
|
+
'Should ingest CSV content via POST /facto/ingest/file',
|
|
1232
|
+
function(fDone)
|
|
1233
|
+
{
|
|
1234
|
+
this.timeout(10000);
|
|
1235
|
+
let tmpCSVContent = 'name,state,population\nAlaska,AK,733391\nDelaware,DE,989948\nVermont,VT,643077';
|
|
1236
|
+
|
|
1237
|
+
_SuperTest
|
|
1238
|
+
.post('/facto/ingest/file')
|
|
1239
|
+
.send(
|
|
1240
|
+
{
|
|
1241
|
+
IDDataset: 1,
|
|
1242
|
+
IDSource: 1,
|
|
1243
|
+
Format: 'csv',
|
|
1244
|
+
Type: 'state-population',
|
|
1245
|
+
Content: tmpCSVContent
|
|
1246
|
+
})
|
|
1247
|
+
.expect(200)
|
|
1248
|
+
.end(
|
|
1249
|
+
(pError, pResponse) =>
|
|
1250
|
+
{
|
|
1251
|
+
if (pError) return fDone(pError);
|
|
1252
|
+
Expect(pResponse.body.Ingested).to.equal(3);
|
|
1253
|
+
Expect(pResponse.body.Errors).to.equal(0);
|
|
1254
|
+
Expect(pResponse.body.Total).to.equal(3);
|
|
1255
|
+
Expect(pResponse.body.Format).to.equal('csv');
|
|
1256
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
1257
|
+
Expect(pResponse.body.Records.length).to.equal(3);
|
|
1258
|
+
// Verify content was stored as JSON
|
|
1259
|
+
let tmpContent = JSON.parse(pResponse.body.Records[0].Content);
|
|
1260
|
+
Expect(tmpContent.name).to.equal('Alaska');
|
|
1261
|
+
Expect(tmpContent.state).to.equal('AK');
|
|
1262
|
+
return fDone();
|
|
1263
|
+
});
|
|
1264
|
+
}
|
|
1265
|
+
);
|
|
1266
|
+
test
|
|
1267
|
+
(
|
|
1268
|
+
'Should ingest JSON array content via POST /facto/ingest/file',
|
|
1269
|
+
function(fDone)
|
|
1270
|
+
{
|
|
1271
|
+
this.timeout(10000);
|
|
1272
|
+
let tmpJSONContent = JSON.stringify([
|
|
1273
|
+
{ county: 'San Diego', state: 'CA', population: 3338330 },
|
|
1274
|
+
{ county: 'Orange', state: 'CA', population: 3186989 }
|
|
1275
|
+
]);
|
|
1276
|
+
|
|
1277
|
+
_SuperTest
|
|
1278
|
+
.post('/facto/ingest/file')
|
|
1279
|
+
.send(
|
|
1280
|
+
{
|
|
1281
|
+
IDDataset: 1,
|
|
1282
|
+
IDSource: 1,
|
|
1283
|
+
Format: 'json',
|
|
1284
|
+
Type: 'county-population',
|
|
1285
|
+
Content: tmpJSONContent
|
|
1286
|
+
})
|
|
1287
|
+
.expect(200)
|
|
1288
|
+
.end(
|
|
1289
|
+
(pError, pResponse) =>
|
|
1290
|
+
{
|
|
1291
|
+
if (pError) return fDone(pError);
|
|
1292
|
+
Expect(pResponse.body.Ingested).to.equal(2);
|
|
1293
|
+
Expect(pResponse.body.Errors).to.equal(0);
|
|
1294
|
+
Expect(pResponse.body.Format).to.equal('json');
|
|
1295
|
+
return fDone();
|
|
1296
|
+
});
|
|
1297
|
+
}
|
|
1298
|
+
);
|
|
1299
|
+
test
|
|
1300
|
+
(
|
|
1301
|
+
'Should auto-detect JSON format',
|
|
1302
|
+
function(fDone)
|
|
1303
|
+
{
|
|
1304
|
+
this.timeout(10000);
|
|
1305
|
+
let tmpContent = JSON.stringify({ data: [{ metric: 'gdp', value: 21433 }] });
|
|
1306
|
+
|
|
1307
|
+
_SuperTest
|
|
1308
|
+
.post('/facto/ingest/file')
|
|
1309
|
+
.send(
|
|
1310
|
+
{
|
|
1311
|
+
IDDataset: 1,
|
|
1312
|
+
IDSource: 1,
|
|
1313
|
+
Type: 'economic',
|
|
1314
|
+
Content: tmpContent
|
|
1315
|
+
})
|
|
1316
|
+
.expect(200)
|
|
1317
|
+
.end(
|
|
1318
|
+
(pError, pResponse) =>
|
|
1319
|
+
{
|
|
1320
|
+
if (pError) return fDone(pError);
|
|
1321
|
+
Expect(pResponse.body.Ingested).to.equal(1);
|
|
1322
|
+
Expect(pResponse.body.Format).to.equal('json');
|
|
1323
|
+
return fDone();
|
|
1324
|
+
});
|
|
1325
|
+
}
|
|
1326
|
+
);
|
|
1327
|
+
test
|
|
1328
|
+
(
|
|
1329
|
+
'Should auto-detect CSV format',
|
|
1330
|
+
function(fDone)
|
|
1331
|
+
{
|
|
1332
|
+
this.timeout(10000);
|
|
1333
|
+
let tmpContent = 'key,value\nalpha,100\nbeta,200';
|
|
1334
|
+
|
|
1335
|
+
_SuperTest
|
|
1336
|
+
.post('/facto/ingest/file')
|
|
1337
|
+
.send(
|
|
1338
|
+
{
|
|
1339
|
+
IDDataset: 1,
|
|
1340
|
+
IDSource: 1,
|
|
1341
|
+
Content: tmpContent
|
|
1342
|
+
})
|
|
1343
|
+
.expect(200)
|
|
1344
|
+
.end(
|
|
1345
|
+
(pError, pResponse) =>
|
|
1346
|
+
{
|
|
1347
|
+
if (pError) return fDone(pError);
|
|
1348
|
+
Expect(pResponse.body.Ingested).to.equal(2);
|
|
1349
|
+
Expect(pResponse.body.Format).to.equal('csv');
|
|
1350
|
+
return fDone();
|
|
1351
|
+
});
|
|
1352
|
+
}
|
|
1353
|
+
);
|
|
1354
|
+
test
|
|
1355
|
+
(
|
|
1356
|
+
'Should return error when Content is missing',
|
|
1357
|
+
function(fDone)
|
|
1358
|
+
{
|
|
1359
|
+
_SuperTest
|
|
1360
|
+
.post('/facto/ingest/file')
|
|
1361
|
+
.send({ IDDataset: 1, IDSource: 1 })
|
|
1362
|
+
.expect(200)
|
|
1363
|
+
.end(
|
|
1364
|
+
(pError, pResponse) =>
|
|
1365
|
+
{
|
|
1366
|
+
if (pError) return fDone(pError);
|
|
1367
|
+
Expect(pResponse.body.Error).to.be.a('string');
|
|
1368
|
+
Expect(pResponse.body.Ingested).to.equal(0);
|
|
1369
|
+
return fDone();
|
|
1370
|
+
});
|
|
1371
|
+
}
|
|
1372
|
+
);
|
|
1373
|
+
}
|
|
1374
|
+
);
|
|
1375
|
+
|
|
1376
|
+
suite
|
|
1377
|
+
(
|
|
1378
|
+
'Programmatic File Ingest',
|
|
1379
|
+
function()
|
|
1380
|
+
{
|
|
1381
|
+
test
|
|
1382
|
+
(
|
|
1383
|
+
'Should ingest a CSV file from disk',
|
|
1384
|
+
function(fDone)
|
|
1385
|
+
{
|
|
1386
|
+
this.timeout(10000);
|
|
1387
|
+
|
|
1388
|
+
// Write a temp CSV file
|
|
1389
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.csv');
|
|
1390
|
+
libFs.writeFileSync(tmpFilePath, 'city,state,zip\nPortland,OR,97201\nSeattle,WA,98101\nBoise,ID,83702\n');
|
|
1391
|
+
|
|
1392
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
1393
|
+
{ type: 'city-data' },
|
|
1394
|
+
(pError, pResult) =>
|
|
1395
|
+
{
|
|
1396
|
+
// Clean up temp file
|
|
1397
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
1398
|
+
|
|
1399
|
+
if (pError) return fDone(pError);
|
|
1400
|
+
Expect(pResult.Ingested).to.equal(3);
|
|
1401
|
+
Expect(pResult.Errors).to.equal(0);
|
|
1402
|
+
Expect(pResult.Format).to.equal('csv');
|
|
1403
|
+
Expect(pResult.Records).to.be.an('array');
|
|
1404
|
+
return fDone();
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1407
|
+
);
|
|
1408
|
+
test
|
|
1409
|
+
(
|
|
1410
|
+
'Should ingest a JSON file from disk',
|
|
1411
|
+
function(fDone)
|
|
1412
|
+
{
|
|
1413
|
+
this.timeout(10000);
|
|
1414
|
+
|
|
1415
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.json');
|
|
1416
|
+
libFs.writeFileSync(tmpFilePath, JSON.stringify([
|
|
1417
|
+
{ region: 'West', count: 50 },
|
|
1418
|
+
{ region: 'East', count: 45 }
|
|
1419
|
+
]));
|
|
1420
|
+
|
|
1421
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
1422
|
+
{ type: 'region-data' },
|
|
1423
|
+
(pError, pResult) =>
|
|
1424
|
+
{
|
|
1425
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
1426
|
+
|
|
1427
|
+
if (pError) return fDone(pError);
|
|
1428
|
+
Expect(pResult.Ingested).to.equal(2);
|
|
1429
|
+
Expect(pResult.Format).to.equal('json');
|
|
1430
|
+
return fDone();
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
);
|
|
1434
|
+
test
|
|
1435
|
+
(
|
|
1436
|
+
'Should return error for non-existent file',
|
|
1437
|
+
function(fDone)
|
|
1438
|
+
{
|
|
1439
|
+
_Fable.RetoldFactoIngestEngine.ingestFile('/tmp/non-existent-file-12345.csv', 1, 1,
|
|
1440
|
+
(pError, pResult) =>
|
|
1441
|
+
{
|
|
1442
|
+
Expect(pError).to.be.an.instanceOf(Error);
|
|
1443
|
+
return fDone();
|
|
1444
|
+
});
|
|
1445
|
+
}
|
|
1446
|
+
);
|
|
1447
|
+
test
|
|
1448
|
+
(
|
|
1449
|
+
'Should return error for unknown file extension',
|
|
1450
|
+
function(fDone)
|
|
1451
|
+
{
|
|
1452
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.xyz');
|
|
1453
|
+
libFs.writeFileSync(tmpFilePath, 'some unknown format data');
|
|
1454
|
+
|
|
1455
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
1456
|
+
(pError, pResult) =>
|
|
1457
|
+
{
|
|
1458
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
1459
|
+
|
|
1460
|
+
Expect(pError).to.be.an.instanceOf(Error);
|
|
1461
|
+
Expect(pError.message).to.contain('Cannot determine format');
|
|
1462
|
+
return fDone();
|
|
1463
|
+
});
|
|
1464
|
+
}
|
|
1465
|
+
);
|
|
1466
|
+
test
|
|
1467
|
+
(
|
|
1468
|
+
'Should handle TSV files with auto-detected tab delimiter',
|
|
1469
|
+
function(fDone)
|
|
1470
|
+
{
|
|
1471
|
+
this.timeout(10000);
|
|
1472
|
+
|
|
1473
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.tsv');
|
|
1474
|
+
libFs.writeFileSync(tmpFilePath, 'name\tscore\nAlice\t95\nBob\t87\n');
|
|
1475
|
+
|
|
1476
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
1477
|
+
{ type: 'scores' },
|
|
1478
|
+
(pError, pResult) =>
|
|
1479
|
+
{
|
|
1480
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
1481
|
+
|
|
1482
|
+
if (pError) return fDone(pError);
|
|
1483
|
+
Expect(pResult.Ingested).to.equal(2);
|
|
1484
|
+
Expect(pResult.Format).to.equal('csv');
|
|
1485
|
+
return fDone();
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
);
|
|
1489
|
+
}
|
|
1490
|
+
);
|
|
1491
|
+
|
|
1492
|
+
suite
|
|
1493
|
+
(
|
|
1494
|
+
'Projection Engine Advanced',
|
|
1495
|
+
function()
|
|
1496
|
+
{
|
|
1497
|
+
test
|
|
1498
|
+
(
|
|
1499
|
+
'Should query records across datasets',
|
|
1500
|
+
function(fDone)
|
|
1501
|
+
{
|
|
1502
|
+
this.timeout(10000);
|
|
1503
|
+
_SuperTest
|
|
1504
|
+
.post('/facto/projections/query')
|
|
1505
|
+
.send({ DatasetIDs: [1], Begin: 0, Cap: 100 })
|
|
1506
|
+
.expect(200)
|
|
1507
|
+
.end(
|
|
1508
|
+
(pError, pResponse) =>
|
|
1509
|
+
{
|
|
1510
|
+
if (pError) return fDone(pError);
|
|
1511
|
+
Expect(pResponse.body.Query).to.be.an('object');
|
|
1512
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
1513
|
+
Expect(pResponse.body.Records.length).to.be.greaterThan(0);
|
|
1514
|
+
Expect(pResponse.body.Count).to.be.greaterThan(0);
|
|
1515
|
+
return fDone();
|
|
1516
|
+
});
|
|
1517
|
+
}
|
|
1518
|
+
);
|
|
1519
|
+
test
|
|
1520
|
+
(
|
|
1521
|
+
'Should query with type filter',
|
|
1522
|
+
function(fDone)
|
|
1523
|
+
{
|
|
1524
|
+
this.timeout(10000);
|
|
1525
|
+
_SuperTest
|
|
1526
|
+
.post('/facto/projections/query')
|
|
1527
|
+
.send({ DatasetIDs: [1], Type: 'census-population', Begin: 0, Cap: 100 })
|
|
1528
|
+
.expect(200)
|
|
1529
|
+
.end(
|
|
1530
|
+
(pError, pResponse) =>
|
|
1531
|
+
{
|
|
1532
|
+
if (pError) return fDone(pError);
|
|
1533
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
1534
|
+
// All returned records should be of the filtered type
|
|
1535
|
+
for (let i = 0; i < pResponse.body.Records.length; i++)
|
|
1536
|
+
{
|
|
1537
|
+
Expect(pResponse.body.Records[i].Type).to.equal('census-population');
|
|
1538
|
+
}
|
|
1539
|
+
return fDone();
|
|
1540
|
+
});
|
|
1541
|
+
}
|
|
1542
|
+
);
|
|
1543
|
+
test
|
|
1544
|
+
(
|
|
1545
|
+
'Should aggregate records by dataset',
|
|
1546
|
+
function(fDone)
|
|
1547
|
+
{
|
|
1548
|
+
this.timeout(10000);
|
|
1549
|
+
_SuperTest
|
|
1550
|
+
.post('/facto/projections/aggregate')
|
|
1551
|
+
.send({ DatasetIDs: [1], GroupBy: 'IDDataset' })
|
|
1552
|
+
.expect(200)
|
|
1553
|
+
.end(
|
|
1554
|
+
(pError, pResponse) =>
|
|
1555
|
+
{
|
|
1556
|
+
if (pError) return fDone(pError);
|
|
1557
|
+
Expect(pResponse.body.Aggregation).to.be.an('array');
|
|
1558
|
+
Expect(pResponse.body.Aggregation.length).to.be.greaterThan(0);
|
|
1559
|
+
Expect(pResponse.body.Total).to.be.greaterThan(0);
|
|
1560
|
+
return fDone();
|
|
1561
|
+
});
|
|
1562
|
+
}
|
|
1563
|
+
);
|
|
1564
|
+
test
|
|
1565
|
+
(
|
|
1566
|
+
'Should query certainty-weighted records',
|
|
1567
|
+
function(fDone)
|
|
1568
|
+
{
|
|
1569
|
+
this.timeout(10000);
|
|
1570
|
+
_SuperTest
|
|
1571
|
+
.post('/facto/projections/certainty')
|
|
1572
|
+
.send({ DatasetIDs: [1], MinCertainty: 0.0, MaxCertainty: 1.0, Begin: 0, Cap: 100 })
|
|
1573
|
+
.expect(200)
|
|
1574
|
+
.end(
|
|
1575
|
+
(pError, pResponse) =>
|
|
1576
|
+
{
|
|
1577
|
+
if (pError) return fDone(pError);
|
|
1578
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
1579
|
+
Expect(pResponse.body.Count).to.be.a('number');
|
|
1580
|
+
return fDone();
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1583
|
+
);
|
|
1584
|
+
test
|
|
1585
|
+
(
|
|
1586
|
+
'Should compare datasets',
|
|
1587
|
+
function(fDone)
|
|
1588
|
+
{
|
|
1589
|
+
this.timeout(10000);
|
|
1590
|
+
_SuperTest
|
|
1591
|
+
.post('/facto/projections/compare')
|
|
1592
|
+
.send({ DatasetIDs: [1, 2] })
|
|
1593
|
+
.expect(200)
|
|
1594
|
+
.end(
|
|
1595
|
+
(pError, pResponse) =>
|
|
1596
|
+
{
|
|
1597
|
+
if (pError) return fDone(pError);
|
|
1598
|
+
Expect(pResponse.body.Datasets).to.be.an('array');
|
|
1599
|
+
Expect(pResponse.body.Datasets.length).to.equal(2);
|
|
1600
|
+
// First dataset should have records, second should have none
|
|
1601
|
+
Expect(pResponse.body.Datasets[0].IDDataset).to.equal(1);
|
|
1602
|
+
Expect(pResponse.body.Datasets[0].RecordCount).to.be.greaterThan(0);
|
|
1603
|
+
Expect(pResponse.body.Datasets[1].IDDataset).to.equal(2);
|
|
1604
|
+
return fDone();
|
|
1605
|
+
});
|
|
1606
|
+
}
|
|
1607
|
+
);
|
|
1608
|
+
test
|
|
1609
|
+
(
|
|
1610
|
+
'Should get warehouse summary statistics',
|
|
1611
|
+
function(fDone)
|
|
1612
|
+
{
|
|
1613
|
+
this.timeout(10000);
|
|
1614
|
+
_SuperTest
|
|
1615
|
+
.get('/facto/projections/summary')
|
|
1616
|
+
.expect(200)
|
|
1617
|
+
.end(
|
|
1618
|
+
(pError, pResponse) =>
|
|
1619
|
+
{
|
|
1620
|
+
if (pError) return fDone(pError);
|
|
1621
|
+
Expect(pResponse.body.Sources).to.be.a('number');
|
|
1622
|
+
Expect(pResponse.body.Sources).to.be.greaterThan(0);
|
|
1623
|
+
Expect(pResponse.body.Datasets).to.be.a('number');
|
|
1624
|
+
Expect(pResponse.body.Datasets).to.be.greaterThan(0);
|
|
1625
|
+
Expect(pResponse.body.Records).to.be.a('number');
|
|
1626
|
+
Expect(pResponse.body.Records).to.be.greaterThan(0);
|
|
1627
|
+
Expect(pResponse.body.DatasetsByType).to.be.an('object');
|
|
1628
|
+
Expect(pResponse.body.DatasetsByType.Raw).to.be.a('number');
|
|
1629
|
+
Expect(pResponse.body.DatasetsByType.Projection).to.be.a('number');
|
|
1630
|
+
return fDone();
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
);
|
|
1634
|
+
test
|
|
1635
|
+
(
|
|
1636
|
+
'Should return empty results for non-existent dataset query',
|
|
1637
|
+
function(fDone)
|
|
1638
|
+
{
|
|
1639
|
+
this.timeout(10000);
|
|
1640
|
+
_SuperTest
|
|
1641
|
+
.post('/facto/projections/query')
|
|
1642
|
+
.send({ DatasetIDs: [999], Begin: 0, Cap: 100 })
|
|
1643
|
+
.expect(200)
|
|
1644
|
+
.end(
|
|
1645
|
+
(pError, pResponse) =>
|
|
1646
|
+
{
|
|
1647
|
+
if (pError) return fDone(pError);
|
|
1648
|
+
Expect(pResponse.body.Records).to.be.an('array');
|
|
1649
|
+
Expect(pResponse.body.Records.length).to.equal(0);
|
|
1650
|
+
return fDone();
|
|
1651
|
+
});
|
|
1652
|
+
}
|
|
1653
|
+
);
|
|
1654
|
+
}
|
|
1655
|
+
);
|
|
1656
|
+
|
|
1657
|
+
suite
|
|
1658
|
+
(
|
|
1659
|
+
'CSV/JSON Parsing Methods',
|
|
1660
|
+
function()
|
|
1661
|
+
{
|
|
1662
|
+
test
|
|
1663
|
+
(
|
|
1664
|
+
'parseCSV should parse a basic CSV string',
|
|
1665
|
+
function(fDone)
|
|
1666
|
+
{
|
|
1667
|
+
_Fable.RetoldFactoIngestEngine.parseCSV('a,b\n1,2\n3,4',
|
|
1668
|
+
(pError, pRecords) =>
|
|
1669
|
+
{
|
|
1670
|
+
Expect(pError).to.not.exist;
|
|
1671
|
+
Expect(pRecords).to.be.an('array');
|
|
1672
|
+
Expect(pRecords.length).to.equal(2);
|
|
1673
|
+
Expect(pRecords[0].a).to.equal('1');
|
|
1674
|
+
Expect(pRecords[0].b).to.equal('2');
|
|
1675
|
+
return fDone();
|
|
1676
|
+
});
|
|
1677
|
+
}
|
|
1678
|
+
);
|
|
1679
|
+
test
|
|
1680
|
+
(
|
|
1681
|
+
'parseJSON should parse an array',
|
|
1682
|
+
function(fDone)
|
|
1683
|
+
{
|
|
1684
|
+
_Fable.RetoldFactoIngestEngine.parseJSON('[{"x":1},{"x":2}]',
|
|
1685
|
+
(pError, pRecords) =>
|
|
1686
|
+
{
|
|
1687
|
+
Expect(pError).to.not.exist;
|
|
1688
|
+
Expect(pRecords).to.be.an('array');
|
|
1689
|
+
Expect(pRecords.length).to.equal(2);
|
|
1690
|
+
return fDone();
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
);
|
|
1694
|
+
test
|
|
1695
|
+
(
|
|
1696
|
+
'parseJSON should extract data key from object',
|
|
1697
|
+
function(fDone)
|
|
1698
|
+
{
|
|
1699
|
+
_Fable.RetoldFactoIngestEngine.parseJSON('{"data":[{"y":10}]}',
|
|
1700
|
+
(pError, pRecords) =>
|
|
1701
|
+
{
|
|
1702
|
+
Expect(pError).to.not.exist;
|
|
1703
|
+
Expect(pRecords).to.be.an('array');
|
|
1704
|
+
Expect(pRecords.length).to.equal(1);
|
|
1705
|
+
Expect(pRecords[0].y).to.equal(10);
|
|
1706
|
+
return fDone();
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
);
|
|
1710
|
+
test
|
|
1711
|
+
(
|
|
1712
|
+
'parseJSON should wrap a single object in an array',
|
|
1713
|
+
function(fDone)
|
|
1714
|
+
{
|
|
1715
|
+
_Fable.RetoldFactoIngestEngine.parseJSON('{"solo":true}',
|
|
1716
|
+
(pError, pRecords) =>
|
|
1717
|
+
{
|
|
1718
|
+
Expect(pError).to.not.exist;
|
|
1719
|
+
Expect(pRecords).to.be.an('array');
|
|
1720
|
+
Expect(pRecords.length).to.equal(1);
|
|
1721
|
+
Expect(pRecords[0].solo).to.equal(true);
|
|
1722
|
+
return fDone();
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
);
|
|
1726
|
+
test
|
|
1727
|
+
(
|
|
1728
|
+
'parseJSON should return error for invalid JSON',
|
|
1729
|
+
function(fDone)
|
|
1730
|
+
{
|
|
1731
|
+
_Fable.RetoldFactoIngestEngine.parseJSON('not valid json {{{',
|
|
1732
|
+
(pError, pRecords) =>
|
|
1733
|
+
{
|
|
1734
|
+
Expect(pError).to.be.an.instanceOf(Error);
|
|
1735
|
+
return fDone();
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
);
|
|
1739
|
+
}
|
|
1740
|
+
);
|
|
1741
|
+
suite
|
|
1742
|
+
(
|
|
1743
|
+
'Multi-Format Ingest',
|
|
1744
|
+
function()
|
|
1745
|
+
{
|
|
1746
|
+
// ========================================================
|
|
1747
|
+
// XML Parsing
|
|
1748
|
+
// ========================================================
|
|
1749
|
+
test
|
|
1750
|
+
(
|
|
1751
|
+
'parseXML should parse a basic XML array',
|
|
1752
|
+
function(fDone)
|
|
1753
|
+
{
|
|
1754
|
+
let tmpXML = '<root><item><name>Alice</name><score>95</score></item><item><name>Bob</name><score>87</score></item></root>';
|
|
1755
|
+
_Fable.RetoldFactoIngestEngine.parseXML(tmpXML,
|
|
1756
|
+
(pError, pRecords) =>
|
|
1757
|
+
{
|
|
1758
|
+
Expect(pError).to.not.exist;
|
|
1759
|
+
Expect(pRecords).to.be.an('array');
|
|
1760
|
+
Expect(pRecords.length).to.equal(2);
|
|
1761
|
+
Expect(pRecords[0].name).to.equal('Alice');
|
|
1762
|
+
Expect(pRecords[1].score).to.equal(87);
|
|
1763
|
+
return fDone();
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
);
|
|
1767
|
+
test
|
|
1768
|
+
(
|
|
1769
|
+
'parseXML should auto-detect nested record array',
|
|
1770
|
+
function(fDone)
|
|
1771
|
+
{
|
|
1772
|
+
let tmpXML = '<?xml version="1.0"?><response><meta><total>2</total></meta><data><record><id>1</id><val>A</val></record><record><id>2</id><val>B</val></record></data></response>';
|
|
1773
|
+
_Fable.RetoldFactoIngestEngine.parseXML(tmpXML,
|
|
1774
|
+
(pError, pRecords) =>
|
|
1775
|
+
{
|
|
1776
|
+
Expect(pError).to.not.exist;
|
|
1777
|
+
Expect(pRecords).to.be.an('array');
|
|
1778
|
+
Expect(pRecords.length).to.equal(2);
|
|
1779
|
+
Expect(pRecords[0].id).to.equal(1);
|
|
1780
|
+
Expect(pRecords[1].val).to.equal('B');
|
|
1781
|
+
return fDone();
|
|
1782
|
+
});
|
|
1783
|
+
}
|
|
1784
|
+
);
|
|
1785
|
+
test
|
|
1786
|
+
(
|
|
1787
|
+
'parseXML should navigate with recordPath option',
|
|
1788
|
+
function(fDone)
|
|
1789
|
+
{
|
|
1790
|
+
let tmpXML = '<api><results><items><item><x>10</x></item><item><x>20</x></item></items></results></api>';
|
|
1791
|
+
_Fable.RetoldFactoIngestEngine.parseXML(tmpXML, { recordPath: 'api.results.items.item' },
|
|
1792
|
+
(pError, pRecords) =>
|
|
1793
|
+
{
|
|
1794
|
+
Expect(pError).to.not.exist;
|
|
1795
|
+
Expect(pRecords).to.be.an('array');
|
|
1796
|
+
Expect(pRecords.length).to.equal(2);
|
|
1797
|
+
Expect(pRecords[0].x).to.equal(10);
|
|
1798
|
+
return fDone();
|
|
1799
|
+
});
|
|
1800
|
+
}
|
|
1801
|
+
);
|
|
1802
|
+
test
|
|
1803
|
+
(
|
|
1804
|
+
'parseXML should return error for malformed XML',
|
|
1805
|
+
function(fDone)
|
|
1806
|
+
{
|
|
1807
|
+
_Fable.RetoldFactoIngestEngine.parseXML('<<<not xml at all>>>',
|
|
1808
|
+
(pError, pRecords) =>
|
|
1809
|
+
{
|
|
1810
|
+
// fast-xml-parser may not error on malformed XML but should at least return something
|
|
1811
|
+
// The key test is that it doesn't crash
|
|
1812
|
+
Expect(pRecords).to.exist;
|
|
1813
|
+
return fDone();
|
|
1814
|
+
});
|
|
1815
|
+
}
|
|
1816
|
+
);
|
|
1817
|
+
test
|
|
1818
|
+
(
|
|
1819
|
+
'parseXML should return error for invalid recordPath',
|
|
1820
|
+
function(fDone)
|
|
1821
|
+
{
|
|
1822
|
+
let tmpXML = '<root><item>test</item></root>';
|
|
1823
|
+
_Fable.RetoldFactoIngestEngine.parseXML(tmpXML, { recordPath: 'root.nonexistent.path' },
|
|
1824
|
+
(pError, pRecords) =>
|
|
1825
|
+
{
|
|
1826
|
+
Expect(pError).to.be.an.instanceOf(Error);
|
|
1827
|
+
Expect(pError.message).to.contain('not found in XML');
|
|
1828
|
+
return fDone();
|
|
1829
|
+
});
|
|
1830
|
+
}
|
|
1831
|
+
);
|
|
1832
|
+
|
|
1833
|
+
// ========================================================
|
|
1834
|
+
// Excel Parsing
|
|
1835
|
+
// ========================================================
|
|
1836
|
+
test
|
|
1837
|
+
(
|
|
1838
|
+
'parseExcel should parse a basic workbook',
|
|
1839
|
+
function(fDone)
|
|
1840
|
+
{
|
|
1841
|
+
let libXLSX = require('xlsx');
|
|
1842
|
+
let tmpWorksheet = libXLSX.utils.aoa_to_sheet([
|
|
1843
|
+
['name', 'score'],
|
|
1844
|
+
['Alice', 95],
|
|
1845
|
+
['Bob', 87]
|
|
1846
|
+
]);
|
|
1847
|
+
let tmpWorkbook = libXLSX.utils.book_new();
|
|
1848
|
+
libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Sheet1');
|
|
1849
|
+
let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
|
|
1850
|
+
|
|
1851
|
+
_Fable.RetoldFactoIngestEngine.parseExcel(tmpBuffer,
|
|
1852
|
+
(pError, pRecords) =>
|
|
1853
|
+
{
|
|
1854
|
+
Expect(pError).to.not.exist;
|
|
1855
|
+
Expect(pRecords).to.be.an('array');
|
|
1856
|
+
Expect(pRecords.length).to.equal(2);
|
|
1857
|
+
Expect(pRecords[0].name).to.equal('Alice');
|
|
1858
|
+
Expect(pRecords[0].score).to.equal(95);
|
|
1859
|
+
return fDone();
|
|
1860
|
+
});
|
|
1861
|
+
}
|
|
1862
|
+
);
|
|
1863
|
+
test
|
|
1864
|
+
(
|
|
1865
|
+
'parseExcel should select sheet by name',
|
|
1866
|
+
function(fDone)
|
|
1867
|
+
{
|
|
1868
|
+
let libXLSX = require('xlsx');
|
|
1869
|
+
let tmpSheet1 = libXLSX.utils.aoa_to_sheet([['a'], [1]]);
|
|
1870
|
+
let tmpSheet2 = libXLSX.utils.aoa_to_sheet([['b'], [2], [3]]);
|
|
1871
|
+
let tmpWorkbook = libXLSX.utils.book_new();
|
|
1872
|
+
libXLSX.utils.book_append_sheet(tmpWorkbook, tmpSheet1, 'First');
|
|
1873
|
+
libXLSX.utils.book_append_sheet(tmpWorkbook, tmpSheet2, 'Second');
|
|
1874
|
+
let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
|
|
1875
|
+
|
|
1876
|
+
_Fable.RetoldFactoIngestEngine.parseExcel(tmpBuffer, { sheet: 'Second' },
|
|
1877
|
+
(pError, pRecords) =>
|
|
1878
|
+
{
|
|
1879
|
+
Expect(pError).to.not.exist;
|
|
1880
|
+
Expect(pRecords).to.be.an('array');
|
|
1881
|
+
Expect(pRecords.length).to.equal(2);
|
|
1882
|
+
Expect(pRecords[0].b).to.equal(2);
|
|
1883
|
+
return fDone();
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
);
|
|
1887
|
+
test
|
|
1888
|
+
(
|
|
1889
|
+
'parseExcel should return empty array for empty sheet',
|
|
1890
|
+
function(fDone)
|
|
1891
|
+
{
|
|
1892
|
+
let libXLSX = require('xlsx');
|
|
1893
|
+
let tmpWorksheet = libXLSX.utils.aoa_to_sheet([]);
|
|
1894
|
+
let tmpWorkbook = libXLSX.utils.book_new();
|
|
1895
|
+
libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Empty');
|
|
1896
|
+
let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
|
|
1897
|
+
|
|
1898
|
+
_Fable.RetoldFactoIngestEngine.parseExcel(tmpBuffer,
|
|
1899
|
+
(pError, pRecords) =>
|
|
1900
|
+
{
|
|
1901
|
+
Expect(pError).to.not.exist;
|
|
1902
|
+
Expect(pRecords).to.be.an('array');
|
|
1903
|
+
Expect(pRecords.length).to.equal(0);
|
|
1904
|
+
return fDone();
|
|
1905
|
+
});
|
|
1906
|
+
}
|
|
1907
|
+
);
|
|
1908
|
+
|
|
1909
|
+
// ========================================================
|
|
1910
|
+
// Fixed-Width Parsing
|
|
1911
|
+
// ========================================================
|
|
1912
|
+
test
|
|
1913
|
+
(
|
|
1914
|
+
'parseFixedWidth should extract columns by position',
|
|
1915
|
+
function(fDone)
|
|
1916
|
+
{
|
|
1917
|
+
let tmpContent = 'ACM12345678 37.7749 -122.4194\nBCN98765432 41.3851 2.1734 \n';
|
|
1918
|
+
_Fable.RetoldFactoIngestEngine.parseFixedWidth(tmpContent,
|
|
1919
|
+
{
|
|
1920
|
+
columns: [
|
|
1921
|
+
{ name: 'ID', start: 1, width: 11 },
|
|
1922
|
+
{ name: 'Lat', start: 13, width: 8 },
|
|
1923
|
+
{ name: 'Lon', start: 22, width: 9 }
|
|
1924
|
+
]
|
|
1925
|
+
},
|
|
1926
|
+
(pError, pRecords) =>
|
|
1927
|
+
{
|
|
1928
|
+
Expect(pError).to.not.exist;
|
|
1929
|
+
Expect(pRecords).to.be.an('array');
|
|
1930
|
+
Expect(pRecords.length).to.equal(2);
|
|
1931
|
+
Expect(pRecords[0].ID).to.equal('ACM12345678');
|
|
1932
|
+
Expect(pRecords[0].Lat).to.equal('37.7749');
|
|
1933
|
+
Expect(pRecords[1].ID).to.equal('BCN98765432');
|
|
1934
|
+
return fDone();
|
|
1935
|
+
});
|
|
1936
|
+
}
|
|
1937
|
+
);
|
|
1938
|
+
test
|
|
1939
|
+
(
|
|
1940
|
+
'parseFixedWidth should skip lines with skipLines option',
|
|
1941
|
+
function(fDone)
|
|
1942
|
+
{
|
|
1943
|
+
let tmpContent = 'HEADER LINE 1\nHEADER LINE 2\nDAT 100\nDAT 200\n';
|
|
1944
|
+
_Fable.RetoldFactoIngestEngine.parseFixedWidth(tmpContent,
|
|
1945
|
+
{
|
|
1946
|
+
columns: [
|
|
1947
|
+
{ name: 'Code', start: 1, width: 3 },
|
|
1948
|
+
{ name: 'Value', start: 5, width: 3 }
|
|
1949
|
+
],
|
|
1950
|
+
skipLines: 2
|
|
1951
|
+
},
|
|
1952
|
+
(pError, pRecords) =>
|
|
1953
|
+
{
|
|
1954
|
+
Expect(pError).to.not.exist;
|
|
1955
|
+
Expect(pRecords).to.be.an('array');
|
|
1956
|
+
Expect(pRecords.length).to.equal(2);
|
|
1957
|
+
Expect(pRecords[0].Code).to.equal('DAT');
|
|
1958
|
+
Expect(pRecords[0].Value).to.equal('100');
|
|
1959
|
+
return fDone();
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1962
|
+
);
|
|
1963
|
+
test
|
|
1964
|
+
(
|
|
1965
|
+
'parseFixedWidth should return error without columns option',
|
|
1966
|
+
function(fDone)
|
|
1967
|
+
{
|
|
1968
|
+
_Fable.RetoldFactoIngestEngine.parseFixedWidth('some data',
|
|
1969
|
+
(pError, pRecords) =>
|
|
1970
|
+
{
|
|
1971
|
+
Expect(pError).to.be.an.instanceOf(Error);
|
|
1972
|
+
Expect(pError.message).to.contain('columns');
|
|
1973
|
+
return fDone();
|
|
1974
|
+
});
|
|
1975
|
+
}
|
|
1976
|
+
);
|
|
1977
|
+
|
|
1978
|
+
// ========================================================
|
|
1979
|
+
// CSV Comment Stripping
|
|
1980
|
+
// ========================================================
|
|
1981
|
+
test
|
|
1982
|
+
(
|
|
1983
|
+
'parseCSV should strip comment lines when stripCommentLines is true',
|
|
1984
|
+
function(fDone)
|
|
1985
|
+
{
|
|
1986
|
+
let tmpContent = '# This is a comment\n# Another comment\nname,score\nAlice,95\nBob,87\n';
|
|
1987
|
+
_Fable.RetoldFactoIngestEngine.parseCSV(tmpContent, { stripCommentLines: true },
|
|
1988
|
+
(pError, pRecords) =>
|
|
1989
|
+
{
|
|
1990
|
+
Expect(pError).to.not.exist;
|
|
1991
|
+
Expect(pRecords).to.be.an('array');
|
|
1992
|
+
Expect(pRecords.length).to.equal(2);
|
|
1993
|
+
Expect(pRecords[0].name).to.equal('Alice');
|
|
1994
|
+
return fDone();
|
|
1995
|
+
});
|
|
1996
|
+
}
|
|
1997
|
+
);
|
|
1998
|
+
|
|
1999
|
+
// ========================================================
|
|
2000
|
+
// ingestFile with new extensions
|
|
2001
|
+
// ========================================================
|
|
2002
|
+
test
|
|
2003
|
+
(
|
|
2004
|
+
'ingestFile should handle .xml files',
|
|
2005
|
+
function(fDone)
|
|
2006
|
+
{
|
|
2007
|
+
this.timeout(10000);
|
|
2008
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.xml');
|
|
2009
|
+
libFs.writeFileSync(tmpFilePath, '<root><item><name>XMLAlice</name><value>100</value></item><item><name>XMLBob</name><value>200</value></item></root>');
|
|
2010
|
+
|
|
2011
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
2012
|
+
(pError, pResult) =>
|
|
2013
|
+
{
|
|
2014
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
2015
|
+
|
|
2016
|
+
Expect(pError).to.not.exist;
|
|
2017
|
+
Expect(pResult.Format).to.equal('xml');
|
|
2018
|
+
Expect(pResult.Ingested).to.equal(2);
|
|
2019
|
+
return fDone();
|
|
2020
|
+
});
|
|
2021
|
+
}
|
|
2022
|
+
);
|
|
2023
|
+
test
|
|
2024
|
+
(
|
|
2025
|
+
'ingestFile should handle .xlsx files',
|
|
2026
|
+
function(fDone)
|
|
2027
|
+
{
|
|
2028
|
+
this.timeout(10000);
|
|
2029
|
+
let libXLSX = require('xlsx');
|
|
2030
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.xlsx');
|
|
2031
|
+
let tmpWorksheet = libXLSX.utils.aoa_to_sheet([
|
|
2032
|
+
['city', 'pop'],
|
|
2033
|
+
['NYC', 8336817],
|
|
2034
|
+
['LA', 3979576]
|
|
2035
|
+
]);
|
|
2036
|
+
let tmpWorkbook = libXLSX.utils.book_new();
|
|
2037
|
+
libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Cities');
|
|
2038
|
+
libXLSX.writeFile(tmpWorkbook, tmpFilePath);
|
|
2039
|
+
|
|
2040
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
2041
|
+
(pError, pResult) =>
|
|
2042
|
+
{
|
|
2043
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
2044
|
+
|
|
2045
|
+
Expect(pError).to.not.exist;
|
|
2046
|
+
Expect(pResult.Format).to.equal('excel');
|
|
2047
|
+
Expect(pResult.Ingested).to.equal(2);
|
|
2048
|
+
return fDone();
|
|
2049
|
+
});
|
|
2050
|
+
}
|
|
2051
|
+
);
|
|
2052
|
+
test
|
|
2053
|
+
(
|
|
2054
|
+
'ingestFile should handle .fw (fixed-width) files',
|
|
2055
|
+
function(fDone)
|
|
2056
|
+
{
|
|
2057
|
+
this.timeout(10000);
|
|
2058
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.fw');
|
|
2059
|
+
libFs.writeFileSync(tmpFilePath, 'AAA 100\nBBB 200\nCCC 300\n');
|
|
2060
|
+
|
|
2061
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
2062
|
+
{
|
|
2063
|
+
columns: [
|
|
2064
|
+
{ name: 'Code', start: 1, width: 3 },
|
|
2065
|
+
{ name: 'Val', start: 5, width: 3 }
|
|
2066
|
+
]
|
|
2067
|
+
},
|
|
2068
|
+
(pError, pResult) =>
|
|
2069
|
+
{
|
|
2070
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
2071
|
+
|
|
2072
|
+
Expect(pError).to.not.exist;
|
|
2073
|
+
Expect(pResult.Format).to.equal('fixed-width');
|
|
2074
|
+
Expect(pResult.Ingested).to.equal(3);
|
|
2075
|
+
return fDone();
|
|
2076
|
+
});
|
|
2077
|
+
}
|
|
2078
|
+
);
|
|
2079
|
+
test
|
|
2080
|
+
(
|
|
2081
|
+
'ingestFile should handle .rdb files as TSV with comment stripping',
|
|
2082
|
+
function(fDone)
|
|
2083
|
+
{
|
|
2084
|
+
this.timeout(10000);
|
|
2085
|
+
let tmpFilePath = libPath.join(__dirname, 'tmp-test-ingest.rdb');
|
|
2086
|
+
let tmpContent = '# USGS comment line\n# Another comment\nagency_cd\tsite_no\tvalue\n5s\t15s\t14n\nUSGS\t01646500\t1234\nUSGS\t01646500\t5678\n';
|
|
2087
|
+
libFs.writeFileSync(tmpFilePath, tmpContent);
|
|
2088
|
+
|
|
2089
|
+
_Fable.RetoldFactoIngestEngine.ingestFile(tmpFilePath, 1, 1,
|
|
2090
|
+
(pError, pResult) =>
|
|
2091
|
+
{
|
|
2092
|
+
try { libFs.unlinkSync(tmpFilePath); } catch(e) {}
|
|
2093
|
+
|
|
2094
|
+
Expect(pError).to.not.exist;
|
|
2095
|
+
Expect(pResult.Format).to.equal('csv');
|
|
2096
|
+
Expect(pResult.Ingested).to.be.greaterThan(0);
|
|
2097
|
+
return fDone();
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
2100
|
+
);
|
|
2101
|
+
|
|
2102
|
+
// ========================================================
|
|
2103
|
+
// POST /facto/ingest/file with new formats
|
|
2104
|
+
// ========================================================
|
|
2105
|
+
test
|
|
2106
|
+
(
|
|
2107
|
+
'POST /facto/ingest/file should auto-detect XML content',
|
|
2108
|
+
function(fDone)
|
|
2109
|
+
{
|
|
2110
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2111
|
+
.send(
|
|
2112
|
+
{
|
|
2113
|
+
Content: '<items><item><name>PostXML1</name></item><item><name>PostXML2</name></item></items>',
|
|
2114
|
+
IDDataset: 1,
|
|
2115
|
+
IDSource: 1,
|
|
2116
|
+
Type: 'xml-test'
|
|
2117
|
+
})
|
|
2118
|
+
.expect(200)
|
|
2119
|
+
.end(
|
|
2120
|
+
(pError, pResponse) =>
|
|
2121
|
+
{
|
|
2122
|
+
Expect(pError).to.not.exist;
|
|
2123
|
+
Expect(pResponse.body.Format).to.equal('xml');
|
|
2124
|
+
Expect(pResponse.body.Ingested).to.equal(2);
|
|
2125
|
+
return fDone();
|
|
2126
|
+
});
|
|
2127
|
+
}
|
|
2128
|
+
);
|
|
2129
|
+
test
|
|
2130
|
+
(
|
|
2131
|
+
'POST /facto/ingest/file should accept explicit Format=xml',
|
|
2132
|
+
function(fDone)
|
|
2133
|
+
{
|
|
2134
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2135
|
+
.send(
|
|
2136
|
+
{
|
|
2137
|
+
Content: '<data><row><a>1</a></row><row><a>2</a></row><row><a>3</a></row></data>',
|
|
2138
|
+
Format: 'xml',
|
|
2139
|
+
IDDataset: 1,
|
|
2140
|
+
IDSource: 1
|
|
2141
|
+
})
|
|
2142
|
+
.expect(200)
|
|
2143
|
+
.end(
|
|
2144
|
+
(pError, pResponse) =>
|
|
2145
|
+
{
|
|
2146
|
+
Expect(pError).to.not.exist;
|
|
2147
|
+
Expect(pResponse.body.Format).to.equal('xml');
|
|
2148
|
+
Expect(pResponse.body.Ingested).to.equal(3);
|
|
2149
|
+
return fDone();
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
);
|
|
2153
|
+
test
|
|
2154
|
+
(
|
|
2155
|
+
'POST /facto/ingest/file should accept Format=fixed-width with Columns',
|
|
2156
|
+
function(fDone)
|
|
2157
|
+
{
|
|
2158
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2159
|
+
.send(
|
|
2160
|
+
{
|
|
2161
|
+
Content: 'AAA 100\nBBB 200\n',
|
|
2162
|
+
Format: 'fixed-width',
|
|
2163
|
+
Columns: [
|
|
2164
|
+
{ name: 'Code', start: 1, width: 3 },
|
|
2165
|
+
{ name: 'Val', start: 5, width: 3 }
|
|
2166
|
+
],
|
|
2167
|
+
IDDataset: 1,
|
|
2168
|
+
IDSource: 1
|
|
2169
|
+
})
|
|
2170
|
+
.expect(200)
|
|
2171
|
+
.end(
|
|
2172
|
+
(pError, pResponse) =>
|
|
2173
|
+
{
|
|
2174
|
+
Expect(pError).to.not.exist;
|
|
2175
|
+
Expect(pResponse.body.Format).to.equal('fixed-width');
|
|
2176
|
+
Expect(pResponse.body.Ingested).to.equal(2);
|
|
2177
|
+
return fDone();
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
);
|
|
2181
|
+
test
|
|
2182
|
+
(
|
|
2183
|
+
'POST /facto/ingest/file should reject fixed-width without Columns',
|
|
2184
|
+
function(fDone)
|
|
2185
|
+
{
|
|
2186
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2187
|
+
.send(
|
|
2188
|
+
{
|
|
2189
|
+
Content: 'AAA 100\nBBB 200\n',
|
|
2190
|
+
Format: 'fixed-width',
|
|
2191
|
+
IDDataset: 1,
|
|
2192
|
+
IDSource: 1
|
|
2193
|
+
})
|
|
2194
|
+
.expect(200)
|
|
2195
|
+
.end(
|
|
2196
|
+
(pError, pResponse) =>
|
|
2197
|
+
{
|
|
2198
|
+
Expect(pError).to.not.exist;
|
|
2199
|
+
Expect(pResponse.body.Error).to.contain('Columns');
|
|
2200
|
+
return fDone();
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
);
|
|
2204
|
+
test
|
|
2205
|
+
(
|
|
2206
|
+
'POST /facto/ingest/file should accept Format=excel with base64 content',
|
|
2207
|
+
function(fDone)
|
|
2208
|
+
{
|
|
2209
|
+
let libXLSX = require('xlsx');
|
|
2210
|
+
let tmpWorksheet = libXLSX.utils.aoa_to_sheet([
|
|
2211
|
+
['x', 'y'],
|
|
2212
|
+
[10, 20],
|
|
2213
|
+
[30, 40]
|
|
2214
|
+
]);
|
|
2215
|
+
let tmpWorkbook = libXLSX.utils.book_new();
|
|
2216
|
+
libXLSX.utils.book_append_sheet(tmpWorkbook, tmpWorksheet, 'Data');
|
|
2217
|
+
let tmpBuffer = libXLSX.write(tmpWorkbook, { type: 'buffer', bookType: 'xlsx' });
|
|
2218
|
+
let tmpBase64 = tmpBuffer.toString('base64');
|
|
2219
|
+
|
|
2220
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2221
|
+
.send(
|
|
2222
|
+
{
|
|
2223
|
+
Content: tmpBase64,
|
|
2224
|
+
Format: 'excel',
|
|
2225
|
+
IDDataset: 1,
|
|
2226
|
+
IDSource: 1,
|
|
2227
|
+
Type: 'excel-test'
|
|
2228
|
+
})
|
|
2229
|
+
.expect(200)
|
|
2230
|
+
.end(
|
|
2231
|
+
(pError, pResponse) =>
|
|
2232
|
+
{
|
|
2233
|
+
Expect(pError).to.not.exist;
|
|
2234
|
+
Expect(pResponse.body.Format).to.equal('excel');
|
|
2235
|
+
Expect(pResponse.body.Ingested).to.equal(2);
|
|
2236
|
+
return fDone();
|
|
2237
|
+
});
|
|
2238
|
+
}
|
|
2239
|
+
);
|
|
2240
|
+
}
|
|
2241
|
+
);
|
|
2242
|
+
suite
|
|
2243
|
+
(
|
|
2244
|
+
'Dataset Versioning',
|
|
2245
|
+
function()
|
|
2246
|
+
{
|
|
2247
|
+
test
|
|
2248
|
+
(
|
|
2249
|
+
'ingest should auto-create IngestJob with DatasetVersion=1',
|
|
2250
|
+
function(fDone)
|
|
2251
|
+
{
|
|
2252
|
+
this.timeout(10000);
|
|
2253
|
+
|
|
2254
|
+
// Create a dedicated dataset for version testing
|
|
2255
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2256
|
+
.send({ Name: 'Version Test Dataset', Type: 'Raw', Description: 'Testing versioning' })
|
|
2257
|
+
.expect(200)
|
|
2258
|
+
.end(
|
|
2259
|
+
(pError, pResponse) =>
|
|
2260
|
+
{
|
|
2261
|
+
Expect(pError).to.not.exist;
|
|
2262
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2263
|
+
Expect(tmpIDDataset).to.be.greaterThan(0);
|
|
2264
|
+
|
|
2265
|
+
// Ingest CSV data
|
|
2266
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2267
|
+
.send(
|
|
2268
|
+
{
|
|
2269
|
+
Content: 'name,value\nAlpha,100\nBeta,200\n',
|
|
2270
|
+
Format: 'csv',
|
|
2271
|
+
IDDataset: tmpIDDataset,
|
|
2272
|
+
IDSource: 1
|
|
2273
|
+
})
|
|
2274
|
+
.expect(200)
|
|
2275
|
+
.end(
|
|
2276
|
+
(pIngestError, pIngestResponse) =>
|
|
2277
|
+
{
|
|
2278
|
+
Expect(pIngestError).to.not.exist;
|
|
2279
|
+
Expect(pIngestResponse.body.Ingested).to.equal(2);
|
|
2280
|
+
Expect(pIngestResponse.body.DatasetVersion).to.equal(1);
|
|
2281
|
+
Expect(pIngestResponse.body.ContentSignature).to.be.a('string');
|
|
2282
|
+
Expect(pIngestResponse.body.ContentSignature.length).to.equal(64);
|
|
2283
|
+
Expect(pIngestResponse.body.IngestJob).to.exist;
|
|
2284
|
+
Expect(pIngestResponse.body.IngestJob.DatasetVersion).to.equal(1);
|
|
2285
|
+
return fDone();
|
|
2286
|
+
});
|
|
2287
|
+
});
|
|
2288
|
+
}
|
|
2289
|
+
);
|
|
2290
|
+
test
|
|
2291
|
+
(
|
|
2292
|
+
'second ingest should auto-increment to DatasetVersion=2',
|
|
2293
|
+
function(fDone)
|
|
2294
|
+
{
|
|
2295
|
+
this.timeout(10000);
|
|
2296
|
+
|
|
2297
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2298
|
+
.send({ Name: 'Version Increment Dataset', Type: 'Raw', Description: 'Testing version increment' })
|
|
2299
|
+
.expect(200)
|
|
2300
|
+
.end(
|
|
2301
|
+
(pError, pResponse) =>
|
|
2302
|
+
{
|
|
2303
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2304
|
+
|
|
2305
|
+
// First ingest
|
|
2306
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2307
|
+
.send(
|
|
2308
|
+
{
|
|
2309
|
+
Content: 'a,b\n1,2\n',
|
|
2310
|
+
Format: 'csv',
|
|
2311
|
+
IDDataset: tmpIDDataset,
|
|
2312
|
+
IDSource: 1
|
|
2313
|
+
})
|
|
2314
|
+
.expect(200)
|
|
2315
|
+
.end(
|
|
2316
|
+
(pErr1, pRes1) =>
|
|
2317
|
+
{
|
|
2318
|
+
Expect(pRes1.body.DatasetVersion).to.equal(1);
|
|
2319
|
+
|
|
2320
|
+
// Second ingest (different content)
|
|
2321
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2322
|
+
.send(
|
|
2323
|
+
{
|
|
2324
|
+
Content: 'a,b\n3,4\n5,6\n',
|
|
2325
|
+
Format: 'csv',
|
|
2326
|
+
IDDataset: tmpIDDataset,
|
|
2327
|
+
IDSource: 1
|
|
2328
|
+
})
|
|
2329
|
+
.expect(200)
|
|
2330
|
+
.end(
|
|
2331
|
+
(pErr2, pRes2) =>
|
|
2332
|
+
{
|
|
2333
|
+
Expect(pRes2.body.DatasetVersion).to.equal(2);
|
|
2334
|
+
Expect(pRes2.body.Ingested).to.equal(2);
|
|
2335
|
+
return fDone();
|
|
2336
|
+
});
|
|
2337
|
+
});
|
|
2338
|
+
});
|
|
2339
|
+
}
|
|
2340
|
+
);
|
|
2341
|
+
test
|
|
2342
|
+
(
|
|
2343
|
+
'records should have correct IDIngestJob FK',
|
|
2344
|
+
function(fDone)
|
|
2345
|
+
{
|
|
2346
|
+
this.timeout(10000);
|
|
2347
|
+
|
|
2348
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2349
|
+
.send({ Name: 'IngestJob FK Dataset', Type: 'Raw', Description: 'Testing IDIngestJob on records' })
|
|
2350
|
+
.expect(200)
|
|
2351
|
+
.end(
|
|
2352
|
+
(pError, pResponse) =>
|
|
2353
|
+
{
|
|
2354
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2355
|
+
|
|
2356
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2357
|
+
.send(
|
|
2358
|
+
{
|
|
2359
|
+
Content: 'x,y\n10,20\n',
|
|
2360
|
+
Format: 'csv',
|
|
2361
|
+
IDDataset: tmpIDDataset,
|
|
2362
|
+
IDSource: 1
|
|
2363
|
+
})
|
|
2364
|
+
.expect(200)
|
|
2365
|
+
.end(
|
|
2366
|
+
(pErr, pRes) =>
|
|
2367
|
+
{
|
|
2368
|
+
Expect(pRes.body.IngestJob).to.exist;
|
|
2369
|
+
let tmpIDIngestJob = pRes.body.IngestJob.IDIngestJob;
|
|
2370
|
+
Expect(tmpIDIngestJob).to.be.greaterThan(0);
|
|
2371
|
+
|
|
2372
|
+
// Check the records have the correct IDIngestJob
|
|
2373
|
+
Expect(pRes.body.Records).to.be.an('array');
|
|
2374
|
+
Expect(pRes.body.Records.length).to.equal(1);
|
|
2375
|
+
Expect(parseInt(pRes.body.Records[0].IDIngestJob, 10)).to.equal(tmpIDIngestJob);
|
|
2376
|
+
return fDone();
|
|
2377
|
+
});
|
|
2378
|
+
});
|
|
2379
|
+
}
|
|
2380
|
+
);
|
|
2381
|
+
}
|
|
2382
|
+
);
|
|
2383
|
+
suite
|
|
2384
|
+
(
|
|
2385
|
+
'Content Signatures',
|
|
2386
|
+
function()
|
|
2387
|
+
{
|
|
2388
|
+
test
|
|
2389
|
+
(
|
|
2390
|
+
'identical content should produce isDuplicate=true on second ingest',
|
|
2391
|
+
function(fDone)
|
|
2392
|
+
{
|
|
2393
|
+
this.timeout(10000);
|
|
2394
|
+
|
|
2395
|
+
let tmpContent = 'col1,col2\nfoo,bar\nbaz,qux\n';
|
|
2396
|
+
|
|
2397
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2398
|
+
.send({ Name: 'Duplicate Sig Dataset', Type: 'Raw', Description: 'Testing duplicate signatures' })
|
|
2399
|
+
.expect(200)
|
|
2400
|
+
.end(
|
|
2401
|
+
(pError, pResponse) =>
|
|
2402
|
+
{
|
|
2403
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2404
|
+
|
|
2405
|
+
// First ingest
|
|
2406
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2407
|
+
.send({ Content: tmpContent, Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2408
|
+
.expect(200)
|
|
2409
|
+
.end(
|
|
2410
|
+
(pErr1, pRes1) =>
|
|
2411
|
+
{
|
|
2412
|
+
Expect(pRes1.body.IsDuplicate).to.equal(false);
|
|
2413
|
+
let tmpSig1 = pRes1.body.ContentSignature;
|
|
2414
|
+
|
|
2415
|
+
// Second ingest with identical content
|
|
2416
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2417
|
+
.send({ Content: tmpContent, Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2418
|
+
.expect(200)
|
|
2419
|
+
.end(
|
|
2420
|
+
(pErr2, pRes2) =>
|
|
2421
|
+
{
|
|
2422
|
+
Expect(pRes2.body.IsDuplicate).to.equal(true);
|
|
2423
|
+
Expect(pRes2.body.ContentSignature).to.equal(tmpSig1);
|
|
2424
|
+
Expect(pRes2.body.DatasetVersion).to.equal(2);
|
|
2425
|
+
return fDone();
|
|
2426
|
+
});
|
|
2427
|
+
});
|
|
2428
|
+
});
|
|
2429
|
+
}
|
|
2430
|
+
);
|
|
2431
|
+
test
|
|
2432
|
+
(
|
|
2433
|
+
'different content should produce isDuplicate=false',
|
|
2434
|
+
function(fDone)
|
|
2435
|
+
{
|
|
2436
|
+
this.timeout(10000);
|
|
2437
|
+
|
|
2438
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2439
|
+
.send({ Name: 'Different Sig Dataset', Type: 'Raw', Description: 'Testing different signatures' })
|
|
2440
|
+
.expect(200)
|
|
2441
|
+
.end(
|
|
2442
|
+
(pError, pResponse) =>
|
|
2443
|
+
{
|
|
2444
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2445
|
+
|
|
2446
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2447
|
+
.send({ Content: 'a,b\n1,2\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2448
|
+
.expect(200)
|
|
2449
|
+
.end(
|
|
2450
|
+
(pErr1, pRes1) =>
|
|
2451
|
+
{
|
|
2452
|
+
let tmpSig1 = pRes1.body.ContentSignature;
|
|
2453
|
+
|
|
2454
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2455
|
+
.send({ Content: 'a,b\n3,4\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2456
|
+
.expect(200)
|
|
2457
|
+
.end(
|
|
2458
|
+
(pErr2, pRes2) =>
|
|
2459
|
+
{
|
|
2460
|
+
Expect(pRes2.body.IsDuplicate).to.equal(false);
|
|
2461
|
+
Expect(pRes2.body.ContentSignature).to.not.equal(tmpSig1);
|
|
2462
|
+
return fDone();
|
|
2463
|
+
});
|
|
2464
|
+
});
|
|
2465
|
+
});
|
|
2466
|
+
}
|
|
2467
|
+
);
|
|
2468
|
+
}
|
|
2469
|
+
);
|
|
2470
|
+
suite
|
|
2471
|
+
(
|
|
2472
|
+
'Version Policy',
|
|
2473
|
+
function()
|
|
2474
|
+
{
|
|
2475
|
+
test
|
|
2476
|
+
(
|
|
2477
|
+
'PUT /facto/dataset/:id/version-policy should set policy',
|
|
2478
|
+
function(fDone)
|
|
2479
|
+
{
|
|
2480
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2481
|
+
.send({ Name: 'Policy Test Dataset', Type: 'Raw', Description: 'Testing VersionPolicy' })
|
|
2482
|
+
.expect(200)
|
|
2483
|
+
.end(
|
|
2484
|
+
(pError, pResponse) =>
|
|
2485
|
+
{
|
|
2486
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2487
|
+
|
|
2488
|
+
_SuperTest.put(`/facto/dataset/${tmpIDDataset}/version-policy`)
|
|
2489
|
+
.send({ VersionPolicy: 'Replace' })
|
|
2490
|
+
.expect(200)
|
|
2491
|
+
.end(
|
|
2492
|
+
(pErr, pRes) =>
|
|
2493
|
+
{
|
|
2494
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2495
|
+
Expect(pRes.body.Dataset.VersionPolicy).to.equal('Replace');
|
|
2496
|
+
return fDone();
|
|
2497
|
+
});
|
|
2498
|
+
});
|
|
2499
|
+
}
|
|
2500
|
+
);
|
|
2501
|
+
test
|
|
2502
|
+
(
|
|
2503
|
+
'PUT /facto/dataset/:id/version-policy should reject invalid policy',
|
|
2504
|
+
function(fDone)
|
|
2505
|
+
{
|
|
2506
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2507
|
+
.send({ Name: 'Invalid Policy Dataset', Type: 'Raw', Description: 'Testing invalid policy' })
|
|
2508
|
+
.expect(200)
|
|
2509
|
+
.end(
|
|
2510
|
+
(pError, pResponse) =>
|
|
2511
|
+
{
|
|
2512
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2513
|
+
|
|
2514
|
+
_SuperTest.put(`/facto/dataset/${tmpIDDataset}/version-policy`)
|
|
2515
|
+
.send({ VersionPolicy: 'InvalidValue' })
|
|
2516
|
+
.expect(200)
|
|
2517
|
+
.end(
|
|
2518
|
+
(pErr, pRes) =>
|
|
2519
|
+
{
|
|
2520
|
+
Expect(pRes.body.Error).to.exist;
|
|
2521
|
+
return fDone();
|
|
2522
|
+
});
|
|
2523
|
+
});
|
|
2524
|
+
}
|
|
2525
|
+
);
|
|
2526
|
+
test
|
|
2527
|
+
(
|
|
2528
|
+
'Replace policy should soft-delete old records on re-import',
|
|
2529
|
+
function(fDone)
|
|
2530
|
+
{
|
|
2531
|
+
this.timeout(10000);
|
|
2532
|
+
|
|
2533
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2534
|
+
.send({ Name: 'Replace Policy Dataset', Type: 'Raw', Description: 'Testing Replace policy' })
|
|
2535
|
+
.expect(200)
|
|
2536
|
+
.end(
|
|
2537
|
+
(pError, pResponse) =>
|
|
2538
|
+
{
|
|
2539
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2540
|
+
|
|
2541
|
+
// Set policy to Replace
|
|
2542
|
+
_SuperTest.put(`/facto/dataset/${tmpIDDataset}/version-policy`)
|
|
2543
|
+
.send({ VersionPolicy: 'Replace' })
|
|
2544
|
+
.expect(200)
|
|
2545
|
+
.end(
|
|
2546
|
+
() =>
|
|
2547
|
+
{
|
|
2548
|
+
// Ingest v1 (3 records)
|
|
2549
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2550
|
+
.send({ Content: 'a,b\n1,2\n3,4\n5,6\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2551
|
+
.expect(200)
|
|
2552
|
+
.end(
|
|
2553
|
+
(pErr1, pRes1) =>
|
|
2554
|
+
{
|
|
2555
|
+
Expect(pRes1.body.Ingested).to.equal(3);
|
|
2556
|
+
Expect(pRes1.body.DatasetVersion).to.equal(1);
|
|
2557
|
+
|
|
2558
|
+
// Ingest v2 (2 records) — v1 records should be soft-deleted
|
|
2559
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2560
|
+
.send({ Content: 'a,b\n7,8\n9,10\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2561
|
+
.expect(200)
|
|
2562
|
+
.end(
|
|
2563
|
+
(pErr2, pRes2) =>
|
|
2564
|
+
{
|
|
2565
|
+
Expect(pRes2.body.Ingested).to.equal(2);
|
|
2566
|
+
Expect(pRes2.body.DatasetVersion).to.equal(2);
|
|
2567
|
+
|
|
2568
|
+
// Count active records — should be 2 (only v2)
|
|
2569
|
+
_SuperTest.get(`/facto/dataset/${tmpIDDataset}/stats`)
|
|
2570
|
+
.expect(200)
|
|
2571
|
+
.end(
|
|
2572
|
+
(pErr3, pRes3) =>
|
|
2573
|
+
{
|
|
2574
|
+
Expect(pRes3.body.RecordCount).to.equal(2);
|
|
2575
|
+
return fDone();
|
|
2576
|
+
});
|
|
2577
|
+
});
|
|
2578
|
+
});
|
|
2579
|
+
});
|
|
2580
|
+
});
|
|
2581
|
+
}
|
|
2582
|
+
);
|
|
2583
|
+
}
|
|
2584
|
+
);
|
|
2585
|
+
suite
|
|
2586
|
+
(
|
|
2587
|
+
'Dataset Version History',
|
|
2588
|
+
function()
|
|
2589
|
+
{
|
|
2590
|
+
test
|
|
2591
|
+
(
|
|
2592
|
+
'GET /facto/dataset/:id/versions should return version history',
|
|
2593
|
+
function(fDone)
|
|
2594
|
+
{
|
|
2595
|
+
this.timeout(10000);
|
|
2596
|
+
|
|
2597
|
+
_SuperTest.post('/1.0/Dataset')
|
|
2598
|
+
.send({ Name: 'Version History Dataset', Type: 'Raw', Description: 'Testing version history' })
|
|
2599
|
+
.expect(200)
|
|
2600
|
+
.end(
|
|
2601
|
+
(pError, pResponse) =>
|
|
2602
|
+
{
|
|
2603
|
+
let tmpIDDataset = pResponse.body.IDDataset;
|
|
2604
|
+
|
|
2605
|
+
// Ingest v1
|
|
2606
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2607
|
+
.send({ Content: 'k,v\na,1\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2608
|
+
.expect(200)
|
|
2609
|
+
.end(
|
|
2610
|
+
() =>
|
|
2611
|
+
{
|
|
2612
|
+
// Ingest v2
|
|
2613
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2614
|
+
.send({ Content: 'k,v\nb,2\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2615
|
+
.expect(200)
|
|
2616
|
+
.end(
|
|
2617
|
+
() =>
|
|
2618
|
+
{
|
|
2619
|
+
// Ingest v3
|
|
2620
|
+
_SuperTest.post('/facto/ingest/file')
|
|
2621
|
+
.send({ Content: 'k,v\nc,3\n', Format: 'csv', IDDataset: tmpIDDataset, IDSource: 1 })
|
|
2622
|
+
.expect(200)
|
|
2623
|
+
.end(
|
|
2624
|
+
() =>
|
|
2625
|
+
{
|
|
2626
|
+
// Get version history
|
|
2627
|
+
_SuperTest.get(`/facto/dataset/${tmpIDDataset}/versions`)
|
|
2628
|
+
.expect(200)
|
|
2629
|
+
.end(
|
|
2630
|
+
(pErr, pRes) =>
|
|
2631
|
+
{
|
|
2632
|
+
Expect(pRes.body.Count).to.equal(3);
|
|
2633
|
+
Expect(pRes.body.Versions).to.be.an('array');
|
|
2634
|
+
Expect(pRes.body.Versions.length).to.equal(3);
|
|
2635
|
+
|
|
2636
|
+
// Should be sorted DESC
|
|
2637
|
+
Expect(parseInt(pRes.body.Versions[0].DatasetVersion, 10)).to.equal(3);
|
|
2638
|
+
Expect(parseInt(pRes.body.Versions[1].DatasetVersion, 10)).to.equal(2);
|
|
2639
|
+
Expect(parseInt(pRes.body.Versions[2].DatasetVersion, 10)).to.equal(1);
|
|
2640
|
+
|
|
2641
|
+
// Each should have a ContentSignature
|
|
2642
|
+
Expect(pRes.body.Versions[0].ContentSignature).to.be.a('string');
|
|
2643
|
+
Expect(pRes.body.Versions[0].ContentSignature.length).to.equal(64);
|
|
2644
|
+
|
|
2645
|
+
return fDone();
|
|
2646
|
+
});
|
|
2647
|
+
});
|
|
2648
|
+
});
|
|
2649
|
+
});
|
|
2650
|
+
});
|
|
2651
|
+
}
|
|
2652
|
+
);
|
|
2653
|
+
}
|
|
2654
|
+
);
|
|
2655
|
+
|
|
2656
|
+
// ================================================================
|
|
2657
|
+
// Source Catalog - CRUD
|
|
2658
|
+
// ================================================================
|
|
2659
|
+
suite
|
|
2660
|
+
(
|
|
2661
|
+
'Source Catalog CRUD',
|
|
2662
|
+
function()
|
|
2663
|
+
{
|
|
2664
|
+
let _CatalogEntryID = 0;
|
|
2665
|
+
|
|
2666
|
+
test
|
|
2667
|
+
(
|
|
2668
|
+
'Create a catalog entry',
|
|
2669
|
+
function(fDone)
|
|
2670
|
+
{
|
|
2671
|
+
_SuperTest
|
|
2672
|
+
.post('/facto/catalog/entry')
|
|
2673
|
+
.send(
|
|
2674
|
+
{
|
|
2675
|
+
Agency: 'US Geological Survey (USGS)',
|
|
2676
|
+
Name: 'USGS Data Portal',
|
|
2677
|
+
Type: 'API',
|
|
2678
|
+
URL: 'https://waterservices.usgs.gov',
|
|
2679
|
+
Protocol: 'HTTPS',
|
|
2680
|
+
Category: 'Science',
|
|
2681
|
+
Region: 'US',
|
|
2682
|
+
UpdateFrequency: 'Continuous',
|
|
2683
|
+
Description: 'Geological and hydrological data',
|
|
2684
|
+
Notes: 'Free, no auth required',
|
|
2685
|
+
Verified: true
|
|
2686
|
+
})
|
|
2687
|
+
.end(
|
|
2688
|
+
(pError, pRes) =>
|
|
2689
|
+
{
|
|
2690
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2691
|
+
Expect(pRes.body.Entry).to.be.an('object');
|
|
2692
|
+
Expect(pRes.body.Entry.IDSourceCatalogEntry).to.be.greaterThan(0);
|
|
2693
|
+
Expect(pRes.body.Entry.Agency).to.equal('US Geological Survey (USGS)');
|
|
2694
|
+
Expect(pRes.body.Entry.Verified).to.equal(1);
|
|
2695
|
+
_CatalogEntryID = pRes.body.Entry.IDSourceCatalogEntry;
|
|
2696
|
+
return fDone();
|
|
2697
|
+
});
|
|
2698
|
+
}
|
|
2699
|
+
);
|
|
2700
|
+
|
|
2701
|
+
test
|
|
2702
|
+
(
|
|
2703
|
+
'Read catalog entry back',
|
|
2704
|
+
function(fDone)
|
|
2705
|
+
{
|
|
2706
|
+
_SuperTest
|
|
2707
|
+
.get(`/facto/catalog/entry/${_CatalogEntryID}`)
|
|
2708
|
+
.end(
|
|
2709
|
+
(pError, pRes) =>
|
|
2710
|
+
{
|
|
2711
|
+
Expect(pRes.body.Entry).to.be.an('object');
|
|
2712
|
+
Expect(pRes.body.Entry.Agency).to.equal('US Geological Survey (USGS)');
|
|
2713
|
+
Expect(pRes.body.Entry.Category).to.equal('Science');
|
|
2714
|
+
Expect(pRes.body.Datasets).to.be.an('array');
|
|
2715
|
+
return fDone();
|
|
2716
|
+
});
|
|
2717
|
+
}
|
|
2718
|
+
);
|
|
2719
|
+
|
|
2720
|
+
test
|
|
2721
|
+
(
|
|
2722
|
+
'Update catalog entry',
|
|
2723
|
+
function(fDone)
|
|
2724
|
+
{
|
|
2725
|
+
_SuperTest
|
|
2726
|
+
.put(`/facto/catalog/entry/${_CatalogEntryID}`)
|
|
2727
|
+
.send({ Category: 'Earth Science', Notes: 'Updated notes' })
|
|
2728
|
+
.end(
|
|
2729
|
+
(pError, pRes) =>
|
|
2730
|
+
{
|
|
2731
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2732
|
+
return fDone();
|
|
2733
|
+
});
|
|
2734
|
+
}
|
|
2735
|
+
);
|
|
2736
|
+
|
|
2737
|
+
test
|
|
2738
|
+
(
|
|
2739
|
+
'List catalog entries',
|
|
2740
|
+
function(fDone)
|
|
2741
|
+
{
|
|
2742
|
+
_SuperTest
|
|
2743
|
+
.get('/facto/catalog/entries')
|
|
2744
|
+
.end(
|
|
2745
|
+
(pError, pRes) =>
|
|
2746
|
+
{
|
|
2747
|
+
Expect(pRes.body.Count).to.be.greaterThan(0);
|
|
2748
|
+
Expect(pRes.body.Entries).to.be.an('array');
|
|
2749
|
+
return fDone();
|
|
2750
|
+
});
|
|
2751
|
+
}
|
|
2752
|
+
);
|
|
2753
|
+
|
|
2754
|
+
test
|
|
2755
|
+
(
|
|
2756
|
+
'Soft-delete catalog entry',
|
|
2757
|
+
function(fDone)
|
|
2758
|
+
{
|
|
2759
|
+
_SuperTest
|
|
2760
|
+
.delete(`/facto/catalog/entry/${_CatalogEntryID}`)
|
|
2761
|
+
.end(
|
|
2762
|
+
(pError, pRes) =>
|
|
2763
|
+
{
|
|
2764
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2765
|
+
Expect(pRes.body.Deleted).to.equal(_CatalogEntryID);
|
|
2766
|
+
|
|
2767
|
+
// Verify it no longer appears in listing
|
|
2768
|
+
_SuperTest
|
|
2769
|
+
.get('/facto/catalog/entries')
|
|
2770
|
+
.end(
|
|
2771
|
+
(pError2, pRes2) =>
|
|
2772
|
+
{
|
|
2773
|
+
let tmpFound = pRes2.body.Entries.filter((e) => e.IDSourceCatalogEntry === _CatalogEntryID);
|
|
2774
|
+
Expect(tmpFound.length).to.equal(0);
|
|
2775
|
+
return fDone();
|
|
2776
|
+
});
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
2779
|
+
);
|
|
2780
|
+
}
|
|
2781
|
+
);
|
|
2782
|
+
|
|
2783
|
+
// ================================================================
|
|
2784
|
+
// Source Catalog - Dataset Definitions
|
|
2785
|
+
// ================================================================
|
|
2786
|
+
suite
|
|
2787
|
+
(
|
|
2788
|
+
'Catalog Dataset Definitions',
|
|
2789
|
+
function()
|
|
2790
|
+
{
|
|
2791
|
+
let _CatalogEntryID = 0;
|
|
2792
|
+
let _CatalogDatasetID = 0;
|
|
2793
|
+
|
|
2794
|
+
test
|
|
2795
|
+
(
|
|
2796
|
+
'Create catalog entry for dataset tests',
|
|
2797
|
+
function(fDone)
|
|
2798
|
+
{
|
|
2799
|
+
_SuperTest
|
|
2800
|
+
.post('/facto/catalog/entry')
|
|
2801
|
+
.send(
|
|
2802
|
+
{
|
|
2803
|
+
Agency: 'NOAA',
|
|
2804
|
+
Name: 'National Weather Service',
|
|
2805
|
+
Type: 'API',
|
|
2806
|
+
URL: 'https://www.weather.gov',
|
|
2807
|
+
Category: 'Weather'
|
|
2808
|
+
})
|
|
2809
|
+
.end(
|
|
2810
|
+
(pError, pRes) =>
|
|
2811
|
+
{
|
|
2812
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2813
|
+
_CatalogEntryID = pRes.body.Entry.IDSourceCatalogEntry;
|
|
2814
|
+
return fDone();
|
|
2815
|
+
});
|
|
2816
|
+
}
|
|
2817
|
+
);
|
|
2818
|
+
|
|
2819
|
+
test
|
|
2820
|
+
(
|
|
2821
|
+
'Add dataset definition to catalog entry',
|
|
2822
|
+
function(fDone)
|
|
2823
|
+
{
|
|
2824
|
+
_SuperTest
|
|
2825
|
+
.post(`/facto/catalog/entry/${_CatalogEntryID}/dataset`)
|
|
2826
|
+
.send(
|
|
2827
|
+
{
|
|
2828
|
+
Name: 'Weather Stations',
|
|
2829
|
+
Format: 'fixed-width',
|
|
2830
|
+
MimeType: 'text/plain',
|
|
2831
|
+
EndpointURL: 'https://www.weather.gov/stations.dat',
|
|
2832
|
+
Description: 'NOAA weather station master list',
|
|
2833
|
+
ParseOptions: JSON.stringify({ columns: [{ name: 'StationID', start: 1, width: 11 }] }),
|
|
2834
|
+
AuthRequirements: 'None',
|
|
2835
|
+
VersionPolicy: 'Append'
|
|
2836
|
+
})
|
|
2837
|
+
.end(
|
|
2838
|
+
(pError, pRes) =>
|
|
2839
|
+
{
|
|
2840
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2841
|
+
Expect(pRes.body.DatasetDefinition).to.be.an('object');
|
|
2842
|
+
Expect(pRes.body.DatasetDefinition.IDCatalogDatasetDefinition).to.be.greaterThan(0);
|
|
2843
|
+
Expect(pRes.body.DatasetDefinition.Format).to.equal('fixed-width');
|
|
2844
|
+
Expect(pRes.body.DatasetDefinition.Provisioned).to.equal(0);
|
|
2845
|
+
_CatalogDatasetID = pRes.body.DatasetDefinition.IDCatalogDatasetDefinition;
|
|
2846
|
+
return fDone();
|
|
2847
|
+
});
|
|
2848
|
+
}
|
|
2849
|
+
);
|
|
2850
|
+
|
|
2851
|
+
test
|
|
2852
|
+
(
|
|
2853
|
+
'List dataset definitions for entry',
|
|
2854
|
+
function(fDone)
|
|
2855
|
+
{
|
|
2856
|
+
_SuperTest
|
|
2857
|
+
.get(`/facto/catalog/entry/${_CatalogEntryID}/datasets`)
|
|
2858
|
+
.end(
|
|
2859
|
+
(pError, pRes) =>
|
|
2860
|
+
{
|
|
2861
|
+
Expect(pRes.body.Count).to.equal(1);
|
|
2862
|
+
Expect(pRes.body.Datasets).to.be.an('array');
|
|
2863
|
+
Expect(pRes.body.Datasets[0].Name).to.equal('Weather Stations');
|
|
2864
|
+
return fDone();
|
|
2865
|
+
});
|
|
2866
|
+
}
|
|
2867
|
+
);
|
|
2868
|
+
|
|
2869
|
+
test
|
|
2870
|
+
(
|
|
2871
|
+
'Update dataset definition',
|
|
2872
|
+
function(fDone)
|
|
2873
|
+
{
|
|
2874
|
+
_SuperTest
|
|
2875
|
+
.put(`/facto/catalog/dataset/${_CatalogDatasetID}`)
|
|
2876
|
+
.send({ Description: 'Updated description', VersionPolicy: 'Replace' })
|
|
2877
|
+
.end(
|
|
2878
|
+
(pError, pRes) =>
|
|
2879
|
+
{
|
|
2880
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2881
|
+
return fDone();
|
|
2882
|
+
});
|
|
2883
|
+
}
|
|
2884
|
+
);
|
|
2885
|
+
|
|
2886
|
+
test
|
|
2887
|
+
(
|
|
2888
|
+
'Soft-delete dataset definition',
|
|
2889
|
+
function(fDone)
|
|
2890
|
+
{
|
|
2891
|
+
_SuperTest
|
|
2892
|
+
.delete(`/facto/catalog/dataset/${_CatalogDatasetID}`)
|
|
2893
|
+
.end(
|
|
2894
|
+
(pError, pRes) =>
|
|
2895
|
+
{
|
|
2896
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2897
|
+
|
|
2898
|
+
// Verify it no longer appears
|
|
2899
|
+
_SuperTest
|
|
2900
|
+
.get(`/facto/catalog/entry/${_CatalogEntryID}/datasets`)
|
|
2901
|
+
.end(
|
|
2902
|
+
(pError2, pRes2) =>
|
|
2903
|
+
{
|
|
2904
|
+
Expect(pRes2.body.Count).to.equal(0);
|
|
2905
|
+
return fDone();
|
|
2906
|
+
});
|
|
2907
|
+
});
|
|
2908
|
+
}
|
|
2909
|
+
);
|
|
2910
|
+
}
|
|
2911
|
+
);
|
|
2912
|
+
|
|
2913
|
+
// ================================================================
|
|
2914
|
+
// Source Catalog - Search
|
|
2915
|
+
// ================================================================
|
|
2916
|
+
suite
|
|
2917
|
+
(
|
|
2918
|
+
'Catalog Search',
|
|
2919
|
+
function()
|
|
2920
|
+
{
|
|
2921
|
+
test
|
|
2922
|
+
(
|
|
2923
|
+
'Create entries for search tests',
|
|
2924
|
+
function(fDone)
|
|
2925
|
+
{
|
|
2926
|
+
_SuperTest
|
|
2927
|
+
.post('/facto/catalog/entry')
|
|
2928
|
+
.send({ Agency: 'EPA', Name: 'Environmental Data', Category: 'Environment', Region: 'US' })
|
|
2929
|
+
.end(
|
|
2930
|
+
(pError, pRes) =>
|
|
2931
|
+
{
|
|
2932
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
2933
|
+
_SuperTest
|
|
2934
|
+
.post('/facto/catalog/entry')
|
|
2935
|
+
.send({ Agency: 'NASA', Name: 'Space Science Data', Category: 'Space', Region: 'Global' })
|
|
2936
|
+
.end(
|
|
2937
|
+
(pError2, pRes2) =>
|
|
2938
|
+
{
|
|
2939
|
+
Expect(pRes2.body.Success).to.equal(true);
|
|
2940
|
+
_SuperTest
|
|
2941
|
+
.post('/facto/catalog/entry')
|
|
2942
|
+
.send({ Agency: 'CDC', Name: 'Health Statistics', Category: 'Health', Region: 'US' })
|
|
2943
|
+
.end(
|
|
2944
|
+
(pError3, pRes3) =>
|
|
2945
|
+
{
|
|
2946
|
+
Expect(pRes3.body.Success).to.equal(true);
|
|
2947
|
+
return fDone();
|
|
2948
|
+
});
|
|
2949
|
+
});
|
|
2950
|
+
});
|
|
2951
|
+
}
|
|
2952
|
+
);
|
|
2953
|
+
|
|
2954
|
+
test
|
|
2955
|
+
(
|
|
2956
|
+
'Search by name',
|
|
2957
|
+
function(fDone)
|
|
2958
|
+
{
|
|
2959
|
+
_SuperTest
|
|
2960
|
+
.get('/facto/catalog/search?q=Space')
|
|
2961
|
+
.end(
|
|
2962
|
+
(pError, pRes) =>
|
|
2963
|
+
{
|
|
2964
|
+
Expect(pRes.body.Query).to.equal('space');
|
|
2965
|
+
Expect(pRes.body.Count).to.be.greaterThan(0);
|
|
2966
|
+
let tmpNames = pRes.body.Entries.map((e) => e.Name);
|
|
2967
|
+
Expect(tmpNames).to.include('Space Science Data');
|
|
2968
|
+
return fDone();
|
|
2969
|
+
});
|
|
2970
|
+
}
|
|
2971
|
+
);
|
|
2972
|
+
|
|
2973
|
+
test
|
|
2974
|
+
(
|
|
2975
|
+
'Search by category',
|
|
2976
|
+
function(fDone)
|
|
2977
|
+
{
|
|
2978
|
+
_SuperTest
|
|
2979
|
+
.get('/facto/catalog/search?q=health')
|
|
2980
|
+
.end(
|
|
2981
|
+
(pError, pRes) =>
|
|
2982
|
+
{
|
|
2983
|
+
Expect(pRes.body.Count).to.be.greaterThan(0);
|
|
2984
|
+
let tmpCategories = pRes.body.Entries.map((e) => e.Category);
|
|
2985
|
+
Expect(tmpCategories).to.include('Health');
|
|
2986
|
+
return fDone();
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2989
|
+
);
|
|
2990
|
+
|
|
2991
|
+
test
|
|
2992
|
+
(
|
|
2993
|
+
'Empty search returns all non-deleted entries',
|
|
2994
|
+
function(fDone)
|
|
2995
|
+
{
|
|
2996
|
+
_SuperTest
|
|
2997
|
+
.get('/facto/catalog/search')
|
|
2998
|
+
.end(
|
|
2999
|
+
(pError, pRes) =>
|
|
3000
|
+
{
|
|
3001
|
+
Expect(pRes.body.Query).to.equal('');
|
|
3002
|
+
// Should include NOAA, EPA, NASA, CDC (not the deleted USGS)
|
|
3003
|
+
Expect(pRes.body.Count).to.be.greaterThan(2);
|
|
3004
|
+
return fDone();
|
|
3005
|
+
});
|
|
3006
|
+
}
|
|
3007
|
+
);
|
|
3008
|
+
}
|
|
3009
|
+
);
|
|
3010
|
+
|
|
3011
|
+
// ================================================================
|
|
3012
|
+
// Source Catalog - Provision
|
|
3013
|
+
// ================================================================
|
|
3014
|
+
suite
|
|
3015
|
+
(
|
|
3016
|
+
'Catalog Provision',
|
|
3017
|
+
function()
|
|
3018
|
+
{
|
|
3019
|
+
let _ProvisionEntryID = 0;
|
|
3020
|
+
let _ProvisionDatasetID = 0;
|
|
3021
|
+
|
|
3022
|
+
test
|
|
3023
|
+
(
|
|
3024
|
+
'Create catalog entry and dataset for provisioning',
|
|
3025
|
+
function(fDone)
|
|
3026
|
+
{
|
|
3027
|
+
_SuperTest
|
|
3028
|
+
.post('/facto/catalog/entry')
|
|
3029
|
+
.send(
|
|
3030
|
+
{
|
|
3031
|
+
Agency: 'Bureau of Labor Statistics',
|
|
3032
|
+
Name: 'BLS Data Portal',
|
|
3033
|
+
Type: 'API',
|
|
3034
|
+
URL: 'https://www.bls.gov',
|
|
3035
|
+
Protocol: 'HTTPS',
|
|
3036
|
+
Category: 'Economics'
|
|
3037
|
+
})
|
|
3038
|
+
.end(
|
|
3039
|
+
(pError, pRes) =>
|
|
3040
|
+
{
|
|
3041
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
3042
|
+
_ProvisionEntryID = pRes.body.Entry.IDSourceCatalogEntry;
|
|
3043
|
+
|
|
3044
|
+
_SuperTest
|
|
3045
|
+
.post(`/facto/catalog/entry/${_ProvisionEntryID}/dataset`)
|
|
3046
|
+
.send(
|
|
3047
|
+
{
|
|
3048
|
+
Name: 'Consumer Price Index',
|
|
3049
|
+
Format: 'csv',
|
|
3050
|
+
MimeType: 'text/csv',
|
|
3051
|
+
EndpointURL: 'https://www.bls.gov/cpi/data.csv',
|
|
3052
|
+
Description: 'Monthly CPI data',
|
|
3053
|
+
ParseOptions: '{}',
|
|
3054
|
+
AuthRequirements: 'None',
|
|
3055
|
+
VersionPolicy: 'Append'
|
|
3056
|
+
})
|
|
3057
|
+
.end(
|
|
3058
|
+
(pError2, pRes2) =>
|
|
3059
|
+
{
|
|
3060
|
+
Expect(pRes2.body.Success).to.equal(true);
|
|
3061
|
+
_ProvisionDatasetID = pRes2.body.DatasetDefinition.IDCatalogDatasetDefinition;
|
|
3062
|
+
return fDone();
|
|
3063
|
+
});
|
|
3064
|
+
});
|
|
3065
|
+
}
|
|
3066
|
+
);
|
|
3067
|
+
|
|
3068
|
+
test
|
|
3069
|
+
(
|
|
3070
|
+
'Provision creates runtime Source, Dataset, and DatasetSource',
|
|
3071
|
+
function(fDone)
|
|
3072
|
+
{
|
|
3073
|
+
_SuperTest
|
|
3074
|
+
.post(`/facto/catalog/dataset/${_ProvisionDatasetID}/provision`)
|
|
3075
|
+
.end(
|
|
3076
|
+
(pError, pRes) =>
|
|
3077
|
+
{
|
|
3078
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
3079
|
+
Expect(pRes.body.Source).to.be.an('object');
|
|
3080
|
+
Expect(pRes.body.Source.Name).to.equal('Bureau of Labor Statistics');
|
|
3081
|
+
Expect(pRes.body.Source.IDSource).to.be.greaterThan(0);
|
|
3082
|
+
Expect(pRes.body.Source.Hash).to.equal('Bureau-of-Labor-Statistics');
|
|
3083
|
+
Expect(pRes.body.Dataset).to.be.an('object');
|
|
3084
|
+
Expect(pRes.body.Dataset.Name).to.equal('Consumer Price Index');
|
|
3085
|
+
Expect(pRes.body.Dataset.IDDataset).to.be.greaterThan(0);
|
|
3086
|
+
Expect(pRes.body.Dataset.Hash).to.equal('Consumer-Price-Index');
|
|
3087
|
+
Expect(pRes.body.DatasetSource).to.be.an('object');
|
|
3088
|
+
return fDone();
|
|
3089
|
+
});
|
|
3090
|
+
}
|
|
3091
|
+
);
|
|
3092
|
+
|
|
3093
|
+
test
|
|
3094
|
+
(
|
|
3095
|
+
'CatalogDatasetDefinition is marked as provisioned',
|
|
3096
|
+
function(fDone)
|
|
3097
|
+
{
|
|
3098
|
+
_SuperTest
|
|
3099
|
+
.get(`/facto/catalog/entry/${_ProvisionEntryID}/datasets`)
|
|
3100
|
+
.end(
|
|
3101
|
+
(pError, pRes) =>
|
|
3102
|
+
{
|
|
3103
|
+
Expect(pRes.body.Count).to.equal(1);
|
|
3104
|
+
let tmpDef = pRes.body.Datasets[0];
|
|
3105
|
+
Expect(tmpDef.Provisioned).to.equal(1);
|
|
3106
|
+
Expect(tmpDef.IDSource).to.be.greaterThan(0);
|
|
3107
|
+
Expect(tmpDef.IDDataset).to.be.greaterThan(0);
|
|
3108
|
+
return fDone();
|
|
3109
|
+
});
|
|
3110
|
+
}
|
|
3111
|
+
);
|
|
3112
|
+
|
|
3113
|
+
test
|
|
3114
|
+
(
|
|
3115
|
+
'Provisioning again is idempotent (reuses existing Source and Dataset)',
|
|
3116
|
+
function(fDone)
|
|
3117
|
+
{
|
|
3118
|
+
_SuperTest
|
|
3119
|
+
.post(`/facto/catalog/dataset/${_ProvisionDatasetID}/provision`)
|
|
3120
|
+
.end(
|
|
3121
|
+
(pError, pRes) =>
|
|
3122
|
+
{
|
|
3123
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
3124
|
+
// Source and Dataset should have the same IDs
|
|
3125
|
+
Expect(pRes.body.Source.Name).to.equal('Bureau of Labor Statistics');
|
|
3126
|
+
Expect(pRes.body.Dataset.Name).to.equal('Consumer Price Index');
|
|
3127
|
+
return fDone();
|
|
3128
|
+
});
|
|
3129
|
+
}
|
|
3130
|
+
);
|
|
3131
|
+
}
|
|
3132
|
+
);
|
|
3133
|
+
|
|
3134
|
+
// ================================================================
|
|
3135
|
+
// Source Catalog - Import / Export
|
|
3136
|
+
// ================================================================
|
|
3137
|
+
suite
|
|
3138
|
+
(
|
|
3139
|
+
'Catalog Import/Export',
|
|
3140
|
+
function()
|
|
3141
|
+
{
|
|
3142
|
+
test
|
|
3143
|
+
(
|
|
3144
|
+
'Import catalog entries with datasets',
|
|
3145
|
+
function(fDone)
|
|
3146
|
+
{
|
|
3147
|
+
let tmpImportData = [
|
|
3148
|
+
{
|
|
3149
|
+
Agency: 'Import Test Agency A',
|
|
3150
|
+
Name: 'Agency A Portal',
|
|
3151
|
+
Type: 'API',
|
|
3152
|
+
Category: 'Test',
|
|
3153
|
+
Datasets: [
|
|
3154
|
+
{
|
|
3155
|
+
Name: 'Dataset Alpha',
|
|
3156
|
+
Format: 'csv',
|
|
3157
|
+
EndpointURL: 'https://example.com/alpha.csv',
|
|
3158
|
+
VersionPolicy: 'Append'
|
|
3159
|
+
},
|
|
3160
|
+
{
|
|
3161
|
+
Name: 'Dataset Beta',
|
|
3162
|
+
Format: 'json',
|
|
3163
|
+
EndpointURL: 'https://example.com/beta.json',
|
|
3164
|
+
VersionPolicy: 'Replace'
|
|
3165
|
+
}
|
|
3166
|
+
]
|
|
3167
|
+
},
|
|
3168
|
+
{
|
|
3169
|
+
Agency: 'Import Test Agency B',
|
|
3170
|
+
Name: 'Agency B Portal',
|
|
3171
|
+
Type: 'File',
|
|
3172
|
+
Category: 'Test',
|
|
3173
|
+
Datasets: [
|
|
3174
|
+
{
|
|
3175
|
+
Name: 'Dataset Gamma',
|
|
3176
|
+
Format: 'xml',
|
|
3177
|
+
EndpointURL: 'https://example.com/gamma.xml'
|
|
3178
|
+
}
|
|
3179
|
+
]
|
|
3180
|
+
}
|
|
3181
|
+
];
|
|
3182
|
+
|
|
3183
|
+
_SuperTest
|
|
3184
|
+
.post('/facto/catalog/import')
|
|
3185
|
+
.send(tmpImportData)
|
|
3186
|
+
.end(
|
|
3187
|
+
(pError, pRes) =>
|
|
3188
|
+
{
|
|
3189
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
3190
|
+
Expect(pRes.body.EntriesCreated).to.equal(2);
|
|
3191
|
+
Expect(pRes.body.DatasetsCreated).to.equal(3);
|
|
3192
|
+
Expect(pRes.body.Errors).to.equal(0);
|
|
3193
|
+
return fDone();
|
|
3194
|
+
});
|
|
3195
|
+
}
|
|
3196
|
+
);
|
|
3197
|
+
|
|
3198
|
+
test
|
|
3199
|
+
(
|
|
3200
|
+
'Export catalog contains imported entries',
|
|
3201
|
+
function(fDone)
|
|
3202
|
+
{
|
|
3203
|
+
_SuperTest
|
|
3204
|
+
.get('/facto/catalog/export')
|
|
3205
|
+
.end(
|
|
3206
|
+
(pError, pRes) =>
|
|
3207
|
+
{
|
|
3208
|
+
Expect(pRes.body.Count).to.be.greaterThan(0);
|
|
3209
|
+
Expect(pRes.body.Entries).to.be.an('array');
|
|
3210
|
+
|
|
3211
|
+
// Find our imported entries
|
|
3212
|
+
let tmpAgencyA = pRes.body.Entries.find((e) => e.Agency === 'Import Test Agency A');
|
|
3213
|
+
Expect(tmpAgencyA).to.be.an('object');
|
|
3214
|
+
Expect(tmpAgencyA.Datasets).to.be.an('array');
|
|
3215
|
+
Expect(tmpAgencyA.Datasets.length).to.equal(2);
|
|
3216
|
+
|
|
3217
|
+
let tmpAgencyB = pRes.body.Entries.find((e) => e.Agency === 'Import Test Agency B');
|
|
3218
|
+
Expect(tmpAgencyB).to.be.an('object');
|
|
3219
|
+
Expect(tmpAgencyB.Datasets.length).to.equal(1);
|
|
3220
|
+
Expect(tmpAgencyB.Datasets[0].Name).to.equal('Dataset Gamma');
|
|
3221
|
+
|
|
3222
|
+
return fDone();
|
|
3223
|
+
});
|
|
3224
|
+
}
|
|
3225
|
+
);
|
|
3226
|
+
}
|
|
3227
|
+
);
|
|
3228
|
+
|
|
3229
|
+
suite
|
|
3230
|
+
(
|
|
3231
|
+
'parseJSON dataPath',
|
|
3232
|
+
() =>
|
|
3233
|
+
{
|
|
3234
|
+
test
|
|
3235
|
+
(
|
|
3236
|
+
'parseJSON with simple dataPath extracts nested array',
|
|
3237
|
+
function(fDone)
|
|
3238
|
+
{
|
|
3239
|
+
let tmpJSON = JSON.stringify({
|
|
3240
|
+
metadata: { page: 1 },
|
|
3241
|
+
data: [
|
|
3242
|
+
{ id: 1, name: 'Alpha' },
|
|
3243
|
+
{ id: 2, name: 'Beta' }
|
|
3244
|
+
]
|
|
3245
|
+
});
|
|
3246
|
+
|
|
3247
|
+
_Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON, { dataPath: 'data' },
|
|
3248
|
+
(pError, pRecords) =>
|
|
3249
|
+
{
|
|
3250
|
+
Expect(pError).to.be.null;
|
|
3251
|
+
Expect(pRecords).to.be.an('array');
|
|
3252
|
+
Expect(pRecords.length).to.equal(2);
|
|
3253
|
+
Expect(pRecords[0].name).to.equal('Alpha');
|
|
3254
|
+
return fDone();
|
|
3255
|
+
});
|
|
3256
|
+
}
|
|
3257
|
+
);
|
|
3258
|
+
|
|
3259
|
+
test
|
|
3260
|
+
(
|
|
3261
|
+
'parseJSON with deep dataPath including array index',
|
|
3262
|
+
function(fDone)
|
|
3263
|
+
{
|
|
3264
|
+
let tmpJSON = JSON.stringify({
|
|
3265
|
+
Results: {
|
|
3266
|
+
series: [
|
|
3267
|
+
{
|
|
3268
|
+
seriesID: 'CUUR0000SA0',
|
|
3269
|
+
data: [
|
|
3270
|
+
{ year: '2024', value: '310.1' },
|
|
3271
|
+
{ year: '2023', value: '305.2' }
|
|
3272
|
+
]
|
|
3273
|
+
}
|
|
3274
|
+
]
|
|
3275
|
+
}
|
|
3276
|
+
});
|
|
3277
|
+
|
|
3278
|
+
_Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON, { dataPath: 'Results.series[0].data' },
|
|
3279
|
+
(pError, pRecords) =>
|
|
3280
|
+
{
|
|
3281
|
+
Expect(pError).to.be.null;
|
|
3282
|
+
Expect(pRecords).to.be.an('array');
|
|
3283
|
+
Expect(pRecords.length).to.equal(2);
|
|
3284
|
+
Expect(pRecords[0].year).to.equal('2024');
|
|
3285
|
+
Expect(pRecords[1].value).to.equal('305.2');
|
|
3286
|
+
return fDone();
|
|
3287
|
+
});
|
|
3288
|
+
}
|
|
3289
|
+
);
|
|
3290
|
+
|
|
3291
|
+
test
|
|
3292
|
+
(
|
|
3293
|
+
'parseJSON backward compatibility (no options)',
|
|
3294
|
+
function(fDone)
|
|
3295
|
+
{
|
|
3296
|
+
let tmpJSON = JSON.stringify({
|
|
3297
|
+
data: [{ x: 1 }, { x: 2 }]
|
|
3298
|
+
});
|
|
3299
|
+
|
|
3300
|
+
_Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON,
|
|
3301
|
+
(pError, pRecords) =>
|
|
3302
|
+
{
|
|
3303
|
+
Expect(pError).to.be.null;
|
|
3304
|
+
Expect(pRecords).to.be.an('array');
|
|
3305
|
+
Expect(pRecords.length).to.equal(2);
|
|
3306
|
+
return fDone();
|
|
3307
|
+
});
|
|
3308
|
+
}
|
|
3309
|
+
);
|
|
3310
|
+
|
|
3311
|
+
test
|
|
3312
|
+
(
|
|
3313
|
+
'parseJSON with invalid dataPath returns error',
|
|
3314
|
+
function(fDone)
|
|
3315
|
+
{
|
|
3316
|
+
let tmpJSON = JSON.stringify({ foo: { bar: 1 } });
|
|
3317
|
+
|
|
3318
|
+
_Fable.RetoldFactoIngestEngine.parseJSON(tmpJSON, { dataPath: 'missing.path' },
|
|
3319
|
+
(pError, pRecords) =>
|
|
3320
|
+
{
|
|
3321
|
+
Expect(pError).to.be.an('error');
|
|
3322
|
+
Expect(pError.message).to.contain('not found');
|
|
3323
|
+
return fDone();
|
|
3324
|
+
});
|
|
3325
|
+
}
|
|
3326
|
+
);
|
|
3327
|
+
|
|
3328
|
+
test
|
|
3329
|
+
(
|
|
3330
|
+
'_resolveDataPath handles empty path',
|
|
3331
|
+
function(fDone)
|
|
3332
|
+
{
|
|
3333
|
+
let tmpObj = { a: 1 };
|
|
3334
|
+
let tmpResult = _Fable.RetoldFactoIngestEngine._resolveDataPath(tmpObj, '');
|
|
3335
|
+
Expect(tmpResult).to.deep.equal(tmpObj);
|
|
3336
|
+
return fDone();
|
|
3337
|
+
}
|
|
3338
|
+
);
|
|
3339
|
+
}
|
|
3340
|
+
);
|
|
3341
|
+
|
|
3342
|
+
suite
|
|
3343
|
+
(
|
|
3344
|
+
'Catalog Fetch',
|
|
3345
|
+
() =>
|
|
3346
|
+
{
|
|
3347
|
+
let _FetchCatalogEntryID;
|
|
3348
|
+
let _FetchCatalogDatasetID;
|
|
3349
|
+
|
|
3350
|
+
test
|
|
3351
|
+
(
|
|
3352
|
+
'Create catalog entry and dataset for fetch testing',
|
|
3353
|
+
function(fDone)
|
|
3354
|
+
{
|
|
3355
|
+
_SuperTest
|
|
3356
|
+
.post('/facto/catalog/entry')
|
|
3357
|
+
.send({
|
|
3358
|
+
Agency: 'Fetch Test Agency',
|
|
3359
|
+
Name: 'Fetch Test Portal',
|
|
3360
|
+
Type: 'API',
|
|
3361
|
+
URL: 'https://example.invalid',
|
|
3362
|
+
Protocol: 'HTTPS',
|
|
3363
|
+
Category: 'Testing'
|
|
3364
|
+
})
|
|
3365
|
+
.end(
|
|
3366
|
+
(pError, pRes) =>
|
|
3367
|
+
{
|
|
3368
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
3369
|
+
_FetchCatalogEntryID = pRes.body.Entry.IDSourceCatalogEntry;
|
|
3370
|
+
|
|
3371
|
+
_SuperTest
|
|
3372
|
+
.post('/facto/catalog/entry/' + _FetchCatalogEntryID + '/dataset')
|
|
3373
|
+
.send({
|
|
3374
|
+
Name: 'Fetch Test Dataset',
|
|
3375
|
+
Format: 'csv',
|
|
3376
|
+
EndpointURL: 'https://example.invalid/data.csv',
|
|
3377
|
+
Description: 'Test dataset for fetch tests',
|
|
3378
|
+
VersionPolicy: 'Append'
|
|
3379
|
+
})
|
|
3380
|
+
.end(
|
|
3381
|
+
(pError2, pRes2) =>
|
|
3382
|
+
{
|
|
3383
|
+
Expect(pRes2.body.Success).to.equal(true);
|
|
3384
|
+
_FetchCatalogDatasetID = pRes2.body.DatasetDefinition.IDCatalogDatasetDefinition;
|
|
3385
|
+
return fDone();
|
|
3386
|
+
});
|
|
3387
|
+
});
|
|
3388
|
+
}
|
|
3389
|
+
);
|
|
3390
|
+
|
|
3391
|
+
test
|
|
3392
|
+
(
|
|
3393
|
+
'Fetch rejects non-provisioned dataset definition',
|
|
3394
|
+
function(fDone)
|
|
3395
|
+
{
|
|
3396
|
+
_SuperTest
|
|
3397
|
+
.post('/facto/catalog/dataset/' + _FetchCatalogDatasetID + '/fetch')
|
|
3398
|
+
.end(
|
|
3399
|
+
(pError, pRes) =>
|
|
3400
|
+
{
|
|
3401
|
+
Expect(pRes.body.Error).to.contain('provisioned');
|
|
3402
|
+
return fDone();
|
|
3403
|
+
});
|
|
3404
|
+
}
|
|
3405
|
+
);
|
|
3406
|
+
|
|
3407
|
+
test
|
|
3408
|
+
(
|
|
3409
|
+
'Provision dataset for fetch testing',
|
|
3410
|
+
function(fDone)
|
|
3411
|
+
{
|
|
3412
|
+
_SuperTest
|
|
3413
|
+
.post('/facto/catalog/dataset/' + _FetchCatalogDatasetID + '/provision')
|
|
3414
|
+
.end(
|
|
3415
|
+
(pError, pRes) =>
|
|
3416
|
+
{
|
|
3417
|
+
Expect(pRes.body.Success).to.equal(true);
|
|
3418
|
+
return fDone();
|
|
3419
|
+
});
|
|
3420
|
+
}
|
|
3421
|
+
);
|
|
3422
|
+
|
|
3423
|
+
test
|
|
3424
|
+
(
|
|
3425
|
+
'Fetch with unreachable URL returns download error',
|
|
3426
|
+
function(fDone)
|
|
3427
|
+
{
|
|
3428
|
+
this.timeout(35000);
|
|
3429
|
+
_SuperTest
|
|
3430
|
+
.post('/facto/catalog/dataset/' + _FetchCatalogDatasetID + '/fetch')
|
|
3431
|
+
.end(
|
|
3432
|
+
(pError, pRes) =>
|
|
3433
|
+
{
|
|
3434
|
+
Expect(pRes.body.Error).to.be.a('string');
|
|
3435
|
+
Expect(pRes.body.Error).to.contain('Download failed');
|
|
3436
|
+
return fDone();
|
|
3437
|
+
});
|
|
3438
|
+
}
|
|
3439
|
+
);
|
|
3440
|
+
|
|
3441
|
+
test
|
|
3442
|
+
(
|
|
3443
|
+
'Fetch rejects invalid dataset definition ID',
|
|
3444
|
+
function(fDone)
|
|
3445
|
+
{
|
|
3446
|
+
_SuperTest
|
|
3447
|
+
.post('/facto/catalog/dataset/99999/fetch')
|
|
3448
|
+
.end(
|
|
3449
|
+
(pError, pRes) =>
|
|
3450
|
+
{
|
|
3451
|
+
Expect(pRes.body.Error).to.be.a('string');
|
|
3452
|
+
return fDone();
|
|
3453
|
+
});
|
|
3454
|
+
}
|
|
3455
|
+
);
|
|
3456
|
+
|
|
3457
|
+
test
|
|
3458
|
+
(
|
|
3459
|
+
'ingestContent works with CSV string directly',
|
|
3460
|
+
function(fDone)
|
|
3461
|
+
{
|
|
3462
|
+
let tmpCSV = 'name,value\nAlpha,100\nBeta,200\nGamma,300';
|
|
3463
|
+
|
|
3464
|
+
_Fable.RetoldFactoIngestEngine.ingestContent(tmpCSV, 1, 1,
|
|
3465
|
+
{ format: 'csv', type: 'test-ingest' },
|
|
3466
|
+
(pError, pResult) =>
|
|
3467
|
+
{
|
|
3468
|
+
Expect(pError).to.be.null;
|
|
3469
|
+
Expect(pResult.Ingested).to.equal(3);
|
|
3470
|
+
Expect(pResult.Format).to.equal('csv');
|
|
3471
|
+
Expect(pResult.DatasetVersion).to.be.a('number');
|
|
3472
|
+
Expect(pResult.ContentSignature).to.be.a('string');
|
|
3473
|
+
Expect(pResult.IngestJob).to.be.an('object');
|
|
3474
|
+
return fDone();
|
|
3475
|
+
});
|
|
3476
|
+
}
|
|
3477
|
+
);
|
|
3478
|
+
|
|
3479
|
+
test
|
|
3480
|
+
(
|
|
3481
|
+
'ingestContent works with JSON string and dataPath',
|
|
3482
|
+
function(fDone)
|
|
3483
|
+
{
|
|
3484
|
+
let tmpJSON = JSON.stringify({
|
|
3485
|
+
metadata: { count: 2 },
|
|
3486
|
+
data: [
|
|
3487
|
+
{ id: 1, val: 'x' },
|
|
3488
|
+
{ id: 2, val: 'y' }
|
|
3489
|
+
]
|
|
3490
|
+
});
|
|
3491
|
+
|
|
3492
|
+
_Fable.RetoldFactoIngestEngine.ingestContent(tmpJSON, 1, 1,
|
|
3493
|
+
{ format: 'json', dataPath: 'data' },
|
|
3494
|
+
(pError, pResult) =>
|
|
3495
|
+
{
|
|
3496
|
+
Expect(pError).to.be.null;
|
|
3497
|
+
Expect(pResult.Ingested).to.equal(2);
|
|
3498
|
+
Expect(pResult.Format).to.equal('json');
|
|
3499
|
+
return fDone();
|
|
3500
|
+
});
|
|
3501
|
+
}
|
|
3502
|
+
);
|
|
3503
|
+
}
|
|
3504
|
+
);
|
|
3505
|
+
|
|
3506
|
+
suite
|
|
3507
|
+
(
|
|
3508
|
+
'Multi-Set Projection Pipeline',
|
|
3509
|
+
function()
|
|
3510
|
+
{
|
|
3511
|
+
// Track IDs for test fixtures
|
|
3512
|
+
let _MultiSetSourceA_ID = 0;
|
|
3513
|
+
let _MultiSetSourceB_ID = 0;
|
|
3514
|
+
let _MultiSetDatasetA_ID = 0;
|
|
3515
|
+
let _MultiSetDatasetB_ID = 0;
|
|
3516
|
+
let _MultiSetProjectionDataset_ID = 0;
|
|
3517
|
+
let _MultiSetMappingA_ID = 0;
|
|
3518
|
+
let _MultiSetMappingB_ID = 0;
|
|
3519
|
+
let _MultiSetProjection_ID = 0;
|
|
3520
|
+
|
|
3521
|
+
test
|
|
3522
|
+
(
|
|
3523
|
+
'Set up multi-set test fixtures: sources, datasets, records',
|
|
3524
|
+
function(fDone)
|
|
3525
|
+
{
|
|
3526
|
+
this.timeout(10000);
|
|
3527
|
+
|
|
3528
|
+
let tmpAnticipate = _Fable.newAnticipate();
|
|
3529
|
+
|
|
3530
|
+
// Create Source A (high reliability)
|
|
3531
|
+
tmpAnticipate.anticipate(
|
|
3532
|
+
(fStepCallback) =>
|
|
3533
|
+
{
|
|
3534
|
+
_SuperTest.post('/1.0/Source')
|
|
3535
|
+
.send({ Name: 'Source A - Census', Type: 'API', Active: 1 })
|
|
3536
|
+
.end((pError, pResponse) =>
|
|
3537
|
+
{
|
|
3538
|
+
_MultiSetSourceA_ID = pResponse.body.IDSource;
|
|
3539
|
+
return fStepCallback();
|
|
3540
|
+
});
|
|
3541
|
+
});
|
|
3542
|
+
|
|
3543
|
+
// Create Source B (lower reliability)
|
|
3544
|
+
tmpAnticipate.anticipate(
|
|
3545
|
+
(fStepCallback) =>
|
|
3546
|
+
{
|
|
3547
|
+
_SuperTest.post('/1.0/Source')
|
|
3548
|
+
.send({ Name: 'Source B - Survey', Type: 'API', Active: 1 })
|
|
3549
|
+
.end((pError, pResponse) =>
|
|
3550
|
+
{
|
|
3551
|
+
_MultiSetSourceB_ID = pResponse.body.IDSource;
|
|
3552
|
+
return fStepCallback();
|
|
3553
|
+
});
|
|
3554
|
+
});
|
|
3555
|
+
|
|
3556
|
+
tmpAnticipate.wait(
|
|
3557
|
+
() =>
|
|
3558
|
+
{
|
|
3559
|
+
let tmpAnticipate2 = _Fable.newAnticipate();
|
|
3560
|
+
|
|
3561
|
+
// Create Dataset A
|
|
3562
|
+
tmpAnticipate2.anticipate(
|
|
3563
|
+
(fStepCallback) =>
|
|
3564
|
+
{
|
|
3565
|
+
_SuperTest.post('/1.0/Dataset')
|
|
3566
|
+
.send({ Name: 'Census Data', Type: 'Raw' })
|
|
3567
|
+
.end((pError, pResponse) =>
|
|
3568
|
+
{
|
|
3569
|
+
_MultiSetDatasetA_ID = pResponse.body.IDDataset;
|
|
3570
|
+
return fStepCallback();
|
|
3571
|
+
});
|
|
3572
|
+
});
|
|
3573
|
+
|
|
3574
|
+
// Create Dataset B
|
|
3575
|
+
tmpAnticipate2.anticipate(
|
|
3576
|
+
(fStepCallback) =>
|
|
3577
|
+
{
|
|
3578
|
+
_SuperTest.post('/1.0/Dataset')
|
|
3579
|
+
.send({ Name: 'Survey Data', Type: 'Raw' })
|
|
3580
|
+
.end((pError, pResponse) =>
|
|
3581
|
+
{
|
|
3582
|
+
_MultiSetDatasetB_ID = pResponse.body.IDDataset;
|
|
3583
|
+
return fStepCallback();
|
|
3584
|
+
});
|
|
3585
|
+
});
|
|
3586
|
+
|
|
3587
|
+
// Create Projection Dataset
|
|
3588
|
+
tmpAnticipate2.anticipate(
|
|
3589
|
+
(fStepCallback) =>
|
|
3590
|
+
{
|
|
3591
|
+
_SuperTest.post('/1.0/Dataset')
|
|
3592
|
+
.send({ Name: 'Multi-Set Projection Target', Type: 'Projection' })
|
|
3593
|
+
.end((pError, pResponse) =>
|
|
3594
|
+
{
|
|
3595
|
+
_MultiSetProjectionDataset_ID = pResponse.body.IDDataset;
|
|
3596
|
+
return fStepCallback();
|
|
3597
|
+
});
|
|
3598
|
+
});
|
|
3599
|
+
|
|
3600
|
+
tmpAnticipate2.wait(
|
|
3601
|
+
() =>
|
|
3602
|
+
{
|
|
3603
|
+
let tmpAnticipate3 = _Fable.newAnticipate();
|
|
3604
|
+
|
|
3605
|
+
// Link DatasetSource A with high weight
|
|
3606
|
+
tmpAnticipate3.anticipate(
|
|
3607
|
+
(fStepCallback) =>
|
|
3608
|
+
{
|
|
3609
|
+
_SuperTest.post('/1.0/DatasetSource')
|
|
3610
|
+
.send({ IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, ReliabilityWeight: 0.9 })
|
|
3611
|
+
.end(() => { return fStepCallback(); });
|
|
3612
|
+
});
|
|
3613
|
+
|
|
3614
|
+
// Link DatasetSource B with lower weight
|
|
3615
|
+
tmpAnticipate3.anticipate(
|
|
3616
|
+
(fStepCallback) =>
|
|
3617
|
+
{
|
|
3618
|
+
_SuperTest.post('/1.0/DatasetSource')
|
|
3619
|
+
.send({ IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, ReliabilityWeight: 0.4 })
|
|
3620
|
+
.end(() => { return fStepCallback(); });
|
|
3621
|
+
});
|
|
3622
|
+
|
|
3623
|
+
// Ingest records for Source A
|
|
3624
|
+
tmpAnticipate3.anticipate(
|
|
3625
|
+
(fStepCallback) =>
|
|
3626
|
+
{
|
|
3627
|
+
let tmpRecords = [
|
|
3628
|
+
{ GUIDRecord: 'PERSON-001', IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, Type: 'person', Content: JSON.stringify({ Name: 'Alice', City: 'Portland', Age: 30 }) },
|
|
3629
|
+
{ GUIDRecord: 'PERSON-002', IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, Type: 'person', Content: JSON.stringify({ Name: 'Bob', City: 'Seattle', Age: 25 }) },
|
|
3630
|
+
{ GUIDRecord: 'PERSON-003', IDDataset: _MultiSetDatasetA_ID, IDSource: _MultiSetSourceA_ID, Type: 'person', Content: JSON.stringify({ Name: 'Charlie', City: 'Denver' }) }
|
|
3631
|
+
];
|
|
3632
|
+
let tmpInner = _Fable.newAnticipate();
|
|
3633
|
+
for (let i = 0; i < tmpRecords.length; i++)
|
|
3634
|
+
{
|
|
3635
|
+
let tmpRec = tmpRecords[i];
|
|
3636
|
+
tmpInner.anticipate(
|
|
3637
|
+
(fInner) =>
|
|
3638
|
+
{
|
|
3639
|
+
_SuperTest.post('/1.0/Record')
|
|
3640
|
+
.send(tmpRec)
|
|
3641
|
+
.end(() => { return fInner(); });
|
|
3642
|
+
});
|
|
3643
|
+
}
|
|
3644
|
+
tmpInner.wait(() => { return fStepCallback(); });
|
|
3645
|
+
});
|
|
3646
|
+
|
|
3647
|
+
// Ingest records for Source B (overlapping GUIDs)
|
|
3648
|
+
tmpAnticipate3.anticipate(
|
|
3649
|
+
(fStepCallback) =>
|
|
3650
|
+
{
|
|
3651
|
+
let tmpRecords = [
|
|
3652
|
+
{ GUIDRecord: 'PERSON-002', IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, Type: 'person', Content: JSON.stringify({ Name: 'Robert', City: 'Tacoma', Age: 26, Phone: '555-0102' }) },
|
|
3653
|
+
{ GUIDRecord: 'PERSON-003', IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, Type: 'person', Content: JSON.stringify({ Name: 'Chuck', City: 'Boulder', Age: 35, Phone: '555-0103' }) },
|
|
3654
|
+
{ GUIDRecord: 'PERSON-004', IDDataset: _MultiSetDatasetB_ID, IDSource: _MultiSetSourceB_ID, Type: 'person', Content: JSON.stringify({ Name: 'Diana', City: 'Austin', Age: 28, Phone: '555-0104' }) }
|
|
3655
|
+
];
|
|
3656
|
+
let tmpInner = _Fable.newAnticipate();
|
|
3657
|
+
for (let i = 0; i < tmpRecords.length; i++)
|
|
3658
|
+
{
|
|
3659
|
+
let tmpRec = tmpRecords[i];
|
|
3660
|
+
tmpInner.anticipate(
|
|
3661
|
+
(fInner) =>
|
|
3662
|
+
{
|
|
3663
|
+
_SuperTest.post('/1.0/Record')
|
|
3664
|
+
.send(tmpRec)
|
|
3665
|
+
.end(() => { return fInner(); });
|
|
3666
|
+
});
|
|
3667
|
+
}
|
|
3668
|
+
tmpInner.wait(() => { return fStepCallback(); });
|
|
3669
|
+
});
|
|
3670
|
+
|
|
3671
|
+
tmpAnticipate3.wait(
|
|
3672
|
+
() =>
|
|
3673
|
+
{
|
|
3674
|
+
return fDone();
|
|
3675
|
+
});
|
|
3676
|
+
});
|
|
3677
|
+
});
|
|
3678
|
+
}
|
|
3679
|
+
);
|
|
3680
|
+
|
|
3681
|
+
test
|
|
3682
|
+
(
|
|
3683
|
+
'Create projection mappings for both sources',
|
|
3684
|
+
function(fDone)
|
|
3685
|
+
{
|
|
3686
|
+
this.timeout(5000);
|
|
3687
|
+
|
|
3688
|
+
let tmpMappingConfigA = JSON.stringify(
|
|
3689
|
+
{
|
|
3690
|
+
Entity: 'Person',
|
|
3691
|
+
GUIDTemplate: '{~D:Record.GUIDRecord~}',
|
|
3692
|
+
Mappings:
|
|
3693
|
+
{
|
|
3694
|
+
Name: '{~D:Record.Name~}',
|
|
3695
|
+
City: '{~D:Record.City~}',
|
|
3696
|
+
Age: '{~D:Record.Age~}',
|
|
3697
|
+
Phone: '{~D:Record.Phone~}'
|
|
3698
|
+
}
|
|
3699
|
+
});
|
|
3700
|
+
|
|
3701
|
+
let tmpMappingConfigB = JSON.stringify(
|
|
3702
|
+
{
|
|
3703
|
+
Entity: 'Person',
|
|
3704
|
+
GUIDTemplate: '{~D:Record.GUIDRecord~}',
|
|
3705
|
+
Mappings:
|
|
3706
|
+
{
|
|
3707
|
+
Name: '{~D:Record.Name~}',
|
|
3708
|
+
City: '{~D:Record.City~}',
|
|
3709
|
+
Age: '{~D:Record.Age~}',
|
|
3710
|
+
Phone: '{~D:Record.Phone~}'
|
|
3711
|
+
}
|
|
3712
|
+
});
|
|
3713
|
+
|
|
3714
|
+
let tmpAnticipate = _Fable.newAnticipate();
|
|
3715
|
+
|
|
3716
|
+
tmpAnticipate.anticipate(
|
|
3717
|
+
(fStepCallback) =>
|
|
3718
|
+
{
|
|
3719
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/mapping`)
|
|
3720
|
+
.send({
|
|
3721
|
+
IDSource: _MultiSetSourceA_ID,
|
|
3722
|
+
Name: 'Census Mapping',
|
|
3723
|
+
MappingConfiguration: tmpMappingConfigA
|
|
3724
|
+
})
|
|
3725
|
+
.end((pError, pResponse) =>
|
|
3726
|
+
{
|
|
3727
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
3728
|
+
_MultiSetMappingA_ID = pResponse.body.Mapping.IDProjectionMapping;
|
|
3729
|
+
return fStepCallback();
|
|
3730
|
+
});
|
|
3731
|
+
});
|
|
3732
|
+
|
|
3733
|
+
tmpAnticipate.anticipate(
|
|
3734
|
+
(fStepCallback) =>
|
|
3735
|
+
{
|
|
3736
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/mapping`)
|
|
3737
|
+
.send({
|
|
3738
|
+
IDSource: _MultiSetSourceB_ID,
|
|
3739
|
+
Name: 'Survey Mapping',
|
|
3740
|
+
MappingConfiguration: tmpMappingConfigB
|
|
3741
|
+
})
|
|
3742
|
+
.end((pError, pResponse) =>
|
|
3743
|
+
{
|
|
3744
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
3745
|
+
_MultiSetMappingB_ID = pResponse.body.Mapping.IDProjectionMapping;
|
|
3746
|
+
return fStepCallback();
|
|
3747
|
+
});
|
|
3748
|
+
});
|
|
3749
|
+
|
|
3750
|
+
tmpAnticipate.wait(
|
|
3751
|
+
() =>
|
|
3752
|
+
{
|
|
3753
|
+
return fDone();
|
|
3754
|
+
});
|
|
3755
|
+
}
|
|
3756
|
+
);
|
|
3757
|
+
|
|
3758
|
+
test
|
|
3759
|
+
(
|
|
3760
|
+
'Create a MultiSetProjection with WriteAll strategy',
|
|
3761
|
+
function(fDone)
|
|
3762
|
+
{
|
|
3763
|
+
this.timeout(5000);
|
|
3764
|
+
|
|
3765
|
+
let tmpPipelineConfig =
|
|
3766
|
+
{
|
|
3767
|
+
Steps:
|
|
3768
|
+
[
|
|
3769
|
+
{
|
|
3770
|
+
IDProjectionMapping: 0, // will be set below
|
|
3771
|
+
Ordinal: 0,
|
|
3772
|
+
MergeStrategy: 'WriteAll',
|
|
3773
|
+
Label: 'Census Source',
|
|
3774
|
+
InputType: 'Records'
|
|
3775
|
+
},
|
|
3776
|
+
{
|
|
3777
|
+
IDProjectionMapping: 0,
|
|
3778
|
+
Ordinal: 1,
|
|
3779
|
+
MergeStrategy: 'WriteAll',
|
|
3780
|
+
Label: 'Survey Source',
|
|
3781
|
+
InputType: 'Records'
|
|
3782
|
+
}
|
|
3783
|
+
],
|
|
3784
|
+
ConfidenceReinforcement:
|
|
3785
|
+
{
|
|
3786
|
+
Enabled: false
|
|
3787
|
+
}
|
|
3788
|
+
};
|
|
3789
|
+
|
|
3790
|
+
// Patch in the mapping IDs
|
|
3791
|
+
tmpPipelineConfig.Steps[0].IDProjectionMapping = _MultiSetMappingA_ID;
|
|
3792
|
+
tmpPipelineConfig.Steps[1].IDProjectionMapping = _MultiSetMappingB_ID;
|
|
3793
|
+
|
|
3794
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
|
|
3795
|
+
.send({
|
|
3796
|
+
Name: 'WriteAll Multi-Set Test',
|
|
3797
|
+
Description: 'Tests WriteAll merge with two sources',
|
|
3798
|
+
PipelineConfiguration: tmpPipelineConfig
|
|
3799
|
+
})
|
|
3800
|
+
.end((pError, pResponse) =>
|
|
3801
|
+
{
|
|
3802
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
3803
|
+
Expect(pResponse.body.MultiSetProjection).to.be.an('object');
|
|
3804
|
+
_MultiSetProjection_ID = pResponse.body.MultiSetProjection.IDMultiSetProjection;
|
|
3805
|
+
return fDone();
|
|
3806
|
+
});
|
|
3807
|
+
}
|
|
3808
|
+
);
|
|
3809
|
+
|
|
3810
|
+
test
|
|
3811
|
+
(
|
|
3812
|
+
'List MultiSetProjections for dataset',
|
|
3813
|
+
function(fDone)
|
|
3814
|
+
{
|
|
3815
|
+
this.timeout(5000);
|
|
3816
|
+
|
|
3817
|
+
_SuperTest.get(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projections`)
|
|
3818
|
+
.end((pError, pResponse) =>
|
|
3819
|
+
{
|
|
3820
|
+
Expect(pResponse.body.Count).to.be.greaterThan(0);
|
|
3821
|
+
Expect(pResponse.body.MultiSetProjections).to.be.an('array');
|
|
3822
|
+
return fDone();
|
|
3823
|
+
});
|
|
3824
|
+
}
|
|
3825
|
+
);
|
|
3826
|
+
|
|
3827
|
+
test
|
|
3828
|
+
(
|
|
3829
|
+
'Get a single MultiSetProjection',
|
|
3830
|
+
function(fDone)
|
|
3831
|
+
{
|
|
3832
|
+
this.timeout(5000);
|
|
3833
|
+
|
|
3834
|
+
_SuperTest.get(`/facto/projection/multi-set-projection/${_MultiSetProjection_ID}`)
|
|
3835
|
+
.end((pError, pResponse) =>
|
|
3836
|
+
{
|
|
3837
|
+
Expect(pResponse.body.MultiSetProjection).to.be.an('object');
|
|
3838
|
+
Expect(pResponse.body.MultiSetProjection.Name).to.equal('WriteAll Multi-Set Test');
|
|
3839
|
+
return fDone();
|
|
3840
|
+
});
|
|
3841
|
+
}
|
|
3842
|
+
);
|
|
3843
|
+
|
|
3844
|
+
test
|
|
3845
|
+
(
|
|
3846
|
+
'Update a MultiSetProjection',
|
|
3847
|
+
function(fDone)
|
|
3848
|
+
{
|
|
3849
|
+
this.timeout(5000);
|
|
3850
|
+
|
|
3851
|
+
_SuperTest.post(`/facto/projection/multi-set-projection/${_MultiSetProjection_ID}/update`)
|
|
3852
|
+
.send({ Description: 'Updated description for WriteAll test' })
|
|
3853
|
+
.end((pError, pResponse) =>
|
|
3854
|
+
{
|
|
3855
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
3856
|
+
Expect(pResponse.body.MultiSetProjection.Description).to.equal('Updated description for WriteAll test');
|
|
3857
|
+
return fDone();
|
|
3858
|
+
});
|
|
3859
|
+
}
|
|
3860
|
+
);
|
|
3861
|
+
|
|
3862
|
+
test
|
|
3863
|
+
(
|
|
3864
|
+
'Create and execute FirstWriteWins pipeline',
|
|
3865
|
+
function(fDone)
|
|
3866
|
+
{
|
|
3867
|
+
this.timeout(15000);
|
|
3868
|
+
|
|
3869
|
+
let tmpPipelineConfig =
|
|
3870
|
+
{
|
|
3871
|
+
Steps:
|
|
3872
|
+
[
|
|
3873
|
+
{
|
|
3874
|
+
IDProjectionMapping: _MultiSetMappingA_ID,
|
|
3875
|
+
Ordinal: 0,
|
|
3876
|
+
MergeStrategy: 'WriteAll',
|
|
3877
|
+
Label: 'Census First',
|
|
3878
|
+
InputType: 'Records'
|
|
3879
|
+
},
|
|
3880
|
+
{
|
|
3881
|
+
IDProjectionMapping: _MultiSetMappingB_ID,
|
|
3882
|
+
Ordinal: 1,
|
|
3883
|
+
MergeStrategy: 'FirstWriteWins',
|
|
3884
|
+
Label: 'Survey Second - FirstWriteWins',
|
|
3885
|
+
InputType: 'Records'
|
|
3886
|
+
}
|
|
3887
|
+
],
|
|
3888
|
+
ConfidenceReinforcement: { Enabled: false }
|
|
3889
|
+
};
|
|
3890
|
+
|
|
3891
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
|
|
3892
|
+
.send({
|
|
3893
|
+
Name: 'FirstWriteWins Test',
|
|
3894
|
+
PipelineConfiguration: tmpPipelineConfig
|
|
3895
|
+
})
|
|
3896
|
+
.end((pError, pResponse) =>
|
|
3897
|
+
{
|
|
3898
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
3899
|
+
let tmpFWW_ID = pResponse.body.MultiSetProjection.IDMultiSetProjection;
|
|
3900
|
+
|
|
3901
|
+
// We cannot fully execute the multi-import without a deployed
|
|
3902
|
+
// projection store, but we can verify the pipeline loads
|
|
3903
|
+
// and the merge logic is applied by checking the response.
|
|
3904
|
+
// For now, test the CRUD path succeeded.
|
|
3905
|
+
Expect(tmpFWW_ID).to.be.greaterThan(0);
|
|
3906
|
+
|
|
3907
|
+
// Verify the pipeline config roundtrips correctly
|
|
3908
|
+
_SuperTest.get(`/facto/projection/multi-set-projection/${tmpFWW_ID}`)
|
|
3909
|
+
.end((pError2, pResponse2) =>
|
|
3910
|
+
{
|
|
3911
|
+
let tmpStored = pResponse2.body.MultiSetProjection;
|
|
3912
|
+
let tmpConfig = JSON.parse(tmpStored.PipelineConfiguration);
|
|
3913
|
+
Expect(tmpConfig.Steps).to.have.length(2);
|
|
3914
|
+
Expect(tmpConfig.Steps[1].MergeStrategy).to.equal('FirstWriteWins');
|
|
3915
|
+
return fDone();
|
|
3916
|
+
});
|
|
3917
|
+
});
|
|
3918
|
+
}
|
|
3919
|
+
);
|
|
3920
|
+
|
|
3921
|
+
test
|
|
3922
|
+
(
|
|
3923
|
+
'Create MergeAndReinforce pipeline with confidence tracking',
|
|
3924
|
+
function(fDone)
|
|
3925
|
+
{
|
|
3926
|
+
this.timeout(5000);
|
|
3927
|
+
|
|
3928
|
+
let tmpPipelineConfig =
|
|
3929
|
+
{
|
|
3930
|
+
Steps:
|
|
3931
|
+
[
|
|
3932
|
+
{
|
|
3933
|
+
IDProjectionMapping: _MultiSetMappingA_ID,
|
|
3934
|
+
Ordinal: 0,
|
|
3935
|
+
MergeStrategy: 'WriteAll',
|
|
3936
|
+
Label: 'Census Base',
|
|
3937
|
+
InputType: 'Records'
|
|
3938
|
+
},
|
|
3939
|
+
{
|
|
3940
|
+
IDProjectionMapping: _MultiSetMappingB_ID,
|
|
3941
|
+
Ordinal: 1,
|
|
3942
|
+
MergeStrategy: 'MergeAndReinforce',
|
|
3943
|
+
Label: 'Survey Reinforcement',
|
|
3944
|
+
InputType: 'Records'
|
|
3945
|
+
}
|
|
3946
|
+
],
|
|
3947
|
+
ConfidenceReinforcement:
|
|
3948
|
+
{
|
|
3949
|
+
Enabled: true,
|
|
3950
|
+
Dimension: 'multi-source',
|
|
3951
|
+
BaseValue: 0.5,
|
|
3952
|
+
IncrementPerConfirmation: 0.15,
|
|
3953
|
+
MaxValue: 1.0
|
|
3954
|
+
}
|
|
3955
|
+
};
|
|
3956
|
+
|
|
3957
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
|
|
3958
|
+
.send({
|
|
3959
|
+
Name: 'MergeAndReinforce Test',
|
|
3960
|
+
PipelineConfiguration: tmpPipelineConfig
|
|
3961
|
+
})
|
|
3962
|
+
.end((pError, pResponse) =>
|
|
3963
|
+
{
|
|
3964
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
3965
|
+
let tmpConfig = JSON.parse(pResponse.body.MultiSetProjection.PipelineConfiguration);
|
|
3966
|
+
Expect(tmpConfig.ConfidenceReinforcement.Enabled).to.equal(true);
|
|
3967
|
+
Expect(tmpConfig.ConfidenceReinforcement.IncrementPerConfirmation).to.equal(0.15);
|
|
3968
|
+
return fDone();
|
|
3969
|
+
});
|
|
3970
|
+
}
|
|
3971
|
+
);
|
|
3972
|
+
|
|
3973
|
+
test
|
|
3974
|
+
(
|
|
3975
|
+
'Create ReliabilityOverwrite pipeline',
|
|
3976
|
+
function(fDone)
|
|
3977
|
+
{
|
|
3978
|
+
this.timeout(5000);
|
|
3979
|
+
|
|
3980
|
+
let tmpPipelineConfig =
|
|
3981
|
+
{
|
|
3982
|
+
Steps:
|
|
3983
|
+
[
|
|
3984
|
+
{
|
|
3985
|
+
IDProjectionMapping: _MultiSetMappingB_ID,
|
|
3986
|
+
Ordinal: 0,
|
|
3987
|
+
MergeStrategy: 'WriteAll',
|
|
3988
|
+
Label: 'Survey First (low reliability)',
|
|
3989
|
+
InputType: 'Records'
|
|
3990
|
+
},
|
|
3991
|
+
{
|
|
3992
|
+
IDProjectionMapping: _MultiSetMappingA_ID,
|
|
3993
|
+
Ordinal: 1,
|
|
3994
|
+
MergeStrategy: 'ReliabilityOverwrite',
|
|
3995
|
+
Label: 'Census Override (high reliability)',
|
|
3996
|
+
InputType: 'Records'
|
|
3997
|
+
}
|
|
3998
|
+
],
|
|
3999
|
+
ConfidenceReinforcement: { Enabled: false }
|
|
4000
|
+
};
|
|
4001
|
+
|
|
4002
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
|
|
4003
|
+
.send({
|
|
4004
|
+
Name: 'ReliabilityOverwrite Test',
|
|
4005
|
+
PipelineConfiguration: tmpPipelineConfig
|
|
4006
|
+
})
|
|
4007
|
+
.end((pError, pResponse) =>
|
|
4008
|
+
{
|
|
4009
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
4010
|
+
let tmpConfig = JSON.parse(pResponse.body.MultiSetProjection.PipelineConfiguration);
|
|
4011
|
+
Expect(tmpConfig.Steps[1].MergeStrategy).to.equal('ReliabilityOverwrite');
|
|
4012
|
+
return fDone();
|
|
4013
|
+
});
|
|
4014
|
+
}
|
|
4015
|
+
);
|
|
4016
|
+
|
|
4017
|
+
test
|
|
4018
|
+
(
|
|
4019
|
+
'Create FieldFillOnly pipeline',
|
|
4020
|
+
function(fDone)
|
|
4021
|
+
{
|
|
4022
|
+
this.timeout(5000);
|
|
4023
|
+
|
|
4024
|
+
let tmpPipelineConfig =
|
|
4025
|
+
{
|
|
4026
|
+
Steps:
|
|
4027
|
+
[
|
|
4028
|
+
{
|
|
4029
|
+
IDProjectionMapping: _MultiSetMappingA_ID,
|
|
4030
|
+
Ordinal: 0,
|
|
4031
|
+
MergeStrategy: 'WriteAll',
|
|
4032
|
+
Label: 'Census Base (some fields missing)',
|
|
4033
|
+
InputType: 'Records'
|
|
4034
|
+
},
|
|
4035
|
+
{
|
|
4036
|
+
IDProjectionMapping: _MultiSetMappingB_ID,
|
|
4037
|
+
Ordinal: 1,
|
|
4038
|
+
MergeStrategy: 'FieldFillOnly',
|
|
4039
|
+
Label: 'Survey Fill (adds Phone field)',
|
|
4040
|
+
InputType: 'Records'
|
|
4041
|
+
}
|
|
4042
|
+
],
|
|
4043
|
+
ConfidenceReinforcement: { Enabled: false }
|
|
4044
|
+
};
|
|
4045
|
+
|
|
4046
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
|
|
4047
|
+
.send({
|
|
4048
|
+
Name: 'FieldFillOnly Test',
|
|
4049
|
+
PipelineConfiguration: tmpPipelineConfig
|
|
4050
|
+
})
|
|
4051
|
+
.end((pError, pResponse) =>
|
|
4052
|
+
{
|
|
4053
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
4054
|
+
let tmpConfig = JSON.parse(pResponse.body.MultiSetProjection.PipelineConfiguration);
|
|
4055
|
+
Expect(tmpConfig.Steps[1].MergeStrategy).to.equal('FieldFillOnly');
|
|
4056
|
+
return fDone();
|
|
4057
|
+
});
|
|
4058
|
+
}
|
|
4059
|
+
);
|
|
4060
|
+
|
|
4061
|
+
test
|
|
4062
|
+
(
|
|
4063
|
+
'Soft-delete a MultiSetProjection',
|
|
4064
|
+
function(fDone)
|
|
4065
|
+
{
|
|
4066
|
+
this.timeout(5000);
|
|
4067
|
+
|
|
4068
|
+
// Create one to delete
|
|
4069
|
+
_SuperTest.post(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projection`)
|
|
4070
|
+
.send({
|
|
4071
|
+
Name: 'To Be Deleted',
|
|
4072
|
+
PipelineConfiguration: { Steps: [] }
|
|
4073
|
+
})
|
|
4074
|
+
.end((pError, pResponse) =>
|
|
4075
|
+
{
|
|
4076
|
+
Expect(pResponse.body.Success).to.equal(true);
|
|
4077
|
+
let tmpDeleteID = pResponse.body.MultiSetProjection.IDMultiSetProjection;
|
|
4078
|
+
|
|
4079
|
+
_SuperTest.del(`/facto/projection/multi-set-projection/${tmpDeleteID}`)
|
|
4080
|
+
.end((pError2, pResponse2) =>
|
|
4081
|
+
{
|
|
4082
|
+
Expect(pResponse2.body.Success).to.equal(true);
|
|
4083
|
+
|
|
4084
|
+
// Verify it's soft-deleted (doesn't appear in list)
|
|
4085
|
+
_SuperTest.get(`/facto/projection/${_MultiSetProjectionDataset_ID}/multi-set-projections`)
|
|
4086
|
+
.end((pError3, pResponse3) =>
|
|
4087
|
+
{
|
|
4088
|
+
let tmpFound = pResponse3.body.MultiSetProjections.filter(
|
|
4089
|
+
(pRec) => { return pRec.IDMultiSetProjection === tmpDeleteID; });
|
|
4090
|
+
Expect(tmpFound.length).to.equal(0);
|
|
4091
|
+
return fDone();
|
|
4092
|
+
});
|
|
4093
|
+
});
|
|
4094
|
+
});
|
|
4095
|
+
}
|
|
4096
|
+
);
|
|
4097
|
+
|
|
4098
|
+
test
|
|
4099
|
+
(
|
|
4100
|
+
'Query certainty log (empty for unexecuted pipeline)',
|
|
4101
|
+
function(fDone)
|
|
4102
|
+
{
|
|
4103
|
+
this.timeout(5000);
|
|
4104
|
+
|
|
4105
|
+
_SuperTest.get(`/facto/projection/multi-set-projection/${_MultiSetProjection_ID}/certainty-log`)
|
|
4106
|
+
.end((pError, pResponse) =>
|
|
4107
|
+
{
|
|
4108
|
+
Expect(pResponse.body.CertaintyLog).to.be.an('array');
|
|
4109
|
+
Expect(pResponse.body.Count).to.equal(0);
|
|
4110
|
+
return fDone();
|
|
4111
|
+
});
|
|
4112
|
+
}
|
|
4113
|
+
);
|
|
4114
|
+
}
|
|
4115
|
+
);
|
|
4116
|
+
}
|
|
4117
|
+
);
|