retold-data-service 2.0.13 → 2.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/launch.json +11 -0
- package/bin/retold-data-service-clone.js +286 -0
- package/package.json +18 -9
- package/source/Retold-Data-Service.js +275 -73
- package/source/services/Retold-Data-Service-ConnectionManager.js +277 -0
- package/source/services/Retold-Data-Service-MeadowEndpoints.js +217 -0
- package/source/services/Retold-Data-Service-ModelManager.js +335 -0
- package/source/services/data-cloner/DataCloner-Command-Connection.js +138 -0
- package/source/services/data-cloner/DataCloner-Command-Headless.js +357 -0
- package/source/services/data-cloner/DataCloner-Command-Schema.js +367 -0
- package/source/services/data-cloner/DataCloner-Command-Session.js +229 -0
- package/source/services/data-cloner/DataCloner-Command-Sync.js +491 -0
- package/source/services/data-cloner/DataCloner-Command-WebUI.js +40 -0
- package/source/services/data-cloner/DataCloner-ProviderRegistry.js +20 -0
- package/source/services/data-cloner/Retold-Data-Service-DataCloner.js +751 -0
- package/source/services/data-cloner/data-cloner-web.html +2706 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Dashboard.js +60 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Integrations.js +132 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Runs.js +93 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Base.js +116 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Bibliograph.js +495 -0
- package/source/services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js +224 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-CSVCheck.js +85 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-CSVTransform.js +180 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionIntersect.js +153 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionPush.js +190 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToArray.js +113 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToCSV.js +211 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-EntityFromTabularFolder.js +244 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-JSONArrayTransform.js +213 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-TSVCheck.js +80 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-TSVTransform.js +166 -0
- package/source/services/meadow-integration/Retold-Data-Service-MeadowIntegration.js +113 -0
- package/source/services/migration-manager/MigrationManager-Command-Connections.js +220 -0
- package/source/services/migration-manager/MigrationManager-Command-DiffMigrate.js +169 -0
- package/source/services/migration-manager/MigrationManager-Command-Schemas.js +532 -0
- package/source/services/migration-manager/MigrationManager-Command-WebUI.js +123 -0
- package/source/services/migration-manager/Retold-Data-Service-MigrationManager.js +357 -0
- package/source/services/stricture/Retold-Data-Service-Stricture.js +303 -0
- package/source/services/stricture/Stricture-Command-Compile.js +39 -0
- package/source/services/stricture/Stricture-Command-Generate-AuthorizationChart.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-DictionaryCSV.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-LaTeX.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-Markdown.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-Meadow.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-ModelGraph.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-MySQL.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-MySQLMigrate.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-Pict.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-TestObjectContainers.js +14 -0
- package/test/RetoldDataService_tests.js +161 -1
- package/debug/data/books.csv +0 -10001
package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Bibliograph.js
ADDED
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration Telemetry — Bibliograph Storage Provider
|
|
3
|
+
*
|
|
4
|
+
* Default out-of-the-box persistence backed by Bibliograph (parime JSON store).
|
|
5
|
+
* Uses sequential keys with hashed RunIDs for file naming.
|
|
6
|
+
*
|
|
7
|
+
* Source naming convention:
|
|
8
|
+
* - Per-tenant: telemetry-runs-{TenantID}
|
|
9
|
+
* - Global: telemetry-runs-global
|
|
10
|
+
*
|
|
11
|
+
* Key scheme: {seq}-{hash8} e.g. "000042-a1b2c3d4"
|
|
12
|
+
*
|
|
13
|
+
* @param {Object} pFable - The Fable instance
|
|
14
|
+
* @param {Object} pOptions - Provider configuration
|
|
15
|
+
*/
|
|
16
|
+
const libIntegrationTelemetryStorageProviderBase = require('./IntegrationTelemetry-StorageProvider-Base.js');
|
|
17
|
+
const libCrypto = require('crypto');
|
|
18
|
+
|
|
19
|
+
class IntegrationTelemetryStorageProviderBibliograph extends libIntegrationTelemetryStorageProviderBase
|
|
20
|
+
{
|
|
21
|
+
constructor(pFable, pOptions)
|
|
22
|
+
{
|
|
23
|
+
super(pFable, pOptions);
|
|
24
|
+
|
|
25
|
+
this._sequenceCounters = {};
|
|
26
|
+
this._sourceInitialized = {};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Get or create the Bibliograph service instance.
|
|
31
|
+
*/
|
|
32
|
+
get bibliograph()
|
|
33
|
+
{
|
|
34
|
+
if (!this.fable.Bibliograph)
|
|
35
|
+
{
|
|
36
|
+
let libBibliograph = require('bibliograph');
|
|
37
|
+
this.fable.serviceManager.addServiceType('Bibliograph', libBibliograph);
|
|
38
|
+
this.fable.serviceManager.instantiateServiceProvider('Bibliograph', this.options.Bibliograph || {});
|
|
39
|
+
}
|
|
40
|
+
return this.fable.Bibliograph;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Build the source hash for a tenant.
|
|
45
|
+
*/
|
|
46
|
+
tenantSource(pTenantID)
|
|
47
|
+
{
|
|
48
|
+
return `telemetry-runs-${pTenantID}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Build a deterministic 8-char hash from a RunID.
|
|
53
|
+
*/
|
|
54
|
+
hashRunID(pRunID)
|
|
55
|
+
{
|
|
56
|
+
return libCrypto.createHash('md5').update(pRunID).digest('hex').substring(0, 8);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Ensure a source exists, calling createSource only once per source.
|
|
61
|
+
*/
|
|
62
|
+
ensureSource(pSourceHash, fCallback)
|
|
63
|
+
{
|
|
64
|
+
if (this._sourceInitialized[pSourceHash])
|
|
65
|
+
{
|
|
66
|
+
return fCallback();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
this.bibliograph.createSource(pSourceHash,
|
|
70
|
+
(pError) =>
|
|
71
|
+
{
|
|
72
|
+
// Ignore "already exists" style errors
|
|
73
|
+
this._sourceInitialized[pSourceHash] = true;
|
|
74
|
+
return fCallback();
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Get the next sequence number for a source.
|
|
80
|
+
*/
|
|
81
|
+
nextSequence(pSourceHash)
|
|
82
|
+
{
|
|
83
|
+
if (!this._sequenceCounters.hasOwnProperty(pSourceHash))
|
|
84
|
+
{
|
|
85
|
+
this._sequenceCounters[pSourceHash] = 0;
|
|
86
|
+
}
|
|
87
|
+
this._sequenceCounters[pSourceHash]++;
|
|
88
|
+
return this._sequenceCounters[pSourceHash].toString().padStart(6, '0');
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Build the record key from a sequence number and RunID hash.
|
|
93
|
+
*/
|
|
94
|
+
buildKey(pSourceHash, pRunID)
|
|
95
|
+
{
|
|
96
|
+
let tmpSeq = this.nextSequence(pSourceHash);
|
|
97
|
+
let tmpHash = this.hashRunID(pRunID);
|
|
98
|
+
return `${tmpSeq}-${tmpHash}`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// ================================================================
|
|
102
|
+
// Write
|
|
103
|
+
// ================================================================
|
|
104
|
+
|
|
105
|
+
writeRun(pTenantID, pRunRecord, fCallback)
|
|
106
|
+
{
|
|
107
|
+
let tmpTenantSource = this.tenantSource(pTenantID);
|
|
108
|
+
let tmpGlobalSource = 'telemetry-runs-global';
|
|
109
|
+
|
|
110
|
+
this.ensureSource(tmpTenantSource,
|
|
111
|
+
(pError) =>
|
|
112
|
+
{
|
|
113
|
+
if (pError)
|
|
114
|
+
{
|
|
115
|
+
this.fable.log.error(`IntegrationTelemetry Bibliograph: Error ensuring tenant source [${tmpTenantSource}]: ${pError}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
this.ensureSource(tmpGlobalSource,
|
|
119
|
+
(pError) =>
|
|
120
|
+
{
|
|
121
|
+
if (pError)
|
|
122
|
+
{
|
|
123
|
+
this.fable.log.error(`IntegrationTelemetry Bibliograph: Error ensuring global source: ${pError}`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let tmpTenantKey = this.buildKey(tmpTenantSource, pRunRecord.RunID);
|
|
127
|
+
let tmpGlobalKey = this.buildKey(tmpGlobalSource, pRunRecord.RunID);
|
|
128
|
+
|
|
129
|
+
// Dual-write: tenant source + global source
|
|
130
|
+
this.bibliograph.write(tmpTenantSource, tmpTenantKey, pRunRecord,
|
|
131
|
+
(pWriteError) =>
|
|
132
|
+
{
|
|
133
|
+
if (pWriteError)
|
|
134
|
+
{
|
|
135
|
+
this.fable.log.error(`IntegrationTelemetry Bibliograph: Error writing tenant run: ${pWriteError}`);
|
|
136
|
+
return fCallback(pWriteError);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
this.bibliograph.write(tmpGlobalSource, tmpGlobalKey, pRunRecord,
|
|
140
|
+
(pGlobalWriteError) =>
|
|
141
|
+
{
|
|
142
|
+
if (pGlobalWriteError)
|
|
143
|
+
{
|
|
144
|
+
this.fable.log.warn(`IntegrationTelemetry Bibliograph: Error writing global run (tenant copy succeeded): ${pGlobalWriteError}`);
|
|
145
|
+
}
|
|
146
|
+
return fCallback();
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ================================================================
|
|
154
|
+
// Read
|
|
155
|
+
// ================================================================
|
|
156
|
+
|
|
157
|
+
readRun(pTenantID, pRunID, fCallback)
|
|
158
|
+
{
|
|
159
|
+
let tmpTenantSource = this.tenantSource(pTenantID);
|
|
160
|
+
|
|
161
|
+
this.bibliograph.readRecordKeys(tmpTenantSource,
|
|
162
|
+
(pError, pKeys) =>
|
|
163
|
+
{
|
|
164
|
+
if (pError || !pKeys)
|
|
165
|
+
{
|
|
166
|
+
// Source doesn't exist yet — no runs recorded
|
|
167
|
+
return fCallback(null, null);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
let tmpTargetHash = this.hashRunID(pRunID);
|
|
171
|
+
let tmpMatchKey = null;
|
|
172
|
+
|
|
173
|
+
for (let i = 0; i < pKeys.length; i++)
|
|
174
|
+
{
|
|
175
|
+
if (pKeys[i].endsWith(`-${tmpTargetHash}`))
|
|
176
|
+
{
|
|
177
|
+
tmpMatchKey = pKeys[i];
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (!tmpMatchKey)
|
|
183
|
+
{
|
|
184
|
+
return fCallback(null, null);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
this.bibliograph.read(tmpTenantSource, tmpMatchKey, fCallback);
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// ================================================================
|
|
192
|
+
// List helpers
|
|
193
|
+
// ================================================================
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Read all records from a source, apply in-memory filters, and return a page.
|
|
197
|
+
*/
|
|
198
|
+
_readAndFilter(pSourceHash, pOptions, fCallback)
|
|
199
|
+
{
|
|
200
|
+
this.bibliograph.readRecordKeys(pSourceHash,
|
|
201
|
+
(pError, pKeys) =>
|
|
202
|
+
{
|
|
203
|
+
if (pError)
|
|
204
|
+
{
|
|
205
|
+
return fCallback(null, []);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (!pKeys || pKeys.length === 0)
|
|
209
|
+
{
|
|
210
|
+
return fCallback(null, []);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Read all records (fine at Bibliograph/parime scale)
|
|
214
|
+
let tmpRecords = [];
|
|
215
|
+
let tmpRemaining = pKeys.length;
|
|
216
|
+
let tmpDone = false;
|
|
217
|
+
|
|
218
|
+
for (let i = 0; i < pKeys.length; i++)
|
|
219
|
+
{
|
|
220
|
+
this.bibliograph.read(pSourceHash, pKeys[i],
|
|
221
|
+
(pReadError, pRecord) =>
|
|
222
|
+
{
|
|
223
|
+
if (tmpDone)
|
|
224
|
+
{
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
if (!pReadError && pRecord)
|
|
229
|
+
{
|
|
230
|
+
tmpRecords.push(pRecord);
|
|
231
|
+
}
|
|
232
|
+
tmpRemaining--;
|
|
233
|
+
|
|
234
|
+
if (tmpRemaining <= 0)
|
|
235
|
+
{
|
|
236
|
+
tmpDone = true;
|
|
237
|
+
// Apply filters
|
|
238
|
+
let tmpFiltered = this._applyFilters(tmpRecords, pOptions);
|
|
239
|
+
return fCallback(null, tmpFiltered);
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Apply filter/sort/paginate to an array of run records.
|
|
248
|
+
*/
|
|
249
|
+
_applyFilters(pRecords, pOptions)
|
|
250
|
+
{
|
|
251
|
+
let tmpOptions = pOptions || {};
|
|
252
|
+
let tmpResult = pRecords.slice();
|
|
253
|
+
|
|
254
|
+
// Filter by Outcome
|
|
255
|
+
if (tmpOptions.Outcome)
|
|
256
|
+
{
|
|
257
|
+
let tmpOutcome = tmpOptions.Outcome.toLowerCase();
|
|
258
|
+
tmpResult = tmpResult.filter(
|
|
259
|
+
(pRecord) =>
|
|
260
|
+
{
|
|
261
|
+
return pRecord.Outcome && pRecord.Outcome.toLowerCase() === tmpOutcome;
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Filter by IntegrationName
|
|
266
|
+
if (tmpOptions.IntegrationName)
|
|
267
|
+
{
|
|
268
|
+
tmpResult = tmpResult.filter(
|
|
269
|
+
(pRecord) =>
|
|
270
|
+
{
|
|
271
|
+
return pRecord.IntegrationName === tmpOptions.IntegrationName;
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// Filter by TenantID
|
|
276
|
+
if (tmpOptions.TenantID)
|
|
277
|
+
{
|
|
278
|
+
tmpResult = tmpResult.filter(
|
|
279
|
+
(pRecord) =>
|
|
280
|
+
{
|
|
281
|
+
return pRecord.TenantID === tmpOptions.TenantID;
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// Filter by date range
|
|
286
|
+
if (tmpOptions.From)
|
|
287
|
+
{
|
|
288
|
+
let tmpFrom = new Date(tmpOptions.From).getTime();
|
|
289
|
+
tmpResult = tmpResult.filter(
|
|
290
|
+
(pRecord) =>
|
|
291
|
+
{
|
|
292
|
+
return pRecord.StartedAt && new Date(pRecord.StartedAt).getTime() >= tmpFrom;
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
if (tmpOptions.To)
|
|
296
|
+
{
|
|
297
|
+
let tmpTo = new Date(tmpOptions.To).getTime();
|
|
298
|
+
tmpResult = tmpResult.filter(
|
|
299
|
+
(pRecord) =>
|
|
300
|
+
{
|
|
301
|
+
return pRecord.StartedAt && new Date(pRecord.StartedAt).getTime() <= tmpTo;
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// Sort by StartedAt descending (most recent first)
|
|
306
|
+
tmpResult.sort(
|
|
307
|
+
(a, b) =>
|
|
308
|
+
{
|
|
309
|
+
let tmpA = a.StartedAt ? new Date(a.StartedAt).getTime() : 0;
|
|
310
|
+
let tmpB = b.StartedAt ? new Date(b.StartedAt).getTime() : 0;
|
|
311
|
+
return tmpB - tmpA;
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
// Paginate
|
|
315
|
+
let tmpOffset = parseInt(tmpOptions.Offset) || 0;
|
|
316
|
+
let tmpLimit = parseInt(tmpOptions.Limit) || 50;
|
|
317
|
+
tmpResult = tmpResult.slice(tmpOffset, tmpOffset + tmpLimit);
|
|
318
|
+
|
|
319
|
+
return tmpResult;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// ================================================================
|
|
323
|
+
// List implementations
|
|
324
|
+
// ================================================================
|
|
325
|
+
|
|
326
|
+
listRuns(pTenantID, pOptions, fCallback)
|
|
327
|
+
{
|
|
328
|
+
let tmpTenantSource = this.tenantSource(pTenantID);
|
|
329
|
+
this._readAndFilter(tmpTenantSource, pOptions, fCallback);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
listRunsByIntegration(pTenantID, pIntegrationName, pOptions, fCallback)
|
|
333
|
+
{
|
|
334
|
+
let tmpTenantSource = this.tenantSource(pTenantID);
|
|
335
|
+
let tmpOptions = Object.assign({}, pOptions, { IntegrationName: pIntegrationName });
|
|
336
|
+
this._readAndFilter(tmpTenantSource, tmpOptions, fCallback);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
listAllTenantRuns(pOptions, fCallback)
|
|
340
|
+
{
|
|
341
|
+
this._readAndFilter('telemetry-runs-global', pOptions, fCallback);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// ================================================================
|
|
345
|
+
// Aggregation
|
|
346
|
+
// ================================================================
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* Build summary stats from an array of run records.
|
|
350
|
+
*/
|
|
351
|
+
_buildSummary(pRecords)
|
|
352
|
+
{
|
|
353
|
+
let tmpTotal = pRecords.length;
|
|
354
|
+
let tmpSuccesses = 0;
|
|
355
|
+
let tmpFailures = 0;
|
|
356
|
+
let tmpPartial = 0;
|
|
357
|
+
let tmpStopped = 0;
|
|
358
|
+
let tmpTotalDuration = 0;
|
|
359
|
+
let tmpTotalRecordsSynced = 0;
|
|
360
|
+
let tmpLatestRun = null;
|
|
361
|
+
|
|
362
|
+
for (let i = 0; i < pRecords.length; i++)
|
|
363
|
+
{
|
|
364
|
+
let tmpRun = pRecords[i];
|
|
365
|
+
|
|
366
|
+
if (tmpRun.Outcome === 'Success') tmpSuccesses++;
|
|
367
|
+
else if (tmpRun.Outcome === 'Error') tmpFailures++;
|
|
368
|
+
else if (tmpRun.Outcome === 'Partial') tmpPartial++;
|
|
369
|
+
else if (tmpRun.Outcome === 'Stopped') tmpStopped++;
|
|
370
|
+
|
|
371
|
+
tmpTotalDuration += (tmpRun.DurationMs || 0);
|
|
372
|
+
if (tmpRun.Summary)
|
|
373
|
+
{
|
|
374
|
+
tmpTotalRecordsSynced += (tmpRun.Summary.RecordsSynced || 0);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
if (!tmpLatestRun || (tmpRun.StartedAt && new Date(tmpRun.StartedAt) > new Date(tmpLatestRun.StartedAt)))
|
|
378
|
+
{
|
|
379
|
+
tmpLatestRun = tmpRun;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
return (
|
|
384
|
+
{
|
|
385
|
+
TotalRuns: tmpTotal,
|
|
386
|
+
Successes: tmpSuccesses,
|
|
387
|
+
Failures: tmpFailures,
|
|
388
|
+
Partial: tmpPartial,
|
|
389
|
+
Stopped: tmpStopped,
|
|
390
|
+
SuccessRate: tmpTotal > 0 ? Math.round((tmpSuccesses / tmpTotal) * 10000) / 100 : 0,
|
|
391
|
+
AverageDurationMs: tmpTotal > 0 ? Math.round(tmpTotalDuration / tmpTotal) : 0,
|
|
392
|
+
TotalRecordsSynced: tmpTotalRecordsSynced,
|
|
393
|
+
LatestRun: tmpLatestRun
|
|
394
|
+
? { RunID: tmpLatestRun.RunID, Outcome: tmpLatestRun.Outcome, StartedAt: tmpLatestRun.StartedAt, CompletedAt: tmpLatestRun.CompletedAt }
|
|
395
|
+
: null
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
getIntegrationSummary(pTenantID, pIntegrationName, fCallback)
|
|
400
|
+
{
|
|
401
|
+
let tmpTenantSource = this.tenantSource(pTenantID);
|
|
402
|
+
this._readAndFilter(tmpTenantSource, { IntegrationName: pIntegrationName, Limit: 10000 },
|
|
403
|
+
(pError, pRecords) =>
|
|
404
|
+
{
|
|
405
|
+
if (pError)
|
|
406
|
+
{
|
|
407
|
+
return fCallback(pError);
|
|
408
|
+
}
|
|
409
|
+
let tmpSummary = this._buildSummary(pRecords);
|
|
410
|
+
tmpSummary.IntegrationName = pIntegrationName;
|
|
411
|
+
tmpSummary.TenantID = pTenantID;
|
|
412
|
+
return fCallback(null, tmpSummary);
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
getDashboardSummary(pTenantID, fCallback)
|
|
417
|
+
{
|
|
418
|
+
let tmpTenantSource = this.tenantSource(pTenantID);
|
|
419
|
+
this._readAndFilter(tmpTenantSource, { Limit: 10000 },
|
|
420
|
+
(pError, pRecords) =>
|
|
421
|
+
{
|
|
422
|
+
if (pError)
|
|
423
|
+
{
|
|
424
|
+
return fCallback(pError);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// Group by IntegrationName
|
|
428
|
+
let tmpByIntegration = {};
|
|
429
|
+
for (let i = 0; i < pRecords.length; i++)
|
|
430
|
+
{
|
|
431
|
+
let tmpName = pRecords[i].IntegrationName || 'Unknown';
|
|
432
|
+
if (!tmpByIntegration[tmpName])
|
|
433
|
+
{
|
|
434
|
+
tmpByIntegration[tmpName] = [];
|
|
435
|
+
}
|
|
436
|
+
tmpByIntegration[tmpName].push(pRecords[i]);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
let tmpIntegrations = [];
|
|
440
|
+
let tmpIntNames = Object.keys(tmpByIntegration);
|
|
441
|
+
for (let i = 0; i < tmpIntNames.length; i++)
|
|
442
|
+
{
|
|
443
|
+
let tmpIntSummary = this._buildSummary(tmpByIntegration[tmpIntNames[i]]);
|
|
444
|
+
tmpIntSummary.IntegrationName = tmpIntNames[i];
|
|
445
|
+
tmpIntegrations.push(tmpIntSummary);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
let tmpOverall = this._buildSummary(pRecords);
|
|
449
|
+
tmpOverall.TenantID = pTenantID;
|
|
450
|
+
tmpOverall.Integrations = tmpIntegrations;
|
|
451
|
+
|
|
452
|
+
return fCallback(null, tmpOverall);
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
getCorporateDashboardSummary(fCallback)
|
|
457
|
+
{
|
|
458
|
+
this._readAndFilter('telemetry-runs-global', { Limit: 10000 },
|
|
459
|
+
(pError, pRecords) =>
|
|
460
|
+
{
|
|
461
|
+
if (pError)
|
|
462
|
+
{
|
|
463
|
+
return fCallback(pError);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// Group by TenantID
|
|
467
|
+
let tmpByTenant = {};
|
|
468
|
+
for (let i = 0; i < pRecords.length; i++)
|
|
469
|
+
{
|
|
470
|
+
let tmpTenantID = pRecords[i].TenantID || 'Unknown';
|
|
471
|
+
if (!tmpByTenant[tmpTenantID])
|
|
472
|
+
{
|
|
473
|
+
tmpByTenant[tmpTenantID] = [];
|
|
474
|
+
}
|
|
475
|
+
tmpByTenant[tmpTenantID].push(pRecords[i]);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
let tmpTenants = [];
|
|
479
|
+
let tmpTenantIDs = Object.keys(tmpByTenant);
|
|
480
|
+
for (let i = 0; i < tmpTenantIDs.length; i++)
|
|
481
|
+
{
|
|
482
|
+
let tmpTenantSummary = this._buildSummary(tmpByTenant[tmpTenantIDs[i]]);
|
|
483
|
+
tmpTenantSummary.TenantID = tmpTenantIDs[i];
|
|
484
|
+
tmpTenants.push(tmpTenantSummary);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
let tmpOverall = this._buildSummary(pRecords);
|
|
488
|
+
tmpOverall.Tenants = tmpTenants;
|
|
489
|
+
|
|
490
|
+
return fCallback(null, tmpOverall);
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
module.exports = IntegrationTelemetryStorageProviderBibliograph;
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retold Data Service - Integration Telemetry Service
|
|
3
|
+
*
|
|
4
|
+
* Fable service that persists integration run metadata longitudinally.
|
|
5
|
+
* Normalises DataCloner sync reports into a compact telemetry record and
|
|
6
|
+
* delegates persistence to a pluggable storage provider.
|
|
7
|
+
*
|
|
8
|
+
* Storage provider resolution:
|
|
9
|
+
* 1. If a custom 'IntegrationTelemetryStorage' service is registered on
|
|
10
|
+
* fable, that instance is used.
|
|
11
|
+
* 2. Otherwise the built-in Bibliograph (parime JSON) provider is used.
|
|
12
|
+
*
|
|
13
|
+
* Two route groups registered via connectRoutes():
|
|
14
|
+
* - /telemetry/runs/* — run history
|
|
15
|
+
* - /telemetry/integrations/* — per-integration views
|
|
16
|
+
* - /telemetry/dashboard/* — tenant + corporate dashboards
|
|
17
|
+
*
|
|
18
|
+
* @author Steven Velozo <steven@velozo.com>
|
|
19
|
+
*/
|
|
20
|
+
const libFableServiceProviderBase = require('fable-serviceproviderbase');
|
|
21
|
+
|
|
22
|
+
const libIntegrationTelemetryStorageProviderBibliograph = require('./IntegrationTelemetry-StorageProvider-Bibliograph.js');
|
|
23
|
+
|
|
24
|
+
const defaultIntegrationTelemetryOptions = (
|
|
25
|
+
{
|
|
26
|
+
RoutePrefix: '/telemetry',
|
|
27
|
+
DefaultTenantID: 'default'
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
class RetoldDataServiceIntegrationTelemetry extends libFableServiceProviderBase
|
|
31
|
+
{
|
|
32
|
+
constructor(pFable, pOptions, pServiceHash)
|
|
33
|
+
{
|
|
34
|
+
let tmpOptions = Object.assign({}, defaultIntegrationTelemetryOptions, pOptions);
|
|
35
|
+
super(pFable, tmpOptions, pServiceHash);
|
|
36
|
+
|
|
37
|
+
this.serviceType = 'RetoldDataServiceIntegrationTelemetry';
|
|
38
|
+
|
|
39
|
+
this._storageProvider = null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The route prefix for all telemetry endpoints.
|
|
44
|
+
*/
|
|
45
|
+
get routePrefix()
|
|
46
|
+
{
|
|
47
|
+
let tmpConfig = this.fable.RetoldDataService
|
|
48
|
+
? (this.fable.RetoldDataService.options.IntegrationTelemetry || {})
|
|
49
|
+
: {};
|
|
50
|
+
return tmpConfig.RoutePrefix || this.options.RoutePrefix || '/telemetry';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Resolve the storage provider.
|
|
55
|
+
*
|
|
56
|
+
* If a custom 'IntegrationTelemetryStorage' service has been wired into
|
|
57
|
+
* fable, use that. Otherwise fall back to the built-in Bibliograph provider.
|
|
58
|
+
*/
|
|
59
|
+
getStorageProvider()
|
|
60
|
+
{
|
|
61
|
+
if (this._storageProvider)
|
|
62
|
+
{
|
|
63
|
+
return this._storageProvider;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Check for a custom provider registered on fable
|
|
67
|
+
if (this.fable.services && this.fable.services.hasOwnProperty('IntegrationTelemetryStorage'))
|
|
68
|
+
{
|
|
69
|
+
this._storageProvider = this.fable.IntegrationTelemetryStorage;
|
|
70
|
+
this.fable.log.info('IntegrationTelemetry: Using custom storage provider [IntegrationTelemetryStorage].');
|
|
71
|
+
return this._storageProvider;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Fall back to Bibliograph
|
|
75
|
+
this._storageProvider = new libIntegrationTelemetryStorageProviderBibliograph(this.fable, this.options);
|
|
76
|
+
this.fable.log.info('IntegrationTelemetry: Using default Bibliograph storage provider.');
|
|
77
|
+
return this._storageProvider;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Normalise a DataCloner sync report into a telemetry record and persist it.
|
|
82
|
+
*
|
|
83
|
+
* @param {Object} pReport - The raw DataCloner sync report
|
|
84
|
+
* @param {string} [pTenantID] - Optional tenant identifier (defaults to options.DefaultTenantID)
|
|
85
|
+
* @param {Function} [fCallback] - Optional (pError) callback
|
|
86
|
+
*/
|
|
87
|
+
recordRun(pReport, pTenantID, fCallback)
|
|
88
|
+
{
|
|
89
|
+
// Support (pReport, fCallback) signature
|
|
90
|
+
if (typeof pTenantID === 'function')
|
|
91
|
+
{
|
|
92
|
+
fCallback = pTenantID;
|
|
93
|
+
pTenantID = null;
|
|
94
|
+
}
|
|
95
|
+
if (!fCallback)
|
|
96
|
+
{
|
|
97
|
+
fCallback = (pError) =>
|
|
98
|
+
{
|
|
99
|
+
if (pError)
|
|
100
|
+
{
|
|
101
|
+
this.fable.log.error(`IntegrationTelemetry: Error recording run: ${pError}`);
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (!pReport)
|
|
107
|
+
{
|
|
108
|
+
return fCallback(new Error('IntegrationTelemetry.recordRun: No report provided.'));
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let tmpTenantID = pTenantID || this.options.DefaultTenantID || 'default';
|
|
112
|
+
|
|
113
|
+
// Normalise the DataCloner report into a telemetry record
|
|
114
|
+
let tmpRecord = this.normaliseReport(pReport, tmpTenantID);
|
|
115
|
+
|
|
116
|
+
this.fable.log.info(`IntegrationTelemetry: Recording run [${tmpRecord.RunID}] for tenant [${tmpTenantID}], outcome [${tmpRecord.Outcome}].`);
|
|
117
|
+
|
|
118
|
+
let tmpProvider = this.getStorageProvider();
|
|
119
|
+
tmpProvider.writeRun(tmpTenantID, tmpRecord, fCallback);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Transform a DataCloner sync report into the normalised telemetry record shape.
|
|
124
|
+
*
|
|
125
|
+
* @param {Object} pReport - The raw DataCloner sync report
|
|
126
|
+
* @param {string} pTenantID - The tenant identifier
|
|
127
|
+
* @return {Object} The normalised telemetry record
|
|
128
|
+
*/
|
|
129
|
+
normaliseReport(pReport, pTenantID)
|
|
130
|
+
{
|
|
131
|
+
let tmpConfig = pReport.Config || {};
|
|
132
|
+
let tmpSummary = pReport.Summary || {};
|
|
133
|
+
let tmpTimestamps = pReport.RunTimestamps || {};
|
|
134
|
+
|
|
135
|
+
// Derive an integration name from the config
|
|
136
|
+
let tmpIntegrationName = 'Unknown';
|
|
137
|
+
if (tmpConfig.RemoteServerURL)
|
|
138
|
+
{
|
|
139
|
+
try
|
|
140
|
+
{
|
|
141
|
+
let tmpURL = new URL(tmpConfig.RemoteServerURL);
|
|
142
|
+
tmpIntegrationName = `${tmpURL.hostname} -> ${tmpConfig.Provider || 'Local'}`;
|
|
143
|
+
}
|
|
144
|
+
catch (pParseError)
|
|
145
|
+
{
|
|
146
|
+
tmpIntegrationName = `${tmpConfig.RemoteServerURL} -> ${tmpConfig.Provider || 'Local'}`;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Map outcome to normalised values
|
|
151
|
+
let tmpOutcome = (pReport.Outcome || 'Unknown').toLowerCase();
|
|
152
|
+
// Normalise to our canonical set
|
|
153
|
+
if (tmpOutcome === 'success') tmpOutcome = 'Success';
|
|
154
|
+
else if (tmpOutcome === 'error') tmpOutcome = 'Error';
|
|
155
|
+
else if (tmpOutcome === 'partial') tmpOutcome = 'Partial';
|
|
156
|
+
else if (tmpOutcome === 'stopped') tmpOutcome = 'Stopped';
|
|
157
|
+
else tmpOutcome = pReport.Outcome || 'Unknown';
|
|
158
|
+
|
|
159
|
+
// Compute duration in milliseconds
|
|
160
|
+
let tmpDurationMs = 0;
|
|
161
|
+
if (tmpTimestamps.Start && tmpTimestamps.End)
|
|
162
|
+
{
|
|
163
|
+
tmpDurationMs = new Date(tmpTimestamps.End).getTime() - new Date(tmpTimestamps.Start).getTime();
|
|
164
|
+
}
|
|
165
|
+
else if (tmpTimestamps.DurationSeconds)
|
|
166
|
+
{
|
|
167
|
+
tmpDurationMs = tmpTimestamps.DurationSeconds * 1000;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return (
|
|
171
|
+
{
|
|
172
|
+
RunID: pReport.RunID || this.fable.getUUID(),
|
|
173
|
+
TenantID: pTenantID,
|
|
174
|
+
IntegrationName: tmpIntegrationName,
|
|
175
|
+
Outcome: tmpOutcome,
|
|
176
|
+
StartedAt: tmpTimestamps.Start || null,
|
|
177
|
+
CompletedAt: tmpTimestamps.End || null,
|
|
178
|
+
DurationMs: tmpDurationMs,
|
|
179
|
+
SyncMode: tmpConfig.SyncMode || 'Initial',
|
|
180
|
+
Summary:
|
|
181
|
+
{
|
|
182
|
+
TotalTables: tmpSummary.TotalTables || 0,
|
|
183
|
+
TablesSucceeded: (tmpSummary.Complete || 0),
|
|
184
|
+
TablesFailed: (tmpSummary.Errors || 0),
|
|
185
|
+
TablesPartial: (tmpSummary.Partial || 0),
|
|
186
|
+
TotalRecords: tmpSummary.TotalRecords || 0,
|
|
187
|
+
RecordsSynced: tmpSummary.TotalSynced || 0,
|
|
188
|
+
RecordsFailed: tmpSummary.TotalErrors || 0,
|
|
189
|
+
RecordsSkipped: tmpSummary.TotalSkipped || 0
|
|
190
|
+
},
|
|
191
|
+
Anomalies: pReport.Anomalies || [],
|
|
192
|
+
Config:
|
|
193
|
+
{
|
|
194
|
+
SourceType: 'Remote API',
|
|
195
|
+
TargetType: tmpConfig.Provider || 'Unknown',
|
|
196
|
+
RemoteServerURL: tmpConfig.RemoteServerURL || null,
|
|
197
|
+
TableCount: tmpConfig.TableCount || 0,
|
|
198
|
+
SyncDeletedRecords: tmpConfig.SyncDeletedRecords || false
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ================================================================
|
|
204
|
+
// Route registration
|
|
205
|
+
// ================================================================
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Register all integration telemetry API routes on the Orator service server.
|
|
209
|
+
*
|
|
210
|
+
* @param {Object} pOratorServiceServer - The Orator ServiceServer instance
|
|
211
|
+
*/
|
|
212
|
+
connectRoutes(pOratorServiceServer)
|
|
213
|
+
{
|
|
214
|
+
require('./IntegrationTelemetry-Command-Runs.js')(this, pOratorServiceServer);
|
|
215
|
+
require('./IntegrationTelemetry-Command-Integrations.js')(this, pOratorServiceServer);
|
|
216
|
+
require('./IntegrationTelemetry-Command-Dashboard.js')(this, pOratorServiceServer);
|
|
217
|
+
|
|
218
|
+
this.fable.log.info('Retold Data Service IntegrationTelemetry API routes registered.');
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
module.exports = RetoldDataServiceIntegrationTelemetry;
|
|
223
|
+
module.exports.serviceType = 'RetoldDataServiceIntegrationTelemetry';
|
|
224
|
+
module.exports.default_configuration = defaultIntegrationTelemetryOptions;
|