retold-data-service 2.0.13 → 2.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/.claude/launch.json +11 -0
  2. package/bin/retold-data-service-clone.js +286 -0
  3. package/package.json +18 -9
  4. package/source/Retold-Data-Service.js +275 -73
  5. package/source/services/Retold-Data-Service-ConnectionManager.js +277 -0
  6. package/source/services/Retold-Data-Service-MeadowEndpoints.js +217 -0
  7. package/source/services/Retold-Data-Service-ModelManager.js +335 -0
  8. package/source/services/data-cloner/DataCloner-Command-Connection.js +138 -0
  9. package/source/services/data-cloner/DataCloner-Command-Headless.js +357 -0
  10. package/source/services/data-cloner/DataCloner-Command-Schema.js +367 -0
  11. package/source/services/data-cloner/DataCloner-Command-Session.js +229 -0
  12. package/source/services/data-cloner/DataCloner-Command-Sync.js +491 -0
  13. package/source/services/data-cloner/DataCloner-Command-WebUI.js +40 -0
  14. package/source/services/data-cloner/DataCloner-ProviderRegistry.js +20 -0
  15. package/source/services/data-cloner/Retold-Data-Service-DataCloner.js +751 -0
  16. package/source/services/data-cloner/data-cloner-web.html +2706 -0
  17. package/source/services/integration-telemetry/IntegrationTelemetry-Command-Dashboard.js +60 -0
  18. package/source/services/integration-telemetry/IntegrationTelemetry-Command-Integrations.js +132 -0
  19. package/source/services/integration-telemetry/IntegrationTelemetry-Command-Runs.js +93 -0
  20. package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Base.js +116 -0
  21. package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Bibliograph.js +495 -0
  22. package/source/services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js +224 -0
  23. package/source/services/meadow-integration/MeadowIntegration-Command-CSVCheck.js +85 -0
  24. package/source/services/meadow-integration/MeadowIntegration-Command-CSVTransform.js +180 -0
  25. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionIntersect.js +153 -0
  26. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionPush.js +190 -0
  27. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToArray.js +113 -0
  28. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToCSV.js +211 -0
  29. package/source/services/meadow-integration/MeadowIntegration-Command-EntityFromTabularFolder.js +244 -0
  30. package/source/services/meadow-integration/MeadowIntegration-Command-JSONArrayTransform.js +213 -0
  31. package/source/services/meadow-integration/MeadowIntegration-Command-TSVCheck.js +80 -0
  32. package/source/services/meadow-integration/MeadowIntegration-Command-TSVTransform.js +166 -0
  33. package/source/services/meadow-integration/Retold-Data-Service-MeadowIntegration.js +113 -0
  34. package/source/services/migration-manager/MigrationManager-Command-Connections.js +220 -0
  35. package/source/services/migration-manager/MigrationManager-Command-DiffMigrate.js +169 -0
  36. package/source/services/migration-manager/MigrationManager-Command-Schemas.js +532 -0
  37. package/source/services/migration-manager/MigrationManager-Command-WebUI.js +123 -0
  38. package/source/services/migration-manager/Retold-Data-Service-MigrationManager.js +357 -0
  39. package/source/services/stricture/Retold-Data-Service-Stricture.js +303 -0
  40. package/source/services/stricture/Stricture-Command-Compile.js +39 -0
  41. package/source/services/stricture/Stricture-Command-Generate-AuthorizationChart.js +14 -0
  42. package/source/services/stricture/Stricture-Command-Generate-DictionaryCSV.js +14 -0
  43. package/source/services/stricture/Stricture-Command-Generate-LaTeX.js +14 -0
  44. package/source/services/stricture/Stricture-Command-Generate-Markdown.js +14 -0
  45. package/source/services/stricture/Stricture-Command-Generate-Meadow.js +14 -0
  46. package/source/services/stricture/Stricture-Command-Generate-ModelGraph.js +14 -0
  47. package/source/services/stricture/Stricture-Command-Generate-MySQL.js +14 -0
  48. package/source/services/stricture/Stricture-Command-Generate-MySQLMigrate.js +14 -0
  49. package/source/services/stricture/Stricture-Command-Generate-Pict.js +14 -0
  50. package/source/services/stricture/Stricture-Command-Generate-TestObjectContainers.js +14 -0
  51. package/test/RetoldDataService_tests.js +161 -1
  52. package/debug/data/books.csv +0 -10001
@@ -0,0 +1,495 @@
1
+ /**
2
+ * Integration Telemetry — Bibliograph Storage Provider
3
+ *
4
+ * Default out-of-the-box persistence backed by Bibliograph (parime JSON store).
5
+ * Uses sequential keys with hashed RunIDs for file naming.
6
+ *
7
+ * Source naming convention:
8
+ * - Per-tenant: telemetry-runs-{TenantID}
9
+ * - Global: telemetry-runs-global
10
+ *
11
+ * Key scheme: {seq}-{hash8} e.g. "000042-a1b2c3d4"
12
+ *
13
+ * @param {Object} pFable - The Fable instance
14
+ * @param {Object} pOptions - Provider configuration
15
+ */
16
+ const libIntegrationTelemetryStorageProviderBase = require('./IntegrationTelemetry-StorageProvider-Base.js');
17
+ const libCrypto = require('crypto');
18
+
19
+ class IntegrationTelemetryStorageProviderBibliograph extends libIntegrationTelemetryStorageProviderBase
20
+ {
21
+ constructor(pFable, pOptions)
22
+ {
23
+ super(pFable, pOptions);
24
+
25
+ this._sequenceCounters = {};
26
+ this._sourceInitialized = {};
27
+ }
28
+
29
+ /**
30
+ * Get or create the Bibliograph service instance.
31
+ */
32
+ get bibliograph()
33
+ {
34
+ if (!this.fable.Bibliograph)
35
+ {
36
+ let libBibliograph = require('bibliograph');
37
+ this.fable.serviceManager.addServiceType('Bibliograph', libBibliograph);
38
+ this.fable.serviceManager.instantiateServiceProvider('Bibliograph', this.options.Bibliograph || {});
39
+ }
40
+ return this.fable.Bibliograph;
41
+ }
42
+
43
+ /**
44
+ * Build the source hash for a tenant.
45
+ */
46
+ tenantSource(pTenantID)
47
+ {
48
+ return `telemetry-runs-${pTenantID}`;
49
+ }
50
+
51
+ /**
52
+ * Build a deterministic 8-char hash from a RunID.
53
+ */
54
+ hashRunID(pRunID)
55
+ {
56
+ return libCrypto.createHash('md5').update(pRunID).digest('hex').substring(0, 8);
57
+ }
58
+
59
+ /**
60
+ * Ensure a source exists, calling createSource only once per source.
61
+ */
62
+ ensureSource(pSourceHash, fCallback)
63
+ {
64
+ if (this._sourceInitialized[pSourceHash])
65
+ {
66
+ return fCallback();
67
+ }
68
+
69
+ this.bibliograph.createSource(pSourceHash,
70
+ (pError) =>
71
+ {
72
+ // Ignore "already exists" style errors
73
+ this._sourceInitialized[pSourceHash] = true;
74
+ return fCallback();
75
+ });
76
+ }
77
+
78
+ /**
79
+ * Get the next sequence number for a source.
80
+ */
81
+ nextSequence(pSourceHash)
82
+ {
83
+ if (!this._sequenceCounters.hasOwnProperty(pSourceHash))
84
+ {
85
+ this._sequenceCounters[pSourceHash] = 0;
86
+ }
87
+ this._sequenceCounters[pSourceHash]++;
88
+ return this._sequenceCounters[pSourceHash].toString().padStart(6, '0');
89
+ }
90
+
91
+ /**
92
+ * Build the record key from a sequence number and RunID hash.
93
+ */
94
+ buildKey(pSourceHash, pRunID)
95
+ {
96
+ let tmpSeq = this.nextSequence(pSourceHash);
97
+ let tmpHash = this.hashRunID(pRunID);
98
+ return `${tmpSeq}-${tmpHash}`;
99
+ }
100
+
101
+ // ================================================================
102
+ // Write
103
+ // ================================================================
104
+
105
+ writeRun(pTenantID, pRunRecord, fCallback)
106
+ {
107
+ let tmpTenantSource = this.tenantSource(pTenantID);
108
+ let tmpGlobalSource = 'telemetry-runs-global';
109
+
110
+ this.ensureSource(tmpTenantSource,
111
+ (pError) =>
112
+ {
113
+ if (pError)
114
+ {
115
+ this.fable.log.error(`IntegrationTelemetry Bibliograph: Error ensuring tenant source [${tmpTenantSource}]: ${pError}`);
116
+ }
117
+
118
+ this.ensureSource(tmpGlobalSource,
119
+ (pError) =>
120
+ {
121
+ if (pError)
122
+ {
123
+ this.fable.log.error(`IntegrationTelemetry Bibliograph: Error ensuring global source: ${pError}`);
124
+ }
125
+
126
+ let tmpTenantKey = this.buildKey(tmpTenantSource, pRunRecord.RunID);
127
+ let tmpGlobalKey = this.buildKey(tmpGlobalSource, pRunRecord.RunID);
128
+
129
+ // Dual-write: tenant source + global source
130
+ this.bibliograph.write(tmpTenantSource, tmpTenantKey, pRunRecord,
131
+ (pWriteError) =>
132
+ {
133
+ if (pWriteError)
134
+ {
135
+ this.fable.log.error(`IntegrationTelemetry Bibliograph: Error writing tenant run: ${pWriteError}`);
136
+ return fCallback(pWriteError);
137
+ }
138
+
139
+ this.bibliograph.write(tmpGlobalSource, tmpGlobalKey, pRunRecord,
140
+ (pGlobalWriteError) =>
141
+ {
142
+ if (pGlobalWriteError)
143
+ {
144
+ this.fable.log.warn(`IntegrationTelemetry Bibliograph: Error writing global run (tenant copy succeeded): ${pGlobalWriteError}`);
145
+ }
146
+ return fCallback();
147
+ });
148
+ });
149
+ });
150
+ });
151
+ }
152
+
153
+ // ================================================================
154
+ // Read
155
+ // ================================================================
156
+
157
+ readRun(pTenantID, pRunID, fCallback)
158
+ {
159
+ let tmpTenantSource = this.tenantSource(pTenantID);
160
+
161
+ this.bibliograph.readRecordKeys(tmpTenantSource,
162
+ (pError, pKeys) =>
163
+ {
164
+ if (pError || !pKeys)
165
+ {
166
+ // Source doesn't exist yet — no runs recorded
167
+ return fCallback(null, null);
168
+ }
169
+
170
+ let tmpTargetHash = this.hashRunID(pRunID);
171
+ let tmpMatchKey = null;
172
+
173
+ for (let i = 0; i < pKeys.length; i++)
174
+ {
175
+ if (pKeys[i].endsWith(`-${tmpTargetHash}`))
176
+ {
177
+ tmpMatchKey = pKeys[i];
178
+ break;
179
+ }
180
+ }
181
+
182
+ if (!tmpMatchKey)
183
+ {
184
+ return fCallback(null, null);
185
+ }
186
+
187
+ this.bibliograph.read(tmpTenantSource, tmpMatchKey, fCallback);
188
+ });
189
+ }
190
+
191
+ // ================================================================
192
+ // List helpers
193
+ // ================================================================
194
+
195
+ /**
196
+ * Read all records from a source, apply in-memory filters, and return a page.
197
+ */
198
+ _readAndFilter(pSourceHash, pOptions, fCallback)
199
+ {
200
+ this.bibliograph.readRecordKeys(pSourceHash,
201
+ (pError, pKeys) =>
202
+ {
203
+ if (pError)
204
+ {
205
+ return fCallback(null, []);
206
+ }
207
+
208
+ if (!pKeys || pKeys.length === 0)
209
+ {
210
+ return fCallback(null, []);
211
+ }
212
+
213
+ // Read all records (fine at Bibliograph/parime scale)
214
+ let tmpRecords = [];
215
+ let tmpRemaining = pKeys.length;
216
+ let tmpDone = false;
217
+
218
+ for (let i = 0; i < pKeys.length; i++)
219
+ {
220
+ this.bibliograph.read(pSourceHash, pKeys[i],
221
+ (pReadError, pRecord) =>
222
+ {
223
+ if (tmpDone)
224
+ {
225
+ return;
226
+ }
227
+
228
+ if (!pReadError && pRecord)
229
+ {
230
+ tmpRecords.push(pRecord);
231
+ }
232
+ tmpRemaining--;
233
+
234
+ if (tmpRemaining <= 0)
235
+ {
236
+ tmpDone = true;
237
+ // Apply filters
238
+ let tmpFiltered = this._applyFilters(tmpRecords, pOptions);
239
+ return fCallback(null, tmpFiltered);
240
+ }
241
+ });
242
+ }
243
+ });
244
+ }
245
+
246
+ /**
247
+ * Apply filter/sort/paginate to an array of run records.
248
+ */
249
+ _applyFilters(pRecords, pOptions)
250
+ {
251
+ let tmpOptions = pOptions || {};
252
+ let tmpResult = pRecords.slice();
253
+
254
+ // Filter by Outcome
255
+ if (tmpOptions.Outcome)
256
+ {
257
+ let tmpOutcome = tmpOptions.Outcome.toLowerCase();
258
+ tmpResult = tmpResult.filter(
259
+ (pRecord) =>
260
+ {
261
+ return pRecord.Outcome && pRecord.Outcome.toLowerCase() === tmpOutcome;
262
+ });
263
+ }
264
+
265
+ // Filter by IntegrationName
266
+ if (tmpOptions.IntegrationName)
267
+ {
268
+ tmpResult = tmpResult.filter(
269
+ (pRecord) =>
270
+ {
271
+ return pRecord.IntegrationName === tmpOptions.IntegrationName;
272
+ });
273
+ }
274
+
275
+ // Filter by TenantID
276
+ if (tmpOptions.TenantID)
277
+ {
278
+ tmpResult = tmpResult.filter(
279
+ (pRecord) =>
280
+ {
281
+ return pRecord.TenantID === tmpOptions.TenantID;
282
+ });
283
+ }
284
+
285
+ // Filter by date range
286
+ if (tmpOptions.From)
287
+ {
288
+ let tmpFrom = new Date(tmpOptions.From).getTime();
289
+ tmpResult = tmpResult.filter(
290
+ (pRecord) =>
291
+ {
292
+ return pRecord.StartedAt && new Date(pRecord.StartedAt).getTime() >= tmpFrom;
293
+ });
294
+ }
295
+ if (tmpOptions.To)
296
+ {
297
+ let tmpTo = new Date(tmpOptions.To).getTime();
298
+ tmpResult = tmpResult.filter(
299
+ (pRecord) =>
300
+ {
301
+ return pRecord.StartedAt && new Date(pRecord.StartedAt).getTime() <= tmpTo;
302
+ });
303
+ }
304
+
305
+ // Sort by StartedAt descending (most recent first)
306
+ tmpResult.sort(
307
+ (a, b) =>
308
+ {
309
+ let tmpA = a.StartedAt ? new Date(a.StartedAt).getTime() : 0;
310
+ let tmpB = b.StartedAt ? new Date(b.StartedAt).getTime() : 0;
311
+ return tmpB - tmpA;
312
+ });
313
+
314
+ // Paginate
315
+ let tmpOffset = parseInt(tmpOptions.Offset) || 0;
316
+ let tmpLimit = parseInt(tmpOptions.Limit) || 50;
317
+ tmpResult = tmpResult.slice(tmpOffset, tmpOffset + tmpLimit);
318
+
319
+ return tmpResult;
320
+ }
321
+
322
+ // ================================================================
323
+ // List implementations
324
+ // ================================================================
325
+
326
+ listRuns(pTenantID, pOptions, fCallback)
327
+ {
328
+ let tmpTenantSource = this.tenantSource(pTenantID);
329
+ this._readAndFilter(tmpTenantSource, pOptions, fCallback);
330
+ }
331
+
332
+ listRunsByIntegration(pTenantID, pIntegrationName, pOptions, fCallback)
333
+ {
334
+ let tmpTenantSource = this.tenantSource(pTenantID);
335
+ let tmpOptions = Object.assign({}, pOptions, { IntegrationName: pIntegrationName });
336
+ this._readAndFilter(tmpTenantSource, tmpOptions, fCallback);
337
+ }
338
+
339
+ listAllTenantRuns(pOptions, fCallback)
340
+ {
341
+ this._readAndFilter('telemetry-runs-global', pOptions, fCallback);
342
+ }
343
+
344
+ // ================================================================
345
+ // Aggregation
346
+ // ================================================================
347
+
348
+ /**
349
+ * Build summary stats from an array of run records.
350
+ */
351
+ _buildSummary(pRecords)
352
+ {
353
+ let tmpTotal = pRecords.length;
354
+ let tmpSuccesses = 0;
355
+ let tmpFailures = 0;
356
+ let tmpPartial = 0;
357
+ let tmpStopped = 0;
358
+ let tmpTotalDuration = 0;
359
+ let tmpTotalRecordsSynced = 0;
360
+ let tmpLatestRun = null;
361
+
362
+ for (let i = 0; i < pRecords.length; i++)
363
+ {
364
+ let tmpRun = pRecords[i];
365
+
366
+ if (tmpRun.Outcome === 'Success') tmpSuccesses++;
367
+ else if (tmpRun.Outcome === 'Error') tmpFailures++;
368
+ else if (tmpRun.Outcome === 'Partial') tmpPartial++;
369
+ else if (tmpRun.Outcome === 'Stopped') tmpStopped++;
370
+
371
+ tmpTotalDuration += (tmpRun.DurationMs || 0);
372
+ if (tmpRun.Summary)
373
+ {
374
+ tmpTotalRecordsSynced += (tmpRun.Summary.RecordsSynced || 0);
375
+ }
376
+
377
+ if (!tmpLatestRun || (tmpRun.StartedAt && new Date(tmpRun.StartedAt) > new Date(tmpLatestRun.StartedAt)))
378
+ {
379
+ tmpLatestRun = tmpRun;
380
+ }
381
+ }
382
+
383
+ return (
384
+ {
385
+ TotalRuns: tmpTotal,
386
+ Successes: tmpSuccesses,
387
+ Failures: tmpFailures,
388
+ Partial: tmpPartial,
389
+ Stopped: tmpStopped,
390
+ SuccessRate: tmpTotal > 0 ? Math.round((tmpSuccesses / tmpTotal) * 10000) / 100 : 0,
391
+ AverageDurationMs: tmpTotal > 0 ? Math.round(tmpTotalDuration / tmpTotal) : 0,
392
+ TotalRecordsSynced: tmpTotalRecordsSynced,
393
+ LatestRun: tmpLatestRun
394
+ ? { RunID: tmpLatestRun.RunID, Outcome: tmpLatestRun.Outcome, StartedAt: tmpLatestRun.StartedAt, CompletedAt: tmpLatestRun.CompletedAt }
395
+ : null
396
+ });
397
+ }
398
+
399
+ getIntegrationSummary(pTenantID, pIntegrationName, fCallback)
400
+ {
401
+ let tmpTenantSource = this.tenantSource(pTenantID);
402
+ this._readAndFilter(tmpTenantSource, { IntegrationName: pIntegrationName, Limit: 10000 },
403
+ (pError, pRecords) =>
404
+ {
405
+ if (pError)
406
+ {
407
+ return fCallback(pError);
408
+ }
409
+ let tmpSummary = this._buildSummary(pRecords);
410
+ tmpSummary.IntegrationName = pIntegrationName;
411
+ tmpSummary.TenantID = pTenantID;
412
+ return fCallback(null, tmpSummary);
413
+ });
414
+ }
415
+
416
+ getDashboardSummary(pTenantID, fCallback)
417
+ {
418
+ let tmpTenantSource = this.tenantSource(pTenantID);
419
+ this._readAndFilter(tmpTenantSource, { Limit: 10000 },
420
+ (pError, pRecords) =>
421
+ {
422
+ if (pError)
423
+ {
424
+ return fCallback(pError);
425
+ }
426
+
427
+ // Group by IntegrationName
428
+ let tmpByIntegration = {};
429
+ for (let i = 0; i < pRecords.length; i++)
430
+ {
431
+ let tmpName = pRecords[i].IntegrationName || 'Unknown';
432
+ if (!tmpByIntegration[tmpName])
433
+ {
434
+ tmpByIntegration[tmpName] = [];
435
+ }
436
+ tmpByIntegration[tmpName].push(pRecords[i]);
437
+ }
438
+
439
+ let tmpIntegrations = [];
440
+ let tmpIntNames = Object.keys(tmpByIntegration);
441
+ for (let i = 0; i < tmpIntNames.length; i++)
442
+ {
443
+ let tmpIntSummary = this._buildSummary(tmpByIntegration[tmpIntNames[i]]);
444
+ tmpIntSummary.IntegrationName = tmpIntNames[i];
445
+ tmpIntegrations.push(tmpIntSummary);
446
+ }
447
+
448
+ let tmpOverall = this._buildSummary(pRecords);
449
+ tmpOverall.TenantID = pTenantID;
450
+ tmpOverall.Integrations = tmpIntegrations;
451
+
452
+ return fCallback(null, tmpOverall);
453
+ });
454
+ }
455
+
456
+ getCorporateDashboardSummary(fCallback)
457
+ {
458
+ this._readAndFilter('telemetry-runs-global', { Limit: 10000 },
459
+ (pError, pRecords) =>
460
+ {
461
+ if (pError)
462
+ {
463
+ return fCallback(pError);
464
+ }
465
+
466
+ // Group by TenantID
467
+ let tmpByTenant = {};
468
+ for (let i = 0; i < pRecords.length; i++)
469
+ {
470
+ let tmpTenantID = pRecords[i].TenantID || 'Unknown';
471
+ if (!tmpByTenant[tmpTenantID])
472
+ {
473
+ tmpByTenant[tmpTenantID] = [];
474
+ }
475
+ tmpByTenant[tmpTenantID].push(pRecords[i]);
476
+ }
477
+
478
+ let tmpTenants = [];
479
+ let tmpTenantIDs = Object.keys(tmpByTenant);
480
+ for (let i = 0; i < tmpTenantIDs.length; i++)
481
+ {
482
+ let tmpTenantSummary = this._buildSummary(tmpByTenant[tmpTenantIDs[i]]);
483
+ tmpTenantSummary.TenantID = tmpTenantIDs[i];
484
+ tmpTenants.push(tmpTenantSummary);
485
+ }
486
+
487
+ let tmpOverall = this._buildSummary(pRecords);
488
+ tmpOverall.Tenants = tmpTenants;
489
+
490
+ return fCallback(null, tmpOverall);
491
+ });
492
+ }
493
+ }
494
+
495
+ module.exports = IntegrationTelemetryStorageProviderBibliograph;
@@ -0,0 +1,224 @@
1
+ /**
2
+ * Retold Data Service - Integration Telemetry Service
3
+ *
4
+ * Fable service that persists integration run metadata longitudinally.
5
+ * Normalises DataCloner sync reports into a compact telemetry record and
6
+ * delegates persistence to a pluggable storage provider.
7
+ *
8
+ * Storage provider resolution:
9
+ * 1. If a custom 'IntegrationTelemetryStorage' service is registered on
10
+ * fable, that instance is used.
11
+ * 2. Otherwise the built-in Bibliograph (parime JSON) provider is used.
12
+ *
13
+ * Two route groups registered via connectRoutes():
14
+ * - /telemetry/runs/* — run history
15
+ * - /telemetry/integrations/* — per-integration views
16
+ * - /telemetry/dashboard/* — tenant + corporate dashboards
17
+ *
18
+ * @author Steven Velozo <steven@velozo.com>
19
+ */
20
+ const libFableServiceProviderBase = require('fable-serviceproviderbase');
21
+
22
+ const libIntegrationTelemetryStorageProviderBibliograph = require('./IntegrationTelemetry-StorageProvider-Bibliograph.js');
23
+
24
+ const defaultIntegrationTelemetryOptions = (
25
+ {
26
+ RoutePrefix: '/telemetry',
27
+ DefaultTenantID: 'default'
28
+ });
29
+
30
+ class RetoldDataServiceIntegrationTelemetry extends libFableServiceProviderBase
31
+ {
32
+ constructor(pFable, pOptions, pServiceHash)
33
+ {
34
+ let tmpOptions = Object.assign({}, defaultIntegrationTelemetryOptions, pOptions);
35
+ super(pFable, tmpOptions, pServiceHash);
36
+
37
+ this.serviceType = 'RetoldDataServiceIntegrationTelemetry';
38
+
39
+ this._storageProvider = null;
40
+ }
41
+
42
+ /**
43
+ * The route prefix for all telemetry endpoints.
44
+ */
45
+ get routePrefix()
46
+ {
47
+ let tmpConfig = this.fable.RetoldDataService
48
+ ? (this.fable.RetoldDataService.options.IntegrationTelemetry || {})
49
+ : {};
50
+ return tmpConfig.RoutePrefix || this.options.RoutePrefix || '/telemetry';
51
+ }
52
+
53
+ /**
54
+ * Resolve the storage provider.
55
+ *
56
+ * If a custom 'IntegrationTelemetryStorage' service has been wired into
57
+ * fable, use that. Otherwise fall back to the built-in Bibliograph provider.
58
+ */
59
+ getStorageProvider()
60
+ {
61
+ if (this._storageProvider)
62
+ {
63
+ return this._storageProvider;
64
+ }
65
+
66
+ // Check for a custom provider registered on fable
67
+ if (this.fable.services && this.fable.services.hasOwnProperty('IntegrationTelemetryStorage'))
68
+ {
69
+ this._storageProvider = this.fable.IntegrationTelemetryStorage;
70
+ this.fable.log.info('IntegrationTelemetry: Using custom storage provider [IntegrationTelemetryStorage].');
71
+ return this._storageProvider;
72
+ }
73
+
74
+ // Fall back to Bibliograph
75
+ this._storageProvider = new libIntegrationTelemetryStorageProviderBibliograph(this.fable, this.options);
76
+ this.fable.log.info('IntegrationTelemetry: Using default Bibliograph storage provider.');
77
+ return this._storageProvider;
78
+ }
79
+
80
+ /**
81
+ * Normalise a DataCloner sync report into a telemetry record and persist it.
82
+ *
83
+ * @param {Object} pReport - The raw DataCloner sync report
84
+ * @param {string} [pTenantID] - Optional tenant identifier (defaults to options.DefaultTenantID)
85
+ * @param {Function} [fCallback] - Optional (pError) callback
86
+ */
87
+ recordRun(pReport, pTenantID, fCallback)
88
+ {
89
+ // Support (pReport, fCallback) signature
90
+ if (typeof pTenantID === 'function')
91
+ {
92
+ fCallback = pTenantID;
93
+ pTenantID = null;
94
+ }
95
+ if (!fCallback)
96
+ {
97
+ fCallback = (pError) =>
98
+ {
99
+ if (pError)
100
+ {
101
+ this.fable.log.error(`IntegrationTelemetry: Error recording run: ${pError}`);
102
+ }
103
+ };
104
+ }
105
+
106
+ if (!pReport)
107
+ {
108
+ return fCallback(new Error('IntegrationTelemetry.recordRun: No report provided.'));
109
+ }
110
+
111
+ let tmpTenantID = pTenantID || this.options.DefaultTenantID || 'default';
112
+
113
+ // Normalise the DataCloner report into a telemetry record
114
+ let tmpRecord = this.normaliseReport(pReport, tmpTenantID);
115
+
116
+ this.fable.log.info(`IntegrationTelemetry: Recording run [${tmpRecord.RunID}] for tenant [${tmpTenantID}], outcome [${tmpRecord.Outcome}].`);
117
+
118
+ let tmpProvider = this.getStorageProvider();
119
+ tmpProvider.writeRun(tmpTenantID, tmpRecord, fCallback);
120
+ }
121
+
122
+ /**
123
+ * Transform a DataCloner sync report into the normalised telemetry record shape.
124
+ *
125
+ * @param {Object} pReport - The raw DataCloner sync report
126
+ * @param {string} pTenantID - The tenant identifier
127
+ * @return {Object} The normalised telemetry record
128
+ */
129
+ normaliseReport(pReport, pTenantID)
130
+ {
131
+ let tmpConfig = pReport.Config || {};
132
+ let tmpSummary = pReport.Summary || {};
133
+ let tmpTimestamps = pReport.RunTimestamps || {};
134
+
135
+ // Derive an integration name from the config
136
+ let tmpIntegrationName = 'Unknown';
137
+ if (tmpConfig.RemoteServerURL)
138
+ {
139
+ try
140
+ {
141
+ let tmpURL = new URL(tmpConfig.RemoteServerURL);
142
+ tmpIntegrationName = `${tmpURL.hostname} -> ${tmpConfig.Provider || 'Local'}`;
143
+ }
144
+ catch (pParseError)
145
+ {
146
+ tmpIntegrationName = `${tmpConfig.RemoteServerURL} -> ${tmpConfig.Provider || 'Local'}`;
147
+ }
148
+ }
149
+
150
+ // Map outcome to normalised values
151
+ let tmpOutcome = (pReport.Outcome || 'Unknown').toLowerCase();
152
+ // Normalise to our canonical set
153
+ if (tmpOutcome === 'success') tmpOutcome = 'Success';
154
+ else if (tmpOutcome === 'error') tmpOutcome = 'Error';
155
+ else if (tmpOutcome === 'partial') tmpOutcome = 'Partial';
156
+ else if (tmpOutcome === 'stopped') tmpOutcome = 'Stopped';
157
+ else tmpOutcome = pReport.Outcome || 'Unknown';
158
+
159
+ // Compute duration in milliseconds
160
+ let tmpDurationMs = 0;
161
+ if (tmpTimestamps.Start && tmpTimestamps.End)
162
+ {
163
+ tmpDurationMs = new Date(tmpTimestamps.End).getTime() - new Date(tmpTimestamps.Start).getTime();
164
+ }
165
+ else if (tmpTimestamps.DurationSeconds)
166
+ {
167
+ tmpDurationMs = tmpTimestamps.DurationSeconds * 1000;
168
+ }
169
+
170
+ return (
171
+ {
172
+ RunID: pReport.RunID || this.fable.getUUID(),
173
+ TenantID: pTenantID,
174
+ IntegrationName: tmpIntegrationName,
175
+ Outcome: tmpOutcome,
176
+ StartedAt: tmpTimestamps.Start || null,
177
+ CompletedAt: tmpTimestamps.End || null,
178
+ DurationMs: tmpDurationMs,
179
+ SyncMode: tmpConfig.SyncMode || 'Initial',
180
+ Summary:
181
+ {
182
+ TotalTables: tmpSummary.TotalTables || 0,
183
+ TablesSucceeded: (tmpSummary.Complete || 0),
184
+ TablesFailed: (tmpSummary.Errors || 0),
185
+ TablesPartial: (tmpSummary.Partial || 0),
186
+ TotalRecords: tmpSummary.TotalRecords || 0,
187
+ RecordsSynced: tmpSummary.TotalSynced || 0,
188
+ RecordsFailed: tmpSummary.TotalErrors || 0,
189
+ RecordsSkipped: tmpSummary.TotalSkipped || 0
190
+ },
191
+ Anomalies: pReport.Anomalies || [],
192
+ Config:
193
+ {
194
+ SourceType: 'Remote API',
195
+ TargetType: tmpConfig.Provider || 'Unknown',
196
+ RemoteServerURL: tmpConfig.RemoteServerURL || null,
197
+ TableCount: tmpConfig.TableCount || 0,
198
+ SyncDeletedRecords: tmpConfig.SyncDeletedRecords || false
199
+ }
200
+ });
201
+ }
202
+
203
+ // ================================================================
204
+ // Route registration
205
+ // ================================================================
206
+
207
+ /**
208
+ * Register all integration telemetry API routes on the Orator service server.
209
+ *
210
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
211
+ */
212
+ connectRoutes(pOratorServiceServer)
213
+ {
214
+ require('./IntegrationTelemetry-Command-Runs.js')(this, pOratorServiceServer);
215
+ require('./IntegrationTelemetry-Command-Integrations.js')(this, pOratorServiceServer);
216
+ require('./IntegrationTelemetry-Command-Dashboard.js')(this, pOratorServiceServer);
217
+
218
+ this.fable.log.info('Retold Data Service IntegrationTelemetry API routes registered.');
219
+ }
220
+ }
221
+
222
+ module.exports = RetoldDataServiceIntegrationTelemetry;
223
+ module.exports.serviceType = 'RetoldDataServiceIntegrationTelemetry';
224
+ module.exports.default_configuration = defaultIntegrationTelemetryOptions;