retold-data-service 2.0.13 → 2.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/.claude/launch.json +11 -0
  2. package/bin/retold-data-service-clone.js +286 -0
  3. package/package.json +18 -9
  4. package/source/Retold-Data-Service.js +275 -73
  5. package/source/services/Retold-Data-Service-ConnectionManager.js +277 -0
  6. package/source/services/Retold-Data-Service-MeadowEndpoints.js +217 -0
  7. package/source/services/Retold-Data-Service-ModelManager.js +335 -0
  8. package/source/services/data-cloner/DataCloner-Command-Connection.js +138 -0
  9. package/source/services/data-cloner/DataCloner-Command-Headless.js +357 -0
  10. package/source/services/data-cloner/DataCloner-Command-Schema.js +367 -0
  11. package/source/services/data-cloner/DataCloner-Command-Session.js +229 -0
  12. package/source/services/data-cloner/DataCloner-Command-Sync.js +491 -0
  13. package/source/services/data-cloner/DataCloner-Command-WebUI.js +40 -0
  14. package/source/services/data-cloner/DataCloner-ProviderRegistry.js +20 -0
  15. package/source/services/data-cloner/Retold-Data-Service-DataCloner.js +751 -0
  16. package/source/services/data-cloner/data-cloner-web.html +2706 -0
  17. package/source/services/integration-telemetry/IntegrationTelemetry-Command-Dashboard.js +60 -0
  18. package/source/services/integration-telemetry/IntegrationTelemetry-Command-Integrations.js +132 -0
  19. package/source/services/integration-telemetry/IntegrationTelemetry-Command-Runs.js +93 -0
  20. package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Base.js +116 -0
  21. package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Bibliograph.js +495 -0
  22. package/source/services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js +224 -0
  23. package/source/services/meadow-integration/MeadowIntegration-Command-CSVCheck.js +85 -0
  24. package/source/services/meadow-integration/MeadowIntegration-Command-CSVTransform.js +180 -0
  25. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionIntersect.js +153 -0
  26. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionPush.js +190 -0
  27. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToArray.js +113 -0
  28. package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToCSV.js +211 -0
  29. package/source/services/meadow-integration/MeadowIntegration-Command-EntityFromTabularFolder.js +244 -0
  30. package/source/services/meadow-integration/MeadowIntegration-Command-JSONArrayTransform.js +213 -0
  31. package/source/services/meadow-integration/MeadowIntegration-Command-TSVCheck.js +80 -0
  32. package/source/services/meadow-integration/MeadowIntegration-Command-TSVTransform.js +166 -0
  33. package/source/services/meadow-integration/Retold-Data-Service-MeadowIntegration.js +113 -0
  34. package/source/services/migration-manager/MigrationManager-Command-Connections.js +220 -0
  35. package/source/services/migration-manager/MigrationManager-Command-DiffMigrate.js +169 -0
  36. package/source/services/migration-manager/MigrationManager-Command-Schemas.js +532 -0
  37. package/source/services/migration-manager/MigrationManager-Command-WebUI.js +123 -0
  38. package/source/services/migration-manager/Retold-Data-Service-MigrationManager.js +357 -0
  39. package/source/services/stricture/Retold-Data-Service-Stricture.js +303 -0
  40. package/source/services/stricture/Stricture-Command-Compile.js +39 -0
  41. package/source/services/stricture/Stricture-Command-Generate-AuthorizationChart.js +14 -0
  42. package/source/services/stricture/Stricture-Command-Generate-DictionaryCSV.js +14 -0
  43. package/source/services/stricture/Stricture-Command-Generate-LaTeX.js +14 -0
  44. package/source/services/stricture/Stricture-Command-Generate-Markdown.js +14 -0
  45. package/source/services/stricture/Stricture-Command-Generate-Meadow.js +14 -0
  46. package/source/services/stricture/Stricture-Command-Generate-ModelGraph.js +14 -0
  47. package/source/services/stricture/Stricture-Command-Generate-MySQL.js +14 -0
  48. package/source/services/stricture/Stricture-Command-Generate-MySQLMigrate.js +14 -0
  49. package/source/services/stricture/Stricture-Command-Generate-Pict.js +14 -0
  50. package/source/services/stricture/Stricture-Command-Generate-TestObjectContainers.js +14 -0
  51. package/test/RetoldDataService_tests.js +161 -1
  52. package/debug/data/books.csv +0 -10001
@@ -0,0 +1,60 @@
1
+ /**
2
+ * IntegrationTelemetry — Dashboard Routes
3
+ *
4
+ * GET {prefix}/dashboard — tenant dashboard summary
5
+ * GET {prefix}/dashboard/corporate — corporate dashboard (all tenants)
6
+ *
7
+ * @param {Object} pTelemetryService - The RetoldDataServiceIntegrationTelemetry instance
8
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
9
+ */
10
+ module.exports = (pTelemetryService, pOratorServiceServer) =>
11
+ {
12
+ let tmpPrefix = pTelemetryService.routePrefix;
13
+
14
+ // ────────────────────────────────────────────────────────────────
15
+ // GET /telemetry/dashboard
16
+ // Aggregated dashboard for a single tenant.
17
+ //
18
+ // Query params:
19
+ // tenant_id — required (defaults to service default)
20
+ // ────────────────────────────────────────────────────────────────
21
+ pOratorServiceServer.get(`${tmpPrefix}/dashboard`,
22
+ (pRequest, pResponse, fNext) =>
23
+ {
24
+ let tmpTenantID = pRequest.params.tenant_id || pTelemetryService.options.DefaultTenantID || 'default';
25
+
26
+ let tmpProvider = pTelemetryService.getStorageProvider();
27
+ tmpProvider.getDashboardSummary(tmpTenantID,
28
+ (pError, pDashboard) =>
29
+ {
30
+ if (pError)
31
+ {
32
+ pResponse.send(500, { Success: false, Error: `${pError}` });
33
+ return fNext();
34
+ }
35
+ pResponse.send(200, { Success: true, Dashboard: pDashboard });
36
+ return fNext();
37
+ });
38
+ });
39
+
40
+ // ────────────────────────────────────────────────────────────────
41
+ // GET /telemetry/dashboard/corporate
42
+ // Aggregated dashboard across all tenants.
43
+ // ────────────────────────────────────────────────────────────────
44
+ pOratorServiceServer.get(`${tmpPrefix}/dashboard/corporate`,
45
+ (pRequest, pResponse, fNext) =>
46
+ {
47
+ let tmpProvider = pTelemetryService.getStorageProvider();
48
+ tmpProvider.getCorporateDashboardSummary(
49
+ (pError, pDashboard) =>
50
+ {
51
+ if (pError)
52
+ {
53
+ pResponse.send(500, { Success: false, Error: `${pError}` });
54
+ return fNext();
55
+ }
56
+ pResponse.send(200, { Success: true, Dashboard: pDashboard });
57
+ return fNext();
58
+ });
59
+ });
60
+ };
@@ -0,0 +1,132 @@
1
+ /**
2
+ * IntegrationTelemetry — Integration-Level Routes
3
+ *
4
+ * GET {prefix}/integrations — list distinct integration names
5
+ * GET {prefix}/integrations/:IntegrationName/runs — runs filtered by integration
6
+ * GET {prefix}/integrations/:IntegrationName/summary — summary stats for an integration
7
+ *
8
+ * @param {Object} pTelemetryService - The RetoldDataServiceIntegrationTelemetry instance
9
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
10
+ */
11
+ module.exports = (pTelemetryService, pOratorServiceServer) =>
12
+ {
13
+ let tmpPrefix = pTelemetryService.routePrefix;
14
+
15
+ // ────────────────────────────────────────────────────────────────
16
+ // GET /telemetry/integrations
17
+ // List distinct integration names for a tenant.
18
+ //
19
+ // Query params:
20
+ // tenant_id — required (defaults to service default)
21
+ // ────────────────────────────────────────────────────────────────
22
+ pOratorServiceServer.get(`${tmpPrefix}/integrations`,
23
+ (pRequest, pResponse, fNext) =>
24
+ {
25
+ let tmpTenantID = pRequest.params.tenant_id || pTelemetryService.options.DefaultTenantID || 'default';
26
+
27
+ let tmpProvider = pTelemetryService.getStorageProvider();
28
+ tmpProvider.listRuns(tmpTenantID, { Limit: 10000 },
29
+ (pError, pRecords) =>
30
+ {
31
+ if (pError)
32
+ {
33
+ pResponse.send(500, { Success: false, Error: `${pError}` });
34
+ return fNext();
35
+ }
36
+
37
+ // Extract unique integration names
38
+ let tmpNames = {};
39
+ for (let i = 0; i < pRecords.length; i++)
40
+ {
41
+ let tmpName = pRecords[i].IntegrationName || 'Unknown';
42
+ if (!tmpNames[tmpName])
43
+ {
44
+ tmpNames[tmpName] = { Name: tmpName, RunCount: 0, LatestRun: null };
45
+ }
46
+ tmpNames[tmpName].RunCount++;
47
+ if (!tmpNames[tmpName].LatestRun || new Date(pRecords[i].StartedAt) > new Date(tmpNames[tmpName].LatestRun))
48
+ {
49
+ tmpNames[tmpName].LatestRun = pRecords[i].StartedAt;
50
+ }
51
+ }
52
+
53
+ let tmpIntegrations = Object.values(tmpNames);
54
+ tmpIntegrations.sort((a, b) => (b.LatestRun || '').localeCompare(a.LatestRun || ''));
55
+
56
+ pResponse.send(200,
57
+ {
58
+ Success: true,
59
+ TenantID: tmpTenantID,
60
+ Count: tmpIntegrations.length,
61
+ Integrations: tmpIntegrations
62
+ });
63
+ return fNext();
64
+ });
65
+ });
66
+
67
+ // ────────────────────────────────────────────────────────────────
68
+ // GET /telemetry/integrations/:IntegrationName/runs
69
+ // List runs filtered by a specific integration name.
70
+ //
71
+ // Query params:
72
+ // tenant_id, limit, offset, outcome, from, to
73
+ // ────────────────────────────────────────────────────────────────
74
+ pOratorServiceServer.get(`${tmpPrefix}/integrations/:IntegrationName/runs`,
75
+ (pRequest, pResponse, fNext) =>
76
+ {
77
+ let tmpTenantID = pRequest.params.tenant_id || pTelemetryService.options.DefaultTenantID || 'default';
78
+ let tmpIntegrationName = decodeURIComponent(pRequest.params.IntegrationName);
79
+ let tmpOptions = (
80
+ {
81
+ Limit: pRequest.params.limit,
82
+ Offset: pRequest.params.offset,
83
+ Outcome: pRequest.params.outcome,
84
+ From: pRequest.params.from,
85
+ To: pRequest.params.to
86
+ });
87
+
88
+ let tmpProvider = pTelemetryService.getStorageProvider();
89
+ tmpProvider.listRunsByIntegration(tmpTenantID, tmpIntegrationName, tmpOptions,
90
+ (pError, pRecords) =>
91
+ {
92
+ if (pError)
93
+ {
94
+ pResponse.send(500, { Success: false, Error: `${pError}` });
95
+ return fNext();
96
+ }
97
+ pResponse.send(200,
98
+ {
99
+ Success: true,
100
+ TenantID: tmpTenantID,
101
+ IntegrationName: tmpIntegrationName,
102
+ Count: pRecords.length,
103
+ Runs: pRecords
104
+ });
105
+ return fNext();
106
+ });
107
+ });
108
+
109
+ // ────────────────────────────────────────────────────────────────
110
+ // GET /telemetry/integrations/:IntegrationName/summary
111
+ // Aggregated summary for a specific integration.
112
+ // ────────────────────────────────────────────────────────────────
113
+ pOratorServiceServer.get(`${tmpPrefix}/integrations/:IntegrationName/summary`,
114
+ (pRequest, pResponse, fNext) =>
115
+ {
116
+ let tmpTenantID = pRequest.params.tenant_id || pTelemetryService.options.DefaultTenantID || 'default';
117
+ let tmpIntegrationName = decodeURIComponent(pRequest.params.IntegrationName);
118
+
119
+ let tmpProvider = pTelemetryService.getStorageProvider();
120
+ tmpProvider.getIntegrationSummary(tmpTenantID, tmpIntegrationName,
121
+ (pError, pSummary) =>
122
+ {
123
+ if (pError)
124
+ {
125
+ pResponse.send(500, { Success: false, Error: `${pError}` });
126
+ return fNext();
127
+ }
128
+ pResponse.send(200, { Success: true, Summary: pSummary });
129
+ return fNext();
130
+ });
131
+ });
132
+ };
@@ -0,0 +1,93 @@
1
+ /**
2
+ * IntegrationTelemetry — Run History Routes
3
+ *
4
+ * GET {prefix}/runs — list runs for a tenant
5
+ * GET {prefix}/runs/:RunID — single run detail
6
+ *
7
+ * @param {Object} pTelemetryService - The RetoldDataServiceIntegrationTelemetry instance
8
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
9
+ */
10
+ module.exports = (pTelemetryService, pOratorServiceServer) =>
11
+ {
12
+ let tmpPrefix = pTelemetryService.routePrefix;
13
+
14
+ // ────────────────────────────────────────────────────────────────
15
+ // GET /telemetry/runs
16
+ // List runs for a tenant with optional filtering.
17
+ //
18
+ // Query params:
19
+ // tenant_id — required (defaults to service default)
20
+ // limit — max records (default 50)
21
+ // offset — pagination offset
22
+ // outcome — filter by outcome (Success, Error, Partial, Stopped)
23
+ // from — ISO date lower bound
24
+ // to — ISO date upper bound
25
+ // ────────────────────────────────────────────────────────────────
26
+ pOratorServiceServer.get(`${tmpPrefix}/runs`,
27
+ (pRequest, pResponse, fNext) =>
28
+ {
29
+ let tmpTenantID = pRequest.params.tenant_id || pTelemetryService.options.DefaultTenantID || 'default';
30
+ let tmpOptions = (
31
+ {
32
+ Limit: pRequest.params.limit,
33
+ Offset: pRequest.params.offset,
34
+ Outcome: pRequest.params.outcome,
35
+ From: pRequest.params.from,
36
+ To: pRequest.params.to
37
+ });
38
+
39
+ let tmpProvider = pTelemetryService.getStorageProvider();
40
+ tmpProvider.listRuns(tmpTenantID, tmpOptions,
41
+ (pError, pRecords) =>
42
+ {
43
+ if (pError)
44
+ {
45
+ pResponse.send(500, { Success: false, Error: `${pError}` });
46
+ return fNext();
47
+ }
48
+ pResponse.send(200,
49
+ {
50
+ Success: true,
51
+ TenantID: tmpTenantID,
52
+ Count: pRecords.length,
53
+ Runs: pRecords
54
+ });
55
+ return fNext();
56
+ });
57
+ });
58
+
59
+ // ────────────────────────────────────────────────────────────────
60
+ // GET /telemetry/runs/:RunID
61
+ // Read a single run record by RunID.
62
+ // ────────────────────────────────────────────────────────────────
63
+ pOratorServiceServer.get(`${tmpPrefix}/runs/:RunID`,
64
+ (pRequest, pResponse, fNext) =>
65
+ {
66
+ let tmpTenantID = pRequest.params.tenant_id || pTelemetryService.options.DefaultTenantID || 'default';
67
+ let tmpRunID = pRequest.params.RunID;
68
+
69
+ if (!tmpRunID)
70
+ {
71
+ pResponse.send(400, { Success: false, Error: 'RunID is required.' });
72
+ return fNext();
73
+ }
74
+
75
+ let tmpProvider = pTelemetryService.getStorageProvider();
76
+ tmpProvider.readRun(tmpTenantID, tmpRunID,
77
+ (pError, pRecord) =>
78
+ {
79
+ if (pError)
80
+ {
81
+ pResponse.send(500, { Success: false, Error: `${pError}` });
82
+ return fNext();
83
+ }
84
+ if (!pRecord)
85
+ {
86
+ pResponse.send(404, { Success: false, Error: `Run [${tmpRunID}] not found for tenant [${tmpTenantID}].` });
87
+ return fNext();
88
+ }
89
+ pResponse.send(200, { Success: true, Run: pRecord });
90
+ return fNext();
91
+ });
92
+ });
93
+ };
@@ -0,0 +1,116 @@
1
+ /**
2
+ * Integration Telemetry — Storage Provider Base
3
+ *
4
+ * Abstract interface for pluggable telemetry persistence. Production
5
+ * deployments supply their own implementation; the built-in Bibliograph
6
+ * provider is used when no custom provider is wired.
7
+ *
8
+ * All methods follow the callback-last (pError, pResult) convention used
9
+ * throughout the retold codebase.
10
+ *
11
+ * @param {Object} pFable - The Fable instance (passed through for logging)
12
+ * @param {Object} pOptions - Provider-specific configuration
13
+ */
14
+ class IntegrationTelemetryStorageProviderBase
15
+ {
16
+ constructor(pFable, pOptions)
17
+ {
18
+ this.fable = pFable;
19
+ this.options = pOptions || {};
20
+ }
21
+
22
+ /**
23
+ * Persist a single telemetry run record.
24
+ *
25
+ * @param {string} pTenantID - Tenant identifier
26
+ * @param {Object} pRunRecord - The normalised telemetry record
27
+ * @param {Function} fCallback - (pError)
28
+ */
29
+ writeRun(pTenantID, pRunRecord, fCallback)
30
+ {
31
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.writeRun is not implemented.'));
32
+ }
33
+
34
+ /**
35
+ * Read a single run by its RunID.
36
+ *
37
+ * @param {string} pTenantID - Tenant identifier
38
+ * @param {string} pRunID - The UUID of the run
39
+ * @param {Function} fCallback - (pError, pRunRecord)
40
+ */
41
+ readRun(pTenantID, pRunID, fCallback)
42
+ {
43
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.readRun is not implemented.'));
44
+ }
45
+
46
+ /**
47
+ * List run records for a tenant with optional filtering.
48
+ *
49
+ * @param {string} pTenantID - Tenant identifier
50
+ * @param {Object} pOptions - { Limit, Offset, Outcome, From, To }
51
+ * @param {Function} fCallback - (pError, pRecords)
52
+ */
53
+ listRuns(pTenantID, pOptions, fCallback)
54
+ {
55
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.listRuns is not implemented.'));
56
+ }
57
+
58
+ /**
59
+ * List run records for a specific integration within a tenant.
60
+ *
61
+ * @param {string} pTenantID - Tenant identifier
62
+ * @param {string} pIntegrationName - Integration name
63
+ * @param {Object} pOptions - { Limit, Offset, Outcome, From, To }
64
+ * @param {Function} fCallback - (pError, pRecords)
65
+ */
66
+ listRunsByIntegration(pTenantID, pIntegrationName, pOptions, fCallback)
67
+ {
68
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.listRunsByIntegration is not implemented.'));
69
+ }
70
+
71
+ /**
72
+ * List runs across all tenants (corporate dashboard).
73
+ *
74
+ * @param {Object} pOptions - { Limit, Offset, Outcome, From, To }
75
+ * @param {Function} fCallback - (pError, pRecords)
76
+ */
77
+ listAllTenantRuns(pOptions, fCallback)
78
+ {
79
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.listAllTenantRuns is not implemented.'));
80
+ }
81
+
82
+ /**
83
+ * Compute an aggregate summary for a specific integration.
84
+ *
85
+ * @param {string} pTenantID - Tenant identifier
86
+ * @param {string} pIntegrationName - Integration name
87
+ * @param {Function} fCallback - (pError, pSummary)
88
+ */
89
+ getIntegrationSummary(pTenantID, pIntegrationName, fCallback)
90
+ {
91
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.getIntegrationSummary is not implemented.'));
92
+ }
93
+
94
+ /**
95
+ * Compute an aggregate dashboard for a single tenant.
96
+ *
97
+ * @param {string} pTenantID - Tenant identifier
98
+ * @param {Function} fCallback - (pError, pDashboard)
99
+ */
100
+ getDashboardSummary(pTenantID, fCallback)
101
+ {
102
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.getDashboardSummary is not implemented.'));
103
+ }
104
+
105
+ /**
106
+ * Compute an aggregate dashboard across all tenants (corporate view).
107
+ *
108
+ * @param {Function} fCallback - (pError, pDashboard)
109
+ */
110
+ getCorporateDashboardSummary(fCallback)
111
+ {
112
+ return fCallback(new Error('IntegrationTelemetryStorageProvider.getCorporateDashboardSummary is not implemented.'));
113
+ }
114
+ }
115
+
116
+ module.exports = IntegrationTelemetryStorageProviderBase;