retold-data-service 2.0.14 → 2.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/launch.json +11 -0
- package/.quackage.json +19 -0
- package/bin/retold-data-service-clone.js +286 -0
- package/package.json +24 -12
- package/source/Retold-Data-Service.js +52 -2
- package/source/services/data-cloner/DataCloner-Command-Connection.js +138 -0
- package/source/services/data-cloner/DataCloner-Command-Headless.js +357 -0
- package/source/services/data-cloner/DataCloner-Command-Schema.js +367 -0
- package/source/services/data-cloner/DataCloner-Command-Session.js +229 -0
- package/source/services/data-cloner/DataCloner-Command-Sync.js +524 -0
- package/source/services/data-cloner/DataCloner-Command-WebUI.js +57 -0
- package/source/services/data-cloner/DataCloner-ProviderRegistry.js +20 -0
- package/source/services/data-cloner/Retold-Data-Service-DataCloner.js +1028 -0
- package/source/services/data-cloner/pict-app/Pict-Application-DataCloner-Configuration.json +9 -0
- package/source/services/data-cloner/pict-app/Pict-Application-DataCloner.js +102 -0
- package/source/services/data-cloner/pict-app/Pict-DataCloner-Bundle.js +6 -0
- package/source/services/data-cloner/pict-app/providers/Pict-Provider-DataCloner.js +974 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Connection.js +407 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Deploy.js +126 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Export.js +483 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Layout.js +390 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Schema.js +241 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Session.js +268 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Sync.js +575 -0
- package/source/services/data-cloner/pict-app/views/PictView-DataCloner-ViewData.js +176 -0
- package/source/services/data-cloner/web/data-cloner.js +7935 -0
- package/source/services/data-cloner/web/data-cloner.js.map +1 -0
- package/source/services/data-cloner/web/data-cloner.min.js +2 -0
- package/source/services/data-cloner/web/data-cloner.min.js.map +1 -0
- package/source/services/data-cloner/web/index.html +17 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Dashboard.js +60 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Integrations.js +132 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Runs.js +93 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Base.js +116 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Bibliograph.js +495 -0
- package/source/services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js +224 -0
- package/test/DataCloner-Integration_tests.js +1205 -0
- package/test/DataCloner-Puppeteer_tests.js +502 -0
- package/test/integration-report.json +311 -0
- package/test/run-integration-tests.js +501 -0
- package/debug/data/books.csv +0 -10001
- package/example_applications/data-cloner/data/cloned.sqlite +0 -0
- package/example_applications/data-cloner/data/cloned.sqlite-shm +0 -0
- package/example_applications/data-cloner/data/cloned.sqlite-wal +0 -0
- package/example_applications/data-cloner/data-cloner-web.html +0 -935
- package/example_applications/data-cloner/data-cloner.js +0 -1047
- package/example_applications/data-cloner/package.json +0 -19
package/.quackage.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"GulpfileConfiguration":
|
|
3
|
+
{
|
|
4
|
+
"EntrypointInputSourceFile": "{~Data:AppData.CWD~}/source/services/data-cloner/pict-app/Pict-DataCloner-Bundle.js",
|
|
5
|
+
"LibraryObjectName": "dataCloner",
|
|
6
|
+
"LibraryOutputFolder": "{~Data:AppData.CWD~}/source/services/data-cloner/web/",
|
|
7
|
+
"LibraryUniminifiedFileName": "data-cloner.{~Data:Record.BuildFileLabel~}js",
|
|
8
|
+
"LibraryMinifiedFileName": "data-cloner.{~Data:Record.BuildFileLabel~}min.js"
|
|
9
|
+
},
|
|
10
|
+
"GulpExecutions":
|
|
11
|
+
[
|
|
12
|
+
{
|
|
13
|
+
"Hash": "default",
|
|
14
|
+
"Name": "Default standard build.",
|
|
15
|
+
"BuildFileLabel": "",
|
|
16
|
+
"BrowsersListRC": "since 2022"
|
|
17
|
+
}
|
|
18
|
+
]
|
|
19
|
+
}
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Retold Data Cloner — CLI Entry Point
|
|
4
|
+
*
|
|
5
|
+
* Starts a retold-data-service instance with the DataCloner execution mode enabled.
|
|
6
|
+
* Supports both a web UI for interactive cloning and a headless pipeline mode
|
|
7
|
+
* (--config + --run) for automated cloning.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* retold-data-clone Start web UI only
|
|
11
|
+
* retold-data-clone --config <path> --run Headless clone from config file
|
|
12
|
+
* retold-data-clone --config-json '{}' --run Headless clone from inline JSON
|
|
13
|
+
*
|
|
14
|
+
* @author Steven Velozo <steven@velozo.com>
|
|
15
|
+
*/
|
|
16
|
+
const libFable = require('fable');
|
|
17
|
+
const libMeadowConnectionSQLite = require('meadow-connection-sqlite');
|
|
18
|
+
const libRetoldDataService = require('../source/Retold-Data-Service.js');
|
|
19
|
+
|
|
20
|
+
const libMeadowCloneRestClient = require('meadow-integration/source/services/clone/Meadow-Service-RestClient');
|
|
21
|
+
const libMeadowSync = require('meadow-integration/source/services/clone/Meadow-Service-Sync');
|
|
22
|
+
|
|
23
|
+
const libFs = require('fs');
|
|
24
|
+
const libPath = require('path');
|
|
25
|
+
|
|
26
|
+
// ================================================================
|
|
27
|
+
// CLI Arguments
|
|
28
|
+
// ================================================================
|
|
29
|
+
|
|
30
|
+
let _CLIConfig = null;
|
|
31
|
+
let _CLIRunHeadless = false;
|
|
32
|
+
let _CLILogPath = null;
|
|
33
|
+
let _CLIMaxRecords = 0;
|
|
34
|
+
let _CLISchemaPath = null;
|
|
35
|
+
let _CLIReportPath = null;
|
|
36
|
+
|
|
37
|
+
for (let i = 2; i < process.argv.length; i++)
|
|
38
|
+
{
|
|
39
|
+
if ((process.argv[i] === '--config' || process.argv[i] === '-c') && process.argv[i + 1])
|
|
40
|
+
{
|
|
41
|
+
let tmpConfigPath = libPath.resolve(process.argv[i + 1]);
|
|
42
|
+
try
|
|
43
|
+
{
|
|
44
|
+
let tmpRaw = libFs.readFileSync(tmpConfigPath, 'utf8');
|
|
45
|
+
_CLIConfig = JSON.parse(tmpRaw);
|
|
46
|
+
console.log(`Data Cloner: Loaded config from ${tmpConfigPath}`);
|
|
47
|
+
}
|
|
48
|
+
catch (pConfigError)
|
|
49
|
+
{
|
|
50
|
+
console.error(`Data Cloner: Failed to load config from ${tmpConfigPath}: ${pConfigError.message}`);
|
|
51
|
+
process.exit(1);
|
|
52
|
+
}
|
|
53
|
+
i++;
|
|
54
|
+
}
|
|
55
|
+
else if (process.argv[i] === '--config-json' && process.argv[i + 1])
|
|
56
|
+
{
|
|
57
|
+
try
|
|
58
|
+
{
|
|
59
|
+
_CLIConfig = JSON.parse(process.argv[i + 1]);
|
|
60
|
+
console.log('Data Cloner: Loaded config from inline JSON');
|
|
61
|
+
}
|
|
62
|
+
catch (pParseError)
|
|
63
|
+
{
|
|
64
|
+
console.error(`Data Cloner: Failed to parse inline JSON: ${pParseError.message}`);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
i++;
|
|
68
|
+
}
|
|
69
|
+
else if (process.argv[i] === '--run' || process.argv[i] === '-r')
|
|
70
|
+
{
|
|
71
|
+
_CLIRunHeadless = true;
|
|
72
|
+
}
|
|
73
|
+
else if ((process.argv[i] === '--port' || process.argv[i] === '-p') && process.argv[i + 1])
|
|
74
|
+
{
|
|
75
|
+
process.env.PORT = process.argv[i + 1];
|
|
76
|
+
i++;
|
|
77
|
+
}
|
|
78
|
+
else if (process.argv[i] === '--log' || process.argv[i] === '-l')
|
|
79
|
+
{
|
|
80
|
+
if (process.argv[i + 1] && !process.argv[i + 1].startsWith('-'))
|
|
81
|
+
{
|
|
82
|
+
_CLILogPath = libPath.resolve(process.argv[i + 1]);
|
|
83
|
+
i++;
|
|
84
|
+
}
|
|
85
|
+
else
|
|
86
|
+
{
|
|
87
|
+
_CLILogPath = `${process.cwd()}/DataCloner-Run-${libFable.generateFileNameDateStamp()}.log`;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
else if ((process.argv[i] === '--max' || process.argv[i] === '-m') && process.argv[i + 1])
|
|
91
|
+
{
|
|
92
|
+
_CLIMaxRecords = parseInt(process.argv[i + 1], 10) || 0;
|
|
93
|
+
i++;
|
|
94
|
+
}
|
|
95
|
+
else if ((process.argv[i] === '--schema' || process.argv[i] === '-s') && process.argv[i + 1])
|
|
96
|
+
{
|
|
97
|
+
_CLISchemaPath = libPath.resolve(process.argv[i + 1]);
|
|
98
|
+
i++;
|
|
99
|
+
}
|
|
100
|
+
else if (process.argv[i] === '--report')
|
|
101
|
+
{
|
|
102
|
+
if (process.argv[i + 1] && !process.argv[i + 1].startsWith('-'))
|
|
103
|
+
{
|
|
104
|
+
_CLIReportPath = libPath.resolve(process.argv[i + 1]);
|
|
105
|
+
i++;
|
|
106
|
+
}
|
|
107
|
+
else
|
|
108
|
+
{
|
|
109
|
+
_CLIReportPath = `${process.cwd()}/DataCloner-Report-${libFable.generateFileNameDateStamp()}.json`;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else if (process.argv[i] === '--help' || process.argv[i] === '-h')
|
|
113
|
+
{
|
|
114
|
+
console.log(`
|
|
115
|
+
Retold Data Cloner
|
|
116
|
+
|
|
117
|
+
Usage:
|
|
118
|
+
retold-data-clone Start web UI only
|
|
119
|
+
retold-data-clone --config <path> --run Headless clone from config file
|
|
120
|
+
retold-data-clone --config-json '{}' --run Headless clone from inline JSON
|
|
121
|
+
|
|
122
|
+
Options:
|
|
123
|
+
--config, -c <path> Path to a JSON config file (generate from the web UI)
|
|
124
|
+
--config-json <json> Inline JSON config string (for one-liner commands)
|
|
125
|
+
--run, -r Auto-run the clone pipeline (requires --config or --config-json)
|
|
126
|
+
--port, -p <port> Override the API server port (default: 8095)
|
|
127
|
+
--log, -l [path] Write log output to a file (default: ./DataCloner-Run-<timestamp>.log)
|
|
128
|
+
--max, -m <n> Limit sync to first n records per entity (for testing)
|
|
129
|
+
--schema, -s <path> Path to a local schema JSON file (skip remote schema fetch)
|
|
130
|
+
--report [path] Write sync report JSON file (default: auto-named; auto-enabled with --log)
|
|
131
|
+
--help, -h Show this help
|
|
132
|
+
`);
|
|
133
|
+
process.exit(0);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (_CLIRunHeadless && !_CLIConfig)
|
|
138
|
+
{
|
|
139
|
+
console.error('Data Cloner: --run requires --config <path> or --config-json <json>');
|
|
140
|
+
process.exit(1);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ================================================================
|
|
144
|
+
// Configuration
|
|
145
|
+
// ================================================================
|
|
146
|
+
|
|
147
|
+
let _Settings = (
|
|
148
|
+
{
|
|
149
|
+
Product: 'RetoldDataCloner',
|
|
150
|
+
ProductVersion: '1.0.0',
|
|
151
|
+
APIServerPort: parseInt(process.env.PORT, 10) || 8095,
|
|
152
|
+
LogStreams:
|
|
153
|
+
[
|
|
154
|
+
{
|
|
155
|
+
streamtype: 'console'
|
|
156
|
+
}
|
|
157
|
+
],
|
|
158
|
+
|
|
159
|
+
SQLite:
|
|
160
|
+
{
|
|
161
|
+
SQLiteFilePath: libPath.join(process.cwd(), 'data', 'cloned.sqlite')
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
if (_CLILogPath)
|
|
166
|
+
{
|
|
167
|
+
_Settings.LogStreams.push(
|
|
168
|
+
{
|
|
169
|
+
loggertype: 'simpleflatfile',
|
|
170
|
+
outputloglinestoconsole: false,
|
|
171
|
+
showtimestamps: true,
|
|
172
|
+
formattedtimestamps: true,
|
|
173
|
+
level: 'trace',
|
|
174
|
+
path: _CLILogPath
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Ensure the data directory exists
|
|
179
|
+
let _DataDir = libPath.join(process.cwd(), 'data');
|
|
180
|
+
if (!libFs.existsSync(_DataDir))
|
|
181
|
+
{
|
|
182
|
+
libFs.mkdirSync(_DataDir, { recursive: true });
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
let _Fable = new libFable(_Settings);
|
|
186
|
+
|
|
187
|
+
// ================================================================
|
|
188
|
+
// SQLite Setup (default — connects automatically)
|
|
189
|
+
// ================================================================
|
|
190
|
+
|
|
191
|
+
_Fable.serviceManager.addServiceType('MeadowSQLiteProvider', libMeadowConnectionSQLite);
|
|
192
|
+
_Fable.serviceManager.instantiateServiceProvider('MeadowSQLiteProvider');
|
|
193
|
+
|
|
194
|
+
_Fable.MeadowSQLiteProvider.connectAsync(
|
|
195
|
+
(pError) =>
|
|
196
|
+
{
|
|
197
|
+
if (pError)
|
|
198
|
+
{
|
|
199
|
+
_Fable.log.error(`SQLite connection error: ${pError}`);
|
|
200
|
+
process.exit(1);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Set default Meadow provider to SQLite
|
|
204
|
+
_Fable.settings.MeadowProvider = 'SQLite';
|
|
205
|
+
|
|
206
|
+
// Register meadow-integration services for clone sync
|
|
207
|
+
_Fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
|
|
208
|
+
_Fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
|
|
209
|
+
|
|
210
|
+
_Fable.serviceManager.addServiceType('RetoldDataService', libRetoldDataService);
|
|
211
|
+
let tmpDataService = _Fable.serviceManager.instantiateServiceProvider('RetoldDataService',
|
|
212
|
+
{
|
|
213
|
+
StorageProvider: 'SQLite',
|
|
214
|
+
StorageProviderModule: 'meadow-connection-sqlite',
|
|
215
|
+
|
|
216
|
+
// No default schema — models loaded at runtime after fetch
|
|
217
|
+
FullMeadowSchemaFilename: false,
|
|
218
|
+
|
|
219
|
+
Endpoints:
|
|
220
|
+
{
|
|
221
|
+
ConnectionManager: true,
|
|
222
|
+
ModelManagerWrite: true,
|
|
223
|
+
Stricture: false,
|
|
224
|
+
MeadowIntegration: false,
|
|
225
|
+
MeadowEndpoints: true,
|
|
226
|
+
MigrationManager: true,
|
|
227
|
+
MigrationManagerWebUI: true,
|
|
228
|
+
DataCloner: true,
|
|
229
|
+
DataClonerWebUI: true,
|
|
230
|
+
IntegrationTelemetry: true
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
// Enable JSON body parsing for POST/PUT requests
|
|
235
|
+
tmpDataService.onBeforeInitialize = (fCallback) =>
|
|
236
|
+
{
|
|
237
|
+
_Fable.OratorServiceServer.server.use(_Fable.OratorServiceServer.bodyParser());
|
|
238
|
+
return fCallback();
|
|
239
|
+
};
|
|
240
|
+
|
|
241
|
+
// SQLite is available but don't presume it's the user's desired provider.
|
|
242
|
+
// The web UI will connect via the auto chain or manual "go" click.
|
|
243
|
+
tmpDataService.onAfterInitialize = (fCallback) =>
|
|
244
|
+
{
|
|
245
|
+
return fCallback();
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
// ================================================================
|
|
249
|
+
// Start the service
|
|
250
|
+
// ================================================================
|
|
251
|
+
|
|
252
|
+
tmpDataService.initializeService(
|
|
253
|
+
(pInitError) =>
|
|
254
|
+
{
|
|
255
|
+
if (pInitError)
|
|
256
|
+
{
|
|
257
|
+
_Fable.log.error(`Initialization error: ${pInitError}`);
|
|
258
|
+
process.exit(1);
|
|
259
|
+
}
|
|
260
|
+
_Fable.log.info(`Data Cloner running on port ${_Settings.APIServerPort}`);
|
|
261
|
+
_Fable.log.info(`Web UI: http://localhost:${_Settings.APIServerPort}/clone/`);
|
|
262
|
+
_Fable.log.info(`Migration Mgr: http://localhost:${_Settings.APIServerPort}/meadow-migrationmanager/`);
|
|
263
|
+
|
|
264
|
+
// ---- Headless auto-run from config file ----
|
|
265
|
+
if (_CLIConfig && _CLIRunHeadless)
|
|
266
|
+
{
|
|
267
|
+
_Fable.RetoldDataServiceDataCloner.runHeadlessPipeline(_CLIConfig,
|
|
268
|
+
{
|
|
269
|
+
logPath: _CLILogPath,
|
|
270
|
+
maxRecords: _CLIMaxRecords,
|
|
271
|
+
schemaPath: _CLISchemaPath,
|
|
272
|
+
serverPort: _Settings.APIServerPort,
|
|
273
|
+
reportPath: _CLIReportPath
|
|
274
|
+
},
|
|
275
|
+
(pPipelineError) =>
|
|
276
|
+
{
|
|
277
|
+
if (pPipelineError)
|
|
278
|
+
{
|
|
279
|
+
_Fable.log.error(`Pipeline error: ${pPipelineError}`);
|
|
280
|
+
process.exit(1);
|
|
281
|
+
}
|
|
282
|
+
process.exit(0);
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
});
|
|
286
|
+
});
|
package/package.json
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "retold-data-service",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.17",
|
|
4
4
|
"description": "Serve up a whole model!",
|
|
5
5
|
"main": "source/Retold-Data-Service.js",
|
|
6
|
+
"bin": {
|
|
7
|
+
"retold-data-service-clone": "bin/retold-data-service-clone.js"
|
|
8
|
+
},
|
|
6
9
|
"scripts": {
|
|
7
|
-
"start": "node
|
|
10
|
+
"start": "node bin/retold-data-service-clone.js",
|
|
8
11
|
"coverage": "npx quack coverage",
|
|
9
12
|
"test": "npx quack test",
|
|
10
13
|
"build": "npx quack build",
|
|
14
|
+
"prepublishOnly": "npx quack build",
|
|
11
15
|
"build-test-model": "cd test && npx stricture -i model/ddl/BookStore.ddl",
|
|
12
16
|
"docker-dev-build": "docker build ./ -f Dockerfile_LUXURYCode -t retold-data-service-image:local",
|
|
13
17
|
"docker-dev-run": "docker run -it -d --name retold-data-service-dev -p 44444:8080 -p 43306:3306 -p 48086:8086 -v \"$PWD/.config:/home/coder/.config\" -v \"$PWD:/home/coder/retold-data-service\" -u \"$(id -u):$(id -g)\" -e \"DOCKER_USER=$USER\" retold-data-service-image:local",
|
|
@@ -15,9 +19,13 @@
|
|
|
15
19
|
"docker-service-build": "docker build ./ -f Dockerfile_Service -t retold-data-service-server-image:local",
|
|
16
20
|
"docker-service-run-test": "docker run -it --init -d --name retold-data-service -p 8086:8086 -p 43306:3306 -v \"$(pwd):/retold-data-service:z\" -u \"$(id -u):$(id -g)\" retold-data-service-server-image:local",
|
|
17
21
|
"docker-service-run": "docker run -it --init -d --name retold-data-service -p 8086:8086 -p 43306:3306 -v \"$(pwd):/retold-data-service:z\" retold-data-service-server-image:local",
|
|
18
|
-
"docker-service-shell": "docker exec -it retold-data-service /bin/bash"
|
|
22
|
+
"docker-service-shell": "docker exec -it retold-data-service /bin/bash",
|
|
23
|
+
"test:integration": "node test/run-integration-tests.js",
|
|
24
|
+
"test:integration:no-browser": "node test/run-integration-tests.js --skip-puppeteer",
|
|
25
|
+
"test:all": "npx quack test && node test/run-integration-tests.js --skip-puppeteer"
|
|
19
26
|
},
|
|
20
27
|
"mocha": {
|
|
28
|
+
"spec": "test/RetoldDataService_tests.js",
|
|
21
29
|
"diff": true,
|
|
22
30
|
"extension": [
|
|
23
31
|
"js"
|
|
@@ -51,24 +59,28 @@
|
|
|
51
59
|
},
|
|
52
60
|
"homepage": "https://github.com/stevenvelozo/retold-data-service",
|
|
53
61
|
"devDependencies": {
|
|
54
|
-
"meadow-connection-sqlite": "^1.0.
|
|
55
|
-
"
|
|
56
|
-
"
|
|
62
|
+
"meadow-connection-sqlite": "^1.0.18",
|
|
63
|
+
"puppeteer": "^24.38.0",
|
|
64
|
+
"quackage": "^1.0.63",
|
|
65
|
+
"stricture": "^4.0.2",
|
|
57
66
|
"supertest": "^7.2.2"
|
|
58
67
|
},
|
|
59
68
|
"dependencies": {
|
|
69
|
+
"bibliograph": "^0.1.4",
|
|
60
70
|
"fable": "^3.1.63",
|
|
61
71
|
"fable-serviceproviderbase": "^3.0.19",
|
|
62
|
-
"meadow": "^2.0.
|
|
63
|
-
"meadow-connection-mysql": "^1.0.
|
|
64
|
-
"meadow-endpoints": "^4.0.
|
|
72
|
+
"meadow": "^2.0.33",
|
|
73
|
+
"meadow-connection-mysql": "^1.0.14",
|
|
74
|
+
"meadow-endpoints": "^4.0.14",
|
|
75
|
+
"meadow-integration": "^1.0.14",
|
|
76
|
+
"meadow-migrationmanager": "^0.0.4",
|
|
65
77
|
"orator": "^6.0.4",
|
|
66
78
|
"orator-http-proxy": "^1.0.5",
|
|
67
79
|
"orator-serviceserver-restify": "^2.0.9",
|
|
68
|
-
"meadow-integration": "^1.0.6",
|
|
69
|
-
"meadow-migrationmanager": "^0.0.4",
|
|
70
80
|
"orator-static-server": "^2.0.4",
|
|
71
81
|
"pict": "^1.0.357",
|
|
72
|
-
"
|
|
82
|
+
"pict-section-histogram": "^1.0.0",
|
|
83
|
+
"pict-sessionmanager": "^1.0.2",
|
|
84
|
+
"stricture": "^4.0.2"
|
|
73
85
|
}
|
|
74
86
|
}
|
|
@@ -16,6 +16,8 @@ const libRetoldDataServiceModelManager = require('./services/Retold-Data-Service
|
|
|
16
16
|
const libRetoldDataServiceStricture = require('./services/stricture/Retold-Data-Service-Stricture.js');
|
|
17
17
|
const libRetoldDataServiceMeadowIntegration = require('./services/meadow-integration/Retold-Data-Service-MeadowIntegration.js');
|
|
18
18
|
const libRetoldDataServiceMigrationManager = require('./services/migration-manager/Retold-Data-Service-MigrationManager.js');
|
|
19
|
+
const libRetoldDataServiceDataCloner = require('./services/data-cloner/Retold-Data-Service-DataCloner.js');
|
|
20
|
+
const libRetoldDataServiceIntegrationTelemetry = require('./services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js');
|
|
19
21
|
|
|
20
22
|
const defaultDataServiceSettings = (
|
|
21
23
|
{
|
|
@@ -45,7 +47,13 @@ const defaultDataServiceSettings = (
|
|
|
45
47
|
// Migration manager API endpoints (/api/*)
|
|
46
48
|
MigrationManager: false,
|
|
47
49
|
// Migration manager web UI (GET /, /lib/*)
|
|
48
|
-
MigrationManagerWebUI: false
|
|
50
|
+
MigrationManagerWebUI: false,
|
|
51
|
+
// Data cloner API endpoints (/clone/*)
|
|
52
|
+
DataCloner: false,
|
|
53
|
+
// Data cloner web UI (GET /clone/)
|
|
54
|
+
DataClonerWebUI: false,
|
|
55
|
+
// Integration telemetry API endpoints (/telemetry/*)
|
|
56
|
+
IntegrationTelemetry: false
|
|
49
57
|
},
|
|
50
58
|
|
|
51
59
|
// Migration manager configuration
|
|
@@ -55,6 +63,22 @@ const defaultDataServiceSettings = (
|
|
|
55
63
|
ModelPath: false,
|
|
56
64
|
// Route prefix for all migration manager endpoints (API + web UI)
|
|
57
65
|
RoutePrefix: '/meadow-migrationmanager'
|
|
66
|
+
},
|
|
67
|
+
|
|
68
|
+
// Data cloner configuration
|
|
69
|
+
DataCloner:
|
|
70
|
+
{
|
|
71
|
+
// Route prefix for all data cloner endpoints (API + web UI)
|
|
72
|
+
RoutePrefix: '/clone'
|
|
73
|
+
},
|
|
74
|
+
|
|
75
|
+
// Integration telemetry configuration
|
|
76
|
+
IntegrationTelemetry:
|
|
77
|
+
{
|
|
78
|
+
// Route prefix for all telemetry endpoints
|
|
79
|
+
RoutePrefix: '/telemetry',
|
|
80
|
+
// Default tenant identifier when none is provided
|
|
81
|
+
DefaultTenantID: 'default'
|
|
58
82
|
}
|
|
59
83
|
});
|
|
60
84
|
|
|
@@ -110,6 +134,14 @@ class RetoldDataService extends libFableServiceProviderBase
|
|
|
110
134
|
this.fable.serviceManager.addServiceType('RetoldDataServiceMigrationManager', libRetoldDataServiceMigrationManager);
|
|
111
135
|
this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceMigrationManager');
|
|
112
136
|
|
|
137
|
+
// Register and instantiate the DataCloner service
|
|
138
|
+
this.fable.serviceManager.addServiceType('RetoldDataServiceDataCloner', libRetoldDataServiceDataCloner);
|
|
139
|
+
this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceDataCloner');
|
|
140
|
+
|
|
141
|
+
// Register and instantiate the IntegrationTelemetry service
|
|
142
|
+
this.fable.serviceManager.addServiceType('RetoldDataServiceIntegrationTelemetry', libRetoldDataServiceIntegrationTelemetry);
|
|
143
|
+
this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceIntegrationTelemetry');
|
|
144
|
+
|
|
113
145
|
// Expose the DAL and MeadowEndpoints from the service on this object and on fable for backward compatibility
|
|
114
146
|
this._DAL = this.fable.RetoldDataServiceMeadowEndpoints._DAL;
|
|
115
147
|
this._MeadowEndpoints = this.fable.RetoldDataServiceMeadowEndpoints._MeadowEndpoints;
|
|
@@ -203,7 +235,7 @@ class RetoldDataService extends libFableServiceProviderBase
|
|
|
203
235
|
this.fable.log.info(`The Retold Data Service is initializing...`);
|
|
204
236
|
|
|
205
237
|
// Log endpoint configuration
|
|
206
|
-
let tmpGroupNames = ['ConnectionManager', 'ModelManagerWrite', 'Stricture', 'MeadowIntegration', 'MeadowEndpoints', 'MigrationManager', 'MigrationManagerWebUI'];
|
|
238
|
+
let tmpGroupNames = ['ConnectionManager', 'ModelManagerWrite', 'Stricture', 'MeadowIntegration', 'MeadowEndpoints', 'MigrationManager', 'MigrationManagerWebUI', 'DataCloner', 'DataClonerWebUI', 'IntegrationTelemetry'];
|
|
207
239
|
let tmpEnabledGroups = [];
|
|
208
240
|
let tmpDisabledGroups = [];
|
|
209
241
|
for (let i = 0; i < tmpGroupNames.length; i++)
|
|
@@ -286,6 +318,24 @@ class RetoldDataService extends libFableServiceProviderBase
|
|
|
286
318
|
this.fable.RetoldDataServiceMigrationManager.connectWebUIRoutes(this.fable.OratorServiceServer);
|
|
287
319
|
}
|
|
288
320
|
|
|
321
|
+
// DataCloner API routes (/clone/*)
|
|
322
|
+
if (this.isEndpointGroupEnabled('DataCloner'))
|
|
323
|
+
{
|
|
324
|
+
this.fable.RetoldDataServiceDataCloner.connectRoutes(this.fable.OratorServiceServer);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// DataCloner Web UI routes (GET /clone/)
|
|
328
|
+
if (this.isEndpointGroupEnabled('DataClonerWebUI'))
|
|
329
|
+
{
|
|
330
|
+
this.fable.RetoldDataServiceDataCloner.connectWebUIRoutes(this.fable.OratorServiceServer);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
// IntegrationTelemetry API routes (/telemetry/*)
|
|
334
|
+
if (this.isEndpointGroupEnabled('IntegrationTelemetry'))
|
|
335
|
+
{
|
|
336
|
+
this.fable.RetoldDataServiceIntegrationTelemetry.connectRoutes(this.fable.OratorServiceServer);
|
|
337
|
+
}
|
|
338
|
+
|
|
289
339
|
return fInitCallback();
|
|
290
340
|
});
|
|
291
341
|
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DataCloner Connection Management Routes
|
|
3
|
+
*
|
|
4
|
+
* Registers /clone/connection/* endpoints for managing the local database
|
|
5
|
+
* connection (status, configure, test).
|
|
6
|
+
*
|
|
7
|
+
* @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
|
|
8
|
+
* @param {Object} pOratorServiceServer - The Orator ServiceServer instance
|
|
9
|
+
*/
|
|
10
|
+
module.exports = (pDataClonerService, pOratorServiceServer) =>
|
|
11
|
+
{
|
|
12
|
+
let tmpFable = pDataClonerService.fable;
|
|
13
|
+
let tmpCloneState = pDataClonerService.cloneState;
|
|
14
|
+
let tmpPrefix = pDataClonerService.routePrefix;
|
|
15
|
+
|
|
16
|
+
// GET /clone/connection/status
|
|
17
|
+
pOratorServiceServer.get(`${tmpPrefix}/connection/status`,
|
|
18
|
+
(pRequest, pResponse, fNext) =>
|
|
19
|
+
{
|
|
20
|
+
pResponse.send(200,
|
|
21
|
+
{
|
|
22
|
+
Provider: tmpCloneState.ConnectionProvider,
|
|
23
|
+
Connected: tmpCloneState.ConnectionConnected,
|
|
24
|
+
Config: tmpCloneState.ConnectionConfig
|
|
25
|
+
});
|
|
26
|
+
return fNext();
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
// POST /clone/connection/configure — Switch the local database provider
|
|
30
|
+
pOratorServiceServer.post(`${tmpPrefix}/connection/configure`,
|
|
31
|
+
(pRequest, pResponse, fNext) =>
|
|
32
|
+
{
|
|
33
|
+
let tmpBody = pRequest.body || {};
|
|
34
|
+
let tmpProvider = tmpBody.Provider;
|
|
35
|
+
let tmpConfig = tmpBody.Config || {};
|
|
36
|
+
|
|
37
|
+
if (!tmpProvider)
|
|
38
|
+
{
|
|
39
|
+
pResponse.send(400, { Success: false, Error: 'Provider is required (e.g. SQLite, MySQL, MSSQL).' });
|
|
40
|
+
return fNext();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
tmpFable.log.info(`Data Cloner: Configuring ${tmpProvider} connection...`);
|
|
44
|
+
|
|
45
|
+
pDataClonerService.connectProvider(tmpProvider, tmpConfig,
|
|
46
|
+
(pConnectError) =>
|
|
47
|
+
{
|
|
48
|
+
if (pConnectError)
|
|
49
|
+
{
|
|
50
|
+
tmpFable.log.error(`Data Cloner: Connection error: ${pConnectError.message}`);
|
|
51
|
+
pResponse.send(500, { Success: false, Error: `Connection failed: ${pConnectError.message}` });
|
|
52
|
+
return fNext();
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
tmpFable.log.info(`Data Cloner: ${tmpProvider} connection established.`);
|
|
56
|
+
pResponse.send(200,
|
|
57
|
+
{
|
|
58
|
+
Success: true,
|
|
59
|
+
Provider: tmpProvider,
|
|
60
|
+
Message: `${tmpProvider} connection established and set as active provider.`
|
|
61
|
+
});
|
|
62
|
+
return fNext();
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// POST /clone/connection/test — Test a connection without making it permanent
|
|
67
|
+
pOratorServiceServer.post(`${tmpPrefix}/connection/test`,
|
|
68
|
+
(pRequest, pResponse, fNext) =>
|
|
69
|
+
{
|
|
70
|
+
let tmpBody = pRequest.body || {};
|
|
71
|
+
let tmpProvider = tmpBody.Provider;
|
|
72
|
+
let tmpConfig = tmpBody.Config || {};
|
|
73
|
+
|
|
74
|
+
if (!tmpProvider)
|
|
75
|
+
{
|
|
76
|
+
pResponse.send(400, { Success: false, Error: 'Provider is required.' });
|
|
77
|
+
return fNext();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
let tmpRegistryEntry = pDataClonerService.providerRegistry[tmpProvider];
|
|
81
|
+
if (!tmpRegistryEntry)
|
|
82
|
+
{
|
|
83
|
+
pResponse.send(400, { Success: false, Error: `Unknown provider: ${tmpProvider}` });
|
|
84
|
+
return fNext();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
tmpFable.log.info(`Data Cloner: Testing ${tmpProvider} connection...`);
|
|
88
|
+
|
|
89
|
+
let tmpModule;
|
|
90
|
+
try
|
|
91
|
+
{
|
|
92
|
+
tmpModule = require(tmpRegistryEntry.moduleName);
|
|
93
|
+
}
|
|
94
|
+
catch (pRequireError)
|
|
95
|
+
{
|
|
96
|
+
pResponse.send(500, { Success: false, Error: `Module not installed: ${tmpRegistryEntry.moduleName}. Run: npm install ${tmpRegistryEntry.moduleName}` });
|
|
97
|
+
return fNext();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Create a temporary fable instance for the test
|
|
101
|
+
let libFable = require('fable');
|
|
102
|
+
let tmpTestFable = new libFable(
|
|
103
|
+
{
|
|
104
|
+
Product: 'DataClonerConnectionTest',
|
|
105
|
+
LogStreams: [{ streamtype: 'console' }],
|
|
106
|
+
[tmpRegistryEntry.configKey]: tmpConfig
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
tmpTestFable.serviceManager.addServiceType(tmpRegistryEntry.serviceName, tmpModule);
|
|
110
|
+
tmpTestFable.serviceManager.instantiateServiceProvider(tmpRegistryEntry.serviceName);
|
|
111
|
+
|
|
112
|
+
tmpTestFable[tmpRegistryEntry.serviceName].connectAsync(
|
|
113
|
+
(pTestError) =>
|
|
114
|
+
{
|
|
115
|
+
if (pTestError)
|
|
116
|
+
{
|
|
117
|
+
tmpFable.log.warn(`Data Cloner: Test connection failed: ${pTestError.message || pTestError}`);
|
|
118
|
+
pResponse.send(200,
|
|
119
|
+
{
|
|
120
|
+
Success: false,
|
|
121
|
+
Provider: tmpProvider,
|
|
122
|
+
Error: `Connection failed: ${pTestError.message || pTestError}`
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
else
|
|
126
|
+
{
|
|
127
|
+
tmpFable.log.info(`Data Cloner: Test connection to ${tmpProvider} succeeded.`);
|
|
128
|
+
pResponse.send(200,
|
|
129
|
+
{
|
|
130
|
+
Success: true,
|
|
131
|
+
Provider: tmpProvider,
|
|
132
|
+
Message: `${tmpProvider} connection test successful.`
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return fNext();
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
};
|