retold-data-service 2.0.13 → 2.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/launch.json +11 -0
- package/bin/retold-data-service-clone.js +286 -0
- package/package.json +18 -9
- package/source/Retold-Data-Service.js +275 -73
- package/source/services/Retold-Data-Service-ConnectionManager.js +277 -0
- package/source/services/Retold-Data-Service-MeadowEndpoints.js +217 -0
- package/source/services/Retold-Data-Service-ModelManager.js +335 -0
- package/source/services/data-cloner/DataCloner-Command-Connection.js +138 -0
- package/source/services/data-cloner/DataCloner-Command-Headless.js +357 -0
- package/source/services/data-cloner/DataCloner-Command-Schema.js +367 -0
- package/source/services/data-cloner/DataCloner-Command-Session.js +229 -0
- package/source/services/data-cloner/DataCloner-Command-Sync.js +491 -0
- package/source/services/data-cloner/DataCloner-Command-WebUI.js +40 -0
- package/source/services/data-cloner/DataCloner-ProviderRegistry.js +20 -0
- package/source/services/data-cloner/Retold-Data-Service-DataCloner.js +751 -0
- package/source/services/data-cloner/data-cloner-web.html +2706 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Dashboard.js +60 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Integrations.js +132 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-Command-Runs.js +93 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Base.js +116 -0
- package/source/services/integration-telemetry/IntegrationTelemetry-StorageProvider-Bibliograph.js +495 -0
- package/source/services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js +224 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-CSVCheck.js +85 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-CSVTransform.js +180 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionIntersect.js +153 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionPush.js +190 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToArray.js +113 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-ComprehensionToCSV.js +211 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-EntityFromTabularFolder.js +244 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-JSONArrayTransform.js +213 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-TSVCheck.js +80 -0
- package/source/services/meadow-integration/MeadowIntegration-Command-TSVTransform.js +166 -0
- package/source/services/meadow-integration/Retold-Data-Service-MeadowIntegration.js +113 -0
- package/source/services/migration-manager/MigrationManager-Command-Connections.js +220 -0
- package/source/services/migration-manager/MigrationManager-Command-DiffMigrate.js +169 -0
- package/source/services/migration-manager/MigrationManager-Command-Schemas.js +532 -0
- package/source/services/migration-manager/MigrationManager-Command-WebUI.js +123 -0
- package/source/services/migration-manager/Retold-Data-Service-MigrationManager.js +357 -0
- package/source/services/stricture/Retold-Data-Service-Stricture.js +303 -0
- package/source/services/stricture/Stricture-Command-Compile.js +39 -0
- package/source/services/stricture/Stricture-Command-Generate-AuthorizationChart.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-DictionaryCSV.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-LaTeX.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-Markdown.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-Meadow.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-ModelGraph.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-MySQL.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-MySQLMigrate.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-Pict.js +14 -0
- package/source/services/stricture/Stricture-Command-Generate-TestObjectContainers.js +14 -0
- package/test/RetoldDataService_tests.js +161 -1
- package/debug/data/books.csv +0 -10001
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MigrationManager Command - Connection Management
|
|
3
|
+
*
|
|
4
|
+
* Routes for database connection CRUD, testing, introspection, and
|
|
5
|
+
* provider listing.
|
|
6
|
+
*
|
|
7
|
+
* GET /api/providers
|
|
8
|
+
* GET /api/connections
|
|
9
|
+
* POST /api/connections
|
|
10
|
+
* DELETE /api/connections/:name
|
|
11
|
+
* POST /api/connections/:name/test
|
|
12
|
+
* POST /api/connections/test
|
|
13
|
+
* POST /api/connections/:name/introspect
|
|
14
|
+
*/
|
|
15
|
+
module.exports = function(pMigrationService, pOratorServiceServer)
|
|
16
|
+
{
|
|
17
|
+
let tmpConnectionLibrary = pMigrationService._connectionLibrary;
|
|
18
|
+
let tmpDatabaseProviderFactory = pMigrationService._databaseProviderFactory;
|
|
19
|
+
let tmpSchemaLibrary = pMigrationService._schemaLibrary;
|
|
20
|
+
let tmpPrefix = pMigrationService.routePrefix;
|
|
21
|
+
|
|
22
|
+
// GET /api/providers — list available database provider types
|
|
23
|
+
pOratorServiceServer.get(tmpPrefix + '/api/providers',
|
|
24
|
+
(pRequest, pResponse, fNext) =>
|
|
25
|
+
{
|
|
26
|
+
try
|
|
27
|
+
{
|
|
28
|
+
pResponse.send(
|
|
29
|
+
{
|
|
30
|
+
Success: true,
|
|
31
|
+
Providers: tmpDatabaseProviderFactory.listAvailableProviders()
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
catch (pError)
|
|
35
|
+
{
|
|
36
|
+
pResponse.send(500, { Success: false, Error: pError.message });
|
|
37
|
+
}
|
|
38
|
+
return fNext();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// GET /api/connections — list all saved connections
|
|
42
|
+
pOratorServiceServer.get(tmpPrefix + '/api/connections',
|
|
43
|
+
(pRequest, pResponse, fNext) =>
|
|
44
|
+
{
|
|
45
|
+
try
|
|
46
|
+
{
|
|
47
|
+
let tmpNames = tmpConnectionLibrary.listConnections();
|
|
48
|
+
let tmpConnections = [];
|
|
49
|
+
|
|
50
|
+
for (let i = 0; i < tmpNames.length; i++)
|
|
51
|
+
{
|
|
52
|
+
let tmpEntry = tmpConnectionLibrary.getConnection(tmpNames[i]);
|
|
53
|
+
tmpConnections.push(
|
|
54
|
+
{
|
|
55
|
+
Name: tmpEntry.Name,
|
|
56
|
+
Type: tmpEntry.Type,
|
|
57
|
+
Config:
|
|
58
|
+
{
|
|
59
|
+
server: tmpEntry.Config.server || tmpEntry.Config.host || '',
|
|
60
|
+
port: tmpEntry.Config.port || '',
|
|
61
|
+
user: tmpEntry.Config.user || '',
|
|
62
|
+
database: tmpEntry.Config.database || ''
|
|
63
|
+
// Intentionally omit password
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
pResponse.send({ Success: true, Connections: tmpConnections });
|
|
69
|
+
}
|
|
70
|
+
catch (pError)
|
|
71
|
+
{
|
|
72
|
+
pResponse.send(500, { Success: false, Error: pError.message });
|
|
73
|
+
}
|
|
74
|
+
return fNext();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
// POST /api/connections — add a new connection
|
|
78
|
+
pOratorServiceServer.post(tmpPrefix + '/api/connections',
|
|
79
|
+
(pRequest, pResponse, fNext) =>
|
|
80
|
+
{
|
|
81
|
+
try
|
|
82
|
+
{
|
|
83
|
+
let tmpName = pRequest.body && pRequest.body.Name;
|
|
84
|
+
let tmpType = pRequest.body && pRequest.body.Type;
|
|
85
|
+
let tmpConfig = pRequest.body && pRequest.body.Config;
|
|
86
|
+
|
|
87
|
+
if (!tmpName || !tmpType || !tmpConfig)
|
|
88
|
+
{
|
|
89
|
+
pResponse.send(400, { Success: false, Error: 'Name, Type, and Config are required.' });
|
|
90
|
+
return fNext();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
tmpConnectionLibrary.addConnection(tmpName, tmpType, tmpConfig);
|
|
94
|
+
|
|
95
|
+
pResponse.send({ Success: true, Name: tmpName });
|
|
96
|
+
}
|
|
97
|
+
catch (pError)
|
|
98
|
+
{
|
|
99
|
+
pResponse.send(500, { Success: false, Error: pError.message });
|
|
100
|
+
}
|
|
101
|
+
return fNext();
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// DELETE /api/connections/:name — remove a connection
|
|
105
|
+
pOratorServiceServer.del(tmpPrefix + '/api/connections/:name',
|
|
106
|
+
(pRequest, pResponse, fNext) =>
|
|
107
|
+
{
|
|
108
|
+
try
|
|
109
|
+
{
|
|
110
|
+
let tmpRemoved = tmpConnectionLibrary.removeConnection(pRequest.params.name);
|
|
111
|
+
|
|
112
|
+
if (!tmpRemoved)
|
|
113
|
+
{
|
|
114
|
+
pResponse.send(404, { Success: false, Error: `Connection [${pRequest.params.name}] not found.` });
|
|
115
|
+
return fNext();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
pResponse.send({ Success: true });
|
|
119
|
+
}
|
|
120
|
+
catch (pError)
|
|
121
|
+
{
|
|
122
|
+
pResponse.send(500, { Success: false, Error: pError.message });
|
|
123
|
+
}
|
|
124
|
+
return fNext();
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
// POST /api/connections/:name/test — test a saved connection
|
|
128
|
+
pOratorServiceServer.post(tmpPrefix + '/api/connections/:name/test',
|
|
129
|
+
(pRequest, pResponse, fNext) =>
|
|
130
|
+
{
|
|
131
|
+
tmpDatabaseProviderFactory.testConnection(pRequest.params.name,
|
|
132
|
+
(pError, pTableList) =>
|
|
133
|
+
{
|
|
134
|
+
if (pError)
|
|
135
|
+
{
|
|
136
|
+
pResponse.send(500, { Success: false, Error: `Connection test failed: ${pError.message || pError}` });
|
|
137
|
+
return fNext();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
pResponse.send(
|
|
141
|
+
{
|
|
142
|
+
Success: true,
|
|
143
|
+
TableCount: pTableList.length,
|
|
144
|
+
Tables: pTableList
|
|
145
|
+
});
|
|
146
|
+
return fNext();
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
// POST /api/connections/test — test an unsaved connection config
|
|
151
|
+
pOratorServiceServer.post(tmpPrefix + '/api/connections/test',
|
|
152
|
+
(pRequest, pResponse, fNext) =>
|
|
153
|
+
{
|
|
154
|
+
let tmpType = pRequest.body && pRequest.body.Type;
|
|
155
|
+
let tmpConfig = pRequest.body && pRequest.body.Config;
|
|
156
|
+
|
|
157
|
+
if (!tmpType || !tmpConfig)
|
|
158
|
+
{
|
|
159
|
+
pResponse.send(400, { Success: false, Error: 'Type and Config are required.' });
|
|
160
|
+
return fNext();
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
tmpDatabaseProviderFactory.testConnectionConfig(tmpType, tmpConfig,
|
|
164
|
+
(pError, pTableList) =>
|
|
165
|
+
{
|
|
166
|
+
if (pError)
|
|
167
|
+
{
|
|
168
|
+
pResponse.send(500, { Success: false, Error: `Connection test failed: ${pError.message || pError}` });
|
|
169
|
+
return fNext();
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
pResponse.send(
|
|
173
|
+
{
|
|
174
|
+
Success: true,
|
|
175
|
+
TableCount: pTableList.length,
|
|
176
|
+
Tables: pTableList
|
|
177
|
+
});
|
|
178
|
+
return fNext();
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// POST /api/connections/:name/introspect — introspect a saved connection
|
|
183
|
+
pOratorServiceServer.post(tmpPrefix + '/api/connections/:name/introspect',
|
|
184
|
+
(pRequest, pResponse, fNext) =>
|
|
185
|
+
{
|
|
186
|
+
tmpDatabaseProviderFactory.introspectConnection(pRequest.params.name,
|
|
187
|
+
(pError, pSchema) =>
|
|
188
|
+
{
|
|
189
|
+
if (pError)
|
|
190
|
+
{
|
|
191
|
+
pResponse.send(500, { Success: false, Error: `Introspection failed: ${pError.message || pError}` });
|
|
192
|
+
return fNext();
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
let tmpSaveAs = pRequest.body && pRequest.body.saveAs;
|
|
196
|
+
|
|
197
|
+
if (tmpSaveAs)
|
|
198
|
+
{
|
|
199
|
+
let tmpEntry = tmpSchemaLibrary.addSchema(tmpSaveAs, '');
|
|
200
|
+
tmpEntry.CompiledSchema = pSchema;
|
|
201
|
+
tmpEntry.LastCompiled = new Date().toJSON();
|
|
202
|
+
|
|
203
|
+
pResponse.send(
|
|
204
|
+
{
|
|
205
|
+
Success: true,
|
|
206
|
+
Schema: pSchema,
|
|
207
|
+
SavedAs: tmpSaveAs
|
|
208
|
+
});
|
|
209
|
+
return fNext();
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
pResponse.send(
|
|
213
|
+
{
|
|
214
|
+
Success: true,
|
|
215
|
+
Schema: pSchema
|
|
216
|
+
});
|
|
217
|
+
return fNext();
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
};
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MigrationManager Command - Diff & Migration
|
|
3
|
+
*
|
|
4
|
+
* Routes for schema diffing and SQL migration script generation.
|
|
5
|
+
*
|
|
6
|
+
* POST /api/schemas/diff
|
|
7
|
+
* POST /api/schemas/generate-migration
|
|
8
|
+
*/
|
|
9
|
+
module.exports = function(pMigrationService, pOratorServiceServer)
|
|
10
|
+
{
|
|
11
|
+
let tmpSchemaLibrary = pMigrationService._schemaLibrary;
|
|
12
|
+
let tmpSchemaDiff = pMigrationService._schemaDiff;
|
|
13
|
+
let tmpMigrationGenerator = pMigrationService._migrationGenerator;
|
|
14
|
+
let tmpDatabaseProviderFactory = pMigrationService._databaseProviderFactory;
|
|
15
|
+
let tmpPrefix = pMigrationService.routePrefix;
|
|
16
|
+
|
|
17
|
+
// POST /api/schemas/diff — diff two schemas (DDL↔DDL, DDL↔DB, DB↔DB)
|
|
18
|
+
pOratorServiceServer.post(tmpPrefix + '/api/schemas/diff',
|
|
19
|
+
(pRequest, pResponse, fNext) =>
|
|
20
|
+
{
|
|
21
|
+
let tmpSourceName = pRequest.body && pRequest.body.source;
|
|
22
|
+
let tmpTargetName = pRequest.body && pRequest.body.target;
|
|
23
|
+
let tmpSourceConnection = pRequest.body && pRequest.body.sourceConnection;
|
|
24
|
+
let tmpTargetConnection = pRequest.body && pRequest.body.targetConnection;
|
|
25
|
+
|
|
26
|
+
// Resolve the source side
|
|
27
|
+
let fResolveSource = (fDone) =>
|
|
28
|
+
{
|
|
29
|
+
if (tmpSourceConnection)
|
|
30
|
+
{
|
|
31
|
+
tmpDatabaseProviderFactory.introspectConnection(tmpSourceConnection,
|
|
32
|
+
(pError, pSchema) =>
|
|
33
|
+
{
|
|
34
|
+
if (pError)
|
|
35
|
+
{
|
|
36
|
+
return fDone(pError);
|
|
37
|
+
}
|
|
38
|
+
return fDone(null, pSchema, `DB:${tmpSourceConnection}`);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
else if (tmpSourceName)
|
|
42
|
+
{
|
|
43
|
+
let tmpSourceEntry = tmpSchemaLibrary.getSchema(tmpSourceName);
|
|
44
|
+
|
|
45
|
+
if (!tmpSourceEntry)
|
|
46
|
+
{
|
|
47
|
+
return fDone(new Error(`Source schema [${tmpSourceName}] not found.`));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (!tmpSourceEntry.CompiledSchema)
|
|
51
|
+
{
|
|
52
|
+
return fDone(new Error(`Source schema [${tmpSourceName}] has not been compiled.`));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return fDone(null, pMigrationService.normalizeSchemaForDiff(tmpSourceEntry.CompiledSchema), tmpSourceName);
|
|
56
|
+
}
|
|
57
|
+
else
|
|
58
|
+
{
|
|
59
|
+
return fDone(new Error('Either source or sourceConnection is required.'));
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
// Resolve the target side
|
|
64
|
+
let fResolveTarget = (fDone) =>
|
|
65
|
+
{
|
|
66
|
+
if (tmpTargetConnection)
|
|
67
|
+
{
|
|
68
|
+
tmpDatabaseProviderFactory.introspectConnection(tmpTargetConnection,
|
|
69
|
+
(pError, pSchema) =>
|
|
70
|
+
{
|
|
71
|
+
if (pError)
|
|
72
|
+
{
|
|
73
|
+
return fDone(pError);
|
|
74
|
+
}
|
|
75
|
+
return fDone(null, pSchema, `DB:${tmpTargetConnection}`);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
else if (tmpTargetName)
|
|
79
|
+
{
|
|
80
|
+
let tmpTargetEntry = tmpSchemaLibrary.getSchema(tmpTargetName);
|
|
81
|
+
|
|
82
|
+
if (!tmpTargetEntry)
|
|
83
|
+
{
|
|
84
|
+
return fDone(new Error(`Target schema [${tmpTargetName}] not found.`));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (!tmpTargetEntry.CompiledSchema)
|
|
88
|
+
{
|
|
89
|
+
return fDone(new Error(`Target schema [${tmpTargetName}] has not been compiled.`));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return fDone(null, pMigrationService.normalizeSchemaForDiff(tmpTargetEntry.CompiledSchema), tmpTargetName);
|
|
93
|
+
}
|
|
94
|
+
else
|
|
95
|
+
{
|
|
96
|
+
return fDone(new Error('Either target or targetConnection is required.'));
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
fResolveSource(
|
|
101
|
+
(pSourceError, pSourceSchema, pSourceLabel) =>
|
|
102
|
+
{
|
|
103
|
+
if (pSourceError)
|
|
104
|
+
{
|
|
105
|
+
pResponse.send(400, { Success: false, Error: pSourceError.message });
|
|
106
|
+
return fNext();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
fResolveTarget(
|
|
110
|
+
(pTargetError, pTargetSchema, pTargetLabel) =>
|
|
111
|
+
{
|
|
112
|
+
if (pTargetError)
|
|
113
|
+
{
|
|
114
|
+
pResponse.send(400, { Success: false, Error: pTargetError.message });
|
|
115
|
+
return fNext();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
try
|
|
119
|
+
{
|
|
120
|
+
let tmpDiffResult = tmpSchemaDiff.diffSchemas(pSourceSchema, pTargetSchema);
|
|
121
|
+
|
|
122
|
+
pResponse.send(
|
|
123
|
+
{
|
|
124
|
+
Success: true,
|
|
125
|
+
Source: pSourceLabel,
|
|
126
|
+
Target: pTargetLabel,
|
|
127
|
+
Diff: tmpDiffResult
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
catch (pDiffError)
|
|
131
|
+
{
|
|
132
|
+
pResponse.send(500, { Success: false, Error: pDiffError.message });
|
|
133
|
+
}
|
|
134
|
+
return fNext();
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// POST /api/schemas/generate-migration — generate SQL migration script
|
|
140
|
+
pOratorServiceServer.post(tmpPrefix + '/api/schemas/generate-migration',
|
|
141
|
+
(pRequest, pResponse, fNext) =>
|
|
142
|
+
{
|
|
143
|
+
try
|
|
144
|
+
{
|
|
145
|
+
let tmpDiff = pRequest.body && pRequest.body.diff;
|
|
146
|
+
let tmpDatabaseType = (pRequest.body && pRequest.body.databaseType) || 'MySQL';
|
|
147
|
+
|
|
148
|
+
if (!tmpDiff)
|
|
149
|
+
{
|
|
150
|
+
pResponse.send(400, { Success: false, Error: 'A diff result object is required.' });
|
|
151
|
+
return fNext();
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let tmpScript = tmpMigrationGenerator.generateMigrationScript(tmpDiff, tmpDatabaseType);
|
|
155
|
+
|
|
156
|
+
pResponse.send(
|
|
157
|
+
{
|
|
158
|
+
Success: true,
|
|
159
|
+
DatabaseType: tmpDatabaseType,
|
|
160
|
+
Script: tmpScript
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
catch (pError)
|
|
164
|
+
{
|
|
165
|
+
pResponse.send(500, { Success: false, Error: pError.message });
|
|
166
|
+
}
|
|
167
|
+
return fNext();
|
|
168
|
+
});
|
|
169
|
+
};
|