retold-data-service 2.0.13 → 2.0.14

package/source/services/meadow-integration/Retold-Data-Service-MeadowIntegration.js

@@ -0,0 +1,113 @@
+/**
+ * Retold Data Service - Meadow Integration Service
+ *
+ * Fable service that exposes meadow-integration capabilities via REST
+ * endpoints under /1.0/Retold/MeadowIntegration/.
+ *
+ * Uses a dedicated Pict instance (which extends Fable with parseTemplate,
+ * ExpressionParser, CSVParser, etc.) for integration operations.
+ *
+ * @author Steven Velozo <steven@velozo.com>
+ */
+const libFableServiceProviderBase = require('fable-serviceproviderbase');
+
+const libPict = require('pict');
+const libPath = require('path');
+
+// Resolve the meadow-integration package root so we can require internal files
+const _MeadowIntegrationBase = libPath.dirname(require.resolve('meadow-integration/package.json'));
+
+// Require meadow-integration service providers from their resolved paths
+const libTabularCheck = require(libPath.join(_MeadowIntegrationBase, 'source/services/tabular/Service-TabularCheck.js'));
+const libTabularTransform = require(libPath.join(_MeadowIntegrationBase, 'source/services/tabular/Service-TabularTransform.js'));
+const libIntegrationAdapter = require(libPath.join(_MeadowIntegrationBase, 'source/Meadow-Service-Integration-Adapter.js'));
+
+const _RoutePrefix = '/1.0/Retold/MeadowIntegration';
+
+class RetoldDataServiceMeadowIntegration extends libFableServiceProviderBase
+{
+	constructor(pFable, pOptions, pServiceHash)
+	{
+		super(pFable, pOptions, pServiceHash);
+
+		this.serviceType = 'RetoldDataServiceMeadowIntegration';
+
+		// Create a dedicated Pict instance for integration operations.
+		// Pict extends Fable with parseTemplate, ExpressionParser, CSVParser,
+		// DataFormat, and other features required by meadow-integration services.
+		this._Pict = new libPict(
+			{
+				Product: 'RetoldDataServiceMeadowIntegration',
+				LogStreams: pFable.settings.LogStreams || [{ streamtype: 'console' }]
+			});
+
+		// Pre-initialize built-in Pict services used by integration operations
+		this._Pict.instantiateServiceProvider('CSVParser');
+		this._Pict.instantiateServiceProvider('FilePersistence');
+		this._Pict.instantiateServiceProvider('DataGeneration');
+
+		// Register meadow-integration service types on the Pict instance
+		this._Pict.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationTabularCheck', libTabularCheck);
+		this._Pict.addAndInstantiateServiceTypeIfNotExists('MeadowIntegrationTabularTransform', libTabularTransform);
+	}
+
+	/**
+	 * The Pict instance used for integration operations.
+	 * Command files use this to access parseTemplate, CSVParser,
+	 * TabularCheck, TabularTransform, etc.
+	 */
+	get pict()
+	{
+		return this._Pict;
+	}
+
+	/**
+	 * The route prefix for all MeadowIntegration endpoints.
+	 */
+	get routePrefix()
+	{
+		return _RoutePrefix;
+	}
+
+	/**
+	 * The Integration Adapter class (used by ComprehensionPush).
+	 */
+	get IntegrationAdapter()
+	{
+		return libIntegrationAdapter;
+	}
+
+	/**
+	 * Register all meadow integration REST routes on the Orator service server.
+	 *
+	 * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+	 */
+	connectRoutes(pOratorServiceServer)
+	{
+		// CSV operations
+		require('./MeadowIntegration-Command-CSVCheck.js')(this, pOratorServiceServer);
+		require('./MeadowIntegration-Command-CSVTransform.js')(this, pOratorServiceServer);
+
+		// TSV operations
+		require('./MeadowIntegration-Command-TSVCheck.js')(this, pOratorServiceServer);
+		require('./MeadowIntegration-Command-TSVTransform.js')(this, pOratorServiceServer);
+
+		// JSON Array operations
+		require('./MeadowIntegration-Command-JSONArrayTransform.js')(this, pOratorServiceServer);
+
+		// Comprehension operations
+		require('./MeadowIntegration-Command-ComprehensionIntersect.js')(this, pOratorServiceServer);
+		require('./MeadowIntegration-Command-ComprehensionToArray.js')(this, pOratorServiceServer);
+		require('./MeadowIntegration-Command-ComprehensionToCSV.js')(this, pOratorServiceServer);
+		require('./MeadowIntegration-Command-ComprehensionPush.js')(this, pOratorServiceServer);
+
+		// Entity operations
+		require('./MeadowIntegration-Command-EntityFromTabularFolder.js')(this, pOratorServiceServer);
+
+		this.fable.log.info('Retold Data Service Meadow Integration routes registered.');
+	}
+}
+
+module.exports = RetoldDataServiceMeadowIntegration;
+module.exports.serviceType = 'RetoldDataServiceMeadowIntegration';
+module.exports.default_configuration = {};

package/source/services/migration-manager/MigrationManager-Command-Connections.js

@@ -0,0 +1,220 @@
+/**
+ * MigrationManager Command - Connection Management
+ *
+ * Routes for database connection CRUD, testing, introspection, and
+ * provider listing.
+ *
+ * GET    /api/providers
+ * GET    /api/connections
+ * POST   /api/connections
+ * DELETE /api/connections/:name
+ * POST   /api/connections/:name/test
+ * POST   /api/connections/test
+ * POST   /api/connections/:name/introspect
+ */
+module.exports = function(pMigrationService, pOratorServiceServer)
+{
+	let tmpConnectionLibrary = pMigrationService._connectionLibrary;
+	let tmpDatabaseProviderFactory = pMigrationService._databaseProviderFactory;
+	let tmpSchemaLibrary = pMigrationService._schemaLibrary;
+	let tmpPrefix = pMigrationService.routePrefix;
+
+	// GET /api/providers — list available database provider types
+	pOratorServiceServer.get(tmpPrefix + '/api/providers',
+		(pRequest, pResponse, fNext) =>
+		{
+			try
+			{
+				pResponse.send(
+				{
+					Success: true,
+					Providers: tmpDatabaseProviderFactory.listAvailableProviders()
+				});
+			}
+			catch (pError)
+			{
+				pResponse.send(500, { Success: false, Error: pError.message });
+			}
+			return fNext();
+		});
+
+	// GET /api/connections — list all saved connections
+	pOratorServiceServer.get(tmpPrefix + '/api/connections',
+		(pRequest, pResponse, fNext) =>
+		{
+			try
+			{
+				let tmpNames = tmpConnectionLibrary.listConnections();
+				let tmpConnections = [];
+
+				for (let i = 0; i < tmpNames.length; i++)
+				{
+					let tmpEntry = tmpConnectionLibrary.getConnection(tmpNames[i]);
+					tmpConnections.push(
+					{
+						Name: tmpEntry.Name,
+						Type: tmpEntry.Type,
+						Config:
+						{
+							server: tmpEntry.Config.server || tmpEntry.Config.host || '',
+							port: tmpEntry.Config.port || '',
+							user: tmpEntry.Config.user || '',
+							database: tmpEntry.Config.database || ''
+							// Intentionally omit password
+						}
+					});
+				}
+
+				pResponse.send({ Success: true, Connections: tmpConnections });
+			}
+			catch (pError)
+			{
+				pResponse.send(500, { Success: false, Error: pError.message });
+			}
+			return fNext();
+		});
+
+	// POST /api/connections — add a new connection
+	pOratorServiceServer.post(tmpPrefix + '/api/connections',
+		(pRequest, pResponse, fNext) =>
+		{
+			try
+			{
+				let tmpName = pRequest.body && pRequest.body.Name;
+				let tmpType = pRequest.body && pRequest.body.Type;
+				let tmpConfig = pRequest.body && pRequest.body.Config;
+
+				if (!tmpName || !tmpType || !tmpConfig)
+				{
+					pResponse.send(400, { Success: false, Error: 'Name, Type, and Config are required.' });
+					return fNext();
+				}
+
+				tmpConnectionLibrary.addConnection(tmpName, tmpType, tmpConfig);
+
+				pResponse.send({ Success: true, Name: tmpName });
+			}
+			catch (pError)
+			{
+				pResponse.send(500, { Success: false, Error: pError.message });
+			}
+			return fNext();
+		});
+
+	// DELETE /api/connections/:name — remove a connection
+	pOratorServiceServer.del(tmpPrefix + '/api/connections/:name',
+		(pRequest, pResponse, fNext) =>
+		{
+			try
+			{
+				let tmpRemoved = tmpConnectionLibrary.removeConnection(pRequest.params.name);
+
+				if (!tmpRemoved)
+				{
+					pResponse.send(404, { Success: false, Error: `Connection [${pRequest.params.name}] not found.` });
+					return fNext();
+				}
+
+				pResponse.send({ Success: true });
+			}
+			catch (pError)
+			{
+				pResponse.send(500, { Success: false, Error: pError.message });
+			}
+			return fNext();
+		});
+
+	// POST /api/connections/:name/test — test a saved connection
+	pOratorServiceServer.post(tmpPrefix + '/api/connections/:name/test',
+		(pRequest, pResponse, fNext) =>
+		{
+			tmpDatabaseProviderFactory.testConnection(pRequest.params.name,
+				(pError, pTableList) =>
+				{
+					if (pError)
+					{
+						pResponse.send(500, { Success: false, Error: `Connection test failed: ${pError.message || pError}` });
+						return fNext();
+					}
+
+					pResponse.send(
+					{
+						Success: true,
+						TableCount: pTableList.length,
+						Tables: pTableList
+					});
+					return fNext();
+				});
+		});
+
+	// POST /api/connections/test — test an unsaved connection config
+	pOratorServiceServer.post(tmpPrefix + '/api/connections/test',
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpType = pRequest.body && pRequest.body.Type;
+			let tmpConfig = pRequest.body && pRequest.body.Config;
+
+			if (!tmpType || !tmpConfig)
+			{
+				pResponse.send(400, { Success: false, Error: 'Type and Config are required.' });
+				return fNext();
+			}
+
+			tmpDatabaseProviderFactory.testConnectionConfig(tmpType, tmpConfig,
+				(pError, pTableList) =>
+				{
+					if (pError)
+					{
+						pResponse.send(500, { Success: false, Error: `Connection test failed: ${pError.message || pError}` });
+						return fNext();
+					}
+
+					pResponse.send(
+					{
+						Success: true,
+						TableCount: pTableList.length,
+						Tables: pTableList
+					});
+					return fNext();
+				});
+		});
+
+	// POST /api/connections/:name/introspect — introspect a saved connection
+	pOratorServiceServer.post(tmpPrefix + '/api/connections/:name/introspect',
+		(pRequest, pResponse, fNext) =>
+		{
+			tmpDatabaseProviderFactory.introspectConnection(pRequest.params.name,
+				(pError, pSchema) =>
+				{
+					if (pError)
+					{
+						pResponse.send(500, { Success: false, Error: `Introspection failed: ${pError.message || pError}` });
+						return fNext();
+					}
+
+					let tmpSaveAs = pRequest.body && pRequest.body.saveAs;
+
+					if (tmpSaveAs)
+					{
+						let tmpEntry = tmpSchemaLibrary.addSchema(tmpSaveAs, '');
+						tmpEntry.CompiledSchema = pSchema;
+						tmpEntry.LastCompiled = new Date().toJSON();
+
+						pResponse.send(
+						{
+							Success: true,
+							Schema: pSchema,
+							SavedAs: tmpSaveAs
+						});
+						return fNext();
+					}
+
+					pResponse.send(
+					{
+						Success: true,
+						Schema: pSchema
+					});
+					return fNext();
+				});
+		});
+};

package/source/services/migration-manager/MigrationManager-Command-DiffMigrate.js

@@ -0,0 +1,169 @@
+/**
+ * MigrationManager Command - Diff & Migration
+ *
+ * Routes for schema diffing and SQL migration script generation.
+ *
+ * POST   /api/schemas/diff
+ * POST   /api/schemas/generate-migration
+ */
+module.exports = function(pMigrationService, pOratorServiceServer)
+{
+	let tmpSchemaLibrary = pMigrationService._schemaLibrary;
+	let tmpSchemaDiff = pMigrationService._schemaDiff;
+	let tmpMigrationGenerator = pMigrationService._migrationGenerator;
+	let tmpDatabaseProviderFactory = pMigrationService._databaseProviderFactory;
+	let tmpPrefix = pMigrationService.routePrefix;
+
+	// POST /api/schemas/diff — diff two schemas (DDL↔DDL, DDL↔DB, DB↔DB)
+	pOratorServiceServer.post(tmpPrefix + '/api/schemas/diff',
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpSourceName = pRequest.body && pRequest.body.source;
+			let tmpTargetName = pRequest.body && pRequest.body.target;
+			let tmpSourceConnection = pRequest.body && pRequest.body.sourceConnection;
+			let tmpTargetConnection = pRequest.body && pRequest.body.targetConnection;
+
+			// Resolve the source side
+			let fResolveSource = (fDone) =>
+			{
+				if (tmpSourceConnection)
+				{
+					tmpDatabaseProviderFactory.introspectConnection(tmpSourceConnection,
+						(pError, pSchema) =>
+						{
+							if (pError)
+							{
+								return fDone(pError);
+							}
+							return fDone(null, pSchema, `DB:${tmpSourceConnection}`);
+						});
+				}
+				else if (tmpSourceName)
+				{
+					let tmpSourceEntry = tmpSchemaLibrary.getSchema(tmpSourceName);
+
+					if (!tmpSourceEntry)
+					{
+						return fDone(new Error(`Source schema [${tmpSourceName}] not found.`));
+					}
+
+					if (!tmpSourceEntry.CompiledSchema)
+					{
+						return fDone(new Error(`Source schema [${tmpSourceName}] has not been compiled.`));
+					}
+
+					return fDone(null, pMigrationService.normalizeSchemaForDiff(tmpSourceEntry.CompiledSchema), tmpSourceName);
+				}
+				else
+				{
+					return fDone(new Error('Either source or sourceConnection is required.'));
+				}
+			};
+
+			// Resolve the target side
+			let fResolveTarget = (fDone) =>
+			{
+				if (tmpTargetConnection)
+				{
+					tmpDatabaseProviderFactory.introspectConnection(tmpTargetConnection,
+						(pError, pSchema) =>
+						{
+							if (pError)
+							{
+								return fDone(pError);
+							}
+							return fDone(null, pSchema, `DB:${tmpTargetConnection}`);
+						});
+				}
+				else if (tmpTargetName)
+				{
+					let tmpTargetEntry = tmpSchemaLibrary.getSchema(tmpTargetName);
+
+					if (!tmpTargetEntry)
+					{
+						return fDone(new Error(`Target schema [${tmpTargetName}] not found.`));
+					}
+
+					if (!tmpTargetEntry.CompiledSchema)
+					{
+						return fDone(new Error(`Target schema [${tmpTargetName}] has not been compiled.`));
+					}
+
+					return fDone(null, pMigrationService.normalizeSchemaForDiff(tmpTargetEntry.CompiledSchema), tmpTargetName);
+				}
+				else
+				{
+					return fDone(new Error('Either target or targetConnection is required.'));
+				}
+			};
+
+			fResolveSource(
+				(pSourceError, pSourceSchema, pSourceLabel) =>
+				{
+					if (pSourceError)
+					{
+						pResponse.send(400, { Success: false, Error: pSourceError.message });
+						return fNext();
+					}
+
+					fResolveTarget(
+						(pTargetError, pTargetSchema, pTargetLabel) =>
+						{
+							if (pTargetError)
+							{
+								pResponse.send(400, { Success: false, Error: pTargetError.message });
+								return fNext();
+							}
+
+							try
+							{
+								let tmpDiffResult = tmpSchemaDiff.diffSchemas(pSourceSchema, pTargetSchema);
+
+								pResponse.send(
+								{
+									Success: true,
+									Source: pSourceLabel,
+									Target: pTargetLabel,
+									Diff: tmpDiffResult
+								});
+							}
+							catch (pDiffError)
+							{
+								pResponse.send(500, { Success: false, Error: pDiffError.message });
+							}
+							return fNext();
+						});
+				});
+		});
+
+	// POST /api/schemas/generate-migration — generate SQL migration script
+	pOratorServiceServer.post(tmpPrefix + '/api/schemas/generate-migration',
+		(pRequest, pResponse, fNext) =>
+		{
+			try
+			{
+				let tmpDiff = pRequest.body && pRequest.body.diff;
+				let tmpDatabaseType = (pRequest.body && pRequest.body.databaseType) || 'MySQL';
+
+				if (!tmpDiff)
+				{
+					pResponse.send(400, { Success: false, Error: 'A diff result object is required.' });
+					return fNext();
+				}
+
+				let tmpScript = tmpMigrationGenerator.generateMigrationScript(tmpDiff, tmpDatabaseType);
+
+				pResponse.send(
+				{
+					Success: true,
+					DatabaseType: tmpDatabaseType,
+					Script: tmpScript
+				});
+			}
+			catch (pError)
+			{
+				pResponse.send(500, { Success: false, Error: pError.message });
+			}
+			return fNext();
+		});
+};