retold-data-service 2.0.14 → 2.0.16

package/.claude/launch.json

@@ -0,0 +1,11 @@
+{
+	"version": "0.0.1",
+	"configurations": [
+		{
+			"name": "data-cloner",
+			"runtimeExecutable": "node",
+			"runtimeArgs": ["bin/retold-data-service-clone.js"],
+			"port": 8095
+		}
+	]
+}

package/bin/retold-data-service-clone.js

@@ -0,0 +1,286 @@
+#!/usr/bin/env node
+/**
+ * Retold Data Cloner — CLI Entry Point
+ *
+ * Starts a retold-data-service instance with the DataCloner execution mode enabled.
+ * Supports both a web UI for interactive cloning and a headless pipeline mode
+ * (--config + --run) for automated cloning.
+ *
+ * Usage:
+ *   retold-data-clone                           Start web UI only
+ *   retold-data-clone --config <path> --run      Headless clone from config file
+ *   retold-data-clone --config-json '{}' --run   Headless clone from inline JSON
+ *
+ * @author Steven Velozo <steven@velozo.com>
+ */
+const libFable = require('fable');
+const libMeadowConnectionSQLite = require('meadow-connection-sqlite');
+const libRetoldDataService = require('../source/Retold-Data-Service.js');
+
+const libMeadowCloneRestClient = require('meadow-integration/source/services/clone/Meadow-Service-RestClient');
+const libMeadowSync = require('meadow-integration/source/services/clone/Meadow-Service-Sync');
+
+const libFs = require('fs');
+const libPath = require('path');
+
+// ================================================================
+// CLI Arguments
+// ================================================================
+
+let _CLIConfig = null;
+let _CLIRunHeadless = false;
+let _CLILogPath = null;
+let _CLIMaxRecords = 0;
+let _CLISchemaPath = null;
+let _CLIReportPath = null;
+
+for (let i = 2; i < process.argv.length; i++)
+{
+	if ((process.argv[i] === '--config' || process.argv[i] === '-c') && process.argv[i + 1])
+	{
+		let tmpConfigPath = libPath.resolve(process.argv[i + 1]);
+		try
+		{
+			let tmpRaw = libFs.readFileSync(tmpConfigPath, 'utf8');
+			_CLIConfig = JSON.parse(tmpRaw);
+			console.log(`Data Cloner: Loaded config from ${tmpConfigPath}`);
+		}
+		catch (pConfigError)
+		{
+			console.error(`Data Cloner: Failed to load config from ${tmpConfigPath}: ${pConfigError.message}`);
+			process.exit(1);
+		}
+		i++;
+	}
+	else if (process.argv[i] === '--config-json' && process.argv[i + 1])
+	{
+		try
+		{
+			_CLIConfig = JSON.parse(process.argv[i + 1]);
+			console.log('Data Cloner: Loaded config from inline JSON');
+		}
+		catch (pParseError)
+		{
+			console.error(`Data Cloner: Failed to parse inline JSON: ${pParseError.message}`);
+			process.exit(1);
+		}
+		i++;
+	}
+	else if (process.argv[i] === '--run' || process.argv[i] === '-r')
+	{
+		_CLIRunHeadless = true;
+	}
+	else if ((process.argv[i] === '--port' || process.argv[i] === '-p') && process.argv[i + 1])
+	{
+		process.env.PORT = process.argv[i + 1];
+		i++;
+	}
+	else if (process.argv[i] === '--log' || process.argv[i] === '-l')
+	{
+		if (process.argv[i + 1] && !process.argv[i + 1].startsWith('-'))
+		{
+			_CLILogPath = libPath.resolve(process.argv[i + 1]);
+			i++;
+		}
+		else
+		{
+			_CLILogPath = `${process.cwd()}/DataCloner-Run-${libFable.generateFileNameDateStamp()}.log`;
+		}
+	}
+	else if ((process.argv[i] === '--max' || process.argv[i] === '-m') && process.argv[i + 1])
+	{
+		_CLIMaxRecords = parseInt(process.argv[i + 1], 10) || 0;
+		i++;
+	}
+	else if ((process.argv[i] === '--schema' || process.argv[i] === '-s') && process.argv[i + 1])
+	{
+		_CLISchemaPath = libPath.resolve(process.argv[i + 1]);
+		i++;
+	}
+	else if (process.argv[i] === '--report')
+	{
+		if (process.argv[i + 1] && !process.argv[i + 1].startsWith('-'))
+		{
+			_CLIReportPath = libPath.resolve(process.argv[i + 1]);
+			i++;
+		}
+		else
+		{
+			_CLIReportPath = `${process.cwd()}/DataCloner-Report-${libFable.generateFileNameDateStamp()}.json`;
+		}
+	}
+	else if (process.argv[i] === '--help' || process.argv[i] === '-h')
+	{
+		console.log(`
+Retold Data Cloner
+
+Usage:
+  retold-data-clone                           Start web UI only
+  retold-data-clone --config <path> --run      Headless clone from config file
+  retold-data-clone --config-json '{}' --run   Headless clone from inline JSON
+
+Options:
+  --config, -c <path>      Path to a JSON config file (generate from the web UI)
+  --config-json <json>     Inline JSON config string (for one-liner commands)
+  --run, -r                Auto-run the clone pipeline (requires --config or --config-json)
+  --port, -p <port>        Override the API server port (default: 8095)
+  --log, -l [path]         Write log output to a file (default: ./DataCloner-Run-<timestamp>.log)
+  --max, -m <n>            Limit sync to first n records per entity (for testing)
+  --schema, -s <path>      Path to a local schema JSON file (skip remote schema fetch)
+  --report [path]          Write sync report JSON file (default: auto-named; auto-enabled with --log)
+  --help, -h               Show this help
+`);
+		process.exit(0);
+	}
+}
+
+if (_CLIRunHeadless && !_CLIConfig)
+{
+	console.error('Data Cloner: --run requires --config <path> or --config-json <json>');
+	process.exit(1);
+}
+
+// ================================================================
+// Configuration
+// ================================================================
+
+let _Settings = (
+	{
+		Product: 'RetoldDataCloner',
+		ProductVersion: '1.0.0',
+		APIServerPort: parseInt(process.env.PORT, 10) || 8095,
+		LogStreams:
+			[
+				{
+					streamtype: 'console'
+				}
+			],
+
+		SQLite:
+			{
+				SQLiteFilePath: libPath.join(process.cwd(), 'data', 'cloned.sqlite')
+			}
+	});
+
+if (_CLILogPath)
+{
+	_Settings.LogStreams.push(
+		{
+			loggertype: 'simpleflatfile',
+			outputloglinestoconsole: false,
+			showtimestamps: true,
+			formattedtimestamps: true,
+			level: 'trace',
+			path: _CLILogPath
+		});
+}
+
+// Ensure the data directory exists
+let _DataDir = libPath.join(process.cwd(), 'data');
+if (!libFs.existsSync(_DataDir))
+{
+	libFs.mkdirSync(_DataDir, { recursive: true });
+}
+
+let _Fable = new libFable(_Settings);
+
+// ================================================================
+// SQLite Setup (default — connects automatically)
+// ================================================================
+
+_Fable.serviceManager.addServiceType('MeadowSQLiteProvider', libMeadowConnectionSQLite);
+_Fable.serviceManager.instantiateServiceProvider('MeadowSQLiteProvider');
+
+_Fable.MeadowSQLiteProvider.connectAsync(
+	(pError) =>
+	{
+		if (pError)
+		{
+			_Fable.log.error(`SQLite connection error: ${pError}`);
+			process.exit(1);
+		}
+
+		// Set default Meadow provider to SQLite
+		_Fable.settings.MeadowProvider = 'SQLite';
+
+		// Register meadow-integration services for clone sync
+		_Fable.serviceManager.addServiceType('MeadowCloneRestClient', libMeadowCloneRestClient);
+		_Fable.serviceManager.addServiceType('MeadowSync', libMeadowSync);
+
+		_Fable.serviceManager.addServiceType('RetoldDataService', libRetoldDataService);
+		let tmpDataService = _Fable.serviceManager.instantiateServiceProvider('RetoldDataService',
+			{
+				StorageProvider: 'SQLite',
+				StorageProviderModule: 'meadow-connection-sqlite',
+
+				// No default schema — models loaded at runtime after fetch
+				FullMeadowSchemaFilename: false,
+
+				Endpoints:
+					{
+						ConnectionManager: true,
+						ModelManagerWrite: true,
+						Stricture: false,
+						MeadowIntegration: false,
+						MeadowEndpoints: true,
+						MigrationManager: true,
+						MigrationManagerWebUI: true,
+						DataCloner: true,
+						DataClonerWebUI: true,
+						IntegrationTelemetry: true
+					}
+			});
+
+		// Enable JSON body parsing for POST/PUT requests
+		tmpDataService.onBeforeInitialize = (fCallback) =>
+		{
+			_Fable.OratorServiceServer.server.use(_Fable.OratorServiceServer.bodyParser());
+			return fCallback();
+		};
+
+		// SQLite is available but don't presume it's the user's desired provider.
+		// The web UI will connect via the auto chain or manual "go" click.
+		tmpDataService.onAfterInitialize = (fCallback) =>
+		{
+			return fCallback();
+		};
+
+		// ================================================================
+		// Start the service
+		// ================================================================
+
+		tmpDataService.initializeService(
+			(pInitError) =>
+			{
+				if (pInitError)
+				{
+					_Fable.log.error(`Initialization error: ${pInitError}`);
+					process.exit(1);
+				}
+				_Fable.log.info(`Data Cloner running on port ${_Settings.APIServerPort}`);
+				_Fable.log.info(`Web UI:          http://localhost:${_Settings.APIServerPort}/clone/`);
+				_Fable.log.info(`Migration Mgr:   http://localhost:${_Settings.APIServerPort}/meadow-migrationmanager/`);
+
+				// ---- Headless auto-run from config file ----
+				if (_CLIConfig && _CLIRunHeadless)
+				{
+					_Fable.RetoldDataServiceDataCloner.runHeadlessPipeline(_CLIConfig,
+						{
+							logPath: _CLILogPath,
+							maxRecords: _CLIMaxRecords,
+							schemaPath: _CLISchemaPath,
+							serverPort: _Settings.APIServerPort,
+							reportPath: _CLIReportPath
+						},
+						(pPipelineError) =>
+						{
+							if (pPipelineError)
+							{
+								_Fable.log.error(`Pipeline error: ${pPipelineError}`);
+								process.exit(1);
+							}
+							process.exit(0);
+						});
+				}
+			});
+	});

package/package.json

@@ -1,8 +1,11 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.14",
+    "version": "2.0.16",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
+    "bin": {
+        "retold-data-service-clone": "bin/retold-data-service-clone.js"
+    },
     "scripts": {
         "start": "node source/Retold-Data-Service.js",
         "coverage": "npx quack coverage",
@@ -51,24 +54,26 @@
     },
     "homepage": "https://github.com/stevenvelozo/retold-data-service",
     "devDependencies": {
-        "meadow-connection-sqlite": "^1.0.17",
-        "quackage": "^1.0.61",
-        "stricture": "^4.0.0",
+        "meadow-connection-sqlite": "^1.0.18",
+        "quackage": "^1.0.63",
+        "stricture": "^4.0.2",
         "supertest": "^7.2.2"
     },
     "dependencies": {
+        "bibliograph": "^0.1.4",
         "fable": "^3.1.63",
         "fable-serviceproviderbase": "^3.0.19",
-        "meadow": "^2.0.28",
-        "meadow-connection-mysql": "^1.0.13",
-        "meadow-endpoints": "^4.0.13",
+        "meadow": "^2.0.31",
+        "meadow-connection-mysql": "^1.0.14",
+        "meadow-endpoints": "^4.0.14",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",
         "orator-serviceserver-restify": "^2.0.9",
-        "meadow-integration": "^1.0.6",
+        "meadow-integration": "^1.0.14",
         "meadow-migrationmanager": "^0.0.4",
         "orator-static-server": "^2.0.4",
         "pict": "^1.0.357",
-        "stricture": "^4.0.0"
+        "pict-sessionmanager": "^1.0.2",
+        "stricture": "^4.0.2"
     }
 }

package/source/Retold-Data-Service.js

@@ -16,6 +16,8 @@ const libRetoldDataServiceModelManager = require('./services/Retold-Data-Service
 const libRetoldDataServiceStricture = require('./services/stricture/Retold-Data-Service-Stricture.js');
 const libRetoldDataServiceMeadowIntegration = require('./services/meadow-integration/Retold-Data-Service-MeadowIntegration.js');
 const libRetoldDataServiceMigrationManager = require('./services/migration-manager/Retold-Data-Service-MigrationManager.js');
+const libRetoldDataServiceDataCloner = require('./services/data-cloner/Retold-Data-Service-DataCloner.js');
+const libRetoldDataServiceIntegrationTelemetry = require('./services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js');
 
 const defaultDataServiceSettings = (
 	{
@@ -45,7 +47,13 @@ const defaultDataServiceSettings = (
 				// Migration manager API endpoints (/api/*)
 				MigrationManager: false,
 				// Migration manager web UI (GET /, /lib/*)
-				MigrationManagerWebUI: false
+				MigrationManagerWebUI: false,
+				// Data cloner API endpoints (/clone/*)
+				DataCloner: false,
+				// Data cloner web UI (GET /clone/)
+				DataClonerWebUI: false,
+				// Integration telemetry API endpoints (/telemetry/*)
+				IntegrationTelemetry: false
 			},
 
 		// Migration manager configuration
@@ -55,6 +63,22 @@ const defaultDataServiceSettings = (
 				ModelPath: false,
 				// Route prefix for all migration manager endpoints (API + web UI)
 				RoutePrefix: '/meadow-migrationmanager'
+			},
+
+		// Data cloner configuration
+		DataCloner:
+			{
+				// Route prefix for all data cloner endpoints (API + web UI)
+				RoutePrefix: '/clone'
+			},
+
+		// Integration telemetry configuration
+		IntegrationTelemetry:
+			{
+				// Route prefix for all telemetry endpoints
+				RoutePrefix: '/telemetry',
+				// Default tenant identifier when none is provided
+				DefaultTenantID: 'default'
 			}
 	});
 
@@ -110,6 +134,14 @@ class RetoldDataService extends libFableServiceProviderBase
 		this.fable.serviceManager.addServiceType('RetoldDataServiceMigrationManager', libRetoldDataServiceMigrationManager);
 		this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceMigrationManager');
 
+		// Register and instantiate the DataCloner service
+		this.fable.serviceManager.addServiceType('RetoldDataServiceDataCloner', libRetoldDataServiceDataCloner);
+		this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceDataCloner');
+
+		// Register and instantiate the IntegrationTelemetry service
+		this.fable.serviceManager.addServiceType('RetoldDataServiceIntegrationTelemetry', libRetoldDataServiceIntegrationTelemetry);
+		this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceIntegrationTelemetry');
+
 		// Expose the DAL and MeadowEndpoints from the service on this object and on fable for backward compatibility
 		this._DAL = this.fable.RetoldDataServiceMeadowEndpoints._DAL;
 		this._MeadowEndpoints = this.fable.RetoldDataServiceMeadowEndpoints._MeadowEndpoints;
@@ -203,7 +235,7 @@ class RetoldDataService extends libFableServiceProviderBase
 			this.fable.log.info(`The Retold Data Service is initializing...`);
 
 			// Log endpoint configuration
-			let tmpGroupNames = ['ConnectionManager', 'ModelManagerWrite', 'Stricture', 'MeadowIntegration', 'MeadowEndpoints', 'MigrationManager', 'MigrationManagerWebUI'];
+			let tmpGroupNames = ['ConnectionManager', 'ModelManagerWrite', 'Stricture', 'MeadowIntegration', 'MeadowEndpoints', 'MigrationManager', 'MigrationManagerWebUI', 'DataCloner', 'DataClonerWebUI', 'IntegrationTelemetry'];
 			let tmpEnabledGroups = [];
 			let tmpDisabledGroups = [];
 			for (let i = 0; i < tmpGroupNames.length; i++)
@@ -286,6 +318,24 @@ class RetoldDataService extends libFableServiceProviderBase
 						this.fable.RetoldDataServiceMigrationManager.connectWebUIRoutes(this.fable.OratorServiceServer);
 					}
 
+					// DataCloner API routes (/clone/*)
+					if (this.isEndpointGroupEnabled('DataCloner'))
+					{
+						this.fable.RetoldDataServiceDataCloner.connectRoutes(this.fable.OratorServiceServer);
+					}
+
+					// DataCloner Web UI routes (GET /clone/)
+					if (this.isEndpointGroupEnabled('DataClonerWebUI'))
+					{
+						this.fable.RetoldDataServiceDataCloner.connectWebUIRoutes(this.fable.OratorServiceServer);
+					}
+
+					// IntegrationTelemetry API routes (/telemetry/*)
+					if (this.isEndpointGroupEnabled('IntegrationTelemetry'))
+					{
+						this.fable.RetoldDataServiceIntegrationTelemetry.connectRoutes(this.fable.OratorServiceServer);
+					}
+
 					return fInitCallback();
 				});
 

package/source/services/data-cloner/DataCloner-Command-Connection.js

@@ -0,0 +1,138 @@
+/**
+ * DataCloner Connection Management Routes
+ *
+ * Registers /clone/connection/* endpoints for managing the local database
+ * connection (status, configure, test).
+ *
+ * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pDataClonerService, pOratorServiceServer) =>
+{
+	let tmpFable = pDataClonerService.fable;
+	let tmpCloneState = pDataClonerService.cloneState;
+	let tmpPrefix = pDataClonerService.routePrefix;
+
+	// GET /clone/connection/status
+	pOratorServiceServer.get(`${tmpPrefix}/connection/status`,
+		(pRequest, pResponse, fNext) =>
+		{
+			pResponse.send(200,
+				{
+					Provider: tmpCloneState.ConnectionProvider,
+					Connected: tmpCloneState.ConnectionConnected,
+					Config: tmpCloneState.ConnectionConfig
+				});
+			return fNext();
+		});
+
+	// POST /clone/connection/configure — Switch the local database provider
+	pOratorServiceServer.post(`${tmpPrefix}/connection/configure`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+			let tmpProvider = tmpBody.Provider;
+			let tmpConfig = tmpBody.Config || {};
+
+			if (!tmpProvider)
+			{
+				pResponse.send(400, { Success: false, Error: 'Provider is required (e.g. SQLite, MySQL, MSSQL).' });
+				return fNext();
+			}
+
+			tmpFable.log.info(`Data Cloner: Configuring ${tmpProvider} connection...`);
+
+			pDataClonerService.connectProvider(tmpProvider, tmpConfig,
+				(pConnectError) =>
+				{
+					if (pConnectError)
+					{
+						tmpFable.log.error(`Data Cloner: Connection error: ${pConnectError.message}`);
+						pResponse.send(500, { Success: false, Error: `Connection failed: ${pConnectError.message}` });
+						return fNext();
+					}
+
+					tmpFable.log.info(`Data Cloner: ${tmpProvider} connection established.`);
+					pResponse.send(200,
+						{
+							Success: true,
+							Provider: tmpProvider,
+							Message: `${tmpProvider} connection established and set as active provider.`
+						});
+					return fNext();
+				});
+		});
+
+	// POST /clone/connection/test — Test a connection without making it permanent
+	pOratorServiceServer.post(`${tmpPrefix}/connection/test`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+			let tmpProvider = tmpBody.Provider;
+			let tmpConfig = tmpBody.Config || {};
+
+			if (!tmpProvider)
+			{
+				pResponse.send(400, { Success: false, Error: 'Provider is required.' });
+				return fNext();
+			}
+
+			let tmpRegistryEntry = pDataClonerService.providerRegistry[tmpProvider];
+			if (!tmpRegistryEntry)
+			{
+				pResponse.send(400, { Success: false, Error: `Unknown provider: ${tmpProvider}` });
+				return fNext();
+			}
+
+			tmpFable.log.info(`Data Cloner: Testing ${tmpProvider} connection...`);
+
+			let tmpModule;
+			try
+			{
+				tmpModule = require(tmpRegistryEntry.moduleName);
+			}
+			catch (pRequireError)
+			{
+				pResponse.send(500, { Success: false, Error: `Module not installed: ${tmpRegistryEntry.moduleName}. Run: npm install ${tmpRegistryEntry.moduleName}` });
+				return fNext();
+			}
+
+			// Create a temporary fable instance for the test
+			let libFable = require('fable');
+			let tmpTestFable = new libFable(
+				{
+					Product: 'DataClonerConnectionTest',
+					LogStreams: [{ streamtype: 'console' }],
+					[tmpRegistryEntry.configKey]: tmpConfig
+				});
+
+			tmpTestFable.serviceManager.addServiceType(tmpRegistryEntry.serviceName, tmpModule);
+			tmpTestFable.serviceManager.instantiateServiceProvider(tmpRegistryEntry.serviceName);
+
+			tmpTestFable[tmpRegistryEntry.serviceName].connectAsync(
+				(pTestError) =>
+				{
+					if (pTestError)
+					{
+						tmpFable.log.warn(`Data Cloner: Test connection failed: ${pTestError.message || pTestError}`);
+						pResponse.send(200,
+							{
+								Success: false,
+								Provider: tmpProvider,
+								Error: `Connection failed: ${pTestError.message || pTestError}`
+							});
+					}
+					else
+					{
+						tmpFable.log.info(`Data Cloner: Test connection to ${tmpProvider} succeeded.`);
+						pResponse.send(200,
+							{
+								Success: true,
+								Provider: tmpProvider,
+								Message: `${tmpProvider} connection test successful.`
+							});
+					}
+					return fNext();
+				});
+		});
+};