retold-data-service 2.0.13 → 2.0.16

package/source/services/data-cloner/DataCloner-Command-Sync.js

@@ -0,0 +1,491 @@
+/**
+ * DataCloner Sync Routes
+ *
+ * Registers /clone/sync/* endpoints for starting, monitoring, and stopping
+ * data synchronization via meadow-integration.
+ *
+ * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pDataClonerService, pOratorServiceServer) =>
+{
+	let tmpFable = pDataClonerService.fable;
+	let tmpCloneState = pDataClonerService.cloneState;
+	let tmpPrefix = pDataClonerService.routePrefix;
+
+	// POST /clone/sync/start
+	pOratorServiceServer.post(`${tmpPrefix}/sync/start`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (tmpCloneState.SyncRunning)
+			{
+				pResponse.send(400, { Success: false, Error: 'Sync is already running.' });
+				return fNext();
+			}
+
+			if (!tmpCloneState.RemoteServerURL)
+			{
+				pResponse.send(400, { Success: false, Error: 'No remote server configured.' });
+				return fNext();
+			}
+
+			if (!tmpFable.MeadowSync || !tmpFable.MeadowSync.MeadowSyncEntities)
+			{
+				pResponse.send(400, { Success: false, Error: 'No sync entities available. Deploy a schema first.' });
+				return fNext();
+			}
+
+			let tmpBody = pRequest.body || {};
+			let tmpSelectedTables = tmpBody.Tables || [];
+			let tmpRequestedMode = tmpBody.SyncMode || 'Initial';
+			let tmpMaxRecords = parseInt(tmpBody.MaxRecordsPerEntity, 10) || 0;
+
+			// Update SyncDeletedRecords from request if provided
+			if (tmpBody.hasOwnProperty('SyncDeletedRecords'))
+			{
+				tmpCloneState.SyncDeletedRecords = !!tmpBody.SyncDeletedRecords;
+				let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+				for (let i = 0; i < tmpEntityNames.length; i++)
+				{
+					tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityNames[i]].SyncDeletedRecords = tmpCloneState.SyncDeletedRecords;
+				}
+			}
+
+			// Update MaxRecordsPerEntity on sync entities
+			if (tmpMaxRecords > 0)
+			{
+				let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+				for (let i = 0; i < tmpEntityNames.length; i++)
+				{
+					tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityNames[i]].MaxRecordsPerEntity = tmpMaxRecords;
+				}
+			}
+
+			// Update DateTimePrecisionMS on MeadowSync and all sync entities
+			if (tmpBody.hasOwnProperty('DateTimePrecisionMS'))
+			{
+				let tmpPrecision = parseInt(tmpBody.DateTimePrecisionMS, 10);
+				if (!isNaN(tmpPrecision))
+				{
+					tmpFable.MeadowSync.DateTimePrecisionMS = tmpPrecision;
+					let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+					for (let i = 0; i < tmpEntityNames.length; i++)
+					{
+						tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityNames[i]].DateTimePrecisionMS = tmpPrecision;
+					}
+				}
+			}
+
+			// If no tables specified, sync all entities
+			if (tmpSelectedTables.length === 0)
+			{
+				tmpSelectedTables = tmpFable.MeadowSync.SyncEntityList || Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+			}
+
+			if (tmpSelectedTables.length === 0)
+			{
+				pResponse.send(400, { Success: false, Error: 'No tables available for sync. Deploy a schema first.' });
+				return fNext();
+			}
+
+			// ---- Handle Sync Mode switching ----
+			let fStartSync = () =>
+			{
+				tmpFable.log.info(`Data Cloner: Starting ${tmpCloneState.SyncMode} sync for ${tmpSelectedTables.length} tables via meadow-integration (SyncDeletedRecords: ${tmpCloneState.SyncDeletedRecords})`);
+
+				// Initialize progress tracking
+				tmpCloneState.SyncRunning = true;
+				tmpCloneState.SyncStopping = false;
+				tmpCloneState.SyncProgress = {};
+				tmpCloneState.SyncRESTErrors = {};
+
+				for (let i = 0; i < tmpSelectedTables.length; i++)
+				{
+					tmpCloneState.SyncProgress[tmpSelectedTables[i]] = (
+						{
+							Status: 'Pending',
+							Total: 0,
+							Synced: 0,
+							Errors: 0,
+							StartTime: null,
+							EndTime: null
+						});
+					tmpCloneState.SyncRESTErrors[tmpSelectedTables[i]] = 0;
+				}
+
+				// Start the sync process asynchronously
+				pDataClonerService.syncTables(tmpSelectedTables);
+
+				pResponse.send(200,
+					{
+						Success: true,
+						Tables: tmpSelectedTables,
+						SyncMode: tmpCloneState.SyncMode,
+						SyncDeletedRecords: tmpCloneState.SyncDeletedRecords,
+						Message: `${tmpCloneState.SyncMode} sync started via meadow-integration.`
+					});
+				return fNext();
+			};
+
+			if (tmpRequestedMode !== tmpCloneState.SyncMode && tmpCloneState.DeployedModelObject)
+			{
+				tmpFable.log.info(`Data Cloner: Switching sync mode from ${tmpCloneState.SyncMode} to ${tmpRequestedMode} — re-creating sync entities...`);
+				tmpCloneState.SyncMode = tmpRequestedMode;
+				tmpFable.MeadowSync.SyncMode = tmpRequestedMode;
+
+				tmpFable.MeadowSync.loadMeadowSchema(tmpCloneState.DeployedModelObject,
+					(pReinitError) =>
+					{
+						if (pReinitError)
+						{
+							tmpFable.log.warn(`Data Cloner: Mode switch schema re-init warning: ${pReinitError}`);
+						}
+						let tmpReinitEntities = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+						tmpFable.log.info(`Data Cloner: Re-created ${tmpReinitEntities.length} sync entities in ${tmpRequestedMode} mode`);
+
+						// Update SyncDeletedRecords and MaxRecordsPerEntity on new entities
+						for (let i = 0; i < tmpReinitEntities.length; i++)
+						{
+							tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].SyncDeletedRecords = tmpCloneState.SyncDeletedRecords;
+							if (tmpMaxRecords > 0)
+							{
+								tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].MaxRecordsPerEntity = tmpMaxRecords;
+							}
+						}
+
+						return fStartSync();
+					});
+			}
+			else
+			{
+				tmpCloneState.SyncMode = tmpRequestedMode;
+				return fStartSync();
+			}
+		});
+
+	// GET /clone/sync/status
+	pOratorServiceServer.get(`${tmpPrefix}/sync/status`,
+		(pRequest, pResponse, fNext) =>
+		{
+			// Update progress from MeadowSync operation trackers
+			if (tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
+			{
+				let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+				for (let i = 0; i < tmpEntityNames.length; i++)
+				{
+					let tmpEntityName = tmpEntityNames[i];
+					let tmpProgress = tmpCloneState.SyncProgress[tmpEntityName];
+					if (tmpProgress && (tmpProgress.Status === 'Syncing' || tmpProgress.Status === 'Pending'))
+					{
+						let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityName];
+						if (tmpSyncEntity && tmpSyncEntity.operation)
+						{
+							let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpEntityName}`];
+							if (tmpTracker)
+							{
+								tmpProgress.Total = tmpTracker.TotalCount || 0;
+								tmpProgress.Synced = Math.max(tmpTracker.CurrentCount || 0, 0);
+							}
+						}
+						let tmpRESTErrors = tmpCloneState.SyncRESTErrors[tmpEntityName] || 0;
+						if (tmpRESTErrors > 0)
+						{
+							tmpProgress.Errors = tmpRESTErrors;
+						}
+					}
+				}
+			}
+
+			pResponse.send(200,
+				{
+					Running: tmpCloneState.SyncRunning,
+					Stopping: tmpCloneState.SyncStopping,
+					SyncMode: tmpCloneState.SyncMode,
+					Tables: tmpCloneState.SyncProgress
+				});
+			return fNext();
+		});
+
+	// GET /clone/sync/live-status
+	// Returns a human-readable narrative of what the data cloner is doing right now.
+	pOratorServiceServer.get(`${tmpPrefix}/sync/live-status`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let fFormatNumber = (pNum) =>
+			{
+				return pNum.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
+			};
+
+			let fFormatDuration = (pMs) =>
+			{
+				let tmpSeconds = Math.floor(pMs / 1000);
+				if (tmpSeconds < 60) return `${tmpSeconds}s`;
+				let tmpMin = Math.floor(tmpSeconds / 60);
+				let tmpSec = tmpSeconds % 60;
+				if (tmpMin < 60) return `${tmpMin}m ${tmpSec}s`;
+				let tmpHr = Math.floor(tmpMin / 60);
+				tmpMin = tmpMin % 60;
+				return `${tmpHr}h ${tmpMin}m`;
+			};
+
+			// Determine overall phase
+			let tmpPhase = 'idle';
+			let tmpMessage = 'Idle';
+			let tmpActiveEntity = null;
+			let tmpActiveProgress = null;
+			let tmpCompleted = [];
+			let tmpPending = [];
+			let tmpErrors = [];
+			let tmpTotalSynced = 0;
+			let tmpTotalRecords = 0;
+			let tmpElapsed = null;
+			let tmpETA = null;
+
+			if (!tmpCloneState.ConnectionConnected)
+			{
+				tmpPhase = 'disconnected';
+				tmpMessage = 'No database connected';
+			}
+			else if (!tmpCloneState.SessionAuthenticated && !tmpCloneState.SessionConfigured)
+			{
+				tmpPhase = 'idle';
+				tmpMessage = `Connected to ${tmpCloneState.ConnectionProvider} — waiting for remote session configuration`;
+			}
+			else if (tmpCloneState.SessionConfigured && !tmpCloneState.SessionAuthenticated)
+			{
+				tmpPhase = 'idle';
+				tmpMessage = `Connected to ${tmpCloneState.ConnectionProvider} — waiting for authentication`;
+			}
+			else if (tmpCloneState.SyncStopping)
+			{
+				tmpPhase = 'stopping';
+				tmpMessage = 'Stopping sync...';
+			}
+			else if (tmpCloneState.SyncRunning)
+			{
+				tmpPhase = 'syncing';
+
+				// Check for pre-count phase
+				if (tmpCloneState.SyncPhase === 'counting')
+				{
+					let tmpPC = tmpCloneState.PreCountProgress || { Counted: 0, TotalTables: 0 };
+					tmpMessage = `Analyzing tables: counted ${tmpPC.Counted} / ${tmpPC.TotalTables}...`;
+
+					pResponse.send(200,
+						{
+							Phase: tmpPhase,
+							Message: tmpMessage,
+							ActiveEntity: null,
+							ActiveProgress: null,
+							Completed: 0,
+							Pending: tmpPC.TotalTables,
+							Errors: 0,
+							TotalTables: tmpPC.TotalTables,
+							TotalSynced: 0,
+							TotalRecords: 0,
+							Elapsed: null,
+							SyncMode: tmpCloneState.SyncMode,
+							ETA: null,
+							PreCountGrandTotal: 0,
+							PreCountProgress: tmpPC
+						});
+					return fNext();
+				}
+
+				// Update progress from MeadowSync operation trackers (same as /sync/status)
+				if (tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
+				{
+					let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+					for (let i = 0; i < tmpEntityNames.length; i++)
+					{
+						let tmpEntityName = tmpEntityNames[i];
+						let tmpProgress = tmpCloneState.SyncProgress[tmpEntityName];
+						if (tmpProgress && (tmpProgress.Status === 'Syncing' || tmpProgress.Status === 'Pending'))
+						{
+							let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityName];
+							if (tmpSyncEntity && tmpSyncEntity.operation)
+							{
+								let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpEntityName}`];
+								if (tmpTracker)
+								{
+									tmpProgress.Total = tmpTracker.TotalCount || 0;
+									tmpProgress.Synced = Math.max(tmpTracker.CurrentCount || 0, 0);
+								}
+							}
+							let tmpRESTErrors = tmpCloneState.SyncRESTErrors[tmpEntityName] || 0;
+							if (tmpRESTErrors > 0)
+							{
+								tmpProgress.Errors = tmpRESTErrors;
+							}
+						}
+					}
+				}
+
+				// Categorize tables
+				let tmpTableNames = Object.keys(tmpCloneState.SyncProgress);
+				for (let i = 0; i < tmpTableNames.length; i++)
+				{
+					let tmpName = tmpTableNames[i];
+					let tmpP = tmpCloneState.SyncProgress[tmpName];
+					tmpTotalSynced += (tmpP.Synced || 0);
+					tmpTotalRecords += (tmpP.Total || 0);
+
+					if (tmpP.Status === 'Syncing')
+					{
+						tmpActiveEntity = tmpName;
+						tmpActiveProgress = tmpP;
+					}
+					else if (tmpP.Status === 'Complete' || tmpP.Status === 'Partial')
+					{
+						tmpCompleted.push({ Name: tmpName, Synced: tmpP.Synced || 0, Total: tmpP.Total || 0, Status: tmpP.Status });
+					}
+					else if (tmpP.Status === 'Error')
+					{
+						tmpErrors.push({ Name: tmpName, Error: tmpP.ErrorMessage || 'Unknown error' });
+					}
+					else if (tmpP.Status === 'Pending')
+					{
+						tmpPending.push(tmpName);
+					}
+				}
+
+				// Build elapsed time and ETA
+				if (tmpCloneState.SyncStartTime)
+				{
+					let tmpElapsedMs = Date.now() - new Date(tmpCloneState.SyncStartTime).getTime();
+					tmpElapsed = fFormatDuration(tmpElapsedMs);
+
+					// Compute ETA using pre-counted grand total (or running total) and records synced so far
+					let tmpETATotalRecords = tmpCloneState.PreCountGrandTotal || tmpTotalRecords;
+					if (tmpETATotalRecords > 0 && tmpTotalSynced > 0 && tmpElapsedMs > 5000)
+					{
+						let tmpRate = tmpTotalSynced / tmpElapsedMs; // records per ms
+						let tmpRemaining = tmpETATotalRecords - tmpTotalSynced;
+						if (tmpRate > 0 && tmpRemaining > 0)
+						{
+							tmpETA = fFormatDuration(tmpRemaining / tmpRate);
+						}
+					}
+				}
+
+				// Build the narrative
+				let tmpParts = [];
+
+				if (tmpActiveEntity && tmpActiveProgress)
+				{
+					let tmpRecordProgress = '';
+					if (tmpActiveProgress.Total > 0)
+					{
+						tmpRecordProgress = ` — record ${fFormatNumber(tmpActiveProgress.Synced)} / ${fFormatNumber(tmpActiveProgress.Total)}`;
+					}
+					else
+					{
+						tmpRecordProgress = ' — counting records...';
+					}
+					tmpParts.push(`${tmpCloneState.SyncMode} sync: ${tmpActiveEntity}${tmpRecordProgress}`);
+				}
+				else
+				{
+					tmpParts.push(`${tmpCloneState.SyncMode} sync in progress`);
+				}
+
+				// Summarize completed tables
+				if (tmpCompleted.length > 0)
+				{
+					// Show a few completed entities by name, then summarize the rest
+					let tmpCompletedSummary = [];
+					let tmpShowCount = Math.min(3, tmpCompleted.length);
+					// Show the most recently completed (last in the list)
+					for (let i = tmpCompleted.length - tmpShowCount; i < tmpCompleted.length; i++)
+					{
+						let tmpC = tmpCompleted[i];
+						tmpCompletedSummary.push(`${tmpC.Name} (${fFormatNumber(tmpC.Synced)})`);
+					}
+					let tmpCompletedStr = tmpCompletedSummary.join(', ');
+					if (tmpCompleted.length > tmpShowCount)
+					{
+						tmpCompletedStr = `${tmpCompleted.length - tmpShowCount} others, ` + tmpCompletedStr;
+					}
+					tmpParts.push(`Synced: ${tmpCompletedStr}`);
+				}
+
+				if (tmpPending.length > 0)
+				{
+					tmpParts.push(`${tmpPending.length} table${tmpPending.length === 1 ? '' : 's'} remaining`);
+				}
+
+				if (tmpErrors.length > 0)
+				{
+					tmpParts.push(`${tmpErrors.length} error${tmpErrors.length === 1 ? '' : 's'}`);
+				}
+
+				tmpMessage = tmpParts.join('. ') + '.';
+			}
+			else if (tmpCloneState.SyncReport)
+			{
+				// Sync finished — show summary
+				tmpPhase = 'complete';
+				let tmpR = tmpCloneState.SyncReport;
+				tmpMessage = `Sync ${tmpR.Outcome.toLowerCase()}: ${fFormatNumber(tmpR.Summary.TotalSynced)} records across ${tmpR.Summary.TotalTables} tables`;
+				if (tmpR.RunTimestamps && tmpR.RunTimestamps.DurationSeconds)
+				{
+					tmpMessage += ` in ${fFormatDuration(tmpR.RunTimestamps.DurationSeconds * 1000)}`;
+				}
+				tmpTotalSynced = tmpR.Summary.TotalSynced;
+				tmpTotalRecords = tmpR.Summary.TotalRecords;
+			}
+			else if (tmpCloneState.SessionAuthenticated)
+			{
+				tmpPhase = 'ready';
+				tmpMessage = `Connected to ${tmpCloneState.ConnectionProvider}, authenticated to ${tmpCloneState.RemoteServerURL || 'remote'} — ready to sync`;
+			}
+
+			pResponse.send(200,
+				{
+					Phase: tmpPhase,
+					Message: tmpMessage,
+					ActiveEntity: tmpActiveEntity,
+					ActiveProgress: tmpActiveProgress ? { Synced: tmpActiveProgress.Synced, Total: tmpActiveProgress.Total } : null,
+					Completed: tmpCompleted.length,
+					Pending: tmpPending.length,
+					Errors: tmpErrors.length,
+					TotalTables: tmpCompleted.length + tmpPending.length + tmpErrors.length + (tmpActiveEntity ? 1 : 0),
+					TotalSynced: tmpTotalSynced,
+					TotalRecords: tmpTotalRecords,
+					Elapsed: tmpElapsed,
+					SyncMode: tmpCloneState.SyncMode,
+					ETA: tmpETA,
+					PreCountGrandTotal: tmpCloneState.PreCountGrandTotal || 0
+				});
+			return fNext();
+		});
+
+	// POST /clone/sync/stop
+	pOratorServiceServer.post(`${tmpPrefix}/sync/stop`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (tmpCloneState.SyncRunning)
+			{
+				tmpCloneState.SyncStopping = true;
+				tmpFable.log.info('Data Cloner: Sync stop requested.');
+			}
+
+			pResponse.send(200, { Success: true, Message: 'Sync stop requested.' });
+			return fNext();
+		});
+
+	// GET /clone/sync/report
+	pOratorServiceServer.get(`${tmpPrefix}/sync/report`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (tmpCloneState.SyncReport)
+			{
+				pResponse.send(200, tmpCloneState.SyncReport);
+			}
+			else
+			{
+				pResponse.send(200, { Success: false, Error: 'No report available. Run a sync first.' });
+			}
+			return fNext();
+		});
+};

package/source/services/data-cloner/DataCloner-Command-WebUI.js

@@ -0,0 +1,40 @@
+/**
+ * DataCloner Web UI Routes
+ *
+ * Serves the data-cloner-web.html file at /clone/ and handles the redirect
+ * from /clone to /clone/.
+ *
+ * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pDataClonerService, pOratorServiceServer) =>
+{
+	let libFs = require('fs');
+	let libPath = require('path');
+
+	let tmpPrefix = pDataClonerService.routePrefix;
+	let tmpHTMLPath = libPath.join(__dirname, 'data-cloner-web.html');
+
+	pOratorServiceServer.get(`${tmpPrefix}/`,
+		(pRequest, pResponse, fNext) =>
+		{
+			try
+			{
+				let tmpHTML = libFs.readFileSync(tmpHTMLPath, 'utf8');
+				pResponse.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+				pResponse.write(tmpHTML);
+				pResponse.end();
+			}
+			catch (pReadError)
+			{
+				pResponse.send(500, { Success: false, Error: 'Failed to load web UI.' });
+			}
+			return fNext();
+		});
+
+	pOratorServiceServer.get(`${tmpPrefix}`,
+		(pRequest, pResponse, fNext) =>
+		{
+			pResponse.redirect(`${tmpPrefix}/`, fNext);
+		});
+};

package/source/services/data-cloner/DataCloner-ProviderRegistry.js

@@ -0,0 +1,20 @@
+/**
+ * DataCloner Provider Registry
+ *
+ * Maps provider names to their module names and fable service names.
+ *
+ * @author Steven Velozo <steven@velozo.com>
+ */
+
+const _ProviderRegistry = {
+	SQLite: { moduleName: 'meadow-connection-sqlite', serviceName: 'MeadowSQLiteProvider', configKey: 'SQLite' },
+	MySQL: { moduleName: 'meadow-connection-mysql', serviceName: 'MeadowMySQLProvider', configKey: 'MySQL' },
+	MSSQL: { moduleName: 'meadow-connection-mssql', serviceName: 'MeadowMSSQLProvider', configKey: 'MSSQL' },
+	PostgreSQL: { moduleName: 'meadow-connection-postgresql', serviceName: 'MeadowConnectionPostgreSQL', configKey: 'PostgreSQL' },
+	Solr: { moduleName: 'meadow-connection-solr', serviceName: 'MeadowConnectionSolr', configKey: 'Solr' },
+	MongoDB: { moduleName: 'meadow-connection-mongodb', serviceName: 'MeadowConnectionMongoDB', configKey: 'MongoDB' },
+	RocksDB: { moduleName: 'meadow-connection-rocksdb', serviceName: 'MeadowConnectionRocksDB', configKey: 'RocksDB' },
+	Bibliograph: { moduleName: 'bibliograph', serviceName: 'Bibliograph', configKey: 'Bibliograph' },
+};
+
+module.exports = _ProviderRegistry;