retold-data-service 2.0.16 → 2.0.18

package/.claude/launch.json

@@ -3,8 +3,8 @@
 	"configurations": [
 		{
 			"name": "data-cloner",
-			"runtimeExecutable": "node",
-			"runtimeArgs": ["bin/retold-data-service-clone.js"],
+			"runtimeExecutable": "npm",
+			"runtimeArgs": ["start"],
 			"port": 8095
 		}
 	]

package/.quackage.json

@@ -0,0 +1,19 @@
+{
+	"GulpfileConfiguration":
+	{
+		"EntrypointInputSourceFile": "{~Data:AppData.CWD~}/source/services/data-cloner/pict-app/Pict-DataCloner-Bundle.js",
+		"LibraryObjectName": "dataCloner",
+		"LibraryOutputFolder": "{~Data:AppData.CWD~}/source/services/data-cloner/web/",
+		"LibraryUniminifiedFileName": "data-cloner.{~Data:Record.BuildFileLabel~}js",
+		"LibraryMinifiedFileName": "data-cloner.{~Data:Record.BuildFileLabel~}min.js"
+	},
+	"GulpExecutions":
+	[
+		{
+			"Hash": "default",
+			"Name": "Default standard build.",
+			"BuildFileLabel": "",
+			"BrowsersListRC": "since 2022"
+		}
+	]
+}

package/package.json

@@ -1,16 +1,17 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.16",
+    "version": "2.0.18",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
         "retold-data-service-clone": "bin/retold-data-service-clone.js"
     },
     "scripts": {
-        "start": "node source/Retold-Data-Service.js",
+        "start": "node bin/retold-data-service-clone.js",
         "coverage": "npx quack coverage",
         "test": "npx quack test",
         "build": "npx quack build",
+        "prepublishOnly": "npx quack build",
         "build-test-model": "cd test && npx stricture -i model/ddl/BookStore.ddl",
         "docker-dev-build": "docker build ./ -f Dockerfile_LUXURYCode -t retold-data-service-image:local",
         "docker-dev-run": "docker run -it -d --name retold-data-service-dev -p 44444:8080 -p 43306:3306 -p 48086:8086 -v \"$PWD/.config:/home/coder/.config\"  -v \"$PWD:/home/coder/retold-data-service\" -u \"$(id -u):$(id -g)\" -e \"DOCKER_USER=$USER\" retold-data-service-image:local",
@@ -18,9 +19,13 @@
         "docker-service-build": "docker build ./ -f Dockerfile_Service -t retold-data-service-server-image:local",
         "docker-service-run-test": "docker run -it --init -d --name retold-data-service -p 8086:8086 -p 43306:3306 -v \"$(pwd):/retold-data-service:z\" -u \"$(id -u):$(id -g)\" retold-data-service-server-image:local",
         "docker-service-run": "docker run -it --init -d --name retold-data-service -p 8086:8086 -p 43306:3306 -v \"$(pwd):/retold-data-service:z\" retold-data-service-server-image:local",
-        "docker-service-shell": "docker exec -it retold-data-service /bin/bash"
+        "docker-service-shell": "docker exec -it retold-data-service /bin/bash",
+        "test:integration": "node test/run-integration-tests.js",
+        "test:integration:no-browser": "node test/run-integration-tests.js --skip-puppeteer",
+        "test:all": "npx quack test && node test/run-integration-tests.js --skip-puppeteer"
     },
     "mocha": {
+        "spec": "test/RetoldDataService_tests.js",
         "diff": true,
         "extension": [
             "js"
@@ -55,6 +60,7 @@
     "homepage": "https://github.com/stevenvelozo/retold-data-service",
     "devDependencies": {
         "meadow-connection-sqlite": "^1.0.18",
+        "puppeteer": "^24.38.0",
         "quackage": "^1.0.63",
         "stricture": "^4.0.2",
         "supertest": "^7.2.2"
@@ -63,16 +69,17 @@
         "bibliograph": "^0.1.4",
         "fable": "^3.1.63",
         "fable-serviceproviderbase": "^3.0.19",
-        "meadow": "^2.0.31",
+        "meadow": "^2.0.33",
         "meadow-connection-mysql": "^1.0.14",
         "meadow-endpoints": "^4.0.14",
+        "meadow-integration": "^1.0.14",
+        "meadow-migrationmanager": "^0.0.4",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",
         "orator-serviceserver-restify": "^2.0.9",
-        "meadow-integration": "^1.0.14",
-        "meadow-migrationmanager": "^0.0.4",
         "orator-static-server": "^2.0.4",
         "pict": "^1.0.357",
+        "pict-section-histogram": "^1.0.0",
         "pict-sessionmanager": "^1.0.2",
         "stricture": "^4.0.2"
     }

package/source/services/data-cloner/DataCloner-Command-Sync.js

@@ -7,6 +7,9 @@
  * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
  * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
  */
+const libFableLog = require('fable-log');
+const libPath = require('path');
+
 module.exports = (pDataClonerService, pOratorServiceServer) =>
 {
 	let tmpFable = pDataClonerService.fable;
@@ -88,6 +91,33 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				return fNext();
 			}
 
+			// ---- Enable log-to-file if requested ----
+			if (tmpBody.LogToFile)
+			{
+				// Remove any previous sync log stream
+				if (tmpCloneState.SyncLogFileLogger)
+				{
+					try { tmpCloneState.SyncLogFileLogger.closeWriter(() => {}); }
+					catch (pErr) { /* ignore */ }
+					tmpCloneState.SyncLogFileLogger = null;
+				}
+
+				let tmpTimestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+				let tmpLogPath = libPath.join(process.cwd(), `DataCloner-Run-${tmpTimestamp}.log`);
+				let tmpFileLogger = new libFableLog.LogProviderFlatfile(
+					{
+						path: tmpLogPath,
+						showtimestamps: true,
+						formattedtimestamps: true,
+						outputloglinestoconsole: false
+					}, tmpFable.log);
+				tmpFileLogger.initialize();
+				tmpFable.log.addLogger(tmpFileLogger, 'trace');
+				tmpCloneState.SyncLogFileLogger = tmpFileLogger;
+				tmpCloneState.SyncLogFilePath = tmpLogPath;
+				tmpFable.log.info(`Data Cloner: Log file enabled — writing to ${tmpLogPath}`);
+			}
+
 			// ---- Handle Sync Mode switching ----
 			let fStartSync = () =>
 			{
@@ -167,33 +197,35 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 	pOratorServiceServer.get(`${tmpPrefix}/sync/status`,
 		(pRequest, pResponse, fNext) =>
 		{
-			// Update progress from MeadowSync operation trackers
-			if (tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
+			// Build a read-only snapshot of progress (never mutate SyncProgress during a poll)
+			let tmpTablesSnapshot = {};
+			let tmpTableNames = Object.keys(tmpCloneState.SyncProgress);
+			for (let i = 0; i < tmpTableNames.length; i++)
 			{
-				let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
-				for (let i = 0; i < tmpEntityNames.length; i++)
+				let tmpName = tmpTableNames[i];
+				let tmpP = tmpCloneState.SyncProgress[tmpName];
+				let tmpSnap = { Status: tmpP.Status, Total: tmpP.Total || 0, Synced: tmpP.Synced || 0, Errors: tmpP.Errors || 0 };
+				if (tmpP.ErrorMessage) tmpSnap.ErrorMessage = tmpP.ErrorMessage;
+
+				if ((tmpP.Status === 'Syncing' || tmpP.Status === 'Pending') && tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
 				{
-					let tmpEntityName = tmpEntityNames[i];
-					let tmpProgress = tmpCloneState.SyncProgress[tmpEntityName];
-					if (tmpProgress && (tmpProgress.Status === 'Syncing' || tmpProgress.Status === 'Pending'))
+					let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpName];
+					if (tmpSyncEntity && tmpSyncEntity.operation)
 					{
-						let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityName];
-						if (tmpSyncEntity && tmpSyncEntity.operation)
+						let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpName}`];
+						if (tmpTracker)
 						{
-							let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpEntityName}`];
-							if (tmpTracker)
-							{
-								tmpProgress.Total = tmpTracker.TotalCount || 0;
-								tmpProgress.Synced = Math.max(tmpTracker.CurrentCount || 0, 0);
-							}
-						}
-						let tmpRESTErrors = tmpCloneState.SyncRESTErrors[tmpEntityName] || 0;
-						if (tmpRESTErrors > 0)
-						{
-							tmpProgress.Errors = tmpRESTErrors;
+							tmpSnap.Total = tmpTracker.TotalCount || tmpSnap.Total;
+							tmpSnap.Synced = Math.max(tmpTracker.CurrentCount || 0, 0);
 						}
 					}
+					let tmpRESTErrors = tmpCloneState.SyncRESTErrors[tmpName] || 0;
+					if (tmpRESTErrors > 0)
+					{
+						tmpSnap.Errors = tmpRESTErrors;
+					}
 				}
+				tmpTablesSnapshot[tmpName] = tmpSnap;
 			}
 
 			pResponse.send(200,
@@ -201,7 +233,7 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 					Running: tmpCloneState.SyncRunning,
 					Stopping: tmpCloneState.SyncStopping,
 					SyncMode: tmpCloneState.SyncMode,
-					Tables: tmpCloneState.SyncProgress
+					Tables: tmpTablesSnapshot
 				});
 			return fNext();
 		});
@@ -287,57 +319,57 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 							SyncMode: tmpCloneState.SyncMode,
 							ETA: null,
 							PreCountGrandTotal: 0,
-							PreCountProgress: tmpPC
+							PreCountProgress: tmpPC,
+							ThroughputSamples: []
 						});
 					return fNext();
 				}
 
-				// Update progress from MeadowSync operation trackers (same as /sync/status)
-				if (tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
+				// Read progress from MeadowSync operation trackers (read-only snapshot — never write back to SyncProgress during a poll)
+				let tmpTableNames = Object.keys(tmpCloneState.SyncProgress);
+				for (let i = 0; i < tmpTableNames.length; i++)
 				{
-					let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
-					for (let i = 0; i < tmpEntityNames.length; i++)
+					let tmpName = tmpTableNames[i];
+					let tmpP = tmpCloneState.SyncProgress[tmpName];
+
+					// Snapshot live tracker values without mutating SyncProgress
+					let tmpLiveTotal = tmpP.Total || 0;
+					let tmpLiveSynced = tmpP.Synced || 0;
+					let tmpLiveErrors = tmpP.Errors || 0;
+
+					if (tmpP.Status === 'Syncing' || tmpP.Status === 'Pending')
 					{
-						let tmpEntityName = tmpEntityNames[i];
-						let tmpProgress = tmpCloneState.SyncProgress[tmpEntityName];
-						if (tmpProgress && (tmpProgress.Status === 'Syncing' || tmpProgress.Status === 'Pending'))
+						if (tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
 						{
-							let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityName];
+							let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpName];
 							if (tmpSyncEntity && tmpSyncEntity.operation)
 							{
-								let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpEntityName}`];
+								let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpName}`];
 								if (tmpTracker)
 								{
-									tmpProgress.Total = tmpTracker.TotalCount || 0;
-									tmpProgress.Synced = Math.max(tmpTracker.CurrentCount || 0, 0);
+									tmpLiveTotal = tmpTracker.TotalCount || tmpLiveTotal;
+									tmpLiveSynced = Math.max(tmpTracker.CurrentCount || 0, 0);
 								}
 							}
-							let tmpRESTErrors = tmpCloneState.SyncRESTErrors[tmpEntityName] || 0;
-							if (tmpRESTErrors > 0)
-							{
-								tmpProgress.Errors = tmpRESTErrors;
-							}
+						}
+						let tmpRESTErrors = tmpCloneState.SyncRESTErrors[tmpName] || 0;
+						if (tmpRESTErrors > 0)
+						{
+							tmpLiveErrors = tmpRESTErrors;
 						}
 					}
-				}
 
-				// Categorize tables
-				let tmpTableNames = Object.keys(tmpCloneState.SyncProgress);
-				for (let i = 0; i < tmpTableNames.length; i++)
-				{
-					let tmpName = tmpTableNames[i];
-					let tmpP = tmpCloneState.SyncProgress[tmpName];
-					tmpTotalSynced += (tmpP.Synced || 0);
-					tmpTotalRecords += (tmpP.Total || 0);
+					tmpTotalSynced += tmpLiveSynced;
+					tmpTotalRecords += tmpLiveTotal;
 
 					if (tmpP.Status === 'Syncing')
 					{
 						tmpActiveEntity = tmpName;
-						tmpActiveProgress = tmpP;
+						tmpActiveProgress = { Synced: tmpLiveSynced, Total: tmpLiveTotal };
 					}
 					else if (tmpP.Status === 'Complete' || tmpP.Status === 'Partial')
 					{
-						tmpCompleted.push({ Name: tmpName, Synced: tmpP.Synced || 0, Total: tmpP.Total || 0, Status: tmpP.Status });
+						tmpCompleted.push({ Name: tmpName, Synced: tmpLiveSynced, Total: tmpLiveTotal, Status: tmpP.Status });
 					}
 					else if (tmpP.Status === 'Error')
 					{
@@ -455,7 +487,8 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 					Elapsed: tmpElapsed,
 					SyncMode: tmpCloneState.SyncMode,
 					ETA: tmpETA,
-					PreCountGrandTotal: tmpCloneState.PreCountGrandTotal || 0
+					PreCountGrandTotal: tmpCloneState.PreCountGrandTotal || 0,
+					ThroughputSamples: tmpCloneState.ThroughputSamples || []
 				});
 			return fNext();
 		});

package/source/services/data-cloner/DataCloner-Command-WebUI.js

@@ -1,8 +1,9 @@
 /**
  * DataCloner Web UI Routes
  *
- * Serves the data-cloner-web.html file at /clone/ and handles the redirect
- * from /clone to /clone/.
+ * Serves the Pict-application-based data cloner web UI.
+ * The HTML entry point loads pict.min.js (from node_modules) and the
+ * Quackage-built application bundle.
  *
  * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
  * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
@@ -13,28 +14,44 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 	let libPath = require('path');
 
 	let tmpPrefix = pDataClonerService.routePrefix;
-	let tmpHTMLPath = libPath.join(__dirname, 'data-cloner-web.html');
+	let tmpWebDir = libPath.join(__dirname, 'web');
 
-	pOratorServiceServer.get(`${tmpPrefix}/`,
-		(pRequest, pResponse, fNext) =>
+	// Helper: serve a static file with the given content type
+	let fServeFile = (pFilePath, pContentType) =>
+	{
+		return (pRequest, pResponse, fNext) =>
 		{
 			try
 			{
-				let tmpHTML = libFs.readFileSync(tmpHTMLPath, 'utf8');
-				pResponse.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
-				pResponse.write(tmpHTML);
+				let tmpContent = libFs.readFileSync(pFilePath, 'utf8');
+				pResponse.writeHead(200, { 'Content-Type': pContentType + '; charset=utf-8' });
+				pResponse.write(tmpContent);
 				pResponse.end();
 			}
 			catch (pReadError)
 			{
-				pResponse.send(500, { Success: false, Error: 'Failed to load web UI.' });
+				pResponse.send(500, { Success: false, Error: 'Failed to load ' + libPath.basename(pFilePath) });
 			}
 			return fNext();
-		});
+		};
+	};
 
+	// ---- HTML ----
+	pOratorServiceServer.get(`${tmpPrefix}/`,
+		fServeFile(libPath.join(tmpWebDir, 'index.html'), 'text/html'));
+
+	// Redirect /clone → /clone/
 	pOratorServiceServer.get(`${tmpPrefix}`,
 		(pRequest, pResponse, fNext) =>
 		{
 			pResponse.redirect(`${tmpPrefix}/`, fNext);
 		});
+
+	// ---- Pict library (from node_modules) ----
+	pOratorServiceServer.get(`${tmpPrefix}/pict.min.js`,
+		fServeFile(require.resolve('pict/dist/pict.min.js'), 'application/javascript'));
+
+	// ---- Application bundle ----
+	pOratorServiceServer.get(`${tmpPrefix}/data-cloner.js`,
+		fServeFile(libPath.join(tmpWebDir, 'data-cloner.js'), 'application/javascript'));
 };