retold-data-service 2.0.19 → 2.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.19",
+    "version": "2.0.21",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
@@ -61,7 +61,7 @@
     "devDependencies": {
         "meadow-connection-sqlite": "^1.0.18",
         "puppeteer": "^24.38.0",
-        "quackage": "^1.0.63",
+        "quackage": "^1.0.64",
         "stricture": "^4.0.2",
         "supertest": "^7.2.2"
     },
@@ -72,7 +72,7 @@
         "meadow": "^2.0.33",
         "meadow-connection-mysql": "^1.0.14",
         "meadow-endpoints": "^4.0.14",
-        "meadow-integration": "^1.0.16",
+        "meadow-integration": "^1.0.18",
         "meadow-migrationmanager": "^0.0.4",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",

package/source/services/data-cloner/DataCloner-Command-Sync.js

@@ -64,6 +64,16 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				}
 			}
 
+			// Update UseAdvancedIDPagination on all sync entities
+			if (tmpBody.hasOwnProperty('UseAdvancedIDPagination'))
+			{
+				let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+				for (let i = 0; i < tmpEntityNames.length; i++)
+				{
+					tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityNames[i]].UseAdvancedIDPagination = !!tmpBody.UseAdvancedIDPagination;
+				}
+			}
+
 			// Update DateTimePrecisionMS on MeadowSync and all sync entities
 			if (tmpBody.hasOwnProperty('DateTimePrecisionMS'))
 			{
@@ -173,7 +183,7 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 						let tmpReinitEntities = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
 						tmpFable.log.info(`Data Cloner: Re-created ${tmpReinitEntities.length} sync entities in ${tmpRequestedMode} mode`);
 
-						// Update SyncDeletedRecords and MaxRecordsPerEntity on new entities
+						// Update SyncDeletedRecords, MaxRecordsPerEntity, and UseAdvancedIDPagination on new entities
 						for (let i = 0; i < tmpReinitEntities.length; i++)
 						{
 							tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].SyncDeletedRecords = tmpCloneState.SyncDeletedRecords;
@@ -181,6 +191,10 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 							{
 								tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].MaxRecordsPerEntity = tmpMaxRecords;
 							}
+							if (tmpBody.hasOwnProperty('UseAdvancedIDPagination'))
+							{
+								tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].UseAdvancedIDPagination = !!tmpBody.UseAdvancedIDPagination;
+							}
 						}
 
 						return fStartSync();

package/source/services/data-cloner/pict-app/providers/Pict-Provider-DataCloner.js

@@ -295,6 +295,12 @@ class DataClonerProvider extends libPictProvider
 		{
 			document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
 		}
+		// Restore advanced ID pagination checkbox
+		let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
+		if (tmpAdvancedIDPagination !== null)
+		{
+			document.getElementById('syncAdvancedIDPagination').checked = tmpAdvancedIDPagination === 'true';
+		}
 	}
 
 	initPersistence()
@@ -345,6 +351,16 @@ class DataClonerProvider extends libPictProvider
 			});
 		}
 
+		// Persist advanced ID pagination checkbox
+		let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
+		if (tmpAdvancedIDPaginationEl)
+		{
+			tmpAdvancedIDPaginationEl.addEventListener('change', function()
+			{
+				localStorage.setItem('dataCloner_syncAdvancedIDPagination', this.checked);
+			});
+		}
+
 		// Persist auto-process checkboxes
 		let tmpAutoIds = ['auto1', 'auto2', 'auto3', 'auto4', 'auto5'];
 		for (let a = 0; a < tmpAutoIds.length; a++)

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Export.js

@@ -101,6 +101,7 @@ class DataClonerExportView extends libPictView
 		if (!isNaN(tmpPrecision) && tmpPrecision !== 1000) tmpConfig.Sync.DateTimePrecisionMS = tmpPrecision;
 		let tmpMaxRecords = parseInt(document.getElementById('syncMaxRecords').value, 10);
 		if (tmpMaxRecords > 0) tmpConfig.Sync.MaxRecords = tmpMaxRecords;
+		if (document.getElementById('syncAdvancedIDPagination').checked) tmpConfig.Sync.UseAdvancedIDPagination = true;
 
 		return tmpConfig;
 	}
@@ -169,6 +170,7 @@ class DataClonerExportView extends libPictView
 		let tmpSelectedTables = this.pict.views['DataCloner-Schema'].getSelectedTables();
 		tmpConfig.Sync.SyncEntityList = tmpSelectedTables.length > 0 ? tmpSelectedTables : [];
 		tmpConfig.Sync.SyncEntityOptions = {};
+		if (document.getElementById('syncAdvancedIDPagination').checked) tmpConfig.Sync.UseAdvancedIDPagination = true;
 
 		// ---- SessionManager ----
 		tmpConfig.SessionManager = { Sessions: {} };

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Sync.js

@@ -17,6 +17,7 @@ class DataClonerSyncView extends libPictView
 		let tmpSyncMode = document.querySelector('input[name="syncMode"]:checked').value;
 		let tmpMaxRecords = parseInt(document.getElementById('syncMaxRecords').value, 10) || 0;
 		let tmpLogToFile = document.getElementById('syncLogFile').checked;
+		let tmpAdvancedIDPagination = document.getElementById('syncAdvancedIDPagination').checked;
 
 		if (tmpSelectedTables.length === 0)
 		{
@@ -32,6 +33,7 @@ class DataClonerSyncView extends libPictView
 		let tmpPostBody = { Tables: tmpSelectedTables, PageSize: tmpPageSize, DateTimePrecisionMS: tmpDateTimePrecisionMS, SyncDeletedRecords: tmpSyncDeletedRecords, SyncMode: tmpSyncMode };
 		if (tmpMaxRecords > 0) tmpPostBody.MaxRecordsPerEntity = tmpMaxRecords;
 		if (tmpLogToFile) tmpPostBody.LogToFile = true;
+		if (tmpAdvancedIDPagination) tmpPostBody.UseAdvancedIDPagination = true;
 		this.pict.providers.DataCloner.api('POST', '/clone/sync/start', tmpPostBody)
 			.then(function(pData)
 			{
@@ -522,6 +524,11 @@ module.exports.default_configuration =
 				<label for="syncDeletedRecords">Sync deleted records (fetch records marked Deleted=1 on source and mirror locally)</label>
 			</div>
 
+			<div class="checkbox-row">
+				<input type="checkbox" id="syncAdvancedIDPagination">
+				<label for="syncAdvancedIDPagination">Use advanced ID pagination (faster for large tables; uses keyset pagination instead of OFFSET)</label>
+			</div>
+
 			<div class="inline-group" style="margin-top:8px; margin-bottom:4px">
 				<div style="flex:0 0 200px">
 					<label for="syncMaxRecords">Max Records per Entity</label>

package/source/services/data-cloner/web/data-cloner.js

@@ -5161,6 +5161,11 @@
           if (tmpSolrSecure !== null) {
             document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
           }
+          // Restore advanced ID pagination checkbox
+          let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
+          if (tmpAdvancedIDPagination !== null) {
+            document.getElementById('syncAdvancedIDPagination').checked = tmpAdvancedIDPagination === 'true';
+          }
         }
         initPersistence() {
           let tmpSelf = this;
@@ -5203,6 +5208,14 @@
             });
           }
 
+          // Persist advanced ID pagination checkbox
+          let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
+          if (tmpAdvancedIDPaginationEl) {
+            tmpAdvancedIDPaginationEl.addEventListener('change', function () {
+              localStorage.setItem('dataCloner_syncAdvancedIDPagination', this.checked);
+            });
+          }
+
           // Persist auto-process checkboxes
           let tmpAutoIds = ['auto1', 'auto2', 'auto3', 'auto4', 'auto5'];
           for (let a = 0; a < tmpAutoIds.length; a++) {
@@ -6330,6 +6343,7 @@
           if (!isNaN(tmpPrecision) && tmpPrecision !== 1000) tmpConfig.Sync.DateTimePrecisionMS = tmpPrecision;
           let tmpMaxRecords = parseInt(document.getElementById('syncMaxRecords').value, 10);
           if (tmpMaxRecords > 0) tmpConfig.Sync.MaxRecords = tmpMaxRecords;
+          if (document.getElementById('syncAdvancedIDPagination').checked) tmpConfig.Sync.UseAdvancedIDPagination = true;
           return tmpConfig;
         }
         buildMeadowIntegrationConfig() {
@@ -6396,6 +6410,7 @@
           let tmpSelectedTables = this.pict.views['DataCloner-Schema'].getSelectedTables();
           tmpConfig.Sync.SyncEntityList = tmpSelectedTables.length > 0 ? tmpSelectedTables : [];
           tmpConfig.Sync.SyncEntityOptions = {};
+          if (document.getElementById('syncAdvancedIDPagination').checked) tmpConfig.Sync.UseAdvancedIDPagination = true;
 
           // ---- SessionManager ----
           tmpConfig.SessionManager = {
@@ -7445,6 +7460,7 @@ select { background: #fff; width: 100%; padding: 8px 12px; border: 1px solid #cc
           let tmpSyncMode = document.querySelector('input[name="syncMode"]:checked').value;
           let tmpMaxRecords = parseInt(document.getElementById('syncMaxRecords').value, 10) || 0;
           let tmpLogToFile = document.getElementById('syncLogFile').checked;
+          let tmpAdvancedIDPagination = document.getElementById('syncAdvancedIDPagination').checked;
           if (tmpSelectedTables.length === 0) {
             this.pict.providers.DataCloner.setStatus('syncStatus', 'No tables selected for sync.', 'error');
             this.pict.providers.DataCloner.setSectionPhase(5, 'error');
@@ -7462,6 +7478,7 @@ select { background: #fff; width: 100%; padding: 8px 12px; border: 1px solid #cc
           };
           if (tmpMaxRecords > 0) tmpPostBody.MaxRecordsPerEntity = tmpMaxRecords;
           if (tmpLogToFile) tmpPostBody.LogToFile = true;
+          if (tmpAdvancedIDPagination) tmpPostBody.UseAdvancedIDPagination = true;
           this.pict.providers.DataCloner.api('POST', '/clone/sync/start', tmpPostBody).then(function (pData) {
             if (pData.Success) {
               let tmpMsg = pData.SyncMode + ' sync started for ' + pData.Tables.length + ' tables.';
@@ -7852,6 +7869,11 @@ select { background: #fff; width: 100%; padding: 8px 12px; border: 1px solid #cc
 				<label for="syncDeletedRecords">Sync deleted records (fetch records marked Deleted=1 on source and mirror locally)</label>
 			</div>
 
+			<div class="checkbox-row">
+				<input type="checkbox" id="syncAdvancedIDPagination">
+				<label for="syncAdvancedIDPagination">Use advanced ID pagination (faster for large tables; uses keyset pagination instead of OFFSET)</label>
+			</div>
+
 			<div class="inline-group" style="margin-top:8px; margin-bottom:4px">
 				<div style="flex:0 0 200px">
 					<label for="syncMaxRecords">Max Records per Entity</label>