retold-data-service 2.0.18 → 2.0.20

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.18",
+    "version": "2.0.20",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
@@ -72,7 +72,7 @@
         "meadow": "^2.0.33",
         "meadow-connection-mysql": "^1.0.14",
         "meadow-endpoints": "^4.0.14",
-        "meadow-integration": "^1.0.14",
+        "meadow-integration": "^1.0.16",
         "meadow-migrationmanager": "^0.0.4",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",

package/source/services/Retold-Data-Service-MeadowEndpoints.js

@@ -137,6 +137,8 @@ class RetoldDataServiceMeadowEndpoints extends libFableServiceProviderBase
 		{
 			let tmpDALEntityName = tmpEntityList[i];
 
+			let tmpRoutesAlreadyConnected = this._MeadowEndpoints.hasOwnProperty(tmpDALEntityName);
+
 			if (this._DAL.hasOwnProperty(tmpDALEntityName))
 			{
 				this.fable.log.warn(`Entity [${tmpDALEntityName}] already exists in the DAL (from another model); overwriting.`);
@@ -152,8 +154,16 @@ class RetoldDataServiceMeadowEndpoints extends libFableServiceProviderBase
 				this._DAL[tmpDALEntityName].setProvider(tmpStorageProvider);
 				this.fable.log.info(`...initializing the ${tmpDALEntityName} Meadow Endpoints`);
 				this._MeadowEndpoints[tmpDALEntityName] = libMeadowEndpoints.new(this._DAL[tmpDALEntityName]);
-				this.fable.log.info(`...mapping the ${tmpDALEntityName} Meadow Endpoints to Orator`);
-				this._MeadowEndpoints[tmpDALEntityName].connectRoutes(this.fable.OratorServiceServer);
+
+				if (!tmpRoutesAlreadyConnected)
+				{
+					this.fable.log.info(`...mapping the ${tmpDALEntityName} Meadow Endpoints to Orator`);
+					this._MeadowEndpoints[tmpDALEntityName].connectRoutes(this.fable.OratorServiceServer);
+				}
+				else
+				{
+					this.fable.log.info(`...routes for ${tmpDALEntityName} already registered; skipping connectRoutes.`);
+				}
 			}
 			catch (pError)
 			{

package/source/services/data-cloner/DataCloner-Command-Schema.js

@@ -274,10 +274,11 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				// Extract the entity name from the URL for error tracking
 				let tmpEntityHint = pURL.split('/')[0].replace(/s$/, '');
 
-				// Use the longer timeout for MAX(column) queries only
-				let tmpIsMaxRequest = (pURL.indexOf('/Max/') > -1);
+				// Use the longer timeout for Max and Count queries (which can be
+				// slow on large tables with millions of rows)
+				let tmpIsHeavyQuery = (pURL.indexOf('/Max/') > -1) || (pURL.match(/\/Count(\/|$)/) !== null);
 				let tmpPreviousTimeout = libHttps.globalAgent.options.timeout;
-				libHttps.globalAgent.options.timeout = tmpIsMaxRequest
+				libHttps.globalAgent.options.timeout = tmpIsHeavyQuery
 					? tmpFable.MeadowCloneRestClient.maxRequestTimeout
 					: tmpFable.MeadowCloneRestClient.requestTimeout;
 

package/source/services/data-cloner/DataCloner-Command-Sync.js

@@ -64,6 +64,16 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				}
 			}
 
+			// Update UseAdvancedIDPagination on all sync entities
+			if (tmpBody.hasOwnProperty('UseAdvancedIDPagination'))
+			{
+				let tmpEntityNames = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+				for (let i = 0; i < tmpEntityNames.length; i++)
+				{
+					tmpFable.MeadowSync.MeadowSyncEntities[tmpEntityNames[i]].UseAdvancedIDPagination = !!tmpBody.UseAdvancedIDPagination;
+				}
+			}
+
 			// Update DateTimePrecisionMS on MeadowSync and all sync entities
 			if (tmpBody.hasOwnProperty('DateTimePrecisionMS'))
 			{
@@ -173,7 +183,7 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 						let tmpReinitEntities = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
 						tmpFable.log.info(`Data Cloner: Re-created ${tmpReinitEntities.length} sync entities in ${tmpRequestedMode} mode`);
 
-						// Update SyncDeletedRecords and MaxRecordsPerEntity on new entities
+						// Update SyncDeletedRecords, MaxRecordsPerEntity, and UseAdvancedIDPagination on new entities
 						for (let i = 0; i < tmpReinitEntities.length; i++)
 						{
 							tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].SyncDeletedRecords = tmpCloneState.SyncDeletedRecords;
@@ -181,6 +191,10 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 							{
 								tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].MaxRecordsPerEntity = tmpMaxRecords;
 							}
+							if (tmpBody.hasOwnProperty('UseAdvancedIDPagination'))
+							{
+								tmpFable.MeadowSync.MeadowSyncEntities[tmpReinitEntities[i]].UseAdvancedIDPagination = !!tmpBody.UseAdvancedIDPagination;
+							}
 						}
 
 						return fStartSync();
@@ -207,6 +221,13 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				let tmpSnap = { Status: tmpP.Status, Total: tmpP.Total || 0, Synced: tmpP.Synced || 0, Errors: tmpP.Errors || 0 };
 				if (tmpP.ErrorMessage) tmpSnap.ErrorMessage = tmpP.ErrorMessage;
 
+				// Include per-record breakdown fields if available (set after entity sync completes)
+				if (tmpP.hasOwnProperty('New')) tmpSnap.New = tmpP.New;
+				if (tmpP.hasOwnProperty('Updated')) tmpSnap.Updated = tmpP.Updated;
+				if (tmpP.hasOwnProperty('Unchanged')) tmpSnap.Unchanged = tmpP.Unchanged;
+				if (tmpP.hasOwnProperty('Deleted')) tmpSnap.Deleted = tmpP.Deleted;
+				if (tmpP.hasOwnProperty('ServerTotal')) tmpSnap.ServerTotal = tmpP.ServerTotal;
+
 				if ((tmpP.Status === 'Syncing' || tmpP.Status === 'Pending') && tmpFable.MeadowSync && tmpFable.MeadowSync.MeadowSyncEntities)
 				{
 					let tmpSyncEntity = tmpFable.MeadowSync.MeadowSyncEntities[tmpName];
@@ -300,9 +321,15 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				// Check for pre-count phase
 				if (tmpCloneState.SyncPhase === 'counting')
 				{
-					let tmpPC = tmpCloneState.PreCountProgress || { Counted: 0, TotalTables: 0 };
+					let tmpPC = tmpCloneState.PreCountProgress || { Counted: 0, TotalTables: 0, Tables: [] };
 					tmpMessage = `Analyzing tables: counted ${tmpPC.Counted} / ${tmpPC.TotalTables}...`;
 
+					let tmpCountElapsed = null;
+					if (tmpPC.StartTime)
+					{
+						tmpCountElapsed = fFormatDuration(Date.now() - tmpPC.StartTime);
+					}
+
 					pResponse.send(200,
 						{
 							Phase: tmpPhase,
@@ -315,10 +342,10 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 							TotalTables: tmpPC.TotalTables,
 							TotalSynced: 0,
 							TotalRecords: 0,
-							Elapsed: null,
+							Elapsed: tmpCountElapsed,
 							SyncMode: tmpCloneState.SyncMode,
 							ETA: null,
-							PreCountGrandTotal: 0,
+							PreCountGrandTotal: tmpCloneState.PreCountGrandTotal || 0,
 							PreCountProgress: tmpPC,
 							ThroughputSamples: []
 						});

package/source/services/data-cloner/DataCloner-ProviderRegistry.js

@@ -10,7 +10,7 @@ const _ProviderRegistry = {
 	SQLite: { moduleName: 'meadow-connection-sqlite', serviceName: 'MeadowSQLiteProvider', configKey: 'SQLite' },
 	MySQL: { moduleName: 'meadow-connection-mysql', serviceName: 'MeadowMySQLProvider', configKey: 'MySQL' },
 	MSSQL: { moduleName: 'meadow-connection-mssql', serviceName: 'MeadowMSSQLProvider', configKey: 'MSSQL' },
-	PostgreSQL: { moduleName: 'meadow-connection-postgresql', serviceName: 'MeadowConnectionPostgreSQL', configKey: 'PostgreSQL' },
+	PostgreSQL: { moduleName: 'meadow-connection-postgresql', serviceName: 'MeadowPostgreSQLProvider', configKey: 'PostgreSQL' },
 	Solr: { moduleName: 'meadow-connection-solr', serviceName: 'MeadowConnectionSolr', configKey: 'Solr' },
 	MongoDB: { moduleName: 'meadow-connection-mongodb', serviceName: 'MeadowConnectionMongoDB', configKey: 'MongoDB' },
 	RocksDB: { moduleName: 'meadow-connection-rocksdb', serviceName: 'MeadowConnectionRocksDB', configKey: 'RocksDB' },

package/source/services/data-cloner/Retold-Data-Service-DataCloner.js

@@ -642,7 +642,12 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 	preCountTables(pTables, fCallback)
 	{
 		this._cloneState.SyncPhase = 'counting';
-		this._cloneState.PreCountProgress = { Counted: 0, TotalTables: pTables.length };
+		this._cloneState.PreCountProgress = {
+			Counted: 0,
+			TotalTables: pTables.length,
+			StartTime: Date.now(),
+			Tables: []
+		};
 		this._cloneState.PreCountGrandTotal = 0;
 
 		this.fable.log.info(`Data Cloner: Pre-counting records for ${pTables.length} tables...`);
@@ -656,6 +661,7 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 					return fNext();
 				}
 
+				let tmpTableStartTime = Date.now();
 				let tmpCountURL = `${pTableName}s/Count`;
 				this.fable.MeadowCloneRestClient.getJSON(tmpCountURL,
 					(pError, pResponse, pBody) =>
@@ -669,7 +675,14 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 						{
 							this._cloneState.SyncProgress[pTableName].Total = tmpCount;
 						}
+						let tmpElapsedMs = Date.now() - tmpTableStartTime;
 						this._cloneState.PreCountProgress.Counted++;
+						this._cloneState.PreCountProgress.Tables.push({
+							Name: pTableName,
+							Count: tmpCount,
+							ElapsedMs: tmpElapsedMs,
+							Error: pError ? true : false
+						});
 						this._cloneState.PreCountGrandTotal += tmpCount;
 						return fNext();
 					});
@@ -866,15 +879,42 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 				if (this._cloneState.SyncLogFileLogger)
 				{
 					let tmpLogPath = this._cloneState.SyncLogFilePath || '';
-					this._cloneState.SyncLogFileLogger.flushBufferToLogFile(() =>
+					let tmpLogger = this._cloneState.SyncLogFileLogger;
+
+					// Log before closing so the message is captured in the file
+					this.fable.log.info(`Data Cloner: Log file closing — ${tmpLogPath}`);
+
+					// Remove the logger from fable.log stream arrays so no writes
+					// are dispatched to it after the underlying stream is closed.
+					let tmpStreamArrays = [
+						'logStreams', 'logStreamsTrace', 'logStreamsDebug',
+						'logStreamsInfo', 'logStreamsWarn', 'logStreamsError', 'logStreamsFatal'
+					];
+					for (let s = 0; s < tmpStreamArrays.length; s++)
 					{
-						this._cloneState.SyncLogFileLogger.closeWriter(() =>
+						let tmpArr = this.fable.log[tmpStreamArrays[s]];
+						if (Array.isArray(tmpArr))
 						{
-							this.fable.log.info(`Data Cloner: Log file closed — ${tmpLogPath}`);
-						});
-					});
+							let tmpIdx = tmpArr.indexOf(tmpLogger);
+							if (tmpIdx > -1) tmpArr.splice(tmpIdx, 1);
+						}
+					}
+
 					this._cloneState.SyncLogFileLogger = null;
 					this._cloneState.SyncLogFilePath = null;
+
+					// Now flush and close safely — no more writes can reach this stream
+					try
+					{
+						tmpLogger.flushBufferToLogFile(() =>
+						{
+							tmpLogger.closeWriter(() => {});
+						});
+					}
+					catch (pCloseErr)
+					{
+						// Ignore close errors
+					}
 				}
 
 				return;
@@ -915,7 +955,7 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 					{
 						let tmpResults = tmpSyncEntity.syncResults;
 						tmpProgress.New = tmpResults.Created || 0;
-						tmpProgress.Updated = 0;
+						tmpProgress.Updated = tmpResults.Updated || 0;
 						tmpProgress.Unchanged = tmpResults.LocalRecordCount || 0;
 						tmpProgress.Deleted = tmpResults.Deleted || 0;
 						tmpProgress.ServerTotal = tmpResults.ServerRecordCount || 0;

package/source/services/data-cloner/pict-app/Pict-Application-DataCloner.js

@@ -73,7 +73,8 @@ class DataClonerApplication extends libPictApplication
 				'solrHost', 'solrPort', 'solrCore', 'solrPath',
 				'mongodbHost', 'mongodbPort', 'mongodbUser', 'mongodbPassword', 'mongodbDatabase', 'mongodbConnectionLimit',
 				'rocksdbFolder',
-				'bibliographFolder'
+				'bibliographFolder',
+				'syncMaxRecords'
 			]
 		};
 

package/source/services/data-cloner/pict-app/providers/Pict-Provider-DataCloner.js

@@ -295,6 +295,12 @@ class DataClonerProvider extends libPictProvider
 		{
 			document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
 		}
+		// Restore advanced ID pagination checkbox
+		let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
+		if (tmpAdvancedIDPagination !== null)
+		{
+			document.getElementById('syncAdvancedIDPagination').checked = tmpAdvancedIDPagination === 'true';
+		}
 	}
 
 	initPersistence()
@@ -345,6 +351,16 @@ class DataClonerProvider extends libPictProvider
 			});
 		}
 
+		// Persist advanced ID pagination checkbox
+		let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
+		if (tmpAdvancedIDPaginationEl)
+		{
+			tmpAdvancedIDPaginationEl.addEventListener('change', function()
+			{
+				localStorage.setItem('dataCloner_syncAdvancedIDPagination', this.checked);
+			});
+		}
+
 		// Persist auto-process checkboxes
 		let tmpAutoIds = ['auto1', 'auto2', 'auto3', 'auto4', 'auto5'];
 		for (let a = 0; a < tmpAutoIds.length; a++)
@@ -447,7 +463,10 @@ class DataClonerProvider extends libPictProvider
 			}
 			if (pData.PreCountProgress && pData.PreCountProgress.Counted < pData.PreCountProgress.TotalTables)
 			{
-				tmpMetaParts.push('<span class="live-status-meta-item">counting: ' + pData.PreCountProgress.Counted + ' / ' + pData.PreCountProgress.TotalTables + '</span>');
+				let tmpCountedSoFar = pData.PreCountGrandTotal > 0
+					? ' (' + pData.PreCountGrandTotal.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',') + ' records found)'
+					: '';
+				tmpMetaParts.push('<span class="live-status-meta-item">counting: ' + pData.PreCountProgress.Counted + ' / ' + pData.PreCountProgress.TotalTables + ' tables' + tmpCountedSoFar + '</span>');
 			}
 			if (pData.Errors > 0)
 			{
@@ -466,7 +485,12 @@ class DataClonerProvider extends libPictProvider
 
 		// Update progress bar
 		let tmpPct = 0;
-		if (pData.Phase === 'syncing' && pData.PreCountGrandTotal > 0 && pData.TotalSynced > 0)
+		if (pData.Phase === 'syncing' && pData.PreCountProgress && pData.PreCountProgress.Counted < pData.PreCountProgress.TotalTables)
+		{
+			// During counting phase, show table counting progress
+			tmpPct = Math.min((pData.PreCountProgress.Counted / pData.PreCountProgress.TotalTables) * 100, 99);
+		}
+		else if (pData.Phase === 'syncing' && pData.PreCountGrandTotal > 0 && pData.TotalSynced > 0)
 		{
 			tmpPct = Math.min((pData.TotalSynced / pData.PreCountGrandTotal) * 100, 99.9);
 		}
@@ -489,6 +513,16 @@ class DataClonerProvider extends libPictProvider
 		}
 		tmpProgressFill.style.width = Math.min(100, Math.round(tmpPct)) + '%';
 
+		// Auto-expand the detail view when sync starts so users see counting progress
+		if ((pData.Phase === 'syncing' || pData.Phase === 'stopping') && !this.pict.AppData.DataCloner.StatusDetailExpanded)
+		{
+			let tmpLayoutView = this.pict.views['DataCloner-Layout'];
+			if (tmpLayoutView && typeof tmpLayoutView.toggleStatusDetail === 'function')
+			{
+				tmpLayoutView.toggleStatusDetail();
+			}
+		}
+
 		// If the detail view is expanded, re-render it with fresh data
 		if (this.pict.AppData.DataCloner.StatusDetailExpanded)
 		{
@@ -558,6 +592,59 @@ class DataClonerProvider extends libPictProvider
 			.catch(function() { /* ignore poll errors */ });
 	}
 
+	renderCountingPhaseDetail(pContainer, pPreCountProgress)
+	{
+		let tmpTables = pPreCountProgress.Tables || [];
+		let tmpCounted = pPreCountProgress.Counted || 0;
+		let tmpTotal = pPreCountProgress.TotalTables || 0;
+		let tmpRunningTotal = 0;
+
+		let tmpHtml = '<div class="status-detail-section">';
+		tmpHtml += '<div class="status-detail-section-title">Counting Records (' + tmpCounted + ' / ' + tmpTotal + ' tables)</div>';
+
+		if (tmpTables.length > 0)
+		{
+			tmpHtml += '<table class="precount-table">';
+			tmpHtml += '<thead><tr><th>Table</th><th style="text-align:right">Records</th><th style="text-align:right">Time</th></tr></thead>';
+			tmpHtml += '<tbody>';
+			for (let i = 0; i < tmpTables.length; i++)
+			{
+				let tmpT = tmpTables[i];
+				tmpRunningTotal += tmpT.Count;
+				let tmpCountFmt = this.formatNumber(tmpT.Count);
+				let tmpTimeFmt = tmpT.ElapsedMs < 1000
+					? tmpT.ElapsedMs + 'ms'
+					: (tmpT.ElapsedMs / 1000).toFixed(1) + 's';
+				let tmpRowClass = tmpT.Error ? ' class="precount-error"' : '';
+				tmpHtml += '<tr' + tmpRowClass + '>';
+				tmpHtml += '<td>' + this.escapeHtml(tmpT.Name) + '</td>';
+				tmpHtml += '<td style="text-align:right; font-variant-numeric:tabular-nums">' + tmpCountFmt + '</td>';
+				tmpHtml += '<td style="text-align:right; font-variant-numeric:tabular-nums; color:#888">' + tmpTimeFmt + '</td>';
+				tmpHtml += '</tr>';
+			}
+			tmpHtml += '</tbody>';
+			tmpHtml += '<tfoot><tr>';
+			tmpHtml += '<td><strong>Total</strong></td>';
+			tmpHtml += '<td style="text-align:right; font-variant-numeric:tabular-nums"><strong>' + this.formatNumber(tmpRunningTotal) + '</strong></td>';
+			tmpHtml += '<td></td>';
+			tmpHtml += '</tr></tfoot>';
+			tmpHtml += '</table>';
+		}
+
+		// Show pending indicator for remaining tables
+		let tmpRemaining = tmpTotal - tmpCounted;
+		if (tmpRemaining > 0)
+		{
+			tmpHtml += '<div class="precount-pending">';
+			tmpHtml += '<span class="precount-spinner"></span> ';
+			tmpHtml += tmpRemaining + ' table' + (tmpRemaining === 1 ? '' : 's') + ' remaining…';
+			tmpHtml += '</div>';
+		}
+
+		tmpHtml += '</div>';
+		pContainer.innerHTML = tmpHtml;
+	}
+
 	renderStatusDetail()
 	{
 		let tmpContainer = document.getElementById('DataCloner-StatusDetail-Container');
@@ -568,6 +655,19 @@ class DataClonerProvider extends libPictProvider
 		let tmpStatusData = tmpAppData.StatusDetailData;
 		let tmpReport = tmpAppData.LastReport;
 
+		// During the counting phase, show per-table counts as they arrive
+		if (tmpLiveStatus && tmpLiveStatus.PreCountProgress
+			&& tmpLiveStatus.PreCountProgress.Tables
+			&& tmpLiveStatus.Phase === 'syncing'
+			&& tmpLiveStatus.PreCountProgress.Counted < tmpLiveStatus.PreCountProgress.TotalTables)
+		{
+			this.renderCountingPhaseDetail(tmpContainer, tmpLiveStatus.PreCountProgress);
+			// Hide histogram during counting
+			let tmpHistContainer = document.getElementById('DataCloner-Throughput-Histogram');
+			if (tmpHistContainer) tmpHistContainer.style.display = 'none';
+			return;
+		}
+
 		// Determine data source: live during sync, report after sync
 		let tmpTables = {};
 		let tmpThroughputSamples = [];

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Export.js

@@ -101,6 +101,7 @@ class DataClonerExportView extends libPictView
 		if (!isNaN(tmpPrecision) && tmpPrecision !== 1000) tmpConfig.Sync.DateTimePrecisionMS = tmpPrecision;
 		let tmpMaxRecords = parseInt(document.getElementById('syncMaxRecords').value, 10);
 		if (tmpMaxRecords > 0) tmpConfig.Sync.MaxRecords = tmpMaxRecords;
+		if (document.getElementById('syncAdvancedIDPagination').checked) tmpConfig.Sync.UseAdvancedIDPagination = true;
 
 		return tmpConfig;
 	}
@@ -169,6 +170,7 @@ class DataClonerExportView extends libPictView
 		let tmpSelectedTables = this.pict.views['DataCloner-Schema'].getSelectedTables();
 		tmpConfig.Sync.SyncEntityList = tmpSelectedTables.length > 0 ? tmpSelectedTables : [];
 		tmpConfig.Sync.SyncEntityOptions = {};
+		if (document.getElementById('syncAdvancedIDPagination').checked) tmpConfig.Sync.UseAdvancedIDPagination = true;
 
 		// ---- SessionManager ----
 		tmpConfig.SessionManager = { Sessions: {} };

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Layout.js

@@ -339,6 +339,34 @@ select { background: #fff; width: 100%; padding: 8px 12px; border: 1px solid #cc
 	font-size: 0.78em; color: #888; margin-top: 4px; padding-left: 18px;
 	font-family: monospace; max-height: 80px; overflow-y: auto;
 }
+
+/* Pre-count Table */
+.precount-table {
+	width: 100%; border-collapse: collapse; font-size: 0.88em;
+}
+.precount-table thead th {
+	text-align: left; font-weight: 600; color: #555; font-size: 0.85em;
+	text-transform: uppercase; letter-spacing: 0.3px;
+	padding: 4px 8px 6px; border-bottom: 2px solid #e9ecef;
+}
+.precount-table tbody td {
+	padding: 4px 8px; border-bottom: 1px solid #f0f0f0;
+}
+.precount-table tbody tr:last-child td { border-bottom: 1px solid #e9ecef; }
+.precount-table tfoot td {
+	padding: 6px 8px 2px; font-size: 0.95em;
+}
+.precount-error td { color: #dc3545; }
+.precount-pending {
+	color: #888; font-size: 0.85em; font-style: italic; padding: 8px 0 2px;
+	display: flex; align-items: center; gap: 6px;
+}
+.precount-spinner {
+	display: inline-block; width: 12px; height: 12px;
+	border: 2px solid #ddd; border-top-color: #4a90d9;
+	border-radius: 50%; animation: precount-spin 0.8s linear infinite;
+}
+@keyframes precount-spin { to { transform: rotate(360deg); } }
 `,
 	Templates:
 	[

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Sync.js

@@ -17,6 +17,7 @@ class DataClonerSyncView extends libPictView
 		let tmpSyncMode = document.querySelector('input[name="syncMode"]:checked').value;
 		let tmpMaxRecords = parseInt(document.getElementById('syncMaxRecords').value, 10) || 0;
 		let tmpLogToFile = document.getElementById('syncLogFile').checked;
+		let tmpAdvancedIDPagination = document.getElementById('syncAdvancedIDPagination').checked;
 
 		if (tmpSelectedTables.length === 0)
 		{
@@ -32,6 +33,7 @@ class DataClonerSyncView extends libPictView
 		let tmpPostBody = { Tables: tmpSelectedTables, PageSize: tmpPageSize, DateTimePrecisionMS: tmpDateTimePrecisionMS, SyncDeletedRecords: tmpSyncDeletedRecords, SyncMode: tmpSyncMode };
 		if (tmpMaxRecords > 0) tmpPostBody.MaxRecordsPerEntity = tmpMaxRecords;
 		if (tmpLogToFile) tmpPostBody.LogToFile = true;
+		if (tmpAdvancedIDPagination) tmpPostBody.UseAdvancedIDPagination = true;
 		this.pict.providers.DataCloner.api('POST', '/clone/sync/start', tmpPostBody)
 			.then(function(pData)
 			{
@@ -522,6 +524,11 @@ module.exports.default_configuration =
 				<label for="syncDeletedRecords">Sync deleted records (fetch records marked Deleted=1 on source and mirror locally)</label>
 			</div>
 
+			<div class="checkbox-row">
+				<input type="checkbox" id="syncAdvancedIDPagination">
+				<label for="syncAdvancedIDPagination">Use advanced ID pagination (faster for large tables; uses keyset pagination instead of OFFSET)</label>
+			</div>
+
 			<div class="inline-group" style="margin-top:8px; margin-bottom:4px">
 				<div style="flex:0 0 200px">
 					<label for="syncMaxRecords">Max Records per Entity</label>