retold-data-service 2.0.37 → 2.0.39

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.37",
+    "version": "2.0.39",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
@@ -73,8 +73,8 @@
         "meadow": "^2.0.33",
         "meadow-connection-mysql": "^1.0.14",
         "meadow-endpoints": "^4.0.15",
-        "meadow-integration": "^1.0.30",
-        "meadow-migrationmanager": "^0.0.9",
+        "meadow-integration": "^1.0.32",
+        "meadow-migrationmanager": "^0.0.10",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",
         "orator-serviceserver-restify": "^2.0.10",

package/source/services/data-cloner/Retold-Data-Service-DataCloner.js

@@ -1018,12 +1018,19 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 						let tmpTracker = tmpSyncEntity.operation.progressTrackers[`FullSync-${tmpTableName}`];
 						if (tmpTracker)
 						{
-							tmpProgress.Total = tmpTracker.TotalCount || 0;
+							// Clamp both to >= 0: Total can be estimated from
+							// Server.RecordCount - Local.RecordCount, which is
+							// negative when local has extra records (e.g. after
+							// server-side deletes).  A negative total in the
+							// progress report is nonsensical.
+							tmpProgress.Total = Math.max(tmpTracker.TotalCount || 0, 0);
 							tmpProgress.Synced = Math.max(tmpTracker.CurrentCount || 0, 0);
 						}
 					}
 
 					// Read per-record breakdown from the sync entity
+					let tmpEntityErrors = 0;
+					let tmpEntitySkipped = 0;
 					if (tmpSyncEntity && tmpSyncEntity.syncResults)
 					{
 						let tmpResults = tmpSyncEntity.syncResults;
@@ -1033,12 +1040,12 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 						tmpProgress.Deleted = tmpResults.Deleted || 0;
 						tmpProgress.ServerTotal = tmpResults.ServerRecordCount || 0;
 						tmpProgress.LocalCountBefore = tmpResults.LocalRecordCount || 0;
+						tmpEntityErrors = tmpResults.Errors || 0;
+						tmpEntitySkipped = tmpResults.Skipped || 0;
 					}
 
 					let tmpRESTErrors = this._cloneState.SyncRESTErrors[tmpTableName] || 0;
-					tmpProgress.Errors = tmpRESTErrors;
-
-					let tmpMissing = tmpProgress.Total - tmpProgress.Synced;
+					tmpProgress.Errors = tmpRESTErrors + tmpEntityErrors;
 
 					if (pError)
 					{
@@ -1048,25 +1055,35 @@ class RetoldDataServiceDataCloner extends libFableServiceProviderBase
 						this.logSyncEvent('TableError', `Sync [${tmpTableName}] — error: ${pError}`,
 							{ Table: tmpTableName, Total: tmpProgress.Total, Synced: tmpProgress.Synced, Error: `${pError}` });
 					}
-					else if (tmpRESTErrors > 0)
-					{
-						tmpProgress.Status = 'Error';
-						tmpProgress.ErrorMessage = `${tmpRESTErrors} REST error(s) during sync`;
-						this.fable.log.warn(`Data Cloner: Sync [${tmpTableName}] — completed with ${tmpRESTErrors} REST error(s). ${tmpProgress.Synced}/${tmpProgress.Total} records synced.`);
-						this.logSyncEvent('TableError', `Sync [${tmpTableName}] — ${tmpRESTErrors} REST error(s).`,
-							{ Table: tmpTableName, Total: tmpProgress.Total, Synced: tmpProgress.Synced, RESTErrors: tmpRESTErrors });
-					}
-					else if (tmpProgress.Total > 0 && tmpMissing > 0)
+					else if (tmpRESTErrors > 0 || tmpEntityErrors > 0)
 					{
-						tmpProgress.Status = 'Partial';
-						tmpProgress.Skipped = tmpMissing;
-						this.fable.log.warn(`Data Cloner: Sync [${tmpTableName}] — partial. ${tmpProgress.Synced}/${tmpProgress.Total} records synced, ${tmpMissing} skipped (GUID conflicts or other errors).`);
-						this.logSyncEvent('TablePartial', `Sync [${tmpTableName}] — partial. ${tmpMissing} skipped.`,
-							{ Table: tmpTableName, Total: tmpProgress.Total, Synced: tmpProgress.Synced, Skipped: tmpMissing });
+						// Only treat the sync as Partial/Error when the sync
+						// entity actually reported errors (REST fetch errors or
+						// per-record commit errors).  A count-mismatch between
+						// estimated Total and actual Synced is not itself an
+						// error — pre-count estimates can be stale (records
+						// were updated rather than added, local already has
+						// records past the server's max ID, etc.), and the
+						// sync code correctly downloads nothing in those
+						// cases.  Reporting "N skipped" for pure estimate
+						// drift is noisy and misleading.
+						let tmpPartialMessage = (tmpRESTErrors > 0 && tmpEntityErrors > 0)
+							? `${tmpRESTErrors} REST error(s) and ${tmpEntityErrors} record error(s) during sync`
+							: (tmpRESTErrors > 0)
+								? `${tmpRESTErrors} REST error(s) during sync`
+								: `${tmpEntityErrors} record error(s) during sync`;
+						tmpProgress.Status = (tmpRESTErrors > 0) ? 'Error' : 'Partial';
+						tmpProgress.ErrorMessage = tmpPartialMessage;
+						tmpProgress.Skipped = tmpEntitySkipped;
+						this.fable.log.warn(`Data Cloner: Sync [${tmpTableName}] — ${tmpPartialMessage}. ${tmpProgress.Synced}/${tmpProgress.Total} records synced.`);
+						this.logSyncEvent(tmpProgress.Status === 'Error' ? 'TableError' : 'TablePartial',
+							`Sync [${tmpTableName}] — ${tmpPartialMessage}.`,
+							{ Table: tmpTableName, Total: tmpProgress.Total, Synced: tmpProgress.Synced, RESTErrors: tmpRESTErrors, RecordErrors: tmpEntityErrors });
 					}
 					else
 					{
 						tmpProgress.Status = 'Complete';
+						tmpProgress.Skipped = tmpEntitySkipped;
 						this.fable.log.info(`Data Cloner: Sync [${tmpTableName}] — complete. ${tmpProgress.Synced}/${tmpProgress.Total} records synced.`);
 						this.logSyncEvent('TableComplete', `Sync [${tmpTableName}] — complete. ${tmpProgress.Synced}/${tmpProgress.Total} records.`,
 							{ Table: tmpTableName, Total: tmpProgress.Total, Synced: tmpProgress.Synced });

package/source/services/data-cloner/pict-app/providers/Pict-Provider-DataCloner.js

@@ -295,6 +295,12 @@ class DataClonerProvider extends libPictProvider
 		{
 			document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
 		}
+		let tmpMssqlLegacyPagination = localStorage.getItem('dataCloner_mssqlLegacyPagination');
+		if (tmpMssqlLegacyPagination !== null)
+		{
+			let tmpEl = document.getElementById('mssqlLegacyPagination');
+			if (tmpEl) tmpEl.checked = tmpMssqlLegacyPagination === 'true';
+		}
 		// Restore advanced ID pagination checkbox
 		let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
 		if (tmpAdvancedIDPagination !== null)
@@ -351,6 +357,16 @@ class DataClonerProvider extends libPictProvider
 			});
 		}
 
+		// Persist MSSQL legacy pagination checkbox
+		let tmpMssqlLegacyPaginationEl = document.getElementById('mssqlLegacyPagination');
+		if (tmpMssqlLegacyPaginationEl)
+		{
+			tmpMssqlLegacyPaginationEl.addEventListener('change', function()
+			{
+				localStorage.setItem('dataCloner_mssqlLegacyPagination', this.checked);
+			});
+		}
+
 		// Persist advanced ID pagination checkbox
 		let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
 		if (tmpAdvancedIDPaginationEl)

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Connection.js

@@ -48,6 +48,11 @@ class DataClonerConnectionView extends libPictView
 			tmpConfig.password = document.getElementById('mssqlPassword').value;
 			tmpConfig.database = document.getElementById('mssqlDatabase').value.trim();
 			tmpConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+			// Use ROW_NUMBER() pagination instead of OFFSET/FETCH for
+			// SQL Server 2008 R2 / 2012 or databases whose compatibility
+			// level is < 110 (the parser rejects OFFSET/FETCH syntax
+			// otherwise).
+			tmpConfig.LegacyPagination = document.getElementById('mssqlLegacyPagination').checked;
 		}
 		else if (tmpProvider === 'PostgreSQL')
 		{
@@ -291,6 +296,10 @@ module.exports.default_configuration =
 					</div>
 					<div></div>
 				</div>
+				<div style="margin-top:8px">
+					<input type="checkbox" id="mssqlLegacyPagination">
+					<label for="mssqlLegacyPagination" title="Enable for SQL Server 2008 R2 / 2012 or databases at compatibility_level &lt; 110. Uses ROW_NUMBER() pagination instead of OFFSET/FETCH.">Legacy pagination (SQL Server &lt; 2012 / compat level &lt; 110)</label>
+				</div>
 			</div>
 
 			<!-- PostgreSQL Config -->

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Export.js

@@ -37,6 +37,10 @@ class DataClonerExportView extends libPictView
 			tmpDbConfig.password = document.getElementById('mssqlPassword').value;
 			tmpDbConfig.database = document.getElementById('mssqlDatabase').value.trim();
 			tmpDbConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+			if (document.getElementById('mssqlLegacyPagination').checked)
+			{
+				tmpDbConfig.LegacyPagination = true;
+			}
 		}
 		else if (tmpProvider === 'PostgreSQL')
 		{
@@ -156,6 +160,10 @@ class DataClonerExportView extends libPictView
 			tmpConfig.Destination.MSSQL.password = document.getElementById('mssqlPassword').value || '';
 			tmpConfig.Destination.MSSQL.database = document.getElementById('mssqlDatabase').value.trim() || 'meadow';
 			tmpConfig.Destination.MSSQL.ConnectionPoolLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+			if (document.getElementById('mssqlLegacyPagination').checked)
+			{
+				tmpConfig.Destination.MSSQL.LegacyPagination = true;
+			}
 		}
 		else
 		{

package/source/services/data-cloner/web/data-cloner.js

@@ -5072,6 +5072,11 @@
           if (tmpSolrSecure !== null) {
             document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
           }
+          let tmpMssqlLegacyPagination = localStorage.getItem('dataCloner_mssqlLegacyPagination');
+          if (tmpMssqlLegacyPagination !== null) {
+            let tmpEl = document.getElementById('mssqlLegacyPagination');
+            if (tmpEl) tmpEl.checked = tmpMssqlLegacyPagination === 'true';
+          }
           // Restore advanced ID pagination checkbox
           let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
           if (tmpAdvancedIDPagination !== null) {
@@ -5119,6 +5124,14 @@
             });
           }
 
+          // Persist MSSQL legacy pagination checkbox
+          let tmpMssqlLegacyPaginationEl = document.getElementById('mssqlLegacyPagination');
+          if (tmpMssqlLegacyPaginationEl) {
+            tmpMssqlLegacyPaginationEl.addEventListener('change', function () {
+              localStorage.setItem('dataCloner_mssqlLegacyPagination', this.checked);
+            });
+          }
+
           // Persist advanced ID pagination checkbox
           let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
           if (tmpAdvancedIDPaginationEl) {
@@ -5851,6 +5864,11 @@
             tmpConfig.password = document.getElementById('mssqlPassword').value;
             tmpConfig.database = document.getElementById('mssqlDatabase').value.trim();
             tmpConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+            // Use ROW_NUMBER() pagination instead of OFFSET/FETCH for
+            // SQL Server 2008 R2 / 2012 or databases whose compatibility
+            // level is < 110 (the parser rejects OFFSET/FETCH syntax
+            // otherwise).
+            tmpConfig.LegacyPagination = document.getElementById('mssqlLegacyPagination').checked;
           } else if (tmpProvider === 'PostgreSQL') {
             tmpConfig.host = document.getElementById('postgresqlHost').value.trim() || '127.0.0.1';
             tmpConfig.port = parseInt(document.getElementById('postgresqlPort').value, 10) || 5432;
@@ -6044,6 +6062,10 @@
 					</div>
 					<div></div>
 				</div>
+				<div style="margin-top:8px">
+					<input type="checkbox" id="mssqlLegacyPagination">
+					<label for="mssqlLegacyPagination" title="Enable for SQL Server 2008 R2 / 2012 or databases at compatibility_level &lt; 110. Uses ROW_NUMBER() pagination instead of OFFSET/FETCH.">Legacy pagination (SQL Server &lt; 2012 / compat level &lt; 110)</label>
+				</div>
 			</div>
 
 			<!-- PostgreSQL Config -->
@@ -6362,6 +6384,9 @@
             tmpDbConfig.password = document.getElementById('mssqlPassword').value;
             tmpDbConfig.database = document.getElementById('mssqlDatabase').value.trim();
             tmpDbConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+            if (document.getElementById('mssqlLegacyPagination').checked) {
+              tmpDbConfig.LegacyPagination = true;
+            }
           } else if (tmpProvider === 'PostgreSQL') {
             tmpDbConfig.host = document.getElementById('postgresqlHost').value.trim() || '127.0.0.1';
             tmpDbConfig.port = parseInt(document.getElementById('postgresqlPort').value, 10) || 5432;
@@ -6465,6 +6490,9 @@
             tmpConfig.Destination.MSSQL.password = document.getElementById('mssqlPassword').value || '';
             tmpConfig.Destination.MSSQL.database = document.getElementById('mssqlDatabase').value.trim() || 'meadow';
             tmpConfig.Destination.MSSQL.ConnectionPoolLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+            if (document.getElementById('mssqlLegacyPagination').checked) {
+              tmpConfig.Destination.MSSQL.LegacyPagination = true;
+            }
           } else {
             // Default to MySQL placeholder for unsupported providers
             tmpConfig.Destination.Provider = 'MySQL';