retold-data-service 2.0.38 → 2.0.42

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.38",
+    "version": "2.0.42",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
@@ -70,11 +70,11 @@
         "bibliograph": "^0.1.4",
         "fable": "^3.1.70",
         "fable-serviceproviderbase": "^3.0.19",
-        "meadow": "^2.0.33",
+        "meadow": "^2.0.35",
         "meadow-connection-mysql": "^1.0.14",
         "meadow-endpoints": "^4.0.15",
-        "meadow-integration": "^1.0.30",
-        "meadow-migrationmanager": "^0.0.9",
+        "meadow-integration": "^1.0.34",
+        "meadow-migrationmanager": "^0.0.12",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",
         "orator-serviceserver-restify": "^2.0.10",

package/source/services/data-cloner/pict-app/Pict-Application-DataCloner.js

@@ -70,6 +70,9 @@ class DataClonerApplication extends libPictApplication
 				'connProvider', 'sqliteFilePath',
 				'mysqlServer', 'mysqlPort', 'mysqlUser', 'mysqlPassword', 'mysqlDatabase', 'mysqlConnectionLimit',
 				'mssqlServer', 'mssqlPort', 'mssqlUser', 'mssqlPassword', 'mssqlDatabase', 'mssqlConnectionLimit',
+				'mssqlRequestTimeoutSec', 'mssqlConnectionTimeoutSec',
+				'mssqlConnectMaxAttempts', 'mssqlDDLMaxAttempts',
+				'mssqlRetryInitialDelaySec', 'mssqlRetryMaxDelaySec',
 				'postgresqlHost', 'postgresqlPort', 'postgresqlUser', 'postgresqlPassword', 'postgresqlDatabase', 'postgresqlConnectionLimit',
 				'solrHost', 'solrPort', 'solrCore', 'solrPath',
 				'mongodbHost', 'mongodbPort', 'mongodbUser', 'mongodbPassword', 'mongodbDatabase', 'mongodbConnectionLimit',

package/source/services/data-cloner/pict-app/providers/Pict-Provider-DataCloner.js

@@ -295,6 +295,12 @@ class DataClonerProvider extends libPictProvider
 		{
 			document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
 		}
+		let tmpMssqlLegacyPagination = localStorage.getItem('dataCloner_mssqlLegacyPagination');
+		if (tmpMssqlLegacyPagination !== null)
+		{
+			let tmpEl = document.getElementById('mssqlLegacyPagination');
+			if (tmpEl) tmpEl.checked = tmpMssqlLegacyPagination === 'true';
+		}
 		// Restore advanced ID pagination checkbox
 		let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
 		if (tmpAdvancedIDPagination !== null)
@@ -351,6 +357,16 @@ class DataClonerProvider extends libPictProvider
 			});
 		}
 
+		// Persist MSSQL legacy pagination checkbox
+		let tmpMssqlLegacyPaginationEl = document.getElementById('mssqlLegacyPagination');
+		if (tmpMssqlLegacyPaginationEl)
+		{
+			tmpMssqlLegacyPaginationEl.addEventListener('change', function()
+			{
+				localStorage.setItem('dataCloner_mssqlLegacyPagination', this.checked);
+			});
+		}
+
 		// Persist advanced ID pagination checkbox
 		let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
 		if (tmpAdvancedIDPaginationEl)

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Connection.js

@@ -48,6 +48,39 @@ class DataClonerConnectionView extends libPictView
 			tmpConfig.password = document.getElementById('mssqlPassword').value;
 			tmpConfig.database = document.getElementById('mssqlDatabase').value.trim();
 			tmpConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+			// Use ROW_NUMBER() pagination instead of OFFSET/FETCH for
+			// SQL Server 2008 R2 / 2012 or databases whose compatibility
+			// level is < 110 (the parser rejects OFFSET/FETCH syntax
+			// otherwise).
+			tmpConfig.LegacyPagination = document.getElementById('mssqlLegacyPagination').checked;
+
+			// Reliability tuning — pull from the advanced settings block.
+			// All are optional; the connection provider falls back to
+			// sensible defaults when a value is missing or zero.
+			let tmpReqSec = parseInt(document.getElementById('mssqlRequestTimeoutSec').value, 10);
+			if (tmpReqSec > 0) tmpConfig.RequestTimeoutMs = tmpReqSec * 1000;
+			let tmpConnSec = parseInt(document.getElementById('mssqlConnectionTimeoutSec').value, 10);
+			if (tmpConnSec > 0) tmpConfig.ConnectionTimeoutMs = tmpConnSec * 1000;
+
+			let tmpConnectAttempts = parseInt(document.getElementById('mssqlConnectMaxAttempts').value, 10);
+			let tmpDDLAttempts = parseInt(document.getElementById('mssqlDDLMaxAttempts').value, 10);
+			let tmpInitialDelaySec = parseInt(document.getElementById('mssqlRetryInitialDelaySec').value, 10);
+			let tmpMaxDelaySec = parseInt(document.getElementById('mssqlRetryMaxDelaySec').value, 10);
+
+			if (tmpConnectAttempts > 0 || tmpInitialDelaySec > 0 || tmpMaxDelaySec > 0)
+			{
+				tmpConfig.ConnectRetryOptions = {};
+				if (tmpConnectAttempts > 0) tmpConfig.ConnectRetryOptions.MaxAttempts = tmpConnectAttempts;
+				if (tmpInitialDelaySec > 0) tmpConfig.ConnectRetryOptions.InitialDelayMs = tmpInitialDelaySec * 1000;
+				if (tmpMaxDelaySec > 0) tmpConfig.ConnectRetryOptions.MaxDelayMs = tmpMaxDelaySec * 1000;
+			}
+			if (tmpDDLAttempts > 0 || tmpInitialDelaySec > 0 || tmpMaxDelaySec > 0)
+			{
+				tmpConfig.DDLRetryOptions = {};
+				if (tmpDDLAttempts > 0) tmpConfig.DDLRetryOptions.MaxAttempts = tmpDDLAttempts;
+				if (tmpInitialDelaySec > 0) tmpConfig.DDLRetryOptions.InitialDelayMs = tmpInitialDelaySec * 1000;
+				if (tmpMaxDelaySec > 0) tmpConfig.DDLRetryOptions.MaxDelayMs = tmpMaxDelaySec * 1000;
+			}
 		}
 		else if (tmpProvider === 'PostgreSQL')
 		{
@@ -291,6 +324,45 @@ module.exports.default_configuration =
 					</div>
 					<div></div>
 				</div>
+				<div style="margin-top:8px">
+					<input type="checkbox" id="mssqlLegacyPagination">
+					<label for="mssqlLegacyPagination" title="Enable for SQL Server 2008 R2 / 2012 or databases at compatibility_level &lt; 110. Uses ROW_NUMBER() pagination instead of OFFSET/FETCH.">Legacy pagination (SQL Server &lt; 2012 / compat level &lt; 110)</label>
+				</div>
+
+				<details style="margin-top:8px">
+					<summary style="cursor:pointer; font-weight:600">Reliability &amp; timeouts (advanced)</summary>
+					<p style="margin:8px 0; font-size:0.9em; color:#555">Tune these for slow / flaky customer networks. Connection and DDL operations will retry with exponential backoff, classifying each failure (NetworkError / RequestTimeout / PoolDegraded / ServerError) in the logs so the failure mode is obvious.</p>
+					<div class="inline-group">
+						<div>
+							<label for="mssqlRequestTimeoutSec">Request timeout (sec)</label>
+							<input type="number" id="mssqlRequestTimeoutSec" placeholder="120" value="120">
+						</div>
+						<div>
+							<label for="mssqlConnectionTimeoutSec">Connection timeout (sec)</label>
+							<input type="number" id="mssqlConnectionTimeoutSec" placeholder="60" value="60">
+						</div>
+					</div>
+					<div class="inline-group">
+						<div>
+							<label for="mssqlConnectMaxAttempts">Connect retries (max attempts)</label>
+							<input type="number" id="mssqlConnectMaxAttempts" placeholder="5" value="5" min="1" max="20">
+						</div>
+						<div>
+							<label for="mssqlDDLMaxAttempts">DDL retries (max attempts)</label>
+							<input type="number" id="mssqlDDLMaxAttempts" placeholder="5" value="5" min="1" max="20">
+						</div>
+					</div>
+					<div class="inline-group">
+						<div>
+							<label for="mssqlRetryInitialDelaySec">Retry initial delay (sec)</label>
+							<input type="number" id="mssqlRetryInitialDelaySec" placeholder="3" value="3" min="1" max="60">
+						</div>
+						<div>
+							<label for="mssqlRetryMaxDelaySec">Retry max delay (sec)</label>
+							<input type="number" id="mssqlRetryMaxDelaySec" placeholder="30" value="30" min="1" max="600">
+						</div>
+					</div>
+				</details>
 			</div>
 
 			<!-- PostgreSQL Config -->

package/source/services/data-cloner/pict-app/views/PictView-DataCloner-Export.js

@@ -37,6 +37,10 @@ class DataClonerExportView extends libPictView
 			tmpDbConfig.password = document.getElementById('mssqlPassword').value;
 			tmpDbConfig.database = document.getElementById('mssqlDatabase').value.trim();
 			tmpDbConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+			if (document.getElementById('mssqlLegacyPagination').checked)
+			{
+				tmpDbConfig.LegacyPagination = true;
+			}
 		}
 		else if (tmpProvider === 'PostgreSQL')
 		{
@@ -156,6 +160,10 @@ class DataClonerExportView extends libPictView
 			tmpConfig.Destination.MSSQL.password = document.getElementById('mssqlPassword').value || '';
 			tmpConfig.Destination.MSSQL.database = document.getElementById('mssqlDatabase').value.trim() || 'meadow';
 			tmpConfig.Destination.MSSQL.ConnectionPoolLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+			if (document.getElementById('mssqlLegacyPagination').checked)
+			{
+				tmpConfig.Destination.MSSQL.LegacyPagination = true;
+			}
 		}
 		else
 		{

package/source/services/data-cloner/web/data-cloner.js

@@ -4803,7 +4803,7 @@
             StatusDetailTimer: null,
             StatusDetailData: null,
             LastLiveStatus: null,
-            PersistFields: ['serverURL', 'authMethod', 'authURI', 'checkURI', 'cookieName', 'cookieValueAddr', 'cookieValueTemplate', 'loginMarker', 'userName', 'password', 'schemaURL', 'pageSize', 'dateTimePrecisionMS', 'connProvider', 'sqliteFilePath', 'mysqlServer', 'mysqlPort', 'mysqlUser', 'mysqlPassword', 'mysqlDatabase', 'mysqlConnectionLimit', 'mssqlServer', 'mssqlPort', 'mssqlUser', 'mssqlPassword', 'mssqlDatabase', 'mssqlConnectionLimit', 'postgresqlHost', 'postgresqlPort', 'postgresqlUser', 'postgresqlPassword', 'postgresqlDatabase', 'postgresqlConnectionLimit', 'solrHost', 'solrPort', 'solrCore', 'solrPath', 'mongodbHost', 'mongodbPort', 'mongodbUser', 'mongodbPassword', 'mongodbDatabase', 'mongodbConnectionLimit', 'rocksdbFolder', 'bibliographFolder', 'syncMaxRecords']
+            PersistFields: ['serverURL', 'authMethod', 'authURI', 'checkURI', 'cookieName', 'cookieValueAddr', 'cookieValueTemplate', 'loginMarker', 'userName', 'password', 'schemaURL', 'pageSize', 'dateTimePrecisionMS', 'connProvider', 'sqliteFilePath', 'mysqlServer', 'mysqlPort', 'mysqlUser', 'mysqlPassword', 'mysqlDatabase', 'mysqlConnectionLimit', 'mssqlServer', 'mssqlPort', 'mssqlUser', 'mssqlPassword', 'mssqlDatabase', 'mssqlConnectionLimit', 'mssqlRequestTimeoutSec', 'mssqlConnectionTimeoutSec', 'mssqlConnectMaxAttempts', 'mssqlDDLMaxAttempts', 'mssqlRetryInitialDelaySec', 'mssqlRetryMaxDelaySec', 'postgresqlHost', 'postgresqlPort', 'postgresqlUser', 'postgresqlPassword', 'postgresqlDatabase', 'postgresqlConnectionLimit', 'solrHost', 'solrPort', 'solrCore', 'solrPath', 'mongodbHost', 'mongodbPort', 'mongodbUser', 'mongodbPassword', 'mongodbDatabase', 'mongodbConnectionLimit', 'rocksdbFolder', 'bibliographFolder', 'syncMaxRecords']
           };
 
           // Make pict available for inline onclick handlers
@@ -5072,6 +5072,11 @@
           if (tmpSolrSecure !== null) {
             document.getElementById('solrSecure').checked = tmpSolrSecure === 'true';
           }
+          let tmpMssqlLegacyPagination = localStorage.getItem('dataCloner_mssqlLegacyPagination');
+          if (tmpMssqlLegacyPagination !== null) {
+            let tmpEl = document.getElementById('mssqlLegacyPagination');
+            if (tmpEl) tmpEl.checked = tmpMssqlLegacyPagination === 'true';
+          }
           // Restore advanced ID pagination checkbox
           let tmpAdvancedIDPagination = localStorage.getItem('dataCloner_syncAdvancedIDPagination');
           if (tmpAdvancedIDPagination !== null) {
@@ -5119,6 +5124,14 @@
             });
           }
 
+          // Persist MSSQL legacy pagination checkbox
+          let tmpMssqlLegacyPaginationEl = document.getElementById('mssqlLegacyPagination');
+          if (tmpMssqlLegacyPaginationEl) {
+            tmpMssqlLegacyPaginationEl.addEventListener('change', function () {
+              localStorage.setItem('dataCloner_mssqlLegacyPagination', this.checked);
+            });
+          }
+
           // Persist advanced ID pagination checkbox
           let tmpAdvancedIDPaginationEl = document.getElementById('syncAdvancedIDPagination');
           if (tmpAdvancedIDPaginationEl) {
@@ -5851,6 +5864,35 @@
             tmpConfig.password = document.getElementById('mssqlPassword').value;
             tmpConfig.database = document.getElementById('mssqlDatabase').value.trim();
             tmpConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+            // Use ROW_NUMBER() pagination instead of OFFSET/FETCH for
+            // SQL Server 2008 R2 / 2012 or databases whose compatibility
+            // level is < 110 (the parser rejects OFFSET/FETCH syntax
+            // otherwise).
+            tmpConfig.LegacyPagination = document.getElementById('mssqlLegacyPagination').checked;
+
+            // Reliability tuning — pull from the advanced settings block.
+            // All are optional; the connection provider falls back to
+            // sensible defaults when a value is missing or zero.
+            let tmpReqSec = parseInt(document.getElementById('mssqlRequestTimeoutSec').value, 10);
+            if (tmpReqSec > 0) tmpConfig.RequestTimeoutMs = tmpReqSec * 1000;
+            let tmpConnSec = parseInt(document.getElementById('mssqlConnectionTimeoutSec').value, 10);
+            if (tmpConnSec > 0) tmpConfig.ConnectionTimeoutMs = tmpConnSec * 1000;
+            let tmpConnectAttempts = parseInt(document.getElementById('mssqlConnectMaxAttempts').value, 10);
+            let tmpDDLAttempts = parseInt(document.getElementById('mssqlDDLMaxAttempts').value, 10);
+            let tmpInitialDelaySec = parseInt(document.getElementById('mssqlRetryInitialDelaySec').value, 10);
+            let tmpMaxDelaySec = parseInt(document.getElementById('mssqlRetryMaxDelaySec').value, 10);
+            if (tmpConnectAttempts > 0 || tmpInitialDelaySec > 0 || tmpMaxDelaySec > 0) {
+              tmpConfig.ConnectRetryOptions = {};
+              if (tmpConnectAttempts > 0) tmpConfig.ConnectRetryOptions.MaxAttempts = tmpConnectAttempts;
+              if (tmpInitialDelaySec > 0) tmpConfig.ConnectRetryOptions.InitialDelayMs = tmpInitialDelaySec * 1000;
+              if (tmpMaxDelaySec > 0) tmpConfig.ConnectRetryOptions.MaxDelayMs = tmpMaxDelaySec * 1000;
+            }
+            if (tmpDDLAttempts > 0 || tmpInitialDelaySec > 0 || tmpMaxDelaySec > 0) {
+              tmpConfig.DDLRetryOptions = {};
+              if (tmpDDLAttempts > 0) tmpConfig.DDLRetryOptions.MaxAttempts = tmpDDLAttempts;
+              if (tmpInitialDelaySec > 0) tmpConfig.DDLRetryOptions.InitialDelayMs = tmpInitialDelaySec * 1000;
+              if (tmpMaxDelaySec > 0) tmpConfig.DDLRetryOptions.MaxDelayMs = tmpMaxDelaySec * 1000;
+            }
           } else if (tmpProvider === 'PostgreSQL') {
             tmpConfig.host = document.getElementById('postgresqlHost').value.trim() || '127.0.0.1';
             tmpConfig.port = parseInt(document.getElementById('postgresqlPort').value, 10) || 5432;
@@ -6044,6 +6086,45 @@
 					</div>
 					<div></div>
 				</div>
+				<div style="margin-top:8px">
+					<input type="checkbox" id="mssqlLegacyPagination">
+					<label for="mssqlLegacyPagination" title="Enable for SQL Server 2008 R2 / 2012 or databases at compatibility_level &lt; 110. Uses ROW_NUMBER() pagination instead of OFFSET/FETCH.">Legacy pagination (SQL Server &lt; 2012 / compat level &lt; 110)</label>
+				</div>
+
+				<details style="margin-top:8px">
+					<summary style="cursor:pointer; font-weight:600">Reliability &amp; timeouts (advanced)</summary>
+					<p style="margin:8px 0; font-size:0.9em; color:#555">Tune these for slow / flaky customer networks. Connection and DDL operations will retry with exponential backoff, classifying each failure (NetworkError / RequestTimeout / PoolDegraded / ServerError) in the logs so the failure mode is obvious.</p>
+					<div class="inline-group">
+						<div>
+							<label for="mssqlRequestTimeoutSec">Request timeout (sec)</label>
+							<input type="number" id="mssqlRequestTimeoutSec" placeholder="120" value="120">
+						</div>
+						<div>
+							<label for="mssqlConnectionTimeoutSec">Connection timeout (sec)</label>
+							<input type="number" id="mssqlConnectionTimeoutSec" placeholder="60" value="60">
+						</div>
+					</div>
+					<div class="inline-group">
+						<div>
+							<label for="mssqlConnectMaxAttempts">Connect retries (max attempts)</label>
+							<input type="number" id="mssqlConnectMaxAttempts" placeholder="5" value="5" min="1" max="20">
+						</div>
+						<div>
+							<label for="mssqlDDLMaxAttempts">DDL retries (max attempts)</label>
+							<input type="number" id="mssqlDDLMaxAttempts" placeholder="5" value="5" min="1" max="20">
+						</div>
+					</div>
+					<div class="inline-group">
+						<div>
+							<label for="mssqlRetryInitialDelaySec">Retry initial delay (sec)</label>
+							<input type="number" id="mssqlRetryInitialDelaySec" placeholder="3" value="3" min="1" max="60">
+						</div>
+						<div>
+							<label for="mssqlRetryMaxDelaySec">Retry max delay (sec)</label>
+							<input type="number" id="mssqlRetryMaxDelaySec" placeholder="30" value="30" min="1" max="600">
+						</div>
+					</div>
+				</details>
 			</div>
 
 			<!-- PostgreSQL Config -->
@@ -6362,6 +6443,9 @@
             tmpDbConfig.password = document.getElementById('mssqlPassword').value;
             tmpDbConfig.database = document.getElementById('mssqlDatabase').value.trim();
             tmpDbConfig.connectionLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+            if (document.getElementById('mssqlLegacyPagination').checked) {
+              tmpDbConfig.LegacyPagination = true;
+            }
           } else if (tmpProvider === 'PostgreSQL') {
             tmpDbConfig.host = document.getElementById('postgresqlHost').value.trim() || '127.0.0.1';
             tmpDbConfig.port = parseInt(document.getElementById('postgresqlPort').value, 10) || 5432;
@@ -6465,6 +6549,9 @@
             tmpConfig.Destination.MSSQL.password = document.getElementById('mssqlPassword').value || '';
             tmpConfig.Destination.MSSQL.database = document.getElementById('mssqlDatabase').value.trim() || 'meadow';
             tmpConfig.Destination.MSSQL.ConnectionPoolLimit = parseInt(document.getElementById('mssqlConnectionLimit').value, 10) || 20;
+            if (document.getElementById('mssqlLegacyPagination').checked) {
+              tmpConfig.Destination.MSSQL.LegacyPagination = true;
+            }
           } else {
             // Default to MySQL placeholder for unsupported providers
             tmpConfig.Destination.Provider = 'MySQL';