retold-data-service 2.1.5 → 2.1.6

package/source/services/data-cloner/DataCloner-Command-Headless.js

@@ -240,6 +240,46 @@ module.exports = (pDataClonerService, pConfig, pCLIOptions, fCallback) =>
 			});
 	};
 
+	// Step 5b: Converge each deployed table's indexes to the clone index policy.
+	//
+	// Historically nothing in the headless pipeline touched indices, so headless
+	// clones ran with just the clustered primary key — leaving the
+	// OngoingEventualConsistency Deleted-filtered range counts clustered-scanning
+	// wide rows, which can stall large clones.  This step declaratively brings the
+	// clone's indexes in line with the desired set (standard operational indexes +
+	// any Sync.IndexPolicy.TableIndexes extras): create missing, then drop
+	// undeclared per prune scope (default 'managed' — only policy-owned / legacy
+	// artifacts, never a hand-authored index).  Create-before-drop and
+	// no-loss-on-failure keep coverage intact.  It must never block the sync, so a
+	// failure here is logged and the pipeline proceeds.  Opt out with
+	// Sync.CreateOperationalIndices=false or Sync.IndexPolicy.Enabled=false.
+	let fStep5b_ConvergeIndices = (fNext) =>
+	{
+		let tmpSync = pConfig.Sync || {};
+		let tmpIndexPolicy = (tmpSync.IndexPolicy && typeof(tmpSync.IndexPolicy) === 'object') ? tmpSync.IndexPolicy : {};
+		if (tmpSync.CreateOperationalIndices === false || tmpIndexPolicy.Enabled === false)
+		{
+			tmpFable.log.info('Headless: Index convergence disabled by config; skipping.');
+			return fNext();
+		}
+
+		let tmpTables = pConfig.Tables || [];
+		tmpFable.log.info(`Headless: Converging indexes on ${tmpTables.length > 0 ? tmpTables.length + ' selected' : 'all'} tables (prune scope: ${tmpIndexPolicy.PruneScope || 'managed'})...`);
+		fPost(`${tmpPrefix}/schema/indices/converge`, { Tables: tmpTables, IndexPolicy: tmpIndexPolicy },
+			(pError, pData) =>
+			{
+				if (pError || !pData || !pData.Success)
+				{
+					tmpFable.log.warn(`Headless: Index convergence reported a problem (continuing to sync): ${pError || (pData && pData.Error) || 'Unknown error'}`);
+				}
+				else
+				{
+					tmpFable.log.info(`Headless: ${pData.Message}`);
+				}
+				return fNext();
+			});
+	};
+
 	// Step 6: Start sync
 	let fStep6_Sync = (fNext) =>
 	{
@@ -254,6 +294,7 @@ module.exports = (pDataClonerService, pConfig, pCLIOptions, fCallback) =>
 				DateTimePrecisionMS: tmpSync.DateTimePrecisionMS,
 				BackSyncTimeLimit: tmpSync.BackSyncTimeLimit,
 				TrueUpPageSize: tmpSync.TrueUpPageSize,
+				SyncRecordConcurrency: tmpSync.SyncRecordConcurrency,
 				SyncEntityOptions: tmpSync.SyncEntityOptions
 			});
 
@@ -268,6 +309,38 @@ module.exports = (pDataClonerService, pConfig, pCLIOptions, fCallback) =>
 				}
 				tmpFable.log.info(`Headless: ${pData.Message}`);
 
+				// Stall detection.  Synced/Total freezes during the (time-budgeted)
+				// delete phase, so a healthy delete pass looks identical to a hung
+				// one from the poll's vantage point.  Only treat it as a stall when
+				// NOTHING in the status changes for well beyond one phase budget —
+				// default max(2x BackSyncTimeLimit, 20 min) — then exit non-zero so
+				// the Docker restart loop can recover (cursor/progress state persists
+				// on the mounted volume).  Configure via Sync.StallTimeoutMs; set it
+				// to 0 to disable.
+				let tmpStallConfigured = parseInt(tmpSync.StallTimeoutMs, 10);
+				let tmpStallTimeoutMs;
+				if (tmpStallConfigured === 0)
+				{
+					tmpStallTimeoutMs = 0;
+				}
+				else if (!isNaN(tmpStallConfigured) && tmpStallConfigured > 0)
+				{
+					tmpStallTimeoutMs = tmpStallConfigured;
+				}
+				else
+				{
+					let tmpBudget = parseInt(tmpSync.BackSyncTimeLimit, 10);
+					if (isNaN(tmpBudget) || tmpBudget <= 0)
+					{
+						tmpBudget = 600000;
+					}
+					tmpStallTimeoutMs = Math.max(2 * tmpBudget, 1200000);
+				}
+				let fLogStall = (typeof(tmpFable.log.fatal) === 'function') ? tmpFable.log.fatal.bind(tmpFable.log) : tmpFable.log.error.bind(tmpFable.log);
+				let tmpLastProgressSignature = null;
+				let tmpLastProgressAtMs = Date.now();
+				tmpFable.log.info(`Headless: Stall detector ${tmpStallTimeoutMs > 0 ? 'armed at ' + (tmpStallTimeoutMs / 60000).toFixed(1) + ' min of no observable progress' : 'disabled'}.`);
+
 				// Poll for completion
 				let fPoll = () =>
 				{
@@ -291,6 +364,27 @@ module.exports = (pDataClonerService, pConfig, pCLIOptions, fCallback) =>
 									let tmpA = tmpTables[tmpActive[0]];
 									tmpFable.log.info(`Headless: [${tmpDone.length}/${tmpNames.length}] Syncing ${tmpActive[0]}: ${tmpA.Synced}/${tmpA.Total}`);
 								}
+
+								// Advance the stall clock only when the observable status
+								// changes (any table's Status or Synced/Total counters).
+								let tmpProgressSignature = tmpNames.slice().sort().map((pName) =>
+									{
+										let tmpT = tmpTables[pName];
+										return `${pName}:${tmpT.Status}:${tmpT.Synced}/${tmpT.Total}`;
+									}).join('|');
+								let tmpNowMs = Date.now();
+								if (tmpProgressSignature !== tmpLastProgressSignature)
+								{
+									tmpLastProgressSignature = tmpProgressSignature;
+									tmpLastProgressAtMs = tmpNowMs;
+								}
+								else if (tmpStallTimeoutMs > 0 && (tmpNowMs - tmpLastProgressAtMs) >= tmpStallTimeoutMs)
+								{
+									let tmpStalledMin = ((tmpNowMs - tmpLastProgressAtMs) / 60000).toFixed(1);
+									fLogStall(`Headless: STALL DETECTED — sync status unchanged for ${tmpStalledMin} min (threshold ${(tmpStallTimeoutMs / 60000).toFixed(1)} min). Active: [${tmpActive.join(', ') || 'none'}], ${tmpDone.length}/${tmpNames.length} tables done. Exiting non-zero so the container restart loop can recover; cursor/progress state persists on the mounted volume.`);
+									return process.exit(1);
+								}
+
 								return setTimeout(fPoll, 5000);
 							}
 
@@ -347,10 +441,13 @@ module.exports = (pDataClonerService, pConfig, pCLIOptions, fCallback) =>
 				{
 					fStep5_Deploy(() =>
 					{
-						fStep6_Sync(() =>
+						fStep5b_ConvergeIndices(() =>
 						{
-							tmpFable.log.info('Headless: Pipeline complete.');
-							return fCallback();
+							fStep6_Sync(() =>
+							{
+								tmpFable.log.info('Headless: Pipeline complete.');
+								return fCallback();
+							});
 						});
 					});
 				});

package/source/services/data-cloner/DataCloner-Command-Schema.js

@@ -16,6 +16,8 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 
 	let libFs = require('fs');
 	let _ProviderRegistry = require('./DataCloner-ProviderRegistry.js');
+	let libIndexPolicy = require('meadow-integration/source/services/clone/Meadow-Service-IndexPolicy.js');
+	let libIndexConvergence = require('meadow-integration/source/services/clone/Meadow-Service-IndexConvergence.js');
 
 	// POST /clone/schema/fetch
 	// Accepts either:
@@ -889,7 +891,7 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 			let tmpTableNames = tmpFilterTables || Object.keys(tmpModelTables);
 			let tmpCreatedIndices = [];
 
-			tmpFable.log.info(`Data Cloner: Creating missing GUID indices for ${tmpTableNames.length} tables...`);
+			tmpFable.log.info(`Data Cloner: Creating missing operational indices (GUID + sync) for ${tmpTableNames.length} tables...`);
 
 			tmpFable.Utility.eachLimit(tmpTableNames, 1,
 				(pTableName, fNextTable) =>
@@ -903,10 +905,15 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 					// Generate all index statements for this table (provider-specific)
 					let tmpAllStatements = tmpSchemaProvider.generateCreateIndexStatements(tmpTableSchema);
 
-					// Filter to GUID indices only (AK_M_GUID*)
-					let tmpGUIDStatements = tmpAllStatements.filter((pStmt) => pStmt.Name && pStmt.Name.indexOf('AK_M_GUID') === 0);
+					// Filter to the clone's operational indices: GUID uniqueness
+					// (AK_M_GUID*) plus the standard sync indices (IX_M_SYNC*) that
+					// keep the OngoingEventualConsistency range-count / delete walks
+					// seekable.  Application ForeignKey / Indexed-column indices are
+					// intentionally left out of the clone.
+					let tmpCloneIndexStatements = tmpAllStatements.filter((pStmt) => pStmt.Name &&
+						(pStmt.Name.indexOf('AK_M_GUID') === 0 || pStmt.Name.indexOf('IX_M_SYNC') === 0));
 
-					if (tmpGUIDStatements.length < 1)
+					if (tmpCloneIndexStatements.length < 1)
 					{
 						return fNextTable();
 					}
@@ -914,12 +921,12 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 					// createIndex is idempotent on all providers — safe to call even if index exists
 					let fCreateNext = (pIndex) =>
 					{
-						if (pIndex >= tmpGUIDStatements.length)
+						if (pIndex >= tmpCloneIndexStatements.length)
 						{
 							return fNextTable();
 						}
 
-						let tmpStmt = tmpGUIDStatements[pIndex];
+						let tmpStmt = tmpCloneIndexStatements[pIndex];
 						tmpSchemaProvider.createIndex(tmpStmt,
 							(pCreateError) =>
 							{
@@ -946,8 +953,8 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 				() =>
 				{
 					let tmpMessage = tmpCreatedIndices.length > 0
-						? `${tmpCreatedIndices.length} GUID index(es) created.`
-						: 'No new GUID indices were needed.';
+						? `${tmpCreatedIndices.length} operational index(es) created.`
+						: 'No new operational indices were needed.';
 
 					tmpFable.log.info(`Data Cloner: ${tmpMessage}`);
 
@@ -960,4 +967,94 @@ module.exports = (pDataClonerService, pOratorServiceServer) =>
 					return fNext();
 				});
 		});
+
+	// POST /clone/schema/indices/converge — Converge each deployed table's indexes
+	// to the clone index policy (create missing, then drop undeclared per prune
+	// scope).  This is the authoritative, declarative index mechanism: the desired
+	// set = standard operational (Deleted composite + non-unique GUID) + any
+	// caller-declared per-table extras (body.IndexPolicy.TableIndexes).
+	pOratorServiceServer.post(`${tmpPrefix}/schema/indices/converge`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpCloneState.DeployedModelObject)
+			{
+				pResponse.send(400, { Success: false, Error: 'No schema deployed. Deploy tables first.' });
+				return fNext();
+			}
+
+			let tmpProviderName = tmpCloneState.ConnectionProvider;
+			let tmpProviderRegistryEntry = _ProviderRegistry[tmpProviderName];
+			let tmpActiveProvider = tmpProviderRegistryEntry ? tmpFable[tmpProviderRegistryEntry.serviceName] : null;
+
+			if (!tmpActiveProvider || !tmpActiveProvider.connected)
+			{
+				pResponse.send(400, { Success: false, Error: 'No connected provider available.' });
+				return fNext();
+			}
+
+			if (typeof(tmpActiveProvider.introspectTableIndices) !== 'function'
+				|| typeof(tmpActiveProvider.generateCreateIndexStatements) !== 'function'
+				|| typeof(tmpActiveProvider.createIndex) !== 'function'
+				|| typeof(tmpActiveProvider.dropIndex) !== 'function')
+			{
+				pResponse.send(400, { Success: false, Error: `Provider ${tmpProviderName} does not support index convergence.` });
+				return fNext();
+			}
+
+			let tmpBody = pRequest.body || {};
+			let tmpIndexConfig = (tmpBody.IndexPolicy && typeof(tmpBody.IndexPolicy) === 'object') ? tmpBody.IndexPolicy : {};
+			let tmpPruneScope = tmpIndexConfig.PruneScope || libIndexConvergence.PRUNE_MANAGED;
+			let tmpFilterTables = Array.isArray(tmpBody.Tables) && tmpBody.Tables.length > 0 ? tmpBody.Tables : null;
+
+			let tmpModelTables = tmpCloneState.DeployedModelObject.Tables || {};
+			let tmpTableNames = tmpFilterTables || Object.keys(tmpModelTables);
+			let tmpResults = [];
+
+			tmpFable.log.info(`Data Cloner: Converging indexes for ${tmpTableNames.length} tables (prune scope: ${tmpPruneScope})...`);
+
+			tmpFable.Utility.eachLimit(tmpTableNames, 1,
+				(pTableName, fNextTable) =>
+				{
+					let tmpTableSchema = tmpModelTables[pTableName];
+					if (!tmpTableSchema)
+					{
+						return fNextTable();
+					}
+
+					let tmpDesired = libIndexPolicy.resolveDesiredIndexes(tmpTableSchema, tmpIndexConfig);
+					libIndexConvergence.convergeTableIndexes(tmpActiveProvider, tmpTableSchema, tmpDesired,
+						{ PruneScope: tmpPruneScope, log: tmpFable.log },
+						(pConvergeError, pResult) =>
+						{
+							if (pConvergeError)
+							{
+								tmpFable.log.warn(`Data Cloner: Index convergence for ${pTableName} failed: ${pConvergeError}`);
+							}
+							else if (pResult && (pResult.created.length > 0 || pResult.dropped.length > 0 || pResult.skipped.length > 0))
+							{
+								tmpFable.log.info(`Data Cloner: ${pTableName} — created [${pResult.created.join(', ')}], dropped [${pResult.dropped.join(', ')}]${pResult.skipped.length > 0 ? `, skipped [${pResult.skipped.join(', ')}]` : ''}`);
+								tmpResults.push(pResult);
+							}
+							return fNextTable();
+						});
+				},
+				() =>
+				{
+					let tmpCreated = tmpResults.reduce((pSum, pR) => { return pSum + pR.created.length; }, 0);
+					let tmpDropped = tmpResults.reduce((pSum, pR) => { return pSum + pR.dropped.length; }, 0);
+					let tmpMessage = (tmpCreated > 0 || tmpDropped > 0)
+						? `Index convergence: ${tmpCreated} created, ${tmpDropped} dropped across ${tmpResults.length} table(s).`
+						: 'Index convergence: all tables already match the desired index set.';
+
+					tmpFable.log.info(`Data Cloner: ${tmpMessage}`);
+
+					pResponse.send(200,
+						{
+							Success: true,
+							Results: tmpResults,
+							Message: tmpMessage
+						});
+					return fNext();
+				});
+		});
 };

package/source/services/data-cloner/DataCloner-Command-Sync.js

@@ -20,6 +20,7 @@ const _RuntimeSyncProperties = {
 	MaxRecordsPerEntity:     (pVal) => { let tmpN = parseInt(pVal, 10); return (!isNaN(tmpN) && tmpN > 0) ? tmpN : null; },
 	DateTimePrecisionMS:     (pVal) => { let tmpN = parseInt(pVal, 10); return !isNaN(tmpN)               ? tmpN : null; },
 	TrueUpPageSize:          (pVal) => { let tmpN = parseInt(pVal, 10); return (!isNaN(tmpN) && tmpN > 0) ? tmpN : null; },
+	SyncRecordConcurrency:   (pVal) => { let tmpN = parseInt(pVal, 10); return (!isNaN(tmpN) && tmpN > 0) ? tmpN : null; },
 	UseAdvancedIDPagination: (pVal) => !!pVal,
 	SyncDeletedRecords:      (pVal) => !!pVal
 };