retold-data-service 2.0.14 → 2.0.16

package/source/services/data-cloner/DataCloner-Command-Schema.js

@@ -0,0 +1,367 @@
+/**
+ * DataCloner Schema Management Routes
+ *
+ * Registers /clone/schema/* endpoints for fetching remote schemas,
+ * deploying tables to the local database, and resetting the database.
+ *
+ * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pDataClonerService, pOratorServiceServer) =>
+{
+	let tmpFable = pDataClonerService.fable;
+	let tmpCloneState = pDataClonerService.cloneState;
+	let tmpPict = pDataClonerService.pict;
+	let tmpPrefix = pDataClonerService.routePrefix;
+
+	let libFs = require('fs');
+
+	// POST /clone/schema/fetch
+	// Accepts either:
+	//   { SchemaURL: "..." }           — fetch from a remote URL
+	//   { Schema: { Tables: {...} } }  — use a pre-loaded schema object directly
+	//   {}                             — default to RemoteServerURL + 'Retold/Models'
+	pOratorServiceServer.post(`${tmpPrefix}/schema/fetch`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+
+			// If a raw Schema object was provided, use it directly (no remote fetch)
+			if (tmpBody.Schema && typeof tmpBody.Schema === 'object')
+			{
+				tmpFable.log.info('Data Cloner: Using pre-loaded schema object...');
+
+				tmpCloneState.RemoteSchema = tmpBody.Schema;
+				tmpCloneState.RemoteModelObject = tmpBody.Schema;
+
+				let tmpTableNames = [];
+				if (tmpBody.Schema.Tables)
+				{
+					tmpTableNames = Object.keys(tmpBody.Schema.Tables);
+				}
+
+				tmpFable.log.info(`Data Cloner: Loaded schema with ${tmpTableNames.length} tables: [${tmpTableNames.join(', ')}]`);
+
+				pResponse.send(200,
+					{
+						Success: true,
+						Source: 'local',
+						TableCount: tmpTableNames.length,
+						Tables: tmpTableNames
+					});
+				return fNext();
+			}
+
+			let tmpSchemaURL = tmpBody.SchemaURL;
+
+			if (!tmpSchemaURL)
+			{
+				// Default to the standard retold model endpoint
+				if (tmpCloneState.RemoteServerURL)
+				{
+					tmpSchemaURL = tmpCloneState.RemoteServerURL + 'Retold/Models';
+				}
+				else
+				{
+					pResponse.send(400, { Success: false, Error: 'SchemaURL is required (or configure a session first).' });
+					return fNext();
+				}
+			}
+
+			tmpFable.log.info(`Data Cloner: Fetching remote schema from ${tmpSchemaURL}...`);
+
+			tmpPict.RestClient.getJSON(tmpSchemaURL,
+				(pError, pHTTPResponse, pData) =>
+				{
+					if (pError)
+					{
+						tmpFable.log.error(`Data Cloner: Schema fetch error: ${pError.message || pError}`);
+						pResponse.send(500, { Success: false, Error: `Schema fetch error: ${pError.message || pError}` });
+						return fNext();
+					}
+
+					if (!pHTTPResponse || pHTTPResponse.statusCode !== 200)
+					{
+						let tmpStatus = pHTTPResponse ? pHTTPResponse.statusCode : 'unknown';
+						tmpFable.log.error(`Data Cloner: Schema fetch returned HTTP ${tmpStatus} — body: ${JSON.stringify(pData)}`);
+						pResponse.send(500, { Success: false, Error: `Schema fetch returned HTTP ${tmpStatus}` });
+						return fNext();
+					}
+
+					tmpCloneState.RemoteSchema = pData;
+					tmpCloneState.RemoteModelObject = pData;
+
+					// Extract table names for the UI
+					let tmpTableNames = [];
+					if (pData && pData.Tables)
+					{
+						tmpTableNames = Object.keys(pData.Tables);
+					}
+
+					tmpFable.log.info(`Data Cloner: Fetched schema with ${tmpTableNames.length} tables: [${tmpTableNames.join(', ')}]`);
+
+					pResponse.send(200,
+						{
+							Success: true,
+							SchemaURL: tmpSchemaURL,
+							TableCount: tmpTableNames.length,
+							Tables: tmpTableNames
+						});
+					return fNext();
+				});
+		});
+
+	// GET /clone/schema
+	pOratorServiceServer.get(`${tmpPrefix}/schema`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpCloneState.RemoteSchema)
+			{
+				pResponse.send(200, { Fetched: false, Tables: [] });
+				return fNext();
+			}
+
+			let tmpTableNames = [];
+			if (tmpCloneState.RemoteSchema.Tables)
+			{
+				tmpTableNames = Object.keys(tmpCloneState.RemoteSchema.Tables);
+			}
+
+			pResponse.send(200,
+				{
+					Fetched: true,
+					TableCount: tmpTableNames.length,
+					Tables: tmpTableNames
+				});
+			return fNext();
+		});
+
+	// POST /clone/reset — Delete the local SQLite database and start fresh
+	pOratorServiceServer.post(`${tmpPrefix}/reset`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpSQLitePath = tmpFable.settings.SQLite && tmpFable.settings.SQLite.SQLiteFilePath;
+			if (!tmpSQLitePath)
+			{
+				pResponse.send(400, { Success: false, Error: 'No SQLite file path configured.' });
+				return fNext();
+			}
+
+			tmpFable.log.info(`Data Cloner: Resetting local database [${tmpSQLitePath}]...`);
+
+			try
+			{
+				// Close the existing SQLite connection and reset the provider state
+				if (tmpFable.MeadowSQLiteProvider)
+				{
+					if (tmpFable.MeadowSQLiteProvider._database)
+					{
+						tmpFable.MeadowSQLiteProvider._database.close();
+					}
+					tmpFable.MeadowSQLiteProvider.connected = false;
+				}
+			}
+			catch (pCloseError)
+			{
+				tmpFable.log.warn(`Data Cloner: Error closing SQLite connection: ${pCloseError}`);
+			}
+
+			try
+			{
+				// Delete the database file
+				if (libFs.existsSync(tmpSQLitePath))
+				{
+					libFs.unlinkSync(tmpSQLitePath);
+					tmpFable.log.info('Data Cloner: SQLite database file deleted.');
+				}
+			}
+			catch (pDeleteError)
+			{
+				tmpFable.log.error(`Data Cloner: Error deleting SQLite file: ${pDeleteError}`);
+				pResponse.send(500, { Success: false, Error: `Failed to delete database: ${pDeleteError}` });
+				return fNext();
+			}
+
+			// Reconnect to create a fresh database
+			tmpFable.MeadowSQLiteProvider.connectAsync(
+				(pReconnectError) =>
+				{
+					if (pReconnectError)
+					{
+						tmpFable.log.error(`Data Cloner: Error reconnecting SQLite: ${pReconnectError}`);
+						pResponse.send(500, { Success: false, Error: `Failed to reconnect: ${pReconnectError}` });
+						return fNext();
+					}
+
+					// Clear sync state
+					tmpCloneState.SyncProgress = {};
+					tmpCloneState.SyncRESTErrors = {};
+
+					tmpFable.log.info('Data Cloner: Database reset complete — fresh SQLite file ready.');
+
+					pResponse.send(200, { Success: true, Message: 'Database reset. Deploy a schema to recreate tables.' });
+					return fNext();
+				});
+		});
+
+	// POST /clone/schema/deploy
+	pOratorServiceServer.post(`${tmpPrefix}/schema/deploy`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpCloneState.RemoteModelObject)
+			{
+				pResponse.send(400, { Success: false, Error: 'No schema fetched. Call POST /clone/schema/fetch first.' });
+				return fNext();
+			}
+
+			let tmpBody = pRequest.body || {};
+			let tmpSelectedTables = tmpBody.Tables || null;
+
+			let tmpFullModel = tmpCloneState.RemoteModelObject;
+
+			// Build a filtered model with ONLY the selected tables
+			let tmpFilteredTables = {};
+			let tmpFilteredSequence = [];
+			let tmpSourceTables = tmpFullModel.Tables || {};
+
+			if (tmpSelectedTables && Array.isArray(tmpSelectedTables) && tmpSelectedTables.length > 0)
+			{
+				for (let i = 0; i < tmpSelectedTables.length; i++)
+				{
+					let tmpTableName = tmpSelectedTables[i];
+					if (tmpSourceTables[tmpTableName])
+					{
+						tmpFilteredTables[tmpTableName] = tmpSourceTables[tmpTableName];
+						tmpFilteredSequence.push(tmpTableName);
+					}
+				}
+			}
+			else
+			{
+				// No selection — use all tables
+				tmpFilteredTables = tmpSourceTables;
+				tmpFilteredSequence = Object.keys(tmpSourceTables);
+			}
+
+			let tmpModelObject = { Tables: tmpFilteredTables, TablesSequence: tmpFilteredSequence };
+
+			let tmpTableNames = Object.keys(tmpModelObject.Tables || {});
+			tmpFable.log.info(`Data Cloner: Deploying ${tmpTableNames.length} tables to local ${tmpCloneState.ConnectionProvider}: [${tmpTableNames.join(', ')}]`);
+
+			// ---- Set up MeadowCloneRestClient ----
+			if (!tmpFable.MeadowCloneRestClient)
+			{
+				tmpFable.serviceManager.instantiateServiceProvider('MeadowCloneRestClient',
+					{
+						ServerURL: tmpCloneState.RemoteServerURL,
+						RequestTimeout: 60000,
+						MaxRequestTimeout: 300000
+					});
+			}
+			else
+			{
+				tmpFable.MeadowCloneRestClient.serverURL = tmpCloneState.RemoteServerURL;
+			}
+
+			// Override getJSON to delegate through pict-sessionmanager's
+			// RestClient, which already handles cookie injection & domain matching.
+			let libHttps = require('https');
+			libHttps.globalAgent.options.timeout = tmpFable.MeadowCloneRestClient.maxRequestTimeout;
+			tmpFable.MeadowCloneRestClient.getJSON = (pURL, fCallback) =>
+			{
+				let tmpFullURL = tmpFable.MeadowCloneRestClient.serverURL + pURL;
+
+				// Extract the entity name from the URL for error tracking
+				let tmpEntityHint = pURL.split('/')[0].replace(/s$/, '');
+
+				// Use the longer timeout for MAX(column) queries only
+				let tmpIsMaxRequest = (pURL.indexOf('/Max/') > -1);
+				let tmpPreviousTimeout = libHttps.globalAgent.options.timeout;
+				libHttps.globalAgent.options.timeout = tmpIsMaxRequest
+					? tmpFable.MeadowCloneRestClient.maxRequestTimeout
+					: tmpFable.MeadowCloneRestClient.requestTimeout;
+
+				tmpPict.RestClient.getJSON(tmpFullURL,
+					(pError, pHTTPResponse, pBody) =>
+					{
+						if (pError)
+						{
+							tmpFable.log.error(`Data Cloner: REST error for ${pURL}: ${pError}`);
+							if (tmpCloneState.SyncRESTErrors[tmpEntityHint] !== undefined)
+							{
+								tmpCloneState.SyncRESTErrors[tmpEntityHint]++;
+							}
+						}
+						else
+						{
+							// Track when the server returns a non-array for list endpoints
+							// Count/FilteredTo returns {"Count":N} (an object, not an array) which is expected
+							if (pURL.indexOf('FilteredTo') > -1 && pURL.indexOf('Count/FilteredTo') < 0 && !Array.isArray(pBody))
+							{
+								let tmpBodyPreview = (typeof(pBody) === 'string') ? pBody.substring(0, 300) : JSON.stringify(pBody).substring(0, 300);
+								tmpFable.log.warn(`Data Cloner: FilteredTo response for ${tmpEntityHint} is not an array — URL: ${pURL} — Response: ${tmpBodyPreview}`);
+								if (tmpCloneState.SyncRESTErrors[tmpEntityHint] !== undefined)
+								{
+									tmpCloneState.SyncRESTErrors[tmpEntityHint]++;
+								}
+							}
+						}
+						libHttps.globalAgent.options.timeout = tmpPreviousTimeout;
+						return fCallback(pError, pHTTPResponse, pBody);
+					});
+			};
+
+			// ---- Set up MeadowSync ----
+			if (!tmpFable.ProgramConfiguration)
+			{
+				tmpFable.ProgramConfiguration = {};
+			}
+
+			tmpFable.serviceManager.instantiateServiceProvider('MeadowSync',
+				{
+					SyncEntityList: tmpTableNames,
+					PageSize: 100,
+					SyncDeletedRecords: tmpCloneState.SyncDeletedRecords
+				});
+
+			tmpFable.MeadowSync.loadMeadowSchema(tmpModelObject,
+				(pSyncInitError) =>
+				{
+					if (pSyncInitError)
+					{
+						tmpFable.log.warn(`Data Cloner: MeadowSync schema init warning: ${pSyncInitError}`);
+					}
+
+					let tmpInitializedEntities = Object.keys(tmpFable.MeadowSync.MeadowSyncEntities);
+					tmpFable.log.info(`Data Cloner: MeadowSync initialized ${tmpInitializedEntities.length} sync entities: [${tmpInitializedEntities.join(', ')}]`);
+
+					// Store the deployed model so sync mode switches can re-create entities
+					tmpCloneState.DeployedModelObject = tmpModelObject;
+
+					tmpFable.log.info(`Data Cloner: Loading model for CRUD endpoints...`);
+
+					// Load the filtered model so CRUD endpoints are available
+					tmpFable.RetoldDataServiceMeadowEndpoints.loadModel('RemoteClone', tmpModelObject, tmpCloneState.ConnectionProvider,
+						(pLoadError) =>
+						{
+							if (pLoadError)
+							{
+								tmpFable.log.error(`Data Cloner: Model load error: ${pLoadError}`);
+							}
+							else
+							{
+								tmpFable.log.info(`Data Cloner: CRUD endpoints available for: [${tmpTableNames.join(', ')}]`);
+							}
+
+							pResponse.send(200,
+								{
+									Success: true,
+									TablesDeployed: tmpTableNames,
+									SyncEntities: tmpInitializedEntities,
+									Message: `${tmpInitializedEntities.length} / ${tmpTableNames.length} tables deployed. meadow-integration sync ready.`
+								});
+							return fNext();
+						});
+				});
+		});
+};

package/source/services/data-cloner/DataCloner-Command-Session.js

@@ -0,0 +1,229 @@
+/**
+ * DataCloner Session Management Routes
+ *
+ * Registers /clone/session/* endpoints for remote session configuration,
+ * authentication, check, and deauthentication via pict-sessionmanager.
+ *
+ * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pDataClonerService, pOratorServiceServer) =>
+{
+	let tmpFable = pDataClonerService.fable;
+	let tmpCloneState = pDataClonerService.cloneState;
+	let tmpPict = pDataClonerService.pict;
+	let tmpPrefix = pDataClonerService.routePrefix;
+
+	// POST /clone/session/configure
+	pOratorServiceServer.post(`${tmpPrefix}/session/configure`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+			let tmpServerURL = tmpBody.ServerURL;
+
+			if (!tmpServerURL)
+			{
+				pResponse.send(400, { Success: false, Error: 'ServerURL is required.' });
+				return fNext();
+			}
+
+			tmpCloneState.RemoteServerURL = tmpServerURL;
+
+			// Remove existing session if reconfiguring
+			if (tmpPict.SessionManager.getSession('Remote'))
+			{
+				tmpPict.SessionManager.deauthenticate('Remote');
+				delete tmpPict.SessionManager.sessions['Remote'];
+			}
+
+			// Extract domain from ServerURL for cookie matching
+			let tmpDomainMatch = tmpBody.DomainMatch;
+			if (!tmpDomainMatch)
+			{
+				try
+				{
+					let tmpURL = new URL(tmpServerURL);
+					tmpDomainMatch = tmpURL.hostname;
+				}
+				catch (pParseError)
+				{
+					tmpDomainMatch = tmpServerURL;
+				}
+			}
+
+			// Helper: ensure a URI template is fully qualified with the server URL
+			let fQualifyURI = (pTemplate, pDefault) =>
+			{
+				let tmpTemplate = pTemplate || pDefault;
+				// Already a full URL — use as-is
+				if (tmpTemplate.indexOf('://') > -1)
+				{
+					return tmpTemplate;
+				}
+				// Strip leading slash
+				let tmpPath = tmpTemplate.replace(/^\//, '');
+				// If the server URL already ends with a version prefix (e.g. /1.0/)
+				// and the path redundantly starts with the same prefix, strip it
+				// to avoid double-prefixing (e.g. /1.0/1.0/Authenticate).
+				let tmpVersionMatch = tmpServerURL.match(/\/(\d+\.\d+)\/?$/);
+				if (tmpVersionMatch)
+				{
+					let tmpURLPrefix = tmpVersionMatch[1] + '/';
+					if (tmpPath.indexOf(tmpURLPrefix) === 0)
+					{
+						tmpPath = tmpPath.substring(tmpURLPrefix.length);
+					}
+				}
+				return tmpServerURL + tmpPath;
+			};
+
+			let tmpSessionConfig = (
+				{
+					Type: 'Cookie',
+
+					// Authentication
+					AuthenticationMethod: tmpBody.AuthenticationMethod || 'get',
+					AuthenticationURITemplate: fQualifyURI(tmpBody.AuthenticationURITemplate, 'Authenticate/{~D:Record.UserName~}/{~D:Record.Password~}'),
+					AuthenticationRetryCount: 2,
+					AuthenticationRetryDebounce: 200,
+
+					// Session check
+					CheckSessionURITemplate: fQualifyURI(tmpBody.CheckSessionURITemplate, 'CheckSession'),
+					CheckSessionLoginMarkerType: tmpBody.CheckSessionLoginMarkerType || 'boolean',
+					CheckSessionLoginMarker: tmpBody.CheckSessionLoginMarker || 'LoggedIn',
+
+					// Cookie injection
+					DomainMatch: tmpDomainMatch,
+					CookieName: tmpBody.CookieName || 'SessionID',
+					CookieValueAddress: tmpBody.CookieValueAddress || 'SessionID',
+					CookieValueTemplate: tmpBody.CookieValueTemplate || false
+				});
+
+			// If authentication is POST-based, set up the request body template
+			if (tmpBody.AuthenticationMethod === 'post')
+			{
+				tmpSessionConfig.AuthenticationRequestBody = tmpBody.AuthenticationRequestBody || (
+					{
+						username: '{~D:Record.UserName~}',
+						password: '{~D:Record.Password~}'
+					});
+			}
+
+			tmpPict.SessionManager.addSession('Remote', tmpSessionConfig);
+			tmpPict.SessionManager.connectToRestClient();
+
+			tmpCloneState.SessionConfigured = true;
+			tmpCloneState.SessionAuthenticated = false;
+
+			tmpFable.log.info(`Data Cloner: Session configured for ${tmpServerURL} (domain: ${tmpDomainMatch})`);
+
+			pResponse.send(200,
+				{
+					Success: true,
+					ServerURL: tmpServerURL,
+					DomainMatch: tmpDomainMatch,
+					AuthenticationMethod: tmpSessionConfig.AuthenticationMethod
+				});
+			return fNext();
+		});
+
+	// POST /clone/session/authenticate
+	pOratorServiceServer.post(`${tmpPrefix}/session/authenticate`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpCloneState.SessionConfigured)
+			{
+				pResponse.send(400, { Success: false, Error: 'Session not configured. Call POST /clone/session/configure first.' });
+				return fNext();
+			}
+
+			let tmpBody = pRequest.body || {};
+			let tmpCredentials = (
+				{
+					UserName: tmpBody.UserName || tmpBody.username,
+					Password: tmpBody.Password || tmpBody.password
+				});
+
+			if (!tmpCredentials.UserName || !tmpCredentials.Password)
+			{
+				pResponse.send(400, { Success: false, Error: 'UserName and Password are required.' });
+				return fNext();
+			}
+
+			tmpFable.log.info(`Data Cloner: Authenticating as ${tmpCredentials.UserName}...`);
+
+			tmpPict.SessionManager.authenticate('Remote', tmpCredentials,
+				(pAuthError, pSessionState) =>
+				{
+					if (pAuthError)
+					{
+						tmpFable.log.error(`Data Cloner: Authentication failed: ${pAuthError.message || pAuthError}`);
+						tmpCloneState.SessionAuthenticated = false;
+						pResponse.send(401,
+							{
+								Success: false,
+								Error: `Authentication failed: ${pAuthError.message || pAuthError}`
+							});
+						return fNext();
+					}
+
+					tmpCloneState.SessionAuthenticated = pSessionState && pSessionState.Authenticated;
+
+					tmpFable.log.info(`Data Cloner: Authentication ${tmpCloneState.SessionAuthenticated ? 'succeeded' : 'failed'}.`);
+
+					pResponse.send(200,
+						{
+							Success: tmpCloneState.SessionAuthenticated,
+							Authenticated: tmpCloneState.SessionAuthenticated,
+							SessionData: pSessionState ? pSessionState.SessionData : {}
+						});
+					return fNext();
+				});
+		});
+
+	// GET /clone/session/check
+	pOratorServiceServer.get(`${tmpPrefix}/session/check`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpCloneState.SessionConfigured)
+			{
+				pResponse.send(200,
+					{
+						Configured: false,
+						Authenticated: false
+					});
+				return fNext();
+			}
+
+			tmpPict.SessionManager.checkSession('Remote',
+				(pCheckError, pAuthenticated, pCheckData) =>
+				{
+					tmpCloneState.SessionAuthenticated = pAuthenticated;
+
+					pResponse.send(200,
+						{
+							Configured: tmpCloneState.SessionConfigured,
+							Authenticated: pAuthenticated,
+							ServerURL: tmpCloneState.RemoteServerURL,
+							CheckData: pCheckData || {}
+						});
+					return fNext();
+				});
+		});
+
+	// POST /clone/session/deauthenticate
+	pOratorServiceServer.post(`${tmpPrefix}/session/deauthenticate`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (tmpCloneState.SessionConfigured)
+			{
+				tmpPict.SessionManager.deauthenticate('Remote');
+			}
+			tmpCloneState.SessionAuthenticated = false;
+
+			tmpFable.log.info('Data Cloner: Session deauthenticated.');
+
+			pResponse.send(200, { Success: true, Authenticated: false });
+			return fNext();
+		});
+};