retold-data-service 2.0.21 → 2.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.21",
+    "version": "2.0.23",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
@@ -61,24 +61,24 @@
     "devDependencies": {
         "meadow-connection-sqlite": "^1.0.18",
         "puppeteer": "^24.38.0",
-        "quackage": "^1.0.64",
+        "quackage": "^1.0.65",
         "stricture": "^4.0.2",
         "supertest": "^7.2.2"
     },
     "dependencies": {
         "bibliograph": "^0.1.4",
-        "fable": "^3.1.63",
+        "fable": "^3.1.67",
         "fable-serviceproviderbase": "^3.0.19",
         "meadow": "^2.0.33",
         "meadow-connection-mysql": "^1.0.14",
-        "meadow-endpoints": "^4.0.14",
-        "meadow-integration": "^1.0.18",
+        "meadow-endpoints": "^4.0.15",
+        "meadow-integration": "^1.0.20",
         "meadow-migrationmanager": "^0.0.4",
         "orator": "^6.0.4",
         "orator-http-proxy": "^1.0.5",
-        "orator-serviceserver-restify": "^2.0.9",
+        "orator-serviceserver-restify": "^2.0.10",
         "orator-static-server": "^2.0.4",
-        "pict": "^1.0.357",
+        "pict": "^1.0.361",
         "pict-section-histogram": "^1.0.0",
         "pict-sessionmanager": "^1.0.2",
         "stricture": "^4.0.2"

package/source/Retold-Data-Service.js

@@ -18,6 +18,7 @@ const libRetoldDataServiceMeadowIntegration = require('./services/meadow-integra
 const libRetoldDataServiceMigrationManager = require('./services/migration-manager/Retold-Data-Service-MigrationManager.js');
 const libRetoldDataServiceDataCloner = require('./services/data-cloner/Retold-Data-Service-DataCloner.js');
 const libRetoldDataServiceIntegrationTelemetry = require('./services/integration-telemetry/Retold-Data-Service-IntegrationTelemetry.js');
+const libRetoldDataServiceComprehensionLoader = require('./services/comprehension-loader/Retold-Data-Service-ComprehensionLoader.js');
 
 const defaultDataServiceSettings = (
 	{
@@ -53,7 +54,11 @@ const defaultDataServiceSettings = (
 				// Data cloner web UI (GET /clone/)
 				DataClonerWebUI: false,
 				// Integration telemetry API endpoints (/telemetry/*)
-				IntegrationTelemetry: false
+				IntegrationTelemetry: false,
+				// Comprehension loader API endpoints (/comprehension_load/*)
+				ComprehensionLoader: false,
+				// Comprehension loader web UI (GET /comprehension_load/)
+				ComprehensionLoaderWebUI: false
 			},
 
 		// Migration manager configuration
@@ -79,6 +84,13 @@ const defaultDataServiceSettings = (
 				RoutePrefix: '/telemetry',
 				// Default tenant identifier when none is provided
 				DefaultTenantID: 'default'
+			},
+
+		// Comprehension loader configuration
+		ComprehensionLoader:
+			{
+				// Route prefix for all comprehension loader endpoints (API + web UI)
+				RoutePrefix: '/comprehension_load'
 			}
 	});
 
@@ -142,6 +154,10 @@ class RetoldDataService extends libFableServiceProviderBase
 		this.fable.serviceManager.addServiceType('RetoldDataServiceIntegrationTelemetry', libRetoldDataServiceIntegrationTelemetry);
 		this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceIntegrationTelemetry');
 
+		// Register and instantiate the ComprehensionLoader service
+		this.fable.serviceManager.addServiceType('RetoldDataServiceComprehensionLoader', libRetoldDataServiceComprehensionLoader);
+		this.fable.serviceManager.instantiateServiceProvider('RetoldDataServiceComprehensionLoader');
+
 		// Expose the DAL and MeadowEndpoints from the service on this object and on fable for backward compatibility
 		this._DAL = this.fable.RetoldDataServiceMeadowEndpoints._DAL;
 		this._MeadowEndpoints = this.fable.RetoldDataServiceMeadowEndpoints._MeadowEndpoints;
@@ -235,7 +251,7 @@ class RetoldDataService extends libFableServiceProviderBase
 			this.fable.log.info(`The Retold Data Service is initializing...`);
 
 			// Log endpoint configuration
-			let tmpGroupNames = ['ConnectionManager', 'ModelManagerWrite', 'Stricture', 'MeadowIntegration', 'MeadowEndpoints', 'MigrationManager', 'MigrationManagerWebUI', 'DataCloner', 'DataClonerWebUI', 'IntegrationTelemetry'];
+			let tmpGroupNames = ['ConnectionManager', 'ModelManagerWrite', 'Stricture', 'MeadowIntegration', 'MeadowEndpoints', 'MigrationManager', 'MigrationManagerWebUI', 'DataCloner', 'DataClonerWebUI', 'IntegrationTelemetry', 'ComprehensionLoader', 'ComprehensionLoaderWebUI'];
 			let tmpEnabledGroups = [];
 			let tmpDisabledGroups = [];
 			for (let i = 0; i < tmpGroupNames.length; i++)
@@ -336,6 +352,18 @@ class RetoldDataService extends libFableServiceProviderBase
 						this.fable.RetoldDataServiceIntegrationTelemetry.connectRoutes(this.fable.OratorServiceServer);
 					}
 
+					// ComprehensionLoader API routes (/comprehension_load/*)
+					if (this.isEndpointGroupEnabled('ComprehensionLoader'))
+					{
+						this.fable.RetoldDataServiceComprehensionLoader.connectRoutes(this.fable.OratorServiceServer);
+					}
+
+					// ComprehensionLoader Web UI routes (GET /comprehension_load/)
+					if (this.isEndpointGroupEnabled('ComprehensionLoaderWebUI'))
+					{
+						this.fable.RetoldDataServiceComprehensionLoader.connectWebUIRoutes(this.fable.OratorServiceServer);
+					}
+
 					return fInitCallback();
 				});
 

package/source/services/comprehension-loader/ComprehensionLoader-Command-Load.js

@@ -0,0 +1,345 @@
+/**
+ * ComprehensionLoader Load Management Routes
+ *
+ * Registers /comprehension_load/comprehension/* and /comprehension_load/load/*
+ * endpoints for receiving comprehension data, starting/stopping the push,
+ * and polling progress.
+ *
+ * @param {Object} pComprehensionLoaderService - The RetoldDataServiceComprehensionLoader instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pComprehensionLoaderService, pOratorServiceServer) =>
+{
+	let tmpFable = pComprehensionLoaderService.fable;
+	let tmpLoadState = pComprehensionLoaderService.loadState;
+	let tmpPict = pComprehensionLoaderService.pict;
+	let tmpPrefix = pComprehensionLoaderService.routePrefix;
+
+	// POST /comprehension_load/comprehension/receive
+	pOratorServiceServer.postWithBodyParser(`${tmpPrefix}/comprehension/receive`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+
+			if (!tmpBody.Comprehension || (typeof tmpBody.Comprehension !== 'object'))
+			{
+				pResponse.send(400, { Success: false, Error: 'No valid Comprehension object provided in request body.' });
+				return fNext();
+			}
+
+			let tmpSummary = pComprehensionLoaderService.loadComprehension(tmpBody.Comprehension);
+
+			tmpFable.log.info(`Comprehension Loader: Received comprehension with ${tmpSummary.EntityList.length} entities, ${tmpSummary.TotalRecords} total records.`);
+
+			pResponse.send(200,
+				{
+					Success: true,
+					EntityCount: tmpSummary.EntityList.length,
+					EntityList: tmpSummary.EntityList,
+					RecordCounts: tmpSummary.RecordCounts,
+					TotalRecords: tmpSummary.TotalRecords
+				});
+			return fNext();
+		});
+
+	// POST /comprehension_load/comprehension/proxy-fetch
+	pOratorServiceServer.postWithBodyParser(`${tmpPrefix}/comprehension/proxy-fetch`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+			let tmpURL = tmpBody.URL;
+
+			if (!tmpURL || typeof tmpURL !== 'string')
+			{
+				pResponse.send(400, { Success: false, Error: 'URL is required.' });
+				return fNext();
+			}
+
+			tmpFable.log.info(`Comprehension Loader: Proxy-fetching comprehension from ${tmpURL}...`);
+
+			tmpPict.RestClient.getJSON(tmpURL,
+				(pError, pHTTPResponse, pData) =>
+				{
+					if (pError)
+					{
+						tmpFable.log.error(`Comprehension Loader: Proxy fetch error: ${pError.message || pError}`);
+						pResponse.send(500, { Success: false, Error: `Proxy fetch error: ${pError.message || pError}` });
+						return fNext();
+					}
+
+					if (!pHTTPResponse || pHTTPResponse.statusCode !== 200)
+					{
+						let tmpStatus = pHTTPResponse ? pHTTPResponse.statusCode : 'unknown';
+						pResponse.send(500, { Success: false, Error: `Proxy fetch returned HTTP ${tmpStatus}` });
+						return fNext();
+					}
+
+					if (!pData || typeof pData !== 'object')
+					{
+						pResponse.send(400, { Success: false, Error: 'Fetched content is not valid JSON.' });
+						return fNext();
+					}
+
+					let tmpSummary = pComprehensionLoaderService.loadComprehension(pData);
+
+					tmpFable.log.info(`Comprehension Loader: Proxy-fetched comprehension with ${tmpSummary.EntityList.length} entities, ${tmpSummary.TotalRecords} total records.`);
+
+					pResponse.send(200,
+						{
+							Success: true,
+							EntityCount: tmpSummary.EntityList.length,
+							EntityList: tmpSummary.EntityList,
+							RecordCounts: tmpSummary.RecordCounts,
+							TotalRecords: tmpSummary.TotalRecords
+						});
+					return fNext();
+				});
+		});
+
+	// GET /comprehension_load/comprehension
+	pOratorServiceServer.get(`${tmpPrefix}/comprehension`,
+		(pRequest, pResponse, fNext) =>
+		{
+			pResponse.send(200,
+				{
+					Loaded: tmpLoadState.ComprehensionData !== null,
+					EntityCount: tmpLoadState.ComprehensionEntityList.length,
+					EntityList: tmpLoadState.ComprehensionEntityList,
+					RecordCounts: tmpLoadState.ComprehensionRecordCounts,
+					TotalRecords: tmpLoadState.ComprehensionTotalRecords
+				});
+			return fNext();
+		});
+
+	// POST /comprehension_load/comprehension/clear
+	pOratorServiceServer.post(`${tmpPrefix}/comprehension/clear`,
+		(pRequest, pResponse, fNext) =>
+		{
+			pComprehensionLoaderService.clearComprehension();
+
+			tmpFable.log.info('Comprehension Loader: Comprehension data cleared.');
+
+			pResponse.send(200, { Success: true });
+			return fNext();
+		});
+
+	// POST /comprehension_load/load/start
+	pOratorServiceServer.post(`${tmpPrefix}/load/start`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (tmpLoadState.LoadRunning)
+			{
+				pResponse.send(409, { Success: false, Error: 'A load is already running.' });
+				return fNext();
+			}
+
+			if (!tmpLoadState.ComprehensionData)
+			{
+				pResponse.send(400, { Success: false, Error: 'No comprehension data loaded. Send data first via /comprehension/receive.' });
+				return fNext();
+			}
+
+			if (!tmpLoadState.SessionAuthenticated)
+			{
+				pResponse.send(400, { Success: false, Error: 'Session not authenticated. Configure and authenticate a session first.' });
+				return fNext();
+			}
+
+			let tmpBody = pRequest.body || {};
+			let tmpEntityList = tmpBody.Entities || tmpLoadState.ComprehensionEntityList;
+
+			// Reset progress
+			tmpLoadState.LoadProgress = {};
+
+			tmpFable.log.info(`Comprehension Loader: Starting load for ${tmpEntityList.length} entities...`);
+
+			// Start push asynchronously
+			pComprehensionLoaderService.pushEntities(tmpEntityList);
+
+			pResponse.send(200,
+				{
+					Success: true,
+					Entities: tmpEntityList,
+					TotalRecords: tmpLoadState.ComprehensionTotalRecords
+				});
+			return fNext();
+		});
+
+	// POST /comprehension_load/load/stop
+	pOratorServiceServer.post(`${tmpPrefix}/load/stop`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpLoadState.LoadRunning)
+			{
+				pResponse.send(200, { Success: true, Message: 'No load is running.' });
+				return fNext();
+			}
+
+			tmpLoadState.LoadStopping = true;
+			tmpFable.log.info('Comprehension Loader: Stop requested.');
+
+			pResponse.send(200, { Success: true, Message: 'Stop signal sent.' });
+			return fNext();
+		});
+
+	// GET /comprehension_load/load/status
+	pOratorServiceServer.get(`${tmpPrefix}/load/status`,
+		(pRequest, pResponse, fNext) =>
+		{
+			pResponse.send(200,
+				{
+					Running: tmpLoadState.LoadRunning,
+					Entities: tmpLoadState.LoadProgress
+				});
+			return fNext();
+		});
+
+	// GET /comprehension_load/load/live-status
+	pOratorServiceServer.get(`${tmpPrefix}/load/live-status`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpEntityNames = Object.keys(tmpLoadState.LoadProgress);
+			let tmpTotalPushed = 0;
+			let tmpTotalRecords = tmpLoadState.ComprehensionTotalRecords;
+			let tmpCompleted = 0;
+			let tmpErrors = 0;
+			let tmpPending = 0;
+			let tmpActiveEntity = null;
+			let tmpActiveProgress = null;
+
+			for (let i = 0; i < tmpEntityNames.length; i++)
+			{
+				let tmpName = tmpEntityNames[i];
+				let tmpP = tmpLoadState.LoadProgress[tmpName];
+				tmpTotalPushed += (tmpP.Pushed || 0);
+
+				if (tmpP.Status === 'Complete') tmpCompleted++;
+				else if (tmpP.Status === 'Error') { tmpErrors++; tmpCompleted++; }
+				else if (tmpP.Status === 'Pushing')
+				{
+					tmpActiveEntity = tmpName;
+					tmpActiveProgress = { Total: tmpP.Total, Pushed: tmpP.Pushed };
+				}
+				else tmpPending++;
+			}
+
+			// Determine phase
+			let tmpPhase = 'idle';
+			let tmpMessage = 'Idle';
+
+			if (!tmpLoadState.ComprehensionData && !tmpLoadState.LoadRunning)
+			{
+				tmpPhase = 'idle';
+				tmpMessage = 'No comprehension data loaded';
+			}
+			else if (tmpLoadState.ComprehensionData && !tmpLoadState.LoadRunning && tmpEntityNames.length === 0)
+			{
+				tmpPhase = 'ready';
+				tmpMessage = `Ready to load ${tmpLoadState.ComprehensionEntityList.length} entities (${tmpTotalRecords} records)`;
+			}
+			else if (tmpLoadState.LoadStopping)
+			{
+				tmpPhase = 'stopping';
+				tmpMessage = 'Stopping...';
+			}
+			else if (tmpLoadState.LoadRunning)
+			{
+				tmpPhase = 'loading';
+				if (tmpActiveEntity)
+				{
+					tmpMessage = `Pushing ${tmpActiveEntity}: ${tmpActiveProgress.Pushed} / ${tmpActiveProgress.Total} records`;
+				}
+				else
+				{
+					tmpMessage = `Loading ${tmpEntityNames.length} entities`;
+				}
+			}
+			else if (tmpEntityNames.length > 0 && !tmpLoadState.LoadRunning)
+			{
+				tmpPhase = 'complete';
+				tmpMessage = `Load complete: ${tmpCompleted} entities, ${tmpTotalPushed} records pushed`;
+			}
+
+			// Calculate elapsed time and ETA
+			let tmpElapsed = null;
+			let tmpETA = null;
+			if (tmpLoadState.LoadStartTime)
+			{
+				let tmpEndTime = tmpLoadState.LoadEndTime ? new Date(tmpLoadState.LoadEndTime).getTime() : Date.now();
+				let tmpElapsedMs = tmpEndTime - new Date(tmpLoadState.LoadStartTime).getTime();
+				let tmpElapsedSec = Math.round(tmpElapsedMs / 1000);
+
+				if (tmpElapsedSec < 60)
+				{
+					tmpElapsed = tmpElapsedSec + 's';
+				}
+				else
+				{
+					let tmpMin = Math.floor(tmpElapsedSec / 60);
+					let tmpSec = tmpElapsedSec % 60;
+					tmpElapsed = tmpMin + ':' + (tmpSec < 10 ? '0' : '') + tmpSec;
+				}
+
+				// ETA based on record-weighted progress
+				if (tmpLoadState.LoadRunning && tmpTotalPushed > 0 && tmpTotalRecords > 0)
+				{
+					let tmpRemainingRecords = tmpTotalRecords - tmpTotalPushed;
+					let tmpMsPerRecord = tmpElapsedMs / tmpTotalPushed;
+					let tmpETAMs = tmpRemainingRecords * tmpMsPerRecord;
+					let tmpETASec = Math.round(tmpETAMs / 1000);
+
+					if (tmpETASec < 60)
+					{
+						tmpETA = tmpETASec + 's';
+					}
+					else
+					{
+						let tmpETAMin = Math.floor(tmpETASec / 60);
+						let tmpETAS = tmpETASec % 60;
+						tmpETA = tmpETAMin + ':' + (tmpETAS < 10 ? '0' : '') + tmpETAS;
+					}
+				}
+			}
+
+			pResponse.send(200,
+				{
+					Phase: tmpPhase,
+					Message: tmpMessage,
+					TotalEntities: tmpEntityNames.length,
+					Completed: tmpCompleted,
+					Errors: tmpErrors,
+					Pending: tmpPending,
+					TotalPushed: tmpTotalPushed,
+					TotalRecords: tmpTotalRecords,
+					ActiveEntity: tmpActiveEntity,
+					ActiveProgress: tmpActiveProgress,
+					Elapsed: tmpElapsed,
+					ETA: tmpETA,
+					ThroughputSamples: tmpLoadState.ThroughputSamples || []
+				});
+			return fNext();
+		});
+
+	// GET /comprehension_load/load/report
+	pOratorServiceServer.get(`${tmpPrefix}/load/report`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpLoadState.LoadReport)
+			{
+				if (tmpLoadState.LoadStartTime && !tmpLoadState.LoadRunning)
+				{
+					pComprehensionLoaderService.generateLoadReport();
+				}
+			}
+
+			if (tmpLoadState.LoadReport)
+			{
+				pResponse.send(200, tmpLoadState.LoadReport);
+			}
+			else
+			{
+				pResponse.send(200, { ReportVersion: null, Message: 'No load report available.' });
+			}
+			return fNext();
+		});
+};

package/source/services/comprehension-loader/ComprehensionLoader-Command-Schema.js

@@ -0,0 +1,97 @@
+/**
+ * ComprehensionLoader Schema Management Routes
+ *
+ * Registers /comprehension_load/schema/* endpoints for fetching remote
+ * schemas to discover valid entities on the target server.
+ *
+ * @param {Object} pComprehensionLoaderService - The RetoldDataServiceComprehensionLoader instance
+ * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
+ */
+module.exports = (pComprehensionLoaderService, pOratorServiceServer) =>
+{
+	let tmpFable = pComprehensionLoaderService.fable;
+	let tmpLoadState = pComprehensionLoaderService.loadState;
+	let tmpPict = pComprehensionLoaderService.pict;
+	let tmpPrefix = pComprehensionLoaderService.routePrefix;
+
+	// POST /comprehension_load/schema/fetch
+	pOratorServiceServer.post(`${tmpPrefix}/schema/fetch`,
+		(pRequest, pResponse, fNext) =>
+		{
+			let tmpBody = pRequest.body || {};
+			let tmpSchemaURL = tmpBody.SchemaURL;
+
+			if (!tmpSchemaURL)
+			{
+				if (tmpLoadState.RemoteServerURL)
+				{
+					tmpSchemaURL = tmpLoadState.RemoteServerURL + 'Retold/Models';
+				}
+				else
+				{
+					pResponse.send(400, { Success: false, Error: 'SchemaURL is required (or configure a session first).' });
+					return fNext();
+				}
+			}
+
+			tmpFable.log.info(`Comprehension Loader: Fetching remote schema from ${tmpSchemaURL}...`);
+
+			tmpPict.RestClient.getJSON(tmpSchemaURL,
+				(pError, pHTTPResponse, pData) =>
+				{
+					if (pError)
+					{
+						tmpFable.log.error(`Comprehension Loader: Schema fetch error: ${pError.message || pError}`);
+						pResponse.send(500, { Success: false, Error: `Schema fetch error: ${pError.message || pError}` });
+						return fNext();
+					}
+
+					if (!pHTTPResponse || pHTTPResponse.statusCode !== 200)
+					{
+						let tmpStatus = pHTTPResponse ? pHTTPResponse.statusCode : 'unknown';
+						tmpFable.log.error(`Comprehension Loader: Schema fetch returned HTTP ${tmpStatus}`);
+						pResponse.send(500, { Success: false, Error: `Schema fetch returned HTTP ${tmpStatus}` });
+						return fNext();
+					}
+
+					tmpLoadState.RemoteSchema = pData;
+
+					let tmpEntityNames = [];
+					if (pData && pData.Tables)
+					{
+						tmpEntityNames = Object.keys(pData.Tables);
+					}
+					tmpLoadState.RemoteEntityList = tmpEntityNames;
+
+					tmpFable.log.info(`Comprehension Loader: Fetched schema with ${tmpEntityNames.length} entities: [${tmpEntityNames.join(', ')}]`);
+
+					pResponse.send(200,
+						{
+							Success: true,
+							SchemaURL: tmpSchemaURL,
+							EntityCount: tmpEntityNames.length,
+							Entities: tmpEntityNames
+						});
+					return fNext();
+				});
+		});
+
+	// GET /comprehension_load/schema
+	pOratorServiceServer.get(`${tmpPrefix}/schema`,
+		(pRequest, pResponse, fNext) =>
+		{
+			if (!tmpLoadState.RemoteSchema)
+			{
+				pResponse.send(200, { Fetched: false, Entities: [] });
+				return fNext();
+			}
+
+			pResponse.send(200,
+				{
+					Fetched: true,
+					EntityCount: tmpLoadState.RemoteEntityList.length,
+					Entities: tmpLoadState.RemoteEntityList
+				});
+			return fNext();
+		});
+};