ultravisor 1.0.22 → 1.0.24

package/source/services/tasks/file-system/Ultravisor-TaskConfigs-FileSystem.cjs

@@ -11,6 +11,7 @@
 
 const libFS = require('fs');
 const libPath = require('path');
+const libFileStream = require('ultravisor-file-stream');
 
 /**
  * Recursively sort all object keys alphabetically (deep).
@@ -424,6 +425,86 @@ module.exports =
 		}
 	},
 
+	// ── chunked-write ─────────────────────────────────────────
+	// Multi-chunk file writer for large transfers (model pushes across
+	// the beacon mesh). Thin adapter over `ultravisor-file-stream`'s
+	// writeChunk primitive — this task type just resolves the staging
+	// path, delegates to the library, and translates the return value
+	// into task-event shape.
+	{
+		Definition: require('./definitions/chunked-write.json'),
+		Execute: function (pTask, pResolvedSettings, pExecutionContext, fCallback)
+		{
+			let tmpFileLocation = pResolvedSettings.FilePath || '';
+			if (!tmpFileLocation)
+			{
+				return fCallback(null, { EventToFire: 'Error', Outputs: {}, Log: ['ChunkedWrite: no FilePath specified.'] });
+			}
+
+			let tmpFilePath = pTask.resolveFilePath(tmpFileLocation, pExecutionContext.StagingPath);
+
+			let tmpResult = libFileStream.writeChunk(
+				Object.assign({}, pResolvedSettings, { TargetPath: tmpFilePath }));
+
+			let tmpChunkIndex = parseInt(pResolvedSettings.ChunkIndex, 10) || 0;
+			let tmpTotalChunks = parseInt(pResolvedSettings.TotalChunks, 10) || 0;
+			let tmpOffset = parseInt(pResolvedSettings.Offset, 10) || 0;
+
+			if (tmpResult.Status === 'Sha256Mismatch')
+			{
+				return fCallback(null, {
+					EventToFire: 'Sha256Mismatch',
+					Outputs:
+					{
+						FilePath: tmpFilePath,
+						PartialFilePath: tmpResult.Result ? tmpResult.Result.PartialFilePath : tmpFilePath + '.part',
+						BytesWritten: tmpResult.Result ? tmpResult.Result.BytesWritten : 0,
+						TotalBytesOnDisk: tmpResult.Result ? tmpResult.Result.TotalBytesOnDisk : 0,
+						IsComplete: false,
+						Sha256Verified: false,
+						ChunkIndex: tmpChunkIndex
+					},
+					Log: [ `ChunkedWrite: ${tmpResult.Error}` ]
+				});
+			}
+			if (tmpResult.Status !== 'Success')
+			{
+				return fCallback(null, {
+					EventToFire: 'Error',
+					Outputs: {},
+					Log: [ `ChunkedWrite: ${tmpResult.Error || 'unknown error'}` ]
+				});
+			}
+
+			let tmpR = tmpResult.Result;
+			let tmpEvent = tmpR.IsComplete ? 'WriteComplete' : 'ChunkWritten';
+			let tmpLogSuffix = tmpR.IsComplete
+				? `final chunk ${tmpChunkIndex}`
+					+ (tmpTotalChunks ? `/${tmpTotalChunks}` : '')
+					+ ` @ offset ${tmpOffset} (${tmpR.BytesWritten} B); ${tmpR.TotalBytesOnDisk} B total; `
+					+ (pResolvedSettings.Sha256 ? `sha256 ${tmpR.Sha256Verified ? 'verified' : 'unverified'}; ` : '')
+					+ `renamed ${tmpFilePath}.part -> ${tmpFilePath}`
+				: `chunk ${tmpChunkIndex}`
+					+ (tmpTotalChunks ? `/${tmpTotalChunks}` : '')
+					+ ` @ offset ${tmpOffset} (${tmpR.BytesWritten} B, ${tmpR.TotalBytesOnDisk} B total on disk)`;
+
+			return fCallback(null, {
+				EventToFire: tmpEvent,
+				Outputs:
+				{
+					FilePath: tmpR.TargetPath,
+					PartialFilePath: tmpR.PartialFilePath,
+					BytesWritten: tmpR.BytesWritten,
+					TotalBytesOnDisk: tmpR.TotalBytesOnDisk,
+					IsComplete: tmpR.IsComplete,
+					Sha256Verified: tmpR.Sha256Verified,
+					ChunkIndex: tmpChunkIndex
+				},
+				Log: [ `ChunkedWrite: ${tmpLogSuffix}` ]
+			});
+		}
+	},
+
 	// ── read-file-buffered ────────────────────────────────────
 	{
 		Definition: require('./definitions/read-file-buffered.json'),

package/source/services/tasks/file-system/definitions/chunked-write.json

@@ -0,0 +1,38 @@
+{
+	"Hash": "chunked-write",
+	"Type": "chunked-write",
+	"Name": "Chunked Write",
+	"Description": "Writes one chunk of a large file at a given byte offset. Intended for multi-chunk transfers (e.g. multi-GB model pushes across the beacon mesh). Chunks are written to a .part sidecar file; the final chunk triggers fsync, optional sha256 verification, and atomic rename to the target path.",
+	"Category": "file-io",
+	"Capability": "File System",
+	"Action": "ChunkedWrite",
+	"Tier": "Platform",
+	"EventInputs": [{ "Name": "BeginChunkWrite", "Description": "Triggers writing this chunk" }],
+	"EventOutputs": [
+		{ "Name": "ChunkWritten", "Description": "Fires after a non-final chunk is written" },
+		{ "Name": "WriteComplete", "Description": "Fires after the final chunk is written, fsynced, verified, and the .part file has been renamed to the target path" },
+		{ "Name": "Sha256Mismatch", "Description": "Fires when IsFinal=true and the computed sha256 of the completed .part file does not match the Sha256 input", "IsError": true },
+		{ "Name": "Error", "Description": "Fires on write/fsync/rename failure", "IsError": true }
+	],
+	"SettingsInputs": [
+		{ "Name": "FilePath", "DataType": "String", "Required": true, "Description": "Final path the completed file will live at. Chunks are written to <FilePath>.part until the final chunk renames it." },
+		{ "Name": "Content", "DataType": "String", "Required": true, "Description": "This chunk's bytes. If ContentEncoding=\"base64\", interpreted as base64 (for binary transport over JSON). Otherwise written as-is per Encoding." },
+		{ "Name": "ContentEncoding", "DataType": "String", "Required": false, "Default": "base64", "Description": "How to interpret Content on the wire. \"base64\" (default) is safe for binary; \"utf8\" for text." },
+		{ "Name": "Offset", "DataType": "Number", "Required": true, "Description": "Byte offset where this chunk begins in the final file. Zero for the first chunk; subsequent chunks advance by the decoded byte length of the prior chunk's content." },
+		{ "Name": "ChunkIndex", "DataType": "Number", "Required": false, "Description": "Zero-based index of this chunk in the sequence (informational; used in logs)" },
+		{ "Name": "TotalChunks", "DataType": "Number", "Required": false, "Description": "Total chunk count the sender expects to send (informational; used in logs)" },
+		{ "Name": "IsFinal", "DataType": "Boolean", "Required": false, "Description": "True for the last chunk in the sequence. Triggers fsync, optional sha256 verify, and atomic rename <FilePath>.part -> <FilePath>." },
+		{ "Name": "Sha256", "DataType": "String", "Required": false, "Description": "Hex-encoded sha256 of the complete final file. When set and IsFinal=true, the task verifies before renaming; mismatch fires Sha256Mismatch and deletes the .part file." },
+		{ "Name": "CreateDirectory", "DataType": "Boolean", "Required": false, "Default": true, "Description": "If true (default), the target directory is created recursively when missing." }
+	],
+	"StateOutputs": [
+		{ "Name": "FilePath", "DataType": "String", "Description": "The fully resolved absolute final path" },
+		{ "Name": "PartialFilePath", "DataType": "String", "Description": "The <FilePath>.part file in progress (or the finalized file after IsFinal)" },
+		{ "Name": "BytesWritten", "DataType": "Number", "Description": "Bytes written for this chunk" },
+		{ "Name": "TotalBytesOnDisk", "DataType": "Number", "Description": "Current size of the .part file (or the final file if complete)" },
+		{ "Name": "IsComplete", "DataType": "Boolean", "Description": "True only after the final chunk successfully renamed the .part file to FilePath" },
+		{ "Name": "Sha256Verified", "DataType": "Boolean", "Description": "True if IsFinal=true, Sha256 was provided, and verification passed" },
+		{ "Name": "ChunkIndex", "DataType": "Number", "Description": "Echo of input ChunkIndex, for convenience in downstream wiring" }
+	],
+	"DefaultSettings": { "FilePath": "", "Content": "", "ContentEncoding": "base64", "Offset": 0, "ChunkIndex": 0, "TotalChunks": 0, "IsFinal": false, "Sha256": "", "CreateDirectory": true }
+}

package/source/web_server/Ultravisor-API-Server.cjs

@@ -28,6 +28,9 @@ class UltravisorAPIServer extends libPictService
 		this._WebSocketSubscriptions = {};
 		// Map of BeaconID -> WebSocket for beacon worker connections
 		this._BeaconWebSockets = {};
+		// Set of WebSocket clients subscribed to the queue.* topic
+		// (broadcast by UltravisorBeaconScheduler).
+		this._QueueSubscribers = new Set();
 	}
 
 	/**
@@ -956,6 +959,42 @@ class UltravisorAPIServer extends libPictService
 				}.bind(this)
 			);
 
+		// --- Retry from checkpoint ---
+		// Re-dispatches the failed node in a completed/errored operation.
+		// All prior node outputs are preserved; only the failed node re-runs.
+		this._OratorServer.post
+			(
+				'/Operation/:RunHash/Retry',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpBody = pRequest.body || {};
+					let tmpEngine = this._getService('UltravisorExecutionEngine');
+					let tmpRunHash = pRequest.params.RunHash;
+
+					tmpEngine.retryFromCheckpoint(tmpRunHash,
+						{
+							NodeHash: tmpBody.NodeHash || null,
+							SettingsOverrides: tmpBody.SettingsOverrides || {}
+						},
+						function (pError, pContext)
+						{
+							if (pError)
+							{
+								pResponse.send(400, { Error: pError.message });
+								return fNext();
+							}
+							pResponse.send({
+								Success: true,
+								Status: pContext.Status,
+								Hash: pContext.Hash,
+								RetryNode: tmpBody.NodeHash || '(auto-detected)',
+								WaitingTasks: Object.keys(pContext.WaitingTasks || {})
+							});
+							return fNext();
+						});
+				}.bind(this)
+			);
+
 		// --- Operation Resume (for value-input tasks) ---
 		this._OratorServer.post
 			(
@@ -1480,6 +1519,267 @@ class UltravisorAPIServer extends libPictService
 				}.bind(this)
 			);
 
+		// --- Run Lifecycle: Start a hub-owned run ---
+		this._OratorServer.post
+			(
+				'/Beacon/Run/Start',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpSession = this._requireSession(pRequest, pResponse, fNext);
+					if (!tmpSession) { return; }
+
+					let tmpRunManager = this._getService('UltravisorBeaconRunManager');
+					if (!tmpRunManager)
+					{
+						pResponse.send(500, { Error: 'BeaconRunManager service not available.' });
+						return fNext();
+					}
+
+					let tmpBody = pRequest.body || {};
+					let tmpIdempotency = pRequest.headers['x-idempotency-key'] || tmpBody.IdempotencyKey || '';
+
+					let tmpRun = tmpRunManager.startRun({
+						IdempotencyKey: tmpIdempotency,
+						SubmitterTag: tmpBody.SubmitterTag || '',
+						Metadata: tmpBody.Metadata || {}
+					});
+
+					pResponse.send({
+						Success: true,
+						RunID: tmpRun.RunID,
+						State: tmpRun.State,
+						StartedAt: tmpRun.StartedAt,
+						IdempotencyKey: tmpRun.IdempotencyKey || ''
+					});
+					return fNext();
+				}.bind(this)
+			);
+
+		// --- Run Lifecycle: End a run explicitly ---
+		this._OratorServer.post
+			(
+				'/Beacon/Run/:RunID/End',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpSession = this._requireSession(pRequest, pResponse, fNext);
+					if (!tmpSession) { return; }
+
+					let tmpRunManager = this._getService('UltravisorBeaconRunManager');
+					if (!tmpRunManager)
+					{
+						pResponse.send(500, { Error: 'BeaconRunManager service not available.' });
+						return fNext();
+					}
+
+					let tmpBody = pRequest.body || {};
+					let tmpState = tmpBody.State || 'Ended';
+					let tmpOK = tmpRunManager.endRun(pRequest.params.RunID, tmpState);
+					if (!tmpOK)
+					{
+						pResponse.send(404, { Error: `Run [${pRequest.params.RunID}] not found.` });
+						return fNext();
+					}
+					pResponse.send({ Success: true, RunID: pRequest.params.RunID, State: tmpState });
+					return fNext();
+				}.bind(this)
+			);
+
+		// --- Async Enqueue: returns the WorkItemHash immediately ---
+		this._OratorServer.post
+			(
+				'/Beacon/Work/Enqueue',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpSession = this._requireSession(pRequest, pResponse, fNext);
+					if (!tmpSession) { return; }
+
+					let tmpCoordinator = this._getService('UltravisorBeaconCoordinator');
+					if (!tmpCoordinator)
+					{
+						pResponse.send(500, { Error: 'BeaconCoordinator service not available.' });
+						return fNext();
+					}
+
+					let tmpBody = pRequest.body || {};
+					if (!tmpBody.Capability)
+					{
+						pResponse.send(400, { Error: 'Capability is required.' });
+						return fNext();
+					}
+
+					// If the client didn't supply a RunID, mint one on the fly so
+					// every enqueue ends up attached to a durable run record.
+					let tmpRunID = tmpBody.RunID || '';
+					if (!tmpRunID)
+					{
+						let tmpRunManager = this._getService('UltravisorBeaconRunManager');
+						if (tmpRunManager)
+						{
+							let tmpIdempotency = pRequest.headers['x-idempotency-key'] || '';
+							let tmpRun = tmpRunManager.startRun({
+								IdempotencyKey: tmpIdempotency,
+								SubmitterTag: tmpBody.SubmitterTag || '',
+								Metadata: tmpBody.Metadata || {}
+							});
+							tmpRunID = tmpRun.RunID;
+						}
+					}
+
+					let tmpWorkItemInfo = {
+						RunID: tmpRunID,
+						RunHash: tmpBody.RunHash || '',
+						NodeHash: tmpBody.NodeHash || '',
+						OperationHash: tmpBody.OperationHash || '',
+						Capability: tmpBody.Capability,
+						Action: tmpBody.Action || 'Execute',
+						Settings: tmpBody.Settings || {},
+						AffinityKey: tmpBody.AffinityKey || '',
+						TimeoutMs: parseInt(tmpBody.TimeoutMs, 10) || 0,
+						Priority: (tmpBody.Priority != null) ? parseInt(tmpBody.Priority, 10) : null
+					};
+
+					let tmpWorkItem = tmpCoordinator.enqueueWorkItem(tmpWorkItemInfo);
+					pResponse.send({
+						Success: true,
+						RunID: tmpRunID,
+						WorkItemHash: tmpWorkItem.WorkItemHash,
+						Status: tmpWorkItem.Status,
+						EnqueuedAt: tmpWorkItem.EnqueuedAt,
+						Priority: tmpWorkItem.Priority
+					});
+					return fNext();
+				}.bind(this)
+			);
+
+		// --- Work Item Cancellation ---
+		this._OratorServer.post
+			(
+				'/Beacon/Work/:WorkItemHash/Cancel',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpSession = this._requireSession(pRequest, pResponse, fNext);
+					if (!tmpSession) { return; }
+
+					let tmpScheduler = this._getService('UltravisorBeaconScheduler');
+					if (!tmpScheduler)
+					{
+						pResponse.send(500, { Error: 'BeaconScheduler service not available.' });
+						return fNext();
+					}
+					let tmpBody = pRequest.body || {};
+					let tmpResult = tmpScheduler.requestCancel(pRequest.params.WorkItemHash,
+						tmpBody.Reason || 'cancel requested');
+					if (tmpResult.Error === 'not found')
+					{
+						pResponse.send(404, { Error: `Work item [${pRequest.params.WorkItemHash}] not found.` });
+						return fNext();
+					}
+					pResponse.send(Object.assign({ Success: !tmpResult.Error, WorkItemHash: pRequest.params.WorkItemHash }, tmpResult));
+					return fNext();
+				}.bind(this)
+			);
+
+		// --- Work Item Reorder (Upcoming bucket) ---
+		this._OratorServer.post
+			(
+				'/Beacon/Work/:WorkItemHash/Reorder',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpSession = this._requireSession(pRequest, pResponse, fNext);
+					if (!tmpSession) { return; }
+
+					let tmpScheduler = this._getService('UltravisorBeaconScheduler');
+					if (!tmpScheduler)
+					{
+						pResponse.send(500, { Error: 'BeaconScheduler service not available.' });
+						return fNext();
+					}
+					let tmpBody = pRequest.body || {};
+					let tmpDirection = tmpBody.Direction;
+					if (tmpDirection !== 'up' && tmpDirection !== 'down')
+					{
+						pResponse.send(400, { Error: 'Direction must be "up" or "down".' });
+						return fNext();
+					}
+					let tmpResult = tmpScheduler.reorderWorkItem(pRequest.params.WorkItemHash, tmpDirection);
+					if (tmpResult.Error === 'not found')
+					{
+						pResponse.send(404, { Error: `Work item [${pRequest.params.WorkItemHash}] not found.` });
+						return fNext();
+					}
+					if (!tmpResult.Reordered)
+					{
+						// 409 for "can't reorder in this state" (at edge, not Upcoming, etc.)
+						pResponse.send(409, tmpResult);
+						return fNext();
+					}
+					pResponse.send(Object.assign({ Success: true }, tmpResult));
+					return fNext();
+				}.bind(this)
+			);
+
+		// --- Per-Work-Item Event Log ---
+		this._OratorServer.get
+			(
+				'/Beacon/Work/:WorkItemHash/Events',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpSession = this._requireSession(pRequest, pResponse, fNext);
+					if (!tmpSession) { return; }
+
+					let tmpStore = this._getService('UltravisorBeaconQueueStore');
+					if (!tmpStore || !tmpStore.isEnabled || !tmpStore.isEnabled())
+					{
+						pResponse.send(503, { Error: 'BeaconQueueStore service not available.' });
+						return fNext();
+					}
+
+					let tmpHash = pRequest.params.WorkItemHash;
+					let tmpLimit = pRequest.query ? parseInt(pRequest.query.limit, 10) || 500 : 500;
+					let tmpItem = tmpStore.getWorkItemByHash(tmpHash);
+					if (!tmpItem)
+					{
+						pResponse.send(404, { Error: `Work item [${tmpHash}] not found.` });
+						return fNext();
+					}
+
+					let tmpEvents = tmpStore.listEventsForWorkItem(tmpHash, tmpLimit);
+					pResponse.send({
+						WorkItemHash: tmpHash,
+						Events: tmpEvents
+					});
+					return fNext();
+				}.bind(this)
+			);
+
+		// --- Queue Snapshot (buckets + summary + items) ---
+		this._OratorServer.get
+			(
+				'/Beacon/Queue',
+				function (pRequest, pResponse, fNext)
+				{
+					let tmpScheduler = this._getService('UltravisorBeaconScheduler');
+					let tmpStore = this._getService('UltravisorBeaconQueueStore');
+					let tmpSummary = tmpScheduler ? tmpScheduler.summarize() : null;
+					let tmpBucket = pRequest.query ? pRequest.query.bucket : null;
+					let tmpLimit = pRequest.query ? parseInt(pRequest.query.limit, 10) || 200 : 200;
+
+					let tmpItems = tmpScheduler ? tmpScheduler.listBuckets(tmpBucket, tmpLimit) : [];
+					let tmpHistorical = null;
+					if (tmpStore && tmpStore.isEnabled && tmpStore.isEnabled()
+						&& pRequest.query && pRequest.query.include === 'history')
+					{
+						tmpHistorical = tmpStore.listWorkItems({ Limit: tmpLimit, OrderBy: 'IDBeaconWorkItem DESC' });
+					}
+					pResponse.send({
+						Summary: tmpSummary,
+						Items: tmpItems,
+						History: tmpHistorical
+					});
+					return fNext();
+				}.bind(this)
+			);
+
 		// --- Direct Dispatch (synchronous) ---
 		this._OratorServer.post
 			(
@@ -2002,6 +2302,30 @@ class UltravisorAPIServer extends libPictService
 						{
 							this._handleBeaconWSDeregister(pWebSocket, tmpData);
 						}
+						else if (tmpData.Action === 'QueueSubscribe')
+						{
+							this._QueueSubscribers.add(pWebSocket);
+							pWebSocket._QueueSubscribed = true;
+							// Send current summary immediately so the UI
+							// doesn't wait a full tick to populate.
+							let tmpSched = this._getService('UltravisorBeaconScheduler');
+							if (tmpSched && typeof tmpSched.summarize === 'function')
+							{
+								try
+								{
+									pWebSocket.send(JSON.stringify({
+										Topic: 'queue.summary',
+										Payload: tmpSched.summarize()
+									}));
+								}
+								catch (pErr) { /* ignore */ }
+							}
+						}
+						else if (tmpData.Action === 'QueueUnsubscribe')
+						{
+							this._QueueSubscribers.delete(pWebSocket);
+							pWebSocket._QueueSubscribed = false;
+						}
 					}.bind(this));
 
 				pWebSocket.on('close',
@@ -2009,6 +2333,7 @@ class UltravisorAPIServer extends libPictService
 					{
 						this._unsubscribeClient(pWebSocket);
 						this._cleanupBeaconWS(pWebSocket);
+						this._QueueSubscribers.delete(pWebSocket);
 					}.bind(this));
 			}.bind(this));
 
@@ -2029,9 +2354,38 @@ class UltravisorAPIServer extends libPictService
 				this.pushWorkItemToBeacon.bind(this));
 		}
 
+		// Wire the scheduler's broadcast hook to the queue.* WebSocket topic.
+		let tmpScheduler = this._getService('UltravisorBeaconScheduler');
+		if (tmpScheduler && typeof tmpScheduler.setBroadcastHandler === 'function')
+		{
+			tmpScheduler.setBroadcastHandler(
+				this._broadcastQueueTopic.bind(this));
+		}
+
 		this.log.info('Ultravisor WebSocket: execution event WebSocket server initialized.');
 	}
 
+	/**
+	 * Fan out a queue.* topic payload to all subscribed WebSocket clients.
+	 *
+	 * @param {string} pTopic - e.g. "queue.enqueued" / "queue.dispatched" / ...
+	 * @param {object} pPayload - topic-specific JSON body
+	 */
+	_broadcastQueueTopic(pTopic, pPayload)
+	{
+		if (!this._QueueSubscribers || this._QueueSubscribers.size === 0) return;
+		let tmpMessage = JSON.stringify({ Topic: pTopic, Payload: pPayload });
+		this._QueueSubscribers.forEach(
+			function (pClient)
+			{
+				if (pClient.readyState === libWebSocket.OPEN)
+				{
+					try { pClient.send(tmpMessage); }
+					catch (pErr) { /* best effort */ }
+				}
+			});
+	}
+
 	/**
 	 * Handle an execution event from the manifest service and broadcast
 	 * it to all WebSocket clients subscribed to that RunHash.