ultravisor 1.0.22 → 1.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/queue-followups/03-config-migration.md +123 -0
- package/docs/queue-followups/04-direct-dispatch-phase-emission.md +128 -0
- package/package.json +3 -1
- package/source/Ultravisor.cjs +4 -0
- package/source/datamodel/Ultravisor-BeaconQueue.json +165 -0
- package/source/services/Ultravisor-Beacon-ActionDefaults.cjs +174 -0
- package/source/services/Ultravisor-Beacon-Coordinator.cjs +382 -6
- package/source/services/Ultravisor-Beacon-RunManager.cjs +169 -0
- package/source/services/Ultravisor-Beacon-Scheduler.cjs +789 -0
- package/source/services/Ultravisor-ExecutionEngine.cjs +242 -6
- package/source/services/Ultravisor-ExecutionManifest.cjs +1 -0
- package/source/services/persistence/Ultravisor-Beacon-QueueStore.cjs +886 -0
- package/source/services/tasks/file-system/Ultravisor-TaskConfigs-FileSystem.cjs +81 -0
- package/source/services/tasks/file-system/definitions/chunked-write.json +38 -0
- package/source/web_server/Ultravisor-API-Server.cjs +354 -0
- package/test/Ultravisor_BeaconQueue_tests.js +502 -0
- package/test/Ultravisor_tests.js +132 -0
|
@@ -0,0 +1,502 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the new beacon queueing system:
|
|
3
|
+
* - UltravisorBeaconQueueStore (SQLite persistence + migrations)
|
|
4
|
+
* - UltravisorBeaconRunManager (hub-assigned RunID + idempotency)
|
|
5
|
+
* - UltravisorBeaconActionDefaults (config normalization)
|
|
6
|
+
* - UltravisorBeaconScheduler (dispatch, health, buckets, cancellation)
|
|
7
|
+
* - Coordinator integration (WorkItem fields, store persistence)
|
|
8
|
+
* - RetoldLabs-QueuePhases emitter (phases.jsonl records)
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const libPict = require('pict');
|
|
12
|
+
const libFS = require('fs');
|
|
13
|
+
const libPath = require('path');
|
|
14
|
+
const libOS = require('os');
|
|
15
|
+
|
|
16
|
+
const Chai = require('chai');
|
|
17
|
+
const Expect = Chai.expect;
|
|
18
|
+
|
|
19
|
+
const libUltravisorBeaconCoordinator = require('../source/services/Ultravisor-Beacon-Coordinator.cjs');
|
|
20
|
+
const libUltravisorBeaconQueueStore = require('../source/services/persistence/Ultravisor-Beacon-QueueStore.cjs');
|
|
21
|
+
const libUltravisorBeaconRunManager = require('../source/services/Ultravisor-Beacon-RunManager.cjs');
|
|
22
|
+
const libUltravisorBeaconActionDefaults = require('../source/services/Ultravisor-Beacon-ActionDefaults.cjs');
|
|
23
|
+
const libUltravisorBeaconScheduler = require('../source/services/Ultravisor-Beacon-Scheduler.cjs');
|
|
24
|
+
const libQueuePhases = require('../../retold-labs/source/RetoldLabs-QueuePhases.cjs');
|
|
25
|
+
|
|
26
|
+
const TEST_BASE = libPath.resolve(__dirname, '..', '.test_staging_queue');
|
|
27
|
+
|
|
28
|
+
function ensureClean(pDir)
|
|
29
|
+
{
|
|
30
|
+
if (libFS.existsSync(pDir))
|
|
31
|
+
{
|
|
32
|
+
libFS.rmSync(pDir, { recursive: true, force: true });
|
|
33
|
+
}
|
|
34
|
+
libFS.mkdirSync(pDir, { recursive: true });
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function buildFable(pStoragePath)
|
|
38
|
+
{
|
|
39
|
+
let tmpFable = new libPict({
|
|
40
|
+
Product: 'Ultravisor-Queue-Test',
|
|
41
|
+
LogLevel: 5,
|
|
42
|
+
UltravisorFileStorePath: pStoragePath,
|
|
43
|
+
UltravisorHubInstanceID: 'testhub'
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
tmpFable.addAndInstantiateServiceTypeIfNotExists('UltravisorBeaconQueueStore', libUltravisorBeaconQueueStore);
|
|
47
|
+
tmpFable.addAndInstantiateServiceTypeIfNotExists('UltravisorBeaconCoordinator', libUltravisorBeaconCoordinator);
|
|
48
|
+
tmpFable.addAndInstantiateServiceTypeIfNotExists('UltravisorBeaconRunManager', libUltravisorBeaconRunManager);
|
|
49
|
+
tmpFable.addAndInstantiateServiceTypeIfNotExists('UltravisorBeaconActionDefaults', libUltravisorBeaconActionDefaults);
|
|
50
|
+
tmpFable.addAndInstantiateServiceTypeIfNotExists('UltravisorBeaconScheduler', libUltravisorBeaconScheduler);
|
|
51
|
+
|
|
52
|
+
let tmpStore = Object.values(tmpFable.servicesMap.UltravisorBeaconQueueStore)[0];
|
|
53
|
+
tmpStore.initialize(pStoragePath);
|
|
54
|
+
|
|
55
|
+
return tmpFable;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function getService(pFable, pName)
|
|
59
|
+
{
|
|
60
|
+
let tmpMap = pFable.servicesMap[pName];
|
|
61
|
+
return tmpMap ? Object.values(tmpMap)[0] : null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function addStubBeacon(pCoordinator, pBeaconID, pCapabilities)
|
|
65
|
+
{
|
|
66
|
+
pCoordinator._Beacons[pBeaconID] = {
|
|
67
|
+
BeaconID: pBeaconID,
|
|
68
|
+
Name: pBeaconID,
|
|
69
|
+
Capabilities: pCapabilities || ['Shell'],
|
|
70
|
+
MaxConcurrent: 2,
|
|
71
|
+
CurrentWorkItems: [],
|
|
72
|
+
Status: 'Online',
|
|
73
|
+
LastHeartbeat: new Date().toISOString()
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
suite('Ultravisor Beacon Queue', () =>
|
|
78
|
+
{
|
|
79
|
+
let _TestDir = '';
|
|
80
|
+
|
|
81
|
+
setup(() =>
|
|
82
|
+
{
|
|
83
|
+
_TestDir = libPath.join(TEST_BASE, `t-${Date.now()}-${Math.floor(Math.random() * 1000)}`);
|
|
84
|
+
ensureClean(_TestDir);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
teardown(() =>
|
|
88
|
+
{
|
|
89
|
+
// Best-effort cleanup — individual stores close their DB.
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
suite('QueueStore', () =>
|
|
93
|
+
{
|
|
94
|
+
test('provisions all tables and is enabled after initialize', () =>
|
|
95
|
+
{
|
|
96
|
+
let tmpFable = buildFable(_TestDir);
|
|
97
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
98
|
+
Expect(tmpStore.isEnabled()).to.equal(true);
|
|
99
|
+
// Hot-lookup paths — the store should at minimum return empty arrays/objects, not throw.
|
|
100
|
+
Expect(tmpStore.listWorkItems({})).to.be.an('array');
|
|
101
|
+
Expect(tmpStore.countByStatus()).to.be.an('object');
|
|
102
|
+
Expect(tmpStore.listActionDefaults()).to.be.an('array');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test('upsertWorkItem / getWorkItemByHash round-trips including nested Settings and Health', () =>
|
|
106
|
+
{
|
|
107
|
+
let tmpFable = buildFable(_TestDir);
|
|
108
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
109
|
+
tmpStore.upsertWorkItem({
|
|
110
|
+
WorkItemHash: 'wi-1',
|
|
111
|
+
RunID: 'rn-test-1',
|
|
112
|
+
Capability: 'Shell',
|
|
113
|
+
Action: 'Execute',
|
|
114
|
+
Settings: { Command: 'ls' },
|
|
115
|
+
Status: 'Queued',
|
|
116
|
+
Priority: 7,
|
|
117
|
+
EnqueuedAt: new Date().toISOString(),
|
|
118
|
+
Health: 0.75,
|
|
119
|
+
HealthLabel: 'Healthy'
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
let tmpRound = tmpStore.getWorkItemByHash('wi-1');
|
|
123
|
+
Expect(tmpRound).to.be.an('object');
|
|
124
|
+
Expect(tmpRound.WorkItemHash).to.equal('wi-1');
|
|
125
|
+
Expect(tmpRound.Priority).to.equal(7);
|
|
126
|
+
Expect(tmpRound.HealthLabel).to.equal('Healthy');
|
|
127
|
+
Expect(tmpRound.Health).to.be.closeTo(0.75, 0.0001);
|
|
128
|
+
Expect(tmpRound.Settings.Command).to.equal('ls');
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
test('updateWorkItem patches only named fields', () =>
|
|
132
|
+
{
|
|
133
|
+
let tmpFable = buildFable(_TestDir);
|
|
134
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
135
|
+
tmpStore.upsertWorkItem({
|
|
136
|
+
WorkItemHash: 'wi-2', RunID: 'rn-2', Capability: 'Shell',
|
|
137
|
+
Status: 'Queued', EnqueuedAt: new Date().toISOString()
|
|
138
|
+
});
|
|
139
|
+
tmpStore.updateWorkItem('wi-2', { Status: 'Dispatched', QueueWaitMs: 123 });
|
|
140
|
+
let tmpOut = tmpStore.getWorkItemByHash('wi-2');
|
|
141
|
+
Expect(tmpOut.Status).to.equal('Dispatched');
|
|
142
|
+
Expect(tmpOut.QueueWaitMs).to.equal(123);
|
|
143
|
+
Expect(tmpOut.Capability).to.equal('Shell'); // unchanged
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test('appendEvent is queryable and preserves payload JSON', () =>
|
|
147
|
+
{
|
|
148
|
+
let tmpFable = buildFable(_TestDir);
|
|
149
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
150
|
+
tmpStore.appendEvent({
|
|
151
|
+
WorkItemHash: 'wi-3', RunID: 'r3', EventType: 'enqueued',
|
|
152
|
+
FromStatus: '', ToStatus: 'Queued', Payload: { Capability: 'Shell', Priority: 2 }
|
|
153
|
+
});
|
|
154
|
+
let tmpEvents = tmpStore.listEventsForWorkItem('wi-3');
|
|
155
|
+
Expect(tmpEvents).to.have.length(1);
|
|
156
|
+
Expect(tmpEvents[0].EventType).to.equal('enqueued');
|
|
157
|
+
Expect(tmpEvents[0].Payload.Capability).to.equal('Shell');
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test('countByStatus aggregates rows by Status', () =>
|
|
161
|
+
{
|
|
162
|
+
let tmpFable = buildFable(_TestDir);
|
|
163
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
164
|
+
let tmpNow = new Date().toISOString();
|
|
165
|
+
tmpStore.upsertWorkItem({ WorkItemHash: 'a', Status: 'Queued', EnqueuedAt: tmpNow, Capability: 'Shell' });
|
|
166
|
+
tmpStore.upsertWorkItem({ WorkItemHash: 'b', Status: 'Queued', EnqueuedAt: tmpNow, Capability: 'Shell' });
|
|
167
|
+
tmpStore.upsertWorkItem({ WorkItemHash: 'c', Status: 'Running', EnqueuedAt: tmpNow, Capability: 'Shell' });
|
|
168
|
+
let tmpCounts = tmpStore.countByStatus();
|
|
169
|
+
Expect(tmpCounts.Queued).to.equal(2);
|
|
170
|
+
Expect(tmpCounts.Running).to.equal(1);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test('upsertAffinityBinding → getAffinityBinding → clearAffinityBinding lifecycle', () =>
|
|
174
|
+
{
|
|
175
|
+
let tmpFable = buildFable(_TestDir);
|
|
176
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
177
|
+
tmpStore.upsertAffinityBinding({ AffinityKey: 'affA', BeaconID: 'b1', ExpiresAt: new Date(Date.now() + 60000).toISOString() });
|
|
178
|
+
let tmpBinding = tmpStore.getAffinityBinding('affA');
|
|
179
|
+
Expect(tmpBinding.BeaconID).to.equal('b1');
|
|
180
|
+
|
|
181
|
+
tmpStore.upsertAffinityBinding({ AffinityKey: 'affA', BeaconID: 'b2', ExpiresAt: new Date(Date.now() + 60000).toISOString() });
|
|
182
|
+
Expect(tmpStore.getAffinityBinding('affA').BeaconID).to.equal('b2');
|
|
183
|
+
|
|
184
|
+
tmpStore.clearAffinityBinding('affA');
|
|
185
|
+
Expect(tmpStore.getAffinityBinding('affA')).to.equal(null);
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
test('action defaults upsert and lookup by (Capability, Action) + wildcard fallback', () =>
|
|
189
|
+
{
|
|
190
|
+
let tmpFable = buildFable(_TestDir);
|
|
191
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
192
|
+
tmpStore.upsertActionDefault({
|
|
193
|
+
Capability: 'Shell', Action: 'Execute',
|
|
194
|
+
TimeoutMs: 60000, MaxAttempts: 3, DefaultPriority: 5, ExpectedWaitP95Ms: 1500
|
|
195
|
+
});
|
|
196
|
+
let tmpRow = tmpStore.getActionDefault('Shell', 'Execute');
|
|
197
|
+
Expect(tmpRow.TimeoutMs).to.equal(60000);
|
|
198
|
+
Expect(tmpRow.MaxAttempts).to.equal(3);
|
|
199
|
+
Expect(tmpRow.ExpectedWaitP95Ms).to.equal(1500);
|
|
200
|
+
});
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
suite('RunManager', () =>
|
|
204
|
+
{
|
|
205
|
+
test('startRun with no IdempotencyKey mints a fresh RunID', () =>
|
|
206
|
+
{
|
|
207
|
+
let tmpFable = buildFable(_TestDir);
|
|
208
|
+
let tmpRM = getService(tmpFable, 'UltravisorBeaconRunManager');
|
|
209
|
+
let tmpRun = tmpRM.startRun({ SubmitterTag: 'test-client' });
|
|
210
|
+
Expect(tmpRun.RunID).to.match(/^rn-testhub-\d+-\d+$/);
|
|
211
|
+
Expect(tmpRun.State).to.equal('Active');
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test('startRun with IdempotencyKey returns the same RunID on replay', () =>
|
|
215
|
+
{
|
|
216
|
+
let tmpFable = buildFable(_TestDir);
|
|
217
|
+
let tmpRM = getService(tmpFable, 'UltravisorBeaconRunManager');
|
|
218
|
+
let tmpA = tmpRM.startRun({ IdempotencyKey: 'key-X', SubmitterTag: 'one' });
|
|
219
|
+
let tmpB = tmpRM.startRun({ IdempotencyKey: 'key-X', SubmitterTag: 'two' });
|
|
220
|
+
Expect(tmpB.RunID).to.equal(tmpA.RunID);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
test('endRun and cancelRun update run state', () =>
|
|
224
|
+
{
|
|
225
|
+
let tmpFable = buildFable(_TestDir);
|
|
226
|
+
let tmpRM = getService(tmpFable, 'UltravisorBeaconRunManager');
|
|
227
|
+
let tmpRun = tmpRM.startRun({});
|
|
228
|
+
Expect(tmpRM.endRun(tmpRun.RunID)).to.equal(true);
|
|
229
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
230
|
+
Expect(tmpStore.getRunByRunID(tmpRun.RunID).State).to.equal('Ended');
|
|
231
|
+
|
|
232
|
+
let tmpRun2 = tmpRM.startRun({});
|
|
233
|
+
tmpRM.cancelRun(tmpRun2.RunID, 'operator cancel');
|
|
234
|
+
let tmpAfter = tmpStore.getRunByRunID(tmpRun2.RunID);
|
|
235
|
+
Expect(tmpAfter.State).to.equal('Canceled');
|
|
236
|
+
Expect(tmpAfter.CancelReason).to.equal('operator cancel');
|
|
237
|
+
});
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
suite('ActionDefaults resolver', () =>
|
|
241
|
+
{
|
|
242
|
+
test('falls back hard-default → Fable setting → per-action row', () =>
|
|
243
|
+
{
|
|
244
|
+
let tmpFable = buildFable(_TestDir);
|
|
245
|
+
tmpFable.settings.UltravisorBeaconWorkItemTimeoutMs = 120000;
|
|
246
|
+
let tmpDefaults = getService(tmpFable, 'UltravisorBeaconActionDefaults');
|
|
247
|
+
|
|
248
|
+
let tmpHard = tmpDefaults.resolve('UnknownCap', 'Unknown');
|
|
249
|
+
// Fable-setting fallback on TimeoutMs kicks in.
|
|
250
|
+
Expect(tmpHard.TimeoutMs).to.equal(120000);
|
|
251
|
+
Expect(tmpHard.MaxAttempts).to.equal(1); // hard default
|
|
252
|
+
|
|
253
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
254
|
+
tmpStore.upsertActionDefault({
|
|
255
|
+
Capability: 'Shell', Action: 'Execute',
|
|
256
|
+
TimeoutMs: 5000, MaxAttempts: 4, DefaultPriority: 3, ExpectedWaitP95Ms: 999
|
|
257
|
+
});
|
|
258
|
+
tmpDefaults.invalidate();
|
|
259
|
+
let tmpRow = tmpDefaults.resolve('Shell', 'Execute');
|
|
260
|
+
Expect(tmpRow.TimeoutMs).to.equal(5000);
|
|
261
|
+
Expect(tmpRow.MaxAttempts).to.equal(4);
|
|
262
|
+
Expect(tmpRow.ExpectedWaitP95Ms).to.equal(999);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test('applyToWorkItem honors per-request Settings over defaults', () =>
|
|
266
|
+
{
|
|
267
|
+
let tmpFable = buildFable(_TestDir);
|
|
268
|
+
let tmpDefaults = getService(tmpFable, 'UltravisorBeaconActionDefaults');
|
|
269
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
270
|
+
tmpStore.upsertActionDefault({
|
|
271
|
+
Capability: 'Shell', Action: 'Execute',
|
|
272
|
+
TimeoutMs: 5000, MaxAttempts: 4, DefaultPriority: 3
|
|
273
|
+
});
|
|
274
|
+
tmpDefaults.invalidate();
|
|
275
|
+
|
|
276
|
+
let tmpItem = { Capability: 'Shell', Action: 'Execute' };
|
|
277
|
+
tmpDefaults.applyToWorkItem(tmpItem, { maxRetries: 7 });
|
|
278
|
+
Expect(tmpItem.MaxAttempts).to.equal(7); // request overrides
|
|
279
|
+
Expect(tmpItem.TimeoutMs).to.equal(5000); // default stands
|
|
280
|
+
Expect(tmpItem.Priority).to.equal(3); // default priority
|
|
281
|
+
});
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
suite('Coordinator integration', () =>
|
|
285
|
+
{
|
|
286
|
+
test('enqueueWorkItem populates new fields and persists to store', () =>
|
|
287
|
+
{
|
|
288
|
+
let tmpFable = buildFable(_TestDir);
|
|
289
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
290
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
291
|
+
|
|
292
|
+
let tmpItem = tmpCoordinator.enqueueWorkItem({
|
|
293
|
+
RunID: 'rn-queue-1',
|
|
294
|
+
Capability: 'Shell',
|
|
295
|
+
Action: 'Execute',
|
|
296
|
+
Priority: 9
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
Expect(tmpItem.Priority).to.equal(9);
|
|
300
|
+
Expect(tmpItem.HealthLabel).to.equal('Unknown');
|
|
301
|
+
Expect(tmpItem.EnqueuedAt).to.be.a('string');
|
|
302
|
+
|
|
303
|
+
let tmpPersisted = tmpStore.getWorkItemByHash(tmpItem.WorkItemHash);
|
|
304
|
+
Expect(tmpPersisted).to.be.an('object');
|
|
305
|
+
Expect(tmpPersisted.RunID).to.equal('rn-queue-1');
|
|
306
|
+
Expect(tmpPersisted.Priority).to.equal(9);
|
|
307
|
+
let tmpEvents = tmpStore.listEventsForWorkItem(tmpItem.WorkItemHash);
|
|
308
|
+
Expect(tmpEvents.map((e) => e.EventType)).to.include('enqueued');
|
|
309
|
+
});
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
suite('Scheduler', () =>
|
|
313
|
+
{
|
|
314
|
+
test('dispatch tick promotes Queued items to Dispatched with QueueMetadata', () =>
|
|
315
|
+
{
|
|
316
|
+
let tmpFable = buildFable(_TestDir);
|
|
317
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
318
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
319
|
+
let tmpStore = getService(tmpFable, 'UltravisorBeaconQueueStore');
|
|
320
|
+
|
|
321
|
+
addStubBeacon(tmpCoordinator, 'b-1', ['Shell']);
|
|
322
|
+
let tmpItem = tmpCoordinator.enqueueWorkItem({
|
|
323
|
+
RunID: 'rn-d-1', Capability: 'Shell', Action: 'Execute'
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
tmpScheduler._dispatchTick();
|
|
327
|
+
|
|
328
|
+
let tmpAfter = tmpCoordinator._WorkQueue[tmpItem.WorkItemHash];
|
|
329
|
+
Expect(tmpAfter.Status).to.equal('Dispatched');
|
|
330
|
+
Expect(tmpAfter.AssignedBeaconID).to.equal('b-1');
|
|
331
|
+
Expect(tmpAfter.DispatchedAt).to.be.a('string');
|
|
332
|
+
Expect(tmpAfter.Settings.QueueMetadata).to.be.an('object');
|
|
333
|
+
Expect(tmpAfter.Settings.QueueMetadata.RunID).to.equal('rn-d-1');
|
|
334
|
+
Expect(typeof tmpAfter.Settings.QueueMetadata.QueueWaitMs).to.equal('number');
|
|
335
|
+
|
|
336
|
+
let tmpStored = tmpStore.getWorkItemByHash(tmpItem.WorkItemHash);
|
|
337
|
+
Expect(tmpStored.Status).to.equal('Dispatched');
|
|
338
|
+
Expect(tmpStored.AttemptNumber).to.equal(1);
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
test('higher priority dispatches ahead of lower priority FIFO', () =>
|
|
342
|
+
{
|
|
343
|
+
let tmpFable = buildFable(_TestDir);
|
|
344
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
345
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
346
|
+
addStubBeacon(tmpCoordinator, 'only', ['Shell']);
|
|
347
|
+
|
|
348
|
+
let tmpLo = tmpCoordinator.enqueueWorkItem({ Capability: 'Shell', Priority: 1 });
|
|
349
|
+
let tmpHi = tmpCoordinator.enqueueWorkItem({ Capability: 'Shell', Priority: 9 });
|
|
350
|
+
|
|
351
|
+
tmpCoordinator._Beacons.only.MaxConcurrent = 1;
|
|
352
|
+
tmpScheduler._dispatchTick();
|
|
353
|
+
|
|
354
|
+
Expect(tmpCoordinator._WorkQueue[tmpHi.WorkItemHash].Status).to.equal('Dispatched');
|
|
355
|
+
Expect(tmpCoordinator._WorkQueue[tmpLo.WorkItemHash].Status).to.not.equal('Dispatched');
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
test('computeHealth returns Unknown with no baseline, Uncertain at mid ratio, Unhealthy near timeout', () =>
|
|
359
|
+
{
|
|
360
|
+
let tmpFable = buildFable(_TestDir);
|
|
361
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
362
|
+
|
|
363
|
+
let tmpFresh = {
|
|
364
|
+
Status: 'Queued', EnqueuedAt: new Date().toISOString(),
|
|
365
|
+
Capability: 'Shell', Action: 'Execute', MaxAttempts: 1, AttemptNumber: 0
|
|
366
|
+
};
|
|
367
|
+
Expect(tmpScheduler.computeHealth(tmpFresh).Label).to.equal('Unknown');
|
|
368
|
+
|
|
369
|
+
// Simulate a running item 80% through its timeout — should score low (<0.3 → Unhealthy)
|
|
370
|
+
let tmpOld = Date.now() - 80000;
|
|
371
|
+
let tmpRunning = {
|
|
372
|
+
Status: 'Running', StartedAt: new Date(tmpOld).toISOString(),
|
|
373
|
+
LastEventAt: new Date(tmpOld).toISOString(),
|
|
374
|
+
Capability: 'Shell', Action: 'Execute', TimeoutMs: 100000,
|
|
375
|
+
MaxAttempts: 1, AttemptNumber: 1
|
|
376
|
+
};
|
|
377
|
+
let tmpHealth = tmpScheduler.computeHealth(tmpRunning);
|
|
378
|
+
Expect(tmpHealth.Label).to.be.oneOf(['Unhealthy', 'Uncertain']);
|
|
379
|
+
Expect(tmpHealth.Score).to.be.lessThan(0.6);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
test('summarize returns bucket counts and per-capability breakdown', () =>
|
|
383
|
+
{
|
|
384
|
+
let tmpFable = buildFable(_TestDir);
|
|
385
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
386
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
387
|
+
tmpCoordinator.enqueueWorkItem({ Capability: 'Shell', Action: 'A' });
|
|
388
|
+
tmpCoordinator.enqueueWorkItem({ Capability: 'Shell', Action: 'A' });
|
|
389
|
+
tmpCoordinator.enqueueWorkItem({ Capability: 'LLM', Action: 'Complete' });
|
|
390
|
+
|
|
391
|
+
let tmpSum = tmpScheduler.summarize();
|
|
392
|
+
Expect(tmpSum.Buckets.Upcoming).to.equal(3);
|
|
393
|
+
Expect(tmpSum.ByCapability).to.be.an('array');
|
|
394
|
+
Expect(tmpSum.ByCapability.length).to.equal(2);
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
test('requestCancel moves Queued item to Canceled', () =>
|
|
398
|
+
{
|
|
399
|
+
let tmpFable = buildFable(_TestDir);
|
|
400
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
401
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
402
|
+
let tmpItem = tmpCoordinator.enqueueWorkItem({ Capability: 'Shell' });
|
|
403
|
+
|
|
404
|
+
let tmpResult = tmpScheduler.requestCancel(tmpItem.WorkItemHash, 'user');
|
|
405
|
+
Expect(tmpResult.Canceled).to.equal(true);
|
|
406
|
+
Expect(tmpCoordinator._WorkQueue[tmpItem.WorkItemHash].Status).to.equal('Canceled');
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
test('requestCancel on Running item sets CancelRequested flag only', () =>
|
|
410
|
+
{
|
|
411
|
+
let tmpFable = buildFable(_TestDir);
|
|
412
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
413
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
414
|
+
let tmpItem = tmpCoordinator.enqueueWorkItem({ Capability: 'Shell' });
|
|
415
|
+
tmpCoordinator._WorkQueue[tmpItem.WorkItemHash].Status = 'Running';
|
|
416
|
+
|
|
417
|
+
let tmpResult = tmpScheduler.requestCancel(tmpItem.WorkItemHash, 'user');
|
|
418
|
+
Expect(tmpResult.Canceled).to.equal(false);
|
|
419
|
+
Expect(tmpResult.CancelRequested).to.equal(true);
|
|
420
|
+
Expect(tmpCoordinator._WorkQueue[tmpItem.WorkItemHash].CancelRequested).to.equal(true);
|
|
421
|
+
});
|
|
422
|
+
|
|
423
|
+
test('broadcast handler receives queue.* topics', () =>
|
|
424
|
+
{
|
|
425
|
+
let tmpFable = buildFable(_TestDir);
|
|
426
|
+
let tmpCoordinator = getService(tmpFable, 'UltravisorBeaconCoordinator');
|
|
427
|
+
let tmpScheduler = getService(tmpFable, 'UltravisorBeaconScheduler');
|
|
428
|
+
let tmpEvents = [];
|
|
429
|
+
tmpScheduler.setBroadcastHandler((pTopic, pPayload) =>
|
|
430
|
+
{
|
|
431
|
+
tmpEvents.push({ Topic: pTopic, Payload: pPayload });
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
addStubBeacon(tmpCoordinator, 'b', ['Shell']);
|
|
435
|
+
tmpCoordinator.enqueueWorkItem({ Capability: 'Shell' });
|
|
436
|
+
tmpScheduler._dispatchTick();
|
|
437
|
+
|
|
438
|
+
let tmpTopics = tmpEvents.map((e) => e.Topic);
|
|
439
|
+
Expect(tmpTopics).to.include('queue.enqueued');
|
|
440
|
+
Expect(tmpTopics).to.include('queue.dispatched');
|
|
441
|
+
});
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
suite('RetoldLabs-QueuePhases', () =>
|
|
445
|
+
{
|
|
446
|
+
test('emitPreWorkerPhases writes queue_wait + worker_spinup lines', () =>
|
|
447
|
+
{
|
|
448
|
+
let tmpDir = libPath.join(_TestDir, 'staging-1');
|
|
449
|
+
libFS.mkdirSync(tmpDir, { recursive: true });
|
|
450
|
+
|
|
451
|
+
let tmpMeta = {
|
|
452
|
+
RunID: 'rn-p-1',
|
|
453
|
+
WorkItemHash: 'wi-p-1',
|
|
454
|
+
EnqueuedAt: new Date(Date.now() - 5000).toISOString(),
|
|
455
|
+
DispatchedAt: new Date(Date.now() - 1000).toISOString(),
|
|
456
|
+
QueueWaitMs: 4000,
|
|
457
|
+
AttemptNumber: 1
|
|
458
|
+
};
|
|
459
|
+
|
|
460
|
+
let tmpOut = libQueuePhases.emitPreWorkerPhases(tmpDir, tmpMeta, { spinupStartMs: Date.now() });
|
|
461
|
+
Expect(tmpOut.QueueWaitMs).to.equal(4000);
|
|
462
|
+
Expect(tmpOut.WorkerSpinupMs).to.be.at.least(0);
|
|
463
|
+
|
|
464
|
+
let tmpText = libFS.readFileSync(libPath.join(tmpDir, 'phases.jsonl'), 'utf8');
|
|
465
|
+
let tmpLines = tmpText.trim().split(/\n/).map((l) => JSON.parse(l));
|
|
466
|
+
Expect(tmpLines).to.have.length(2);
|
|
467
|
+
Expect(tmpLines[0].name).to.equal('queue_wait');
|
|
468
|
+
Expect(tmpLines[0].run_id).to.equal('rn-p-1');
|
|
469
|
+
Expect(tmpLines[0].duration_ms).to.equal(4000);
|
|
470
|
+
Expect(tmpLines[1].name).to.equal('worker_spinup');
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
test('emitAssetCapturePhase appends the post-worker phase', () =>
|
|
474
|
+
{
|
|
475
|
+
let tmpDir = libPath.join(_TestDir, 'staging-2');
|
|
476
|
+
libFS.mkdirSync(tmpDir, { recursive: true });
|
|
477
|
+
|
|
478
|
+
let tmpMeta = { RunID: 'rn-p-2', WorkItemHash: 'wi-p-2', AttemptNumber: 1 };
|
|
479
|
+
let tmpStart = Date.now() - 250;
|
|
480
|
+
libQueuePhases.emitAssetCapturePhase(tmpDir, tmpMeta, tmpStart, Date.now());
|
|
481
|
+
|
|
482
|
+
let tmpText = libFS.readFileSync(libPath.join(tmpDir, 'phases.jsonl'), 'utf8');
|
|
483
|
+
let tmpLines = tmpText.trim().split(/\n/).map((l) => JSON.parse(l));
|
|
484
|
+
Expect(tmpLines).to.have.length(1);
|
|
485
|
+
Expect(tmpLines[0].name).to.equal('asset_capture');
|
|
486
|
+
Expect(tmpLines[0].duration_ms).to.be.at.least(0);
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
test('envForWorker merges RETOLD_RUN_ID without clobbering existing env', () =>
|
|
490
|
+
{
|
|
491
|
+
let tmpEnv = libQueuePhases.envForWorker({ EXISTING: '1' },
|
|
492
|
+
{ RunID: 'rn-env', WorkItemHash: 'wi-env' });
|
|
493
|
+
Expect(tmpEnv.EXISTING).to.equal('1');
|
|
494
|
+
Expect(tmpEnv.RETOLD_RUN_ID).to.equal('rn-env');
|
|
495
|
+
Expect(tmpEnv.RETOLD_WORK_ITEM_HASH).to.equal('wi-env');
|
|
496
|
+
|
|
497
|
+
let tmpNoClobber = libQueuePhases.envForWorker({ RETOLD_RUN_ID: 'already-set' },
|
|
498
|
+
{ RunID: 'new-id' });
|
|
499
|
+
Expect(tmpNoClobber.RETOLD_RUN_ID).to.equal('already-set');
|
|
500
|
+
});
|
|
501
|
+
});
|
|
502
|
+
});
|
package/test/Ultravisor_tests.js
CHANGED
|
@@ -760,6 +760,138 @@ suite
|
|
|
760
760
|
});
|
|
761
761
|
}
|
|
762
762
|
);
|
|
763
|
+
|
|
764
|
+
test
|
|
765
|
+
(
|
|
766
|
+
'State connection Data.StateKey should override target port name when resolving settings.',
|
|
767
|
+
function()
|
|
768
|
+
{
|
|
769
|
+
// The storyboard — long-form video operation wires
|
|
770
|
+
// a value-input's InputValue state output into a
|
|
771
|
+
// parameter-sweep task's `ParameterSets` setting
|
|
772
|
+
// via an event-trigger target port. The target
|
|
773
|
+
// port name can't match the setting name in that
|
|
774
|
+
// shape, so the connection declares
|
|
775
|
+
// `Data.StateKey: "ParameterSets"` and the engine
|
|
776
|
+
// has to honor it. Without the StateKey override,
|
|
777
|
+
// the value would land on `tmpSettings[<port name>]`
|
|
778
|
+
// and the sweep task's `pResolvedSettings.ParameterSets`
|
|
779
|
+
// would be undefined, causing "ParameterSets must be
|
|
780
|
+
// a JSON array." at runtime.
|
|
781
|
+
let tmpFable = createTestFable();
|
|
782
|
+
let tmpEngine = Object.values(tmpFable.servicesMap['UltravisorExecutionEngine'])[0];
|
|
783
|
+
|
|
784
|
+
let tmpNode = {
|
|
785
|
+
Hash: 'sweep-node',
|
|
786
|
+
Type: 'parameter-sweep',
|
|
787
|
+
Data: {},
|
|
788
|
+
Settings: {},
|
|
789
|
+
Ports:
|
|
790
|
+
[
|
|
791
|
+
{ Direction: 'input', Hash: 'sweep-node-ei-begin', Label: 'BeginSweep', Side: 'left-bottom' }
|
|
792
|
+
]
|
|
793
|
+
};
|
|
794
|
+
|
|
795
|
+
let tmpContext = {
|
|
796
|
+
TaskOutputs: {
|
|
797
|
+
'value-input-node': { InputValue: [ { prompt: 'beat 1' }, { prompt: 'beat 2' } ] }
|
|
798
|
+
},
|
|
799
|
+
_ConnectionMap: {
|
|
800
|
+
stateTargets:
|
|
801
|
+
{
|
|
802
|
+
'sweep-node':
|
|
803
|
+
[
|
|
804
|
+
{
|
|
805
|
+
Hash: 'state-conn',
|
|
806
|
+
ConnectionType: 'state',
|
|
807
|
+
SourceNodeHash: 'value-input-node',
|
|
808
|
+
SourcePortHash: 'value-input-node-so-InputValue',
|
|
809
|
+
TargetNodeHash: 'sweep-node',
|
|
810
|
+
TargetPortHash: 'sweep-node-ei-begin',
|
|
811
|
+
Data: { StateKey: 'ParameterSets' }
|
|
812
|
+
}
|
|
813
|
+
]
|
|
814
|
+
}
|
|
815
|
+
},
|
|
816
|
+
_PortLabelMap:
|
|
817
|
+
{
|
|
818
|
+
'value-input-node-so-InputValue': 'InputValue',
|
|
819
|
+
'sweep-node-ei-begin': 'begin'
|
|
820
|
+
}
|
|
821
|
+
};
|
|
822
|
+
|
|
823
|
+
let tmpResolved = tmpEngine._resolveStateConnections('sweep-node', tmpNode, tmpContext);
|
|
824
|
+
|
|
825
|
+
// The StateKey override routes InputValue into the
|
|
826
|
+
// setting named ParameterSets, not into the target
|
|
827
|
+
// port's label ("begin").
|
|
828
|
+
Expect(Array.isArray(tmpResolved.ParameterSets)).to.equal(true);
|
|
829
|
+
Expect(tmpResolved.ParameterSets.length).to.equal(2);
|
|
830
|
+
Expect(tmpResolved.ParameterSets[0].prompt).to.equal('beat 1');
|
|
831
|
+
// The target port's label-named key should NOT
|
|
832
|
+
// have been populated when StateKey is present.
|
|
833
|
+
Expect(tmpResolved.begin).to.equal(undefined);
|
|
834
|
+
}
|
|
835
|
+
);
|
|
836
|
+
|
|
837
|
+
test
|
|
838
|
+
(
|
|
839
|
+
'State connection without Data.StateKey should still route by target port name.',
|
|
840
|
+
function()
|
|
841
|
+
{
|
|
842
|
+
// Regression guard for the StateKey fallback: when
|
|
843
|
+
// the state connection has no StateKey, the engine
|
|
844
|
+
// must continue to use the target port name as the
|
|
845
|
+
// settings key (backward compatibility for every
|
|
846
|
+
// operation wired the old way, including the
|
|
847
|
+
// template-transform test above).
|
|
848
|
+
let tmpFable = createTestFable();
|
|
849
|
+
let tmpEngine = Object.values(tmpFable.servicesMap['UltravisorExecutionEngine'])[0];
|
|
850
|
+
|
|
851
|
+
let tmpNode = {
|
|
852
|
+
Hash: 'write-node',
|
|
853
|
+
Type: 'write-file',
|
|
854
|
+
Data: {},
|
|
855
|
+
Settings: {},
|
|
856
|
+
Ports:
|
|
857
|
+
[
|
|
858
|
+
{ Direction: 'input', Hash: 'write-node-si-Content', Label: 'Content', Side: 'left-top' }
|
|
859
|
+
]
|
|
860
|
+
};
|
|
861
|
+
|
|
862
|
+
let tmpContext = {
|
|
863
|
+
TaskOutputs: {
|
|
864
|
+
'read-node': { FileContent: 'hello world' }
|
|
865
|
+
},
|
|
866
|
+
_ConnectionMap: {
|
|
867
|
+
stateTargets:
|
|
868
|
+
{
|
|
869
|
+
'write-node':
|
|
870
|
+
[
|
|
871
|
+
{
|
|
872
|
+
Hash: 'legacy-state-conn',
|
|
873
|
+
ConnectionType: 'state',
|
|
874
|
+
SourceNodeHash: 'read-node',
|
|
875
|
+
SourcePortHash: 'read-node-so-FileContent',
|
|
876
|
+
TargetNodeHash: 'write-node',
|
|
877
|
+
TargetPortHash: 'write-node-si-Content'
|
|
878
|
+
// No Data.StateKey — fall through
|
|
879
|
+
}
|
|
880
|
+
]
|
|
881
|
+
}
|
|
882
|
+
},
|
|
883
|
+
_PortLabelMap:
|
|
884
|
+
{
|
|
885
|
+
'read-node-so-FileContent': 'FileContent',
|
|
886
|
+
'write-node-si-Content': 'Content'
|
|
887
|
+
}
|
|
888
|
+
};
|
|
889
|
+
|
|
890
|
+
let tmpResolved = tmpEngine._resolveStateConnections('write-node', tmpNode, tmpContext);
|
|
891
|
+
|
|
892
|
+
Expect(tmpResolved.Content).to.equal('hello world');
|
|
893
|
+
}
|
|
894
|
+
);
|
|
763
895
|
}
|
|
764
896
|
);
|
|
765
897
|
|