ultravisor 1.0.17 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/source/cli/Ultravisor-CLIProgram.cjs +28 -0
- package/source/services/Ultravisor-Beacon-Coordinator.cjs +15 -0
- package/source/services/Ultravisor-Beacon-Reachability.cjs +206 -1
- package/source/services/tasks/platform/Ultravisor-TaskConfigs-Platform.cjs +189 -26
- package/source/services/tasks/platform/definitions/file-transfer.json +11 -8
- package/source/services/tasks/platform/definitions/resolve-address.json +4 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ultravisor",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.19",
|
|
4
4
|
"description": "Cyclic process execution with ai integration.",
|
|
5
5
|
"main": "source/Ultravisor.cjs",
|
|
6
6
|
"bin": {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"pict": "^1.0.361",
|
|
35
35
|
"pict-service-commandlineutility": "^1.0.19",
|
|
36
36
|
"pict-serviceproviderbase": "^1.0.4",
|
|
37
|
-
"ultravisor-beacon": "^0.0.
|
|
37
|
+
"ultravisor-beacon": "^0.0.9",
|
|
38
38
|
"ws": "^8.20.0"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
@@ -116,6 +116,34 @@ if (_LogFilePath)
|
|
|
116
116
|
console.log(`[Ultravisor] Logging to file: ${_LogFilePath}`);
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
+
// Apply LogNoisiness from RETOLD_LOG_NOISINESS env var. Pict-style log
|
|
120
|
+
// noisiness is a 0-5 scale where 0 is silent (production default) and 5 shows
|
|
121
|
+
// everything. Diagnostic log statements throughout Ultravisor (especially the
|
|
122
|
+
// shared-fs reachability auto-detect path and the platform tasks) are gated
|
|
123
|
+
// with `if (this.fable.LogNoisiness >= N)` so they're free at level 0 and
|
|
124
|
+
// explosively detailed at level 4-5.
|
|
125
|
+
//
|
|
126
|
+
// Useful values:
|
|
127
|
+
// 1 — high-level decisions (auto-detected shared-fs peer X)
|
|
128
|
+
// 2 — entry points and decisions in shared-fs / dispatch paths
|
|
129
|
+
// 3 — per-candidate iteration in reachability
|
|
130
|
+
// 4 — per-mount comparison details
|
|
131
|
+
// 5 — everything
|
|
132
|
+
//
|
|
133
|
+
// In stack-mode the launcher inherits process.env into the child Ultravisor
|
|
134
|
+
// process automatically, so setting RETOLD_LOG_NOISINESS once on the host
|
|
135
|
+
// (or in the docker-compose `environment:` block) lights up both processes.
|
|
136
|
+
let _LogNoisiness = parseInt(process.env.RETOLD_LOG_NOISINESS, 10);
|
|
137
|
+
if (!isNaN(_LogNoisiness) && _LogNoisiness > 0)
|
|
138
|
+
{
|
|
139
|
+
_Ultravisor_Pict.LogNoisiness = _LogNoisiness;
|
|
140
|
+
if (_Ultravisor_Pict.fable && _Ultravisor_Pict.fable !== _Ultravisor_Pict)
|
|
141
|
+
{
|
|
142
|
+
_Ultravisor_Pict.fable.LogNoisiness = _LogNoisiness;
|
|
143
|
+
}
|
|
144
|
+
console.log(`[Ultravisor] LogNoisiness=${_LogNoisiness} (verbose diagnostics enabled).`);
|
|
145
|
+
}
|
|
146
|
+
|
|
119
147
|
// If a config file override was passed via --config / -c, apply it on top of the gathered config
|
|
120
148
|
if (_ConfigFileOverride)
|
|
121
149
|
{
|
|
@@ -231,6 +231,16 @@ class UltravisorBeaconCoordinator extends libPictService
|
|
|
231
231
|
{
|
|
232
232
|
tmpExistingBeacon.BindAddresses = pBeaconInfo.BindAddresses;
|
|
233
233
|
}
|
|
234
|
+
// Refresh shared-fs identity on reconnect — host id can change between
|
|
235
|
+
// container restarts and a reconnecting beacon may have new mounts.
|
|
236
|
+
if (typeof pBeaconInfo.HostID === 'string' && pBeaconInfo.HostID.length > 0)
|
|
237
|
+
{
|
|
238
|
+
tmpExistingBeacon.HostID = pBeaconInfo.HostID;
|
|
239
|
+
}
|
|
240
|
+
if (Array.isArray(pBeaconInfo.SharedMounts))
|
|
241
|
+
{
|
|
242
|
+
tmpExistingBeacon.SharedMounts = pBeaconInfo.SharedMounts;
|
|
243
|
+
}
|
|
234
244
|
|
|
235
245
|
this.log.info(`BeaconCoordinator: reconnected beacon [${tmpExistingBeacon.BeaconID}] "${tmpName}" with session [${tmpExistingBeacon.SessionID}].`);
|
|
236
246
|
|
|
@@ -272,6 +282,11 @@ class UltravisorBeaconCoordinator extends libPictService
|
|
|
272
282
|
Tags: pBeaconInfo.Tags || {},
|
|
273
283
|
Contexts: pBeaconInfo.Contexts || {},
|
|
274
284
|
BindAddresses: Array.isArray(pBeaconInfo.BindAddresses) ? pBeaconInfo.BindAddresses : [],
|
|
285
|
+
// Shared-fs identity. Both fields are optional for backwards compatibility:
|
|
286
|
+
// older beacons that don't send them just get null/[] and the shared-fs
|
|
287
|
+
// strategy is silently skipped for them.
|
|
288
|
+
HostID: (typeof pBeaconInfo.HostID === 'string' && pBeaconInfo.HostID.length > 0) ? pBeaconInfo.HostID : null,
|
|
289
|
+
SharedMounts: Array.isArray(pBeaconInfo.SharedMounts) ? pBeaconInfo.SharedMounts : [],
|
|
275
290
|
RegisteredAt: new Date().toISOString()
|
|
276
291
|
};
|
|
277
292
|
|
|
@@ -362,9 +362,17 @@ class UltravisorBeaconReachability extends libPictService
|
|
|
362
362
|
* Determine the transfer strategy between a source beacon (file
|
|
363
363
|
* owner) and a requesting beacon (the one that needs the file).
|
|
364
364
|
*
|
|
365
|
+
* Strategy ordering (most efficient first):
|
|
366
|
+
* local — same beacon, no transport at all
|
|
367
|
+
* shared-fs — different beacons but on the same host with overlapping
|
|
368
|
+
* filesystem mounts, so the requesting beacon can read the
|
|
369
|
+
* source's file path directly with no copy
|
|
370
|
+
* direct — HTTP fetch from the source beacon's bind address
|
|
371
|
+
* proxy — HTTP fetch via the coordinator
|
|
372
|
+
*
|
|
365
373
|
* @param {string} pSourceBeaconID - Beacon that owns the resource
|
|
366
374
|
* @param {string} pRequestingBeaconID - Beacon that wants the resource
|
|
367
|
-
* @returns {object} { Strategy, DirectURL }
|
|
375
|
+
* @returns {object} { Strategy, DirectURL, SharedMountRoot? }
|
|
368
376
|
*/
|
|
369
377
|
resolveStrategy(pSourceBeaconID, pRequestingBeaconID)
|
|
370
378
|
{
|
|
@@ -386,6 +394,28 @@ class UltravisorBeaconReachability extends libPictService
|
|
|
386
394
|
return { Strategy: 'proxy', DirectURL: '' };
|
|
387
395
|
}
|
|
388
396
|
|
|
397
|
+
// Shared-fs detection — check whether both beacons live on the same host
|
|
398
|
+
// AND advertise at least one shared filesystem mount in common. When they
|
|
399
|
+
// do, the requesting beacon can read the source file directly from the
|
|
400
|
+
// shared mount without an HTTP transfer.
|
|
401
|
+
let tmpRequestingBeacon = tmpCoordinator.getBeacon(pRequestingBeaconID);
|
|
402
|
+
if (tmpRequestingBeacon
|
|
403
|
+
&& tmpSourceBeacon.HostID
|
|
404
|
+
&& tmpRequestingBeacon.HostID
|
|
405
|
+
&& tmpSourceBeacon.HostID === tmpRequestingBeacon.HostID)
|
|
406
|
+
{
|
|
407
|
+
let tmpSharedMount = this._findSharedMount(
|
|
408
|
+
tmpSourceBeacon.SharedMounts, tmpRequestingBeacon.SharedMounts);
|
|
409
|
+
if (tmpSharedMount)
|
|
410
|
+
{
|
|
411
|
+
return {
|
|
412
|
+
Strategy: 'shared-fs',
|
|
413
|
+
DirectURL: '',
|
|
414
|
+
SharedMountRoot: tmpSharedMount.Root
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
389
419
|
// Check matrix
|
|
390
420
|
let tmpEntry = this.getReachability(pRequestingBeaconID, pSourceBeaconID);
|
|
391
421
|
|
|
@@ -398,6 +428,181 @@ class UltravisorBeaconReachability extends libPictService
|
|
|
398
428
|
// unreachable or untested — fall back to proxy
|
|
399
429
|
return { Strategy: 'proxy', DirectURL: '' };
|
|
400
430
|
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Find the first MountID that appears in both beacons' SharedMounts arrays.
|
|
434
|
+
*
|
|
435
|
+
* The MountID is derived from `stat.dev` + the resolved root path on the
|
|
436
|
+
* beacon side, so two beacons that bind-mount the same host directory get
|
|
437
|
+
* the same ID, while two unrelated /media directories on different machines
|
|
438
|
+
* (or in different containers without shared mounts) get different IDs.
|
|
439
|
+
*
|
|
440
|
+
* @param {Array} pSourceMounts - SharedMounts reported by the source beacon
|
|
441
|
+
* @param {Array} pRequestingMounts - SharedMounts reported by the requester
|
|
442
|
+
* @returns {object|null} The matching mount entry from the source beacon,
|
|
443
|
+
* or null if no overlap exists
|
|
444
|
+
*/
|
|
445
|
+
_findSharedMount(pSourceMounts, pRequestingMounts)
|
|
446
|
+
{
|
|
447
|
+
if (!Array.isArray(pSourceMounts) || pSourceMounts.length === 0)
|
|
448
|
+
{
|
|
449
|
+
return null;
|
|
450
|
+
}
|
|
451
|
+
if (!Array.isArray(pRequestingMounts) || pRequestingMounts.length === 0)
|
|
452
|
+
{
|
|
453
|
+
return null;
|
|
454
|
+
}
|
|
455
|
+
for (let i = 0; i < pSourceMounts.length; i++)
|
|
456
|
+
{
|
|
457
|
+
let tmpSrc = pSourceMounts[i];
|
|
458
|
+
if (!tmpSrc || !tmpSrc.MountID)
|
|
459
|
+
{
|
|
460
|
+
continue;
|
|
461
|
+
}
|
|
462
|
+
for (let j = 0; j < pRequestingMounts.length; j++)
|
|
463
|
+
{
|
|
464
|
+
let tmpReq = pRequestingMounts[j];
|
|
465
|
+
if (!tmpReq || !tmpReq.MountID)
|
|
466
|
+
{
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
if (tmpSrc.MountID === tmpReq.MountID)
|
|
470
|
+
{
|
|
471
|
+
return tmpSrc;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
return null;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* Walk all online beacons looking for one that shares a filesystem with
|
|
480
|
+
* the given source beacon. Returns the first peer that:
|
|
481
|
+
* 1. is not the source beacon itself
|
|
482
|
+
* 2. has the same HostID as the source
|
|
483
|
+
* 3. has at least one MountID overlap with the source
|
|
484
|
+
*
|
|
485
|
+
* This is used by the resolve-address task to auto-detect when a shared-fs
|
|
486
|
+
* fast path is available even when the caller did not explicitly identify
|
|
487
|
+
* a "requesting beacon". The typical case: retold-remote dispatches a media
|
|
488
|
+
* operation, the file lives on retold-remote, and orator-conversion (which
|
|
489
|
+
* will eventually consume the file) runs in the same process / on the same
|
|
490
|
+
* host with the same content mount. The auto-detection finds orator-conversion
|
|
491
|
+
* as a peer of retold-remote and reports the shared-fs strategy so the
|
|
492
|
+
* file-transfer task can short-circuit.
|
|
493
|
+
*
|
|
494
|
+
* Diagnostic logging is gated by Fable.LogNoisiness (set via the
|
|
495
|
+
* RETOLD_LOG_NOISINESS env var, 0-5):
|
|
496
|
+
* >= 2: log entry, source beacon summary, final decision
|
|
497
|
+
* >= 3: per-candidate iteration with rejection reasons
|
|
498
|
+
* >= 4: per-mount comparison details
|
|
499
|
+
*
|
|
500
|
+
* @param {string} pSourceBeaconID
|
|
501
|
+
* @returns {object|null} { Peer, Mount } or null if no shared-fs peer exists
|
|
502
|
+
*/
|
|
503
|
+
findSharedFsPeer(pSourceBeaconID)
|
|
504
|
+
{
|
|
505
|
+
let tmpNoisy = (this.fable && this.fable.LogNoisiness) || 0;
|
|
506
|
+
|
|
507
|
+
let tmpCoordinator = this._getService('UltravisorBeaconCoordinator');
|
|
508
|
+
if (!tmpCoordinator)
|
|
509
|
+
{
|
|
510
|
+
if (tmpNoisy >= 2)
|
|
511
|
+
{
|
|
512
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): no UltravisorBeaconCoordinator service registered, returning null.`);
|
|
513
|
+
}
|
|
514
|
+
return null;
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
let tmpSource = tmpCoordinator.getBeacon(pSourceBeaconID);
|
|
518
|
+
if (!tmpSource)
|
|
519
|
+
{
|
|
520
|
+
if (tmpNoisy >= 2)
|
|
521
|
+
{
|
|
522
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): source beacon not in coordinator registry, returning null.`);
|
|
523
|
+
}
|
|
524
|
+
return null;
|
|
525
|
+
}
|
|
526
|
+
if (!tmpSource.HostID)
|
|
527
|
+
{
|
|
528
|
+
if (tmpNoisy >= 2)
|
|
529
|
+
{
|
|
530
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): source beacon has no HostID (legacy beacon), returning null.`);
|
|
531
|
+
}
|
|
532
|
+
return null;
|
|
533
|
+
}
|
|
534
|
+
if (!Array.isArray(tmpSource.SharedMounts) || tmpSource.SharedMounts.length === 0)
|
|
535
|
+
{
|
|
536
|
+
if (tmpNoisy >= 2)
|
|
537
|
+
{
|
|
538
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): source beacon advertises no SharedMounts, returning null.`);
|
|
539
|
+
}
|
|
540
|
+
return null;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
if (tmpNoisy >= 2)
|
|
544
|
+
{
|
|
545
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): source HostID=${tmpSource.HostID}, ${tmpSource.SharedMounts.length} mount(s)=${JSON.stringify(tmpSource.SharedMounts.map((m) => m.MountID))}`);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
let tmpAllBeacons = tmpCoordinator.listBeacons();
|
|
549
|
+
if (tmpNoisy >= 2)
|
|
550
|
+
{
|
|
551
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): scanning ${tmpAllBeacons.length} registered beacon(s) for shared-fs peers...`);
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
for (let i = 0; i < tmpAllBeacons.length; i++)
|
|
555
|
+
{
|
|
556
|
+
let tmpPeer = tmpAllBeacons[i];
|
|
557
|
+
if (!tmpPeer || tmpPeer.BeaconID === pSourceBeaconID)
|
|
558
|
+
{
|
|
559
|
+
if (tmpNoisy >= 3)
|
|
560
|
+
{
|
|
561
|
+
this.log.info(`[Reachability] skip [${tmpPeer ? tmpPeer.BeaconID : '(null)'}]: ${tmpPeer ? 'self' : 'null entry'}`);
|
|
562
|
+
}
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
if (tmpPeer.Status && tmpPeer.Status !== 'Online')
|
|
566
|
+
{
|
|
567
|
+
if (tmpNoisy >= 3)
|
|
568
|
+
{
|
|
569
|
+
this.log.info(`[Reachability] skip [${tmpPeer.BeaconID}]: status=${tmpPeer.Status} (not Online)`);
|
|
570
|
+
}
|
|
571
|
+
continue;
|
|
572
|
+
}
|
|
573
|
+
if (!tmpPeer.HostID || tmpPeer.HostID !== tmpSource.HostID)
|
|
574
|
+
{
|
|
575
|
+
if (tmpNoisy >= 3)
|
|
576
|
+
{
|
|
577
|
+
this.log.info(`[Reachability] skip [${tmpPeer.BeaconID}]: HostID=${tmpPeer.HostID || '(none)'} != source ${tmpSource.HostID}`);
|
|
578
|
+
}
|
|
579
|
+
continue;
|
|
580
|
+
}
|
|
581
|
+
if (tmpNoisy >= 4)
|
|
582
|
+
{
|
|
583
|
+
this.log.info(`[Reachability] compare mounts for [${tmpPeer.BeaconID}]: source=${JSON.stringify(tmpSource.SharedMounts.map((m) => m.MountID))} vs peer=${JSON.stringify((tmpPeer.SharedMounts || []).map((m) => m.MountID))}`);
|
|
584
|
+
}
|
|
585
|
+
let tmpSharedMount = this._findSharedMount(tmpSource.SharedMounts, tmpPeer.SharedMounts);
|
|
586
|
+
if (tmpSharedMount)
|
|
587
|
+
{
|
|
588
|
+
if (tmpNoisy >= 2)
|
|
589
|
+
{
|
|
590
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): MATCH peer=[${tmpPeer.BeaconID}] mount=[${tmpSharedMount.MountID}] root=${tmpSharedMount.Root}`);
|
|
591
|
+
}
|
|
592
|
+
return { Peer: tmpPeer, Mount: tmpSharedMount };
|
|
593
|
+
}
|
|
594
|
+
if (tmpNoisy >= 3)
|
|
595
|
+
{
|
|
596
|
+
this.log.info(`[Reachability] skip [${tmpPeer.BeaconID}]: same HostID but no overlapping MountID`);
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
if (tmpNoisy >= 2)
|
|
601
|
+
{
|
|
602
|
+
this.log.info(`[Reachability] findSharedFsPeer(${pSourceBeaconID}): no shared-fs peer found among ${tmpAllBeacons.length} beacon(s).`);
|
|
603
|
+
}
|
|
604
|
+
return null;
|
|
605
|
+
}
|
|
401
606
|
}
|
|
402
607
|
|
|
403
608
|
module.exports = UltravisorBeaconReachability;
|
|
@@ -25,6 +25,18 @@ function _getService(pTask, pTypeName)
|
|
|
25
25
|
: null;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Get the LogNoisiness level (0-5) from the Fable instance attached to a task.
|
|
30
|
+
* Used to gate verbose diagnostic logging in the platform tasks. The user
|
|
31
|
+
* controls this via the RETOLD_LOG_NOISINESS environment variable, which the
|
|
32
|
+
* stack launcher applies to both the retold-remote Fable and the Ultravisor
|
|
33
|
+
* Pict instance at startup.
|
|
34
|
+
*/
|
|
35
|
+
function _getNoisiness(pTask)
|
|
36
|
+
{
|
|
37
|
+
return (pTask && pTask.fable && pTask.fable.LogNoisiness) || 0;
|
|
38
|
+
}
|
|
39
|
+
|
|
28
40
|
|
|
29
41
|
module.exports =
|
|
30
42
|
[
|
|
@@ -60,7 +72,7 @@ module.exports =
|
|
|
60
72
|
{
|
|
61
73
|
return fCallback(null, {
|
|
62
74
|
EventToFire: 'Error',
|
|
63
|
-
Outputs: { URL: '', BeaconID: '', BeaconName: '', Context: '', Path: '', Filename: '' },
|
|
75
|
+
Outputs: { URL: '', BeaconID: '', BeaconName: '', Context: '', Path: '', Filename: '', LocalPath: '' },
|
|
64
76
|
Log: [`Resolve Address: could not resolve "${tmpAddress}". Beacon may be offline or context not registered.`]
|
|
65
77
|
});
|
|
66
78
|
}
|
|
@@ -74,7 +86,8 @@ module.exports =
|
|
|
74
86
|
Filename: tmpResolved.Filename,
|
|
75
87
|
Strategy: 'direct',
|
|
76
88
|
DirectURL: '',
|
|
77
|
-
ProxyURL: ''
|
|
89
|
+
ProxyURL: '',
|
|
90
|
+
LocalPath: ''
|
|
78
91
|
};
|
|
79
92
|
|
|
80
93
|
// Helper: build a full URL from a beacon's bind address + context path + resource path
|
|
@@ -105,29 +118,115 @@ module.exports =
|
|
|
105
118
|
return pDirectBaseURL.replace(/\/$/, '') + tmpContextPath + tmpEncodedPath;
|
|
106
119
|
};
|
|
107
120
|
|
|
121
|
+
// Helper: compute the absolute path on the source beacon's filesystem
|
|
122
|
+
// for the resolved resource. Returns null if the source beacon does
|
|
123
|
+
// not have a BasePath registered for this context.
|
|
124
|
+
let _computeSharedFsLocalPath = function ()
|
|
125
|
+
{
|
|
126
|
+
let tmpSourceBeacon = tmpCoordinator.getBeacon(tmpResolved.BeaconID);
|
|
127
|
+
let tmpCtx = tmpSourceBeacon && tmpSourceBeacon.Contexts
|
|
128
|
+
? tmpSourceBeacon.Contexts[tmpResolved.Context]
|
|
129
|
+
: null;
|
|
130
|
+
if (tmpCtx && tmpCtx.BasePath)
|
|
131
|
+
{
|
|
132
|
+
return libPath.join(tmpCtx.BasePath, tmpResolved.Path);
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
let tmpReachability = _getService(pTask, 'UltravisorBeaconReachability');
|
|
138
|
+
let tmpNoisy = _getNoisiness(pTask);
|
|
139
|
+
|
|
140
|
+
if (tmpNoisy >= 2)
|
|
141
|
+
{
|
|
142
|
+
pTask.log.info(`[ResolveAddress] entry: address=${tmpAddress} sourceBeacon=${tmpResolved.BeaconID} requestingBeacon=${pResolvedSettings.RequestingBeaconID || '(none)'} reachability=${tmpReachability ? 'present' : 'missing'}`);
|
|
143
|
+
}
|
|
144
|
+
|
|
108
145
|
// Resolve transfer strategy when a requesting beacon is specified
|
|
109
146
|
let tmpRequestingBeaconID = pResolvedSettings.RequestingBeaconID;
|
|
110
|
-
if (tmpRequestingBeaconID)
|
|
147
|
+
if (tmpRequestingBeaconID && tmpReachability)
|
|
111
148
|
{
|
|
112
|
-
|
|
113
|
-
if (tmpReachability)
|
|
149
|
+
if (tmpNoisy >= 2)
|
|
114
150
|
{
|
|
115
|
-
|
|
116
|
-
|
|
151
|
+
pTask.log.info(`[ResolveAddress] explicit RequestingBeaconID=${tmpRequestingBeaconID} provided — calling resolveStrategy directly.`);
|
|
152
|
+
}
|
|
153
|
+
let tmpStrategyResult = tmpReachability.resolveStrategy(tmpResolved.BeaconID, tmpRequestingBeaconID);
|
|
154
|
+
tmpOutputs.Strategy = tmpStrategyResult.Strategy;
|
|
117
155
|
|
|
118
|
-
|
|
156
|
+
if (tmpStrategyResult.Strategy === 'shared-fs')
|
|
157
|
+
{
|
|
158
|
+
// Both beacons see the same filesystem mount. Look up the
|
|
159
|
+
// source beacon's context BasePath and join it with the inner
|
|
160
|
+
// resource path to get an absolute path that's also valid on
|
|
161
|
+
// the requesting beacon (because they share the mount).
|
|
162
|
+
let tmpAbsPath = _computeSharedFsLocalPath();
|
|
163
|
+
if (tmpAbsPath)
|
|
164
|
+
{
|
|
165
|
+
tmpOutputs.LocalPath = tmpAbsPath;
|
|
166
|
+
// URL is intentionally left as the original (relative) URL
|
|
167
|
+
// — file-transfer will see LocalPath and short-circuit, so
|
|
168
|
+
// the URL is never actually fetched.
|
|
169
|
+
}
|
|
170
|
+
else
|
|
171
|
+
{
|
|
172
|
+
// No BasePath available — fall back to direct so the
|
|
173
|
+
// transfer still works via HTTP.
|
|
174
|
+
pTask.log.warn(`Resolve Address: shared-fs strategy chosen but no BasePath available for context [${tmpResolved.Context}] on beacon [${tmpResolved.BeaconID}], falling back to direct.`);
|
|
175
|
+
tmpOutputs.Strategy = 'direct';
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
else if (tmpStrategyResult.Strategy === 'direct' && tmpStrategyResult.DirectURL)
|
|
179
|
+
{
|
|
180
|
+
tmpOutputs.DirectURL = _buildDirectURL(tmpStrategyResult.DirectURL);
|
|
181
|
+
tmpOutputs.URL = tmpOutputs.DirectURL;
|
|
182
|
+
}
|
|
183
|
+
else if (tmpStrategyResult.Strategy === 'proxy')
|
|
184
|
+
{
|
|
185
|
+
// Proxy URL: Ultravisor's own endpoint serves the file
|
|
186
|
+
tmpOutputs.ProxyURL = tmpResolved.URL;
|
|
187
|
+
tmpOutputs.URL = tmpResolved.URL;
|
|
188
|
+
}
|
|
189
|
+
// 'local' strategy: URL stays as the context BaseURL (same host)
|
|
190
|
+
}
|
|
191
|
+
else if (tmpReachability)
|
|
192
|
+
{
|
|
193
|
+
// Auto-detect a shared-fs peer when no RequestingBeaconID was passed.
|
|
194
|
+
// This is the common case for retold-remote: it dispatches a media
|
|
195
|
+
// operation, the file lives on the retold-remote beacon, and an
|
|
196
|
+
// orator-conversion beacon on the same host shares the mount. The
|
|
197
|
+
// auto-detection finds the orator-conversion peer and lets us
|
|
198
|
+
// short-circuit the file-transfer entirely.
|
|
199
|
+
if (tmpNoisy >= 2)
|
|
200
|
+
{
|
|
201
|
+
pTask.log.info(`[ResolveAddress] no RequestingBeaconID — entering auto-detect path for source ${tmpResolved.BeaconID}`);
|
|
202
|
+
}
|
|
203
|
+
let tmpPeerInfo = tmpReachability.findSharedFsPeer(tmpResolved.BeaconID);
|
|
204
|
+
if (tmpPeerInfo)
|
|
205
|
+
{
|
|
206
|
+
let tmpAbsPath = _computeSharedFsLocalPath();
|
|
207
|
+
if (tmpAbsPath)
|
|
119
208
|
{
|
|
120
|
-
tmpOutputs.
|
|
121
|
-
tmpOutputs.
|
|
209
|
+
tmpOutputs.Strategy = 'shared-fs';
|
|
210
|
+
tmpOutputs.LocalPath = tmpAbsPath;
|
|
211
|
+
pTask.log.info(`Resolve Address: auto-detected shared-fs peer [${tmpPeerInfo.Peer.BeaconID}] for source [${tmpResolved.BeaconID}] via mount [${tmpPeerInfo.Mount.MountID}].`);
|
|
212
|
+
if (tmpNoisy >= 2)
|
|
213
|
+
{
|
|
214
|
+
pTask.log.info(`[ResolveAddress] auto-detect SUCCESS: LocalPath=${tmpAbsPath} (peer=${tmpPeerInfo.Peer.BeaconID}, mount=${tmpPeerInfo.Mount.MountID})`);
|
|
215
|
+
}
|
|
122
216
|
}
|
|
123
|
-
else if (
|
|
217
|
+
else if (tmpNoisy >= 2)
|
|
124
218
|
{
|
|
125
|
-
|
|
126
|
-
tmpOutputs.ProxyURL = tmpResolved.URL;
|
|
127
|
-
tmpOutputs.URL = tmpResolved.URL;
|
|
219
|
+
pTask.log.info(`[ResolveAddress] auto-detect found peer [${tmpPeerInfo.Peer.BeaconID}] but source beacon has no BasePath for context [${tmpResolved.Context}] — cannot use shared-fs.`);
|
|
128
220
|
}
|
|
129
|
-
// 'local' strategy: URL stays as the context BaseURL (same host)
|
|
130
221
|
}
|
|
222
|
+
else if (tmpNoisy >= 2)
|
|
223
|
+
{
|
|
224
|
+
pTask.log.info(`[ResolveAddress] auto-detect found NO shared-fs peer for source ${tmpResolved.BeaconID} — falling through to default direct/proxy strategy.`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
else if (tmpNoisy >= 2)
|
|
228
|
+
{
|
|
229
|
+
pTask.log.info(`[ResolveAddress] reachability service not available — skipping auto-detect, default strategy will be used.`);
|
|
131
230
|
}
|
|
132
231
|
|
|
133
232
|
// If the URL is still relative (no protocol), use the beacon's first
|
|
@@ -158,19 +257,26 @@ module.exports =
|
|
|
158
257
|
tmpStateWrites[pResolvedSettings.Destination] = tmpOutputs;
|
|
159
258
|
}
|
|
160
259
|
|
|
161
|
-
|
|
260
|
+
let tmpResolvedDest = tmpOutputs.LocalPath || tmpOutputs.URL;
|
|
261
|
+
pTask.log.info(`Resolve Address: ${tmpAddress} → ${tmpResolvedDest} [${tmpOutputs.Strategy}] (beacon: ${tmpResolved.BeaconName})`);
|
|
262
|
+
|
|
263
|
+
let tmpLogLines = [
|
|
264
|
+
`Resolved: ${tmpAddress}`,
|
|
265
|
+
`URL: ${tmpOutputs.URL}`,
|
|
266
|
+
`Strategy: ${tmpOutputs.Strategy}`,
|
|
267
|
+
`Beacon: ${tmpResolved.BeaconName} (${tmpResolved.BeaconID})`,
|
|
268
|
+
`Context: ${tmpResolved.Context}, Path: ${tmpResolved.Path}`
|
|
269
|
+
];
|
|
270
|
+
if (tmpOutputs.LocalPath)
|
|
271
|
+
{
|
|
272
|
+
tmpLogLines.push(`LocalPath: ${tmpOutputs.LocalPath} (shared filesystem — no transfer needed)`);
|
|
273
|
+
}
|
|
162
274
|
|
|
163
275
|
return fCallback(null, {
|
|
164
276
|
EventToFire: 'Complete',
|
|
165
277
|
Outputs: tmpOutputs,
|
|
166
278
|
StateWrites: tmpStateWrites,
|
|
167
|
-
Log:
|
|
168
|
-
`Resolved: ${tmpAddress}`,
|
|
169
|
-
`URL: ${tmpOutputs.URL}`,
|
|
170
|
-
`Strategy: ${tmpOutputs.Strategy}`,
|
|
171
|
-
`Beacon: ${tmpResolved.BeaconName} (${tmpResolved.BeaconID})`,
|
|
172
|
-
`Context: ${tmpResolved.Context}, Path: ${tmpResolved.Path}`
|
|
173
|
-
]
|
|
279
|
+
Log: tmpLogLines
|
|
174
280
|
});
|
|
175
281
|
}
|
|
176
282
|
},
|
|
@@ -182,13 +288,70 @@ module.exports =
|
|
|
182
288
|
Execute: function (pTask, pResolvedSettings, pExecutionContext, fCallback)
|
|
183
289
|
{
|
|
184
290
|
let tmpSourceURL = pResolvedSettings.SourceURL;
|
|
291
|
+
let tmpSourceLocalPath = pResolvedSettings.SourceLocalPath;
|
|
185
292
|
let tmpFilename = pResolvedSettings.Filename;
|
|
293
|
+
let tmpNoisy = _getNoisiness(pTask);
|
|
294
|
+
|
|
295
|
+
if (tmpNoisy >= 2)
|
|
296
|
+
{
|
|
297
|
+
pTask.log.info(`[FileTransfer] entry: SourceURL=${tmpSourceURL ? tmpSourceURL.substring(0, 80) : '(none)'} SourceLocalPath=${tmpSourceLocalPath || '(none)'} Filename=${tmpFilename || '(none)'}`);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Shared-filesystem fast path: if the source beacon and the requesting
|
|
301
|
+
// beacon both report the same MountID, resolve-address will populate
|
|
302
|
+
// SourceLocalPath with the absolute on-disk path. Both beacons see the
|
|
303
|
+
// same file at the same path because they share the mount, so we hand
|
|
304
|
+
// it back as the LocalPath without copying anything. This is the
|
|
305
|
+
// difference between "instant" and "374 MB download to staging" on a
|
|
306
|
+
// stack-mode deployment where retold-remote and orator-conversion live
|
|
307
|
+
// in the same container.
|
|
308
|
+
if (tmpSourceLocalPath)
|
|
309
|
+
{
|
|
310
|
+
if (libFS.existsSync(tmpSourceLocalPath))
|
|
311
|
+
{
|
|
312
|
+
let tmpStat;
|
|
313
|
+
try
|
|
314
|
+
{
|
|
315
|
+
tmpStat = libFS.statSync(tmpSourceLocalPath);
|
|
316
|
+
}
|
|
317
|
+
catch (pStatError)
|
|
318
|
+
{
|
|
319
|
+
tmpStat = null;
|
|
320
|
+
}
|
|
321
|
+
pTask.log.info(`File Transfer: shared-fs hit, using ${tmpSourceLocalPath} directly (${tmpStat ? tmpStat.size : '?'} bytes, no copy).`);
|
|
322
|
+
return fCallback(null, {
|
|
323
|
+
EventToFire: 'Complete',
|
|
324
|
+
Outputs: {
|
|
325
|
+
LocalPath: tmpSourceLocalPath,
|
|
326
|
+
BytesTransferred: 0,
|
|
327
|
+
DurationMs: 0,
|
|
328
|
+
Strategy: 'shared-fs'
|
|
329
|
+
},
|
|
330
|
+
Log: [
|
|
331
|
+
`Shared filesystem detected — no transfer needed.`,
|
|
332
|
+
`Source path: ${tmpSourceLocalPath}`,
|
|
333
|
+
`Bytes transferred: 0 (zero-copy)`
|
|
334
|
+
]
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
// Path was provided but doesn't exist on this beacon — log and fall
|
|
338
|
+
// through to the HTTP path so we still satisfy the request.
|
|
339
|
+
pTask.log.warn(`File Transfer: SourceLocalPath [${tmpSourceLocalPath}] does not exist on this beacon, falling back to HTTP transfer.`);
|
|
340
|
+
if (tmpNoisy >= 2)
|
|
341
|
+
{
|
|
342
|
+
pTask.log.info(`[FileTransfer] SourceLocalPath was set but file is missing on this beacon — the requesting beacon doesn't actually share this filesystem at the expected path.`);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
else if (tmpNoisy >= 2)
|
|
346
|
+
{
|
|
347
|
+
pTask.log.info(`[FileTransfer] no SourceLocalPath in settings — running standard HTTP download path. (Either resolve-address chose 'direct'/'proxy', or the operation graph isn't wiring resolve.LocalPath → transfer.SourceLocalPath.)`);
|
|
348
|
+
}
|
|
186
349
|
|
|
187
350
|
if (!tmpSourceURL)
|
|
188
351
|
{
|
|
189
352
|
return fCallback(null, {
|
|
190
353
|
EventToFire: 'Error',
|
|
191
|
-
Outputs: { LocalPath: '', BytesTransferred: 0, DurationMs: 0 },
|
|
354
|
+
Outputs: { LocalPath: '', BytesTransferred: 0, DurationMs: 0, Strategy: '' },
|
|
192
355
|
Log: ['File Transfer: no SourceURL provided.']
|
|
193
356
|
});
|
|
194
357
|
}
|
|
@@ -197,7 +360,7 @@ module.exports =
|
|
|
197
360
|
{
|
|
198
361
|
return fCallback(null, {
|
|
199
362
|
EventToFire: 'Error',
|
|
200
|
-
Outputs: { LocalPath: '', BytesTransferred: 0, DurationMs: 0 },
|
|
363
|
+
Outputs: { LocalPath: '', BytesTransferred: 0, DurationMs: 0, Strategy: '' },
|
|
201
364
|
Log: ['File Transfer: no Filename provided.']
|
|
202
365
|
});
|
|
203
366
|
}
|
|
@@ -467,7 +630,7 @@ function _pipeToFile(pResponse, pOutputPath, pStartTime, pFilename, pTask, fCall
|
|
|
467
630
|
|
|
468
631
|
return fCallback(null, {
|
|
469
632
|
EventToFire: 'Complete',
|
|
470
|
-
Outputs: { LocalPath: pOutputPath, BytesTransferred: tmpBytes, DurationMs: tmpDuration },
|
|
633
|
+
Outputs: { LocalPath: pOutputPath, BytesTransferred: tmpBytes, DurationMs: tmpDuration, Strategy: 'http' },
|
|
471
634
|
Log: [
|
|
472
635
|
`Downloaded: ${pFilename}`,
|
|
473
636
|
`Size: ${tmpBytes} bytes`,
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"Hash": "file-transfer",
|
|
3
3
|
"Type": "file-transfer",
|
|
4
4
|
"Name": "File Transfer",
|
|
5
|
-
"Description": "Download a file from a URL to the operation's staging directory. Streams the download — no base64 encoding. Logs bytes transferred and duration for visibility in the manifest.",
|
|
5
|
+
"Description": "Download a file from a URL to the operation's staging directory. Streams the download — no base64 encoding. When SourceLocalPath is provided (set by resolve-address with a shared-fs strategy), the task short-circuits and returns that path directly without copying anything. Logs bytes transferred and duration for visibility in the manifest.",
|
|
6
6
|
"Category": "platform",
|
|
7
7
|
"Capability": "Data Transfer",
|
|
8
8
|
"Action": "Download",
|
|
@@ -13,14 +13,17 @@
|
|
|
13
13
|
{ "Name": "Error", "IsError": true }
|
|
14
14
|
],
|
|
15
15
|
"SettingsInputs": [
|
|
16
|
-
{ "Name": "SourceURL", "DataType": "String", "Required":
|
|
17
|
-
{ "Name": "
|
|
18
|
-
{ "Name": "
|
|
16
|
+
{ "Name": "SourceURL", "DataType": "String", "Required": false, "Description": "URL to download from. Required unless SourceLocalPath is provided." },
|
|
17
|
+
{ "Name": "SourceLocalPath", "DataType": "String", "Required": false, "Description": "Absolute filesystem path to read directly. When set and the file exists, the HTTP download is skipped entirely and BytesTransferred reports 0. Used by the shared-fs reachability strategy." },
|
|
18
|
+
{ "Name": "Filename", "DataType": "String", "Required": false, "Description": "Filename to save as in the staging directory. Required unless SourceLocalPath is provided." },
|
|
19
|
+
{ "Name": "AffinityKey", "DataType": "String", "Required": false, "Description": "Affinity key for caching (same key skips re-download)" },
|
|
20
|
+
{ "Name": "TimeoutMs", "DataType": "Number", "Required": false, "Description": "HTTP download timeout in milliseconds (default 300000)" }
|
|
19
21
|
],
|
|
20
22
|
"StateOutputs": [
|
|
21
|
-
{ "Name": "LocalPath", "DataType": "String", "Description": "Absolute path to the downloaded
|
|
22
|
-
{ "Name": "BytesTransferred", "DataType": "Number", "Description": "Size of the downloaded file in bytes" },
|
|
23
|
-
{ "Name": "DurationMs", "DataType": "Number", "Description": "Download duration in milliseconds" }
|
|
23
|
+
{ "Name": "LocalPath", "DataType": "String", "Description": "Absolute path to the file (downloaded to staging, or pointing at the shared-fs source when no copy was needed)" },
|
|
24
|
+
{ "Name": "BytesTransferred", "DataType": "Number", "Description": "Size of the downloaded file in bytes (0 when shared-fs zero-copy was used)" },
|
|
25
|
+
{ "Name": "DurationMs", "DataType": "Number", "Description": "Download duration in milliseconds" },
|
|
26
|
+
{ "Name": "Strategy", "DataType": "String", "Description": "Which path was taken: 'shared-fs' (zero-copy) or 'http' (downloaded)" }
|
|
24
27
|
],
|
|
25
|
-
"DefaultSettings": { "SourceURL": "", "Filename": "", "AffinityKey": "" }
|
|
28
|
+
"DefaultSettings": { "SourceURL": "", "SourceLocalPath": "", "Filename": "", "AffinityKey": "", "TimeoutMs": 300000 }
|
|
26
29
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"Hash": "resolve-address",
|
|
3
3
|
"Type": "resolve-address",
|
|
4
4
|
"Name": "Resolve Address",
|
|
5
|
-
"Description": "Resolve a universal address (>beacon/context/path) to a concrete URL and metadata, with optional transfer strategy resolution (local/direct/proxy) when a requesting beacon is specified.",
|
|
5
|
+
"Description": "Resolve a universal address (>beacon/context/path) to a concrete URL and metadata, with optional transfer strategy resolution (local/shared-fs/direct/proxy) when a requesting beacon is specified.",
|
|
6
6
|
"Category": "platform",
|
|
7
7
|
"Capability": "Address Resolution",
|
|
8
8
|
"Action": "Resolve",
|
|
@@ -24,9 +24,10 @@
|
|
|
24
24
|
{ "Name": "Context", "DataType": "String", "Description": "Context name (e.g. File, Cache)" },
|
|
25
25
|
{ "Name": "Path", "DataType": "String", "Description": "Path within the context" },
|
|
26
26
|
{ "Name": "Filename", "DataType": "String", "Description": "Filename portion of the path" },
|
|
27
|
-
{ "Name": "Strategy", "DataType": "String", "Description": "Transfer strategy: local, direct, or proxy" },
|
|
27
|
+
{ "Name": "Strategy", "DataType": "String", "Description": "Transfer strategy: local, shared-fs, direct, or proxy" },
|
|
28
28
|
{ "Name": "DirectURL", "DataType": "String", "Description": "Direct beacon-to-beacon URL (when strategy is direct)" },
|
|
29
|
-
{ "Name": "ProxyURL", "DataType": "String", "Description": "Proxy URL through Ultravisor (when strategy is proxy)" }
|
|
29
|
+
{ "Name": "ProxyURL", "DataType": "String", "Description": "Proxy URL through Ultravisor (when strategy is proxy)" },
|
|
30
|
+
{ "Name": "LocalPath", "DataType": "String", "Description": "Absolute path to the resource on the shared filesystem (when strategy is shared-fs); empty otherwise" }
|
|
30
31
|
],
|
|
31
32
|
"DefaultSettings": { "Address": "", "Destination": "", "RequestingBeaconID": "" }
|
|
32
33
|
}
|