wenay-common2 1.0.69 → 1.0.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/demo/client.ts +222 -0
- package/demo/index.html +58 -0
- package/demo/server.ts +93 -0
- package/doc/INTENT.md +31 -0
- package/doc/ROADMAP.md +19 -1
- package/doc/changes/1.0.70.md +4 -0
- package/doc/changes/1.0.71.md +9 -0
- package/doc/changes/1.0.72.md +6 -0
- package/doc/changes/1.0.73.md +6 -0
- package/doc/wenay-common2-rare.md +29 -8
- package/doc/wenay-common2.md +13 -2
- package/lib/Common/media/media-index.d.ts +1 -0
- package/lib/Common/media/media-index.js +1 -0
- package/lib/Common/media/media-source.d.ts +3 -0
- package/lib/Common/media/media-source.js +136 -27
- package/lib/Common/media/media-view.d.ts +42 -0
- package/lib/Common/media/media-view.js +191 -0
- package/lib/Common/peer/peer-client.d.ts +12 -5
- package/lib/Common/peer/peer-client.js +53 -3
- package/lib/Common/peer/peer-host.d.ts +8 -4
- package/lib/Common/peer/peer-host.js +2 -2
- package/lib/Common/peer/peer-relay.d.ts +10 -2
- package/lib/Common/peer/peer-relay.js +40 -18
- package/observe/PLAN.md +131 -0
- package/observe/README.md +226 -0
- package/observe/hot-write.test.ts +95 -0
- package/observe/listen-core.test.ts +66 -0
- package/observe/listen-store.test.ts +56 -0
- package/observe/listen.test.ts +92 -0
- package/observe/reactive.test.ts +270 -0
- package/observe/reactive.ts +1 -0
- package/observe/store-manager.test.ts +118 -0
- package/observe/store-mirror.example.ts +74 -0
- package/observe/store.test.ts +235 -0
- package/observe/store.ts +1 -0
- package/observe/usage-real-socket.ts +174 -0
- package/observe/usage.ts +200 -0
- package/oracle/PASSED.md +27 -0
- package/oracle/README.md +12 -0
- package/oracle/fixes-primitives.spec.ts +90 -0
- package/oracle/realsocket/_rs.ts +106 -0
- package/oracle/realsocket/auth.spec.ts +91 -0
- package/oracle/realsocket/callbacks.spec.ts +122 -0
- package/oracle/realsocket/caps.spec.ts +124 -0
- package/oracle/realsocket/core.spec.ts +49 -0
- package/oracle/realsocket/dedupe.spec.ts +142 -0
- package/oracle/realsocket/errors.spec.ts +121 -0
- package/oracle/realsocket/lifecycle.spec.ts +128 -0
- package/oracle/realsocket/limits.spec.ts +98 -0
- package/oracle/realsocket/pipe.spec.ts +101 -0
- package/oracle/realsocket/shape.spec.ts +124 -0
- package/oracle/realsocket/slimv2.spec.ts +116 -0
- package/oracle/realsocket/stress.spec.ts +132 -0
- package/oracle/regression/_clientapiall-replay.fixture.ts +57 -0
- package/oracle/regression/async-queues.spec.ts +254 -0
- package/oracle/regression/bytestream.spec.ts +152 -0
- package/oracle/regression/clientapiall-replay-types.spec.ts +47 -0
- package/oracle/regression/core-clone-equal.spec.ts +124 -0
- package/oracle/regression/data-structures.spec.ts +135 -0
- package/oracle/regression/listen-events.spec.ts +206 -0
- package/oracle/regression/observe-core.spec.ts +278 -0
- package/oracle/regression/package-export.spec.ts +120 -0
- package/oracle/regression/rpc-dedupe-callbacks.spec.ts +195 -0
- package/oracle/regression/rpc-lifecycle.spec.ts +150 -0
- package/oracle/regression/store-each.spec.ts +209 -0
- package/package.json +6 -1
- package/replay/PLAN.md +171 -0
- package/replay/canvas-socket.test.ts +187 -0
- package/replay/coalesce.test.ts +260 -0
- package/replay/conflate-socket.test.ts +288 -0
- package/replay/conflate.test.ts +225 -0
- package/replay/history.test.ts +222 -0
- package/replay/media-socket.test.ts +157 -0
- package/replay/media-view.test.ts +164 -0
- package/replay/offline-store-socket.test.ts +290 -0
- package/replay/offline-store.test.ts +156 -0
- package/replay/peer-repair.test.ts +184 -0
- package/replay/peer-sdk.test.ts +228 -0
- package/replay/replay-listen.test.ts +156 -0
- package/replay/route-coordinator.test.ts +314 -0
- package/replay/route-handoff.test.ts +150 -0
- package/replay/route-webrtc.test.ts +321 -0
- package/replay/rpc-auto.test.ts +256 -0
- package/replay/socket-replay.test.ts +199 -0
- package/replay/staleness.test.ts +176 -0
- package/replay/store-replay.test.ts +151 -0
- package/replay/video-socket.demo.ts +200 -0
- package/doc/changes/1.0.63.md +0 -8
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MEDIA_FRAME_HEADER_BYTES = exports.MEDIA_FRAME_VERSION = exports.MEDIA_FRAME_MAGIC = void 0;
|
|
4
|
+
exports.toBytes = toBytes;
|
|
4
5
|
exports.encodeMediaFrame = encodeMediaFrame;
|
|
5
6
|
exports.decodeMediaFrame = decodeMediaFrame;
|
|
6
7
|
exports.createAudioSource = createAudioSource;
|
|
@@ -356,7 +357,7 @@ function createAudioSource(opts = {}) {
|
|
|
356
357
|
async function start() {
|
|
357
358
|
try {
|
|
358
359
|
ensureSocketTransport(transport);
|
|
359
|
-
if (!hasGetUserMedia()) {
|
|
360
|
+
if (!opts.stream && !hasGetUserMedia()) {
|
|
360
361
|
shell.setState('no-device');
|
|
361
362
|
return shell.state;
|
|
362
363
|
}
|
|
@@ -364,7 +365,7 @@ function createAudioSource(opts = {}) {
|
|
|
364
365
|
shell.setState('requesting');
|
|
365
366
|
shell.stats.startedAt = nowMono();
|
|
366
367
|
const constraints = { audio: { deviceId: deviceId ? { exact: deviceId } : undefined, channelCount: opts.channels, sampleRate: opts.sampleRate } };
|
|
367
|
-
stream = await globalThis.navigator.mediaDevices.getUserMedia(constraints);
|
|
368
|
+
stream = await resolveMediaStream(opts.stream, () => globalThis.navigator.mediaDevices.getUserMedia(constraints));
|
|
368
369
|
if (opts.mode == 'record')
|
|
369
370
|
startRecord(stream);
|
|
370
371
|
else
|
|
@@ -395,6 +396,50 @@ function createAudioSource(opts = {}) {
|
|
|
395
396
|
};
|
|
396
397
|
return attachControl([shell.emit, shell.listenApi], control);
|
|
397
398
|
}
|
|
399
|
+
async function resolveMediaStream(injected, fallback) {
|
|
400
|
+
if (!injected)
|
|
401
|
+
return fallback();
|
|
402
|
+
return typeof injected == 'function' ? await injected() : injected;
|
|
403
|
+
}
|
|
404
|
+
function videoEncodeWorkerCode() {
|
|
405
|
+
return `
|
|
406
|
+
const canvas = new OffscreenCanvas(1, 1)
|
|
407
|
+
const ctx = canvas.getContext('2d')
|
|
408
|
+
onmessage = async function onEncodeRequest(ev) {
|
|
409
|
+
const req = ev.data
|
|
410
|
+
try {
|
|
411
|
+
canvas.width = req.w
|
|
412
|
+
canvas.height = req.h
|
|
413
|
+
ctx.drawImage(req.bitmap, 0, 0, req.w, req.h)
|
|
414
|
+
req.bitmap.close()
|
|
415
|
+
const blob = await canvas.convertToBlob({type: req.mime, quality: req.quality})
|
|
416
|
+
const buf = await blob.arrayBuffer()
|
|
417
|
+
postMessage({buf}, [buf])
|
|
418
|
+
} catch (e) {
|
|
419
|
+
postMessage({error: String((e && e.message) || e)})
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
`;
|
|
423
|
+
}
|
|
424
|
+
function startCaptureTicker(ms, tick, useWorker) {
|
|
425
|
+
const g = globalThis;
|
|
426
|
+
if (useWorker && g.Worker && g.Blob && g.URL) {
|
|
427
|
+
try {
|
|
428
|
+
const blob = new g.Blob([`setInterval(function () { postMessage(0) }, ${ms})`], { type: 'text/javascript' });
|
|
429
|
+
const url = g.URL.createObjectURL(blob);
|
|
430
|
+
const worker = new g.Worker(url);
|
|
431
|
+
worker.onmessage = function onCaptureTick() { tick(); };
|
|
432
|
+
return function stopWorkerTicker() {
|
|
433
|
+
worker.terminate();
|
|
434
|
+
g.URL.revokeObjectURL(url);
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
catch {
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
const timer = setInterval(tick, ms);
|
|
441
|
+
return function stopIntervalTicker() { clearInterval(timer); };
|
|
442
|
+
}
|
|
398
443
|
function mimeForVideo(codec) {
|
|
399
444
|
if (codec == 'png')
|
|
400
445
|
return 'image/png';
|
|
@@ -422,9 +467,37 @@ function createVideoSource(opts = {}) {
|
|
|
422
467
|
let video = opts.video;
|
|
423
468
|
let canvas = opts.canvas;
|
|
424
469
|
let ctx = null;
|
|
425
|
-
let
|
|
470
|
+
let stopTicker = null;
|
|
471
|
+
let imageCapture = null;
|
|
472
|
+
let encodeWorker = null;
|
|
426
473
|
let busy = false;
|
|
427
474
|
let seq = 0;
|
|
475
|
+
async function grabBitmap() {
|
|
476
|
+
if (imageCapture) {
|
|
477
|
+
try {
|
|
478
|
+
return await imageCapture.grabFrame();
|
|
479
|
+
}
|
|
480
|
+
catch { }
|
|
481
|
+
}
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
function emitVideoFrame(payload, codec, w, h) {
|
|
485
|
+
shell.emitFrame(encodeMediaFrame({
|
|
486
|
+
kind: 'video-frame',
|
|
487
|
+
codec,
|
|
488
|
+
seq: ++seq,
|
|
489
|
+
tMono: nowMono(),
|
|
490
|
+
width: w,
|
|
491
|
+
height: h,
|
|
492
|
+
}, payload), seq);
|
|
493
|
+
}
|
|
494
|
+
function encodeInWorker(bitmap, w, h, mime, quality) {
|
|
495
|
+
return new Promise(function encodeRoundtrip(resolve, reject) {
|
|
496
|
+
encodeWorker.onmessage = (ev) => ev.data?.error ? reject(new Error(ev.data.error)) : resolve(ev.data.buf);
|
|
497
|
+
encodeWorker.onerror = (e) => reject(new Error(String(e?.message ?? 'video encode worker error')));
|
|
498
|
+
encodeWorker.postMessage({ bitmap, w, h, mime, quality }, [bitmap]);
|
|
499
|
+
});
|
|
500
|
+
}
|
|
428
501
|
async function captureOne() {
|
|
429
502
|
if (busy || shell.state != 'live') {
|
|
430
503
|
shell.stats.dropped++;
|
|
@@ -432,26 +505,40 @@ function createVideoSource(opts = {}) {
|
|
|
432
505
|
}
|
|
433
506
|
busy = true;
|
|
434
507
|
try {
|
|
435
|
-
const
|
|
436
|
-
const
|
|
437
|
-
|
|
508
|
+
const bitmap = await grabBitmap();
|
|
509
|
+
const srcW = (bitmap ? bitmap.width : video.videoWidth ?? video.width) ?? 0;
|
|
510
|
+
const srcH = (bitmap ? bitmap.height : video.videoHeight ?? video.height) ?? 0;
|
|
511
|
+
let w = opts.width ?? srcW;
|
|
512
|
+
let h = opts.height ?? srcH;
|
|
513
|
+
if (srcW && srcH) {
|
|
514
|
+
if (opts.width && opts.height == undefined) {
|
|
515
|
+
w = Math.min(opts.width, srcW);
|
|
516
|
+
h = Math.round(w * srcH / srcW);
|
|
517
|
+
}
|
|
518
|
+
else if (opts.height && opts.width == undefined) {
|
|
519
|
+
h = Math.min(opts.height, srcH);
|
|
520
|
+
w = Math.round(h * srcW / srcH);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
if (!w || !h) {
|
|
524
|
+
bitmap?.close?.();
|
|
438
525
|
return;
|
|
526
|
+
}
|
|
527
|
+
const codec = opts.codec ?? 'jpeg';
|
|
528
|
+
const quality = opts.quality ?? 0.82;
|
|
529
|
+
if (encodeWorker && bitmap) {
|
|
530
|
+
const buf = await encodeInWorker(bitmap, w, h, mimeForVideo(codec), quality);
|
|
531
|
+
emitVideoFrame(new Uint8Array(buf), codec, w, h);
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
439
534
|
if (canvas.width != w)
|
|
440
535
|
canvas.width = w;
|
|
441
536
|
if (canvas.height != h)
|
|
442
537
|
canvas.height = h;
|
|
443
|
-
ctx.drawImage(video, 0, 0, w, h);
|
|
444
|
-
|
|
445
|
-
const blob = await canvasToBlob(canvas, mimeForVideo(codec),
|
|
446
|
-
|
|
447
|
-
shell.emitFrame(encodeMediaFrame({
|
|
448
|
-
kind: 'video-frame',
|
|
449
|
-
codec,
|
|
450
|
-
seq: ++seq,
|
|
451
|
-
tMono: nowMono(),
|
|
452
|
-
width: w,
|
|
453
|
-
height: h,
|
|
454
|
-
}, payload), seq);
|
|
538
|
+
ctx.drawImage(bitmap ?? video, 0, 0, w, h);
|
|
539
|
+
bitmap?.close?.();
|
|
540
|
+
const blob = await canvasToBlob(canvas, mimeForVideo(codec), quality);
|
|
541
|
+
emitVideoFrame(new Uint8Array(await blob.arrayBuffer()), codec, w, h);
|
|
455
542
|
}
|
|
456
543
|
catch (e) {
|
|
457
544
|
shell.setState('error', e);
|
|
@@ -461,10 +548,11 @@ function createVideoSource(opts = {}) {
|
|
|
461
548
|
}
|
|
462
549
|
}
|
|
463
550
|
function stop() {
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
551
|
+
stopTicker?.();
|
|
552
|
+
stopTicker = null;
|
|
553
|
+
imageCapture = null;
|
|
554
|
+
encodeWorker?.terminate?.();
|
|
555
|
+
encodeWorker = null;
|
|
468
556
|
stopTracks(stream);
|
|
469
557
|
stream = null;
|
|
470
558
|
if (!opts.video && video) {
|
|
@@ -478,7 +566,7 @@ function createVideoSource(opts = {}) {
|
|
|
478
566
|
async function start() {
|
|
479
567
|
try {
|
|
480
568
|
ensureSocketTransport(transport);
|
|
481
|
-
if (!hasGetUserMedia()) {
|
|
569
|
+
if (!opts.stream && !hasGetUserMedia()) {
|
|
482
570
|
shell.setState('no-device');
|
|
483
571
|
return shell.state;
|
|
484
572
|
}
|
|
@@ -490,26 +578,47 @@ function createVideoSource(opts = {}) {
|
|
|
490
578
|
stop();
|
|
491
579
|
shell.setState('requesting');
|
|
492
580
|
shell.stats.startedAt = nowMono();
|
|
493
|
-
stream = await globalThis.navigator.mediaDevices.getUserMedia({
|
|
581
|
+
stream = await resolveMediaStream(opts.stream, () => globalThis.navigator.mediaDevices.getUserMedia({
|
|
494
582
|
video: {
|
|
495
583
|
deviceId: deviceId ? { exact: deviceId } : undefined,
|
|
496
584
|
width: opts.width,
|
|
497
585
|
height: opts.height,
|
|
498
586
|
},
|
|
499
|
-
});
|
|
587
|
+
}));
|
|
500
588
|
video = video ?? doc.createElement('video');
|
|
501
589
|
video.muted = true;
|
|
502
590
|
video.playsInline = true;
|
|
503
591
|
video.srcObject = stream;
|
|
504
592
|
await video.play?.();
|
|
505
|
-
|
|
593
|
+
const track = stream?.getVideoTracks?.()?.[0];
|
|
594
|
+
const ImageCaptureCtor = globalThis.ImageCapture;
|
|
595
|
+
try {
|
|
596
|
+
imageCapture = track && typeof ImageCaptureCtor == 'function' ? new ImageCaptureCtor(track) : null;
|
|
597
|
+
}
|
|
598
|
+
catch {
|
|
599
|
+
imageCapture = null;
|
|
600
|
+
}
|
|
601
|
+
const g = globalThis;
|
|
602
|
+
if (opts.worker != false && imageCapture && g.Worker && g.Blob && g.URL && g.OffscreenCanvas) {
|
|
603
|
+
try {
|
|
604
|
+
const blob = new g.Blob([videoEncodeWorkerCode()], { type: 'text/javascript' });
|
|
605
|
+
const url = g.URL.createObjectURL(blob);
|
|
606
|
+
encodeWorker = new g.Worker(url);
|
|
607
|
+
g.URL.revokeObjectURL(url);
|
|
608
|
+
}
|
|
609
|
+
catch {
|
|
610
|
+
encodeWorker = null;
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
const OffscreenCanvasCtor = g.OffscreenCanvas;
|
|
614
|
+
canvas = canvas ?? (OffscreenCanvasCtor ? new OffscreenCanvasCtor(1, 1) : doc.createElement('canvas'));
|
|
506
615
|
ctx = canvas.getContext('2d');
|
|
507
616
|
if (!ctx)
|
|
508
617
|
throw new Error('2d canvas context is not available');
|
|
509
618
|
shell.setState('live');
|
|
510
619
|
const ms = Math.max(1, Math.round(1000 / (opts.fps ?? 3)));
|
|
511
620
|
await captureOne();
|
|
512
|
-
|
|
621
|
+
stopTicker = startCaptureTicker(ms, captureOne, opts.worker != false);
|
|
513
622
|
return shell.state;
|
|
514
623
|
}
|
|
515
624
|
catch (e) {
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
type tMediaLine = {
|
|
2
|
+
on(cb: (frame: any, sentAt?: number) => void): () => void;
|
|
3
|
+
};
|
|
4
|
+
export type AttachVideoCanvasOpts = {
|
|
5
|
+
createBitmap?: (blob: any) => Promise<any>;
|
|
6
|
+
onError?: (e: unknown) => void;
|
|
7
|
+
};
|
|
8
|
+
export declare function attachVideoCanvas(line: tMediaLine, canvas: any, opts?: AttachVideoCanvasOpts): {
|
|
9
|
+
stats: () => {
|
|
10
|
+
frames: number;
|
|
11
|
+
drawn: number;
|
|
12
|
+
perSec: number;
|
|
13
|
+
ageMs: number;
|
|
14
|
+
width: number;
|
|
15
|
+
height: number;
|
|
16
|
+
};
|
|
17
|
+
off: () => void;
|
|
18
|
+
};
|
|
19
|
+
export type AttachAudioPlayerOpts = {
|
|
20
|
+
maxBacklogSec?: number;
|
|
21
|
+
audioContext?: () => any;
|
|
22
|
+
onError?: (e: unknown) => void;
|
|
23
|
+
};
|
|
24
|
+
export declare function attachAudioPlayer(line: tMediaLine, opts?: AttachAudioPlayerOpts): {
|
|
25
|
+
enable(): void;
|
|
26
|
+
disable(): void;
|
|
27
|
+
readonly enabled: boolean;
|
|
28
|
+
stats: () => {
|
|
29
|
+
frames: number;
|
|
30
|
+
played: number;
|
|
31
|
+
dropped: number;
|
|
32
|
+
perSec: number;
|
|
33
|
+
ageMs: number;
|
|
34
|
+
};
|
|
35
|
+
off: () => void;
|
|
36
|
+
};
|
|
37
|
+
export type PipeMediaPublishOpts = {
|
|
38
|
+
stamp?: boolean;
|
|
39
|
+
onError?: (e: unknown) => void;
|
|
40
|
+
};
|
|
41
|
+
export declare function pipeMediaPublish(line: tMediaLine, publish: (frame: Uint8Array, sentAt?: number) => unknown, opts?: PipeMediaPublishOpts): () => void;
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.attachVideoCanvas = attachVideoCanvas;
|
|
4
|
+
exports.attachAudioPlayer = attachAudioPlayer;
|
|
5
|
+
exports.pipeMediaPublish = pipeMediaPublish;
|
|
6
|
+
const media_source_1 = require("./media-source");
|
|
7
|
+
function mimeForCodec(codec) {
|
|
8
|
+
if (codec == 'png')
|
|
9
|
+
return 'image/png';
|
|
10
|
+
if (codec == 'webp')
|
|
11
|
+
return 'image/webp';
|
|
12
|
+
return 'image/jpeg';
|
|
13
|
+
}
|
|
14
|
+
function createAgeMeter() {
|
|
15
|
+
let ageMs = 0;
|
|
16
|
+
return {
|
|
17
|
+
note(sentAt) {
|
|
18
|
+
if (typeof sentAt != 'number')
|
|
19
|
+
return;
|
|
20
|
+
const age = Date.now() - sentAt;
|
|
21
|
+
ageMs = ageMs ? Math.round(ageMs * 0.8 + age * 0.2) : age;
|
|
22
|
+
},
|
|
23
|
+
get ageMs() { return ageMs; },
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
function createRateMeter() {
|
|
27
|
+
let windowStart = 0;
|
|
28
|
+
let windowCount = 0;
|
|
29
|
+
let perSec = 0;
|
|
30
|
+
return {
|
|
31
|
+
note() {
|
|
32
|
+
const now = Date.now();
|
|
33
|
+
if (now - windowStart >= 1000) {
|
|
34
|
+
perSec = windowStart ? windowCount : 0;
|
|
35
|
+
windowStart = now;
|
|
36
|
+
windowCount = 0;
|
|
37
|
+
}
|
|
38
|
+
windowCount++;
|
|
39
|
+
},
|
|
40
|
+
get perSec() { return perSec; },
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function attachVideoCanvas(line, canvas, opts = {}) {
|
|
44
|
+
const ctx = canvas.getContext('2d');
|
|
45
|
+
if (!ctx)
|
|
46
|
+
throw new Error('2d canvas context is not available');
|
|
47
|
+
const makeBitmap = opts.createBitmap ?? function defaultBitmap(blob) {
|
|
48
|
+
const create = globalThis.createImageBitmap;
|
|
49
|
+
if (typeof create != 'function')
|
|
50
|
+
throw new Error('createImageBitmap is not available');
|
|
51
|
+
return create(blob);
|
|
52
|
+
};
|
|
53
|
+
const age = createAgeMeter();
|
|
54
|
+
const rate = createRateMeter();
|
|
55
|
+
let frames = 0;
|
|
56
|
+
let drawn = 0;
|
|
57
|
+
let width = 0;
|
|
58
|
+
let height = 0;
|
|
59
|
+
let busy = false;
|
|
60
|
+
const off = line.on(async function onVideoFrame(raw, sentAt) {
|
|
61
|
+
frames++;
|
|
62
|
+
age.note(sentAt);
|
|
63
|
+
rate.note();
|
|
64
|
+
if (busy)
|
|
65
|
+
return;
|
|
66
|
+
busy = true;
|
|
67
|
+
try {
|
|
68
|
+
const f = (0, media_source_1.decodeMediaFrame)((0, media_source_1.toBytes)(raw));
|
|
69
|
+
if (f.kind != 'video-frame' || !f.width || !f.height)
|
|
70
|
+
return;
|
|
71
|
+
const bitmap = await makeBitmap(new Blob([f.payload.slice()], { type: mimeForCodec(f.codec) }));
|
|
72
|
+
if (canvas.width != f.width)
|
|
73
|
+
canvas.width = f.width;
|
|
74
|
+
if (canvas.height != f.height)
|
|
75
|
+
canvas.height = f.height;
|
|
76
|
+
ctx.drawImage(bitmap, 0, 0);
|
|
77
|
+
bitmap?.close?.();
|
|
78
|
+
width = f.width;
|
|
79
|
+
height = f.height;
|
|
80
|
+
drawn++;
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
opts.onError?.(e);
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
busy = false;
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
return {
|
|
90
|
+
stats: () => ({ frames, drawn, perSec: rate.perSec, ageMs: age.ageMs, width, height }),
|
|
91
|
+
off,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function attachAudioPlayer(line, opts = {}) {
|
|
95
|
+
const maxBacklogSec = opts.maxBacklogSec ?? 0.35;
|
|
96
|
+
const age = createAgeMeter();
|
|
97
|
+
const rate = createRateMeter();
|
|
98
|
+
let audioCtx = null;
|
|
99
|
+
let playhead = 0;
|
|
100
|
+
let frames = 0;
|
|
101
|
+
let played = 0;
|
|
102
|
+
let dropped = 0;
|
|
103
|
+
function makeContext() {
|
|
104
|
+
if (opts.audioContext)
|
|
105
|
+
return opts.audioContext();
|
|
106
|
+
const Ctor = globalThis.AudioContext ?? globalThis.webkitAudioContext;
|
|
107
|
+
if (!Ctor)
|
|
108
|
+
throw new Error('AudioContext is not available');
|
|
109
|
+
return new Ctor();
|
|
110
|
+
}
|
|
111
|
+
function toSamples(codec, payload) {
|
|
112
|
+
const copy = payload.slice();
|
|
113
|
+
if (codec == 'float32')
|
|
114
|
+
return new Float32Array(copy.buffer);
|
|
115
|
+
if (codec != 'pcm16')
|
|
116
|
+
return null;
|
|
117
|
+
const pcm = new Int16Array(copy.buffer);
|
|
118
|
+
const out = new Float32Array(pcm.length);
|
|
119
|
+
for (let i = 0; i < pcm.length; i++)
|
|
120
|
+
out[i] = pcm[i] / 0x8000;
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
function push(raw) {
|
|
124
|
+
const f = (0, media_source_1.decodeMediaFrame)((0, media_source_1.toBytes)(raw));
|
|
125
|
+
if (f.kind != 'audio-pcm')
|
|
126
|
+
return;
|
|
127
|
+
const samples = toSamples(f.codec, f.payload);
|
|
128
|
+
if (!samples)
|
|
129
|
+
return;
|
|
130
|
+
const channels = f.channels || 1;
|
|
131
|
+
const sampleRate = f.sampleRate || 48000;
|
|
132
|
+
const nFrames = Math.floor(samples.length / channels);
|
|
133
|
+
if (!nFrames)
|
|
134
|
+
return;
|
|
135
|
+
if (playhead - audioCtx.currentTime > maxBacklogSec) {
|
|
136
|
+
playhead = audioCtx.currentTime + 0.05;
|
|
137
|
+
dropped++;
|
|
138
|
+
}
|
|
139
|
+
const buf = audioCtx.createBuffer(channels, nFrames, sampleRate);
|
|
140
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
141
|
+
const chan = buf.getChannelData(ch);
|
|
142
|
+
for (let i = 0; i < nFrames; i++)
|
|
143
|
+
chan[i] = samples[i * channels + ch];
|
|
144
|
+
}
|
|
145
|
+
const node = audioCtx.createBufferSource();
|
|
146
|
+
node.buffer = buf;
|
|
147
|
+
node.connect(audioCtx.destination);
|
|
148
|
+
const at = Math.max(audioCtx.currentTime + 0.05, playhead);
|
|
149
|
+
node.start(at);
|
|
150
|
+
playhead = at + nFrames / sampleRate;
|
|
151
|
+
played++;
|
|
152
|
+
}
|
|
153
|
+
const off = line.on(function onAudioFrame(raw, sentAt) {
|
|
154
|
+
frames++;
|
|
155
|
+
age.note(sentAt);
|
|
156
|
+
rate.note();
|
|
157
|
+
if (!audioCtx)
|
|
158
|
+
return;
|
|
159
|
+
try {
|
|
160
|
+
push(raw);
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
opts.onError?.(e);
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
return {
|
|
167
|
+
enable() {
|
|
168
|
+
audioCtx = audioCtx ?? makeContext();
|
|
169
|
+
playhead = 0;
|
|
170
|
+
void audioCtx.resume?.();
|
|
171
|
+
},
|
|
172
|
+
disable() {
|
|
173
|
+
void audioCtx?.close?.();
|
|
174
|
+
audioCtx = null;
|
|
175
|
+
},
|
|
176
|
+
get enabled() { return !!audioCtx; },
|
|
177
|
+
stats: () => ({ frames, played, dropped, perSec: rate.perSec, ageMs: age.ageMs }),
|
|
178
|
+
off,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function pipeMediaPublish(line, publish, opts = {}) {
|
|
182
|
+
return line.on(function publishFrame(frame) {
|
|
183
|
+
try {
|
|
184
|
+
Promise.resolve(publish(frame, opts.stamp == false ? undefined : Date.now()))
|
|
185
|
+
.catch(function onPublishFail(e) { opts.onError?.(e); });
|
|
186
|
+
}
|
|
187
|
+
catch (e) {
|
|
188
|
+
opts.onError?.(e);
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
}
|
|
@@ -2,7 +2,7 @@ import { StoreDrain, StorePatch } from '../Observe/store';
|
|
|
2
2
|
import { ReplayRemote } from '../events/replay-wire';
|
|
3
3
|
import { RoutePolicy } from '../events/route-coordinator';
|
|
4
4
|
import { RtcPeerConnection, SignalEnvelope } from '../events/route-signal-webrtc';
|
|
5
|
-
import { PatchEnvelope } from './peer-relay';
|
|
5
|
+
import { PatchEnvelope, RelayPushResult, tRelayGap } from './peer-relay';
|
|
6
6
|
export type PeerRemote = {
|
|
7
7
|
signal: {
|
|
8
8
|
send: (env: SignalEnvelope) => Promise<boolean | void>;
|
|
@@ -10,10 +10,13 @@ export type PeerRemote = {
|
|
|
10
10
|
on: (cb: (env: SignalEnvelope) => void) => any;
|
|
11
11
|
};
|
|
12
12
|
};
|
|
13
|
-
publish: (env: PatchEnvelope) => Promise<
|
|
14
|
-
peers: Record<string, ReplayRemote<[StorePatch]
|
|
13
|
+
publish: (env: PatchEnvelope) => Promise<RelayPushResult | void> | RelayPushResult | void;
|
|
14
|
+
peers: Record<string, ReplayRemote<[StorePatch]> & {
|
|
15
|
+
seq?: () => number | Promise<number>;
|
|
16
|
+
}>;
|
|
15
17
|
};
|
|
16
|
-
export type
|
|
18
|
+
export type tPublishRepair<J extends tRelayGap = 'resume'> = J extends 'sacred' ? 'tail' : 'tail' | 'keyframe';
|
|
19
|
+
export type PeerClientDeps<T extends object, J extends tRelayGap = 'resume'> = {
|
|
17
20
|
remote: PeerRemote;
|
|
18
21
|
account: string;
|
|
19
22
|
initial: T;
|
|
@@ -22,10 +25,13 @@ export type PeerClientDeps<T extends object> = {
|
|
|
22
25
|
accept?: (env: SignalEnvelope) => boolean | Promise<boolean>;
|
|
23
26
|
policy?: RoutePolicy;
|
|
24
27
|
peerInitial?: () => T;
|
|
28
|
+
journal?: J;
|
|
29
|
+
repair?: tPublishRepair<J>;
|
|
30
|
+
onPublishError?: (e: unknown) => void;
|
|
25
31
|
history?: number;
|
|
26
32
|
drain?: StoreDrain;
|
|
27
33
|
};
|
|
28
|
-
export declare function createPeerClient<T extends object>(deps: PeerClientDeps<T>): {
|
|
34
|
+
export declare function createPeerClient<T extends object, J extends tRelayGap = 'resume'>(deps: PeerClientDeps<T, J>): {
|
|
29
35
|
store: import("../Observe/store").Store<T>;
|
|
30
36
|
peer: (other: string) => {
|
|
31
37
|
account: string;
|
|
@@ -41,6 +47,7 @@ export declare function createPeerClient<T extends object>(deps: PeerClientDeps<
|
|
|
41
47
|
close(): void;
|
|
42
48
|
};
|
|
43
49
|
onRoute: (cb: (ev: import("../events/route-coordinator").RouteChangeEvent) => void) => import("../..").ListenOff;
|
|
50
|
+
resync: () => Promise<void>;
|
|
44
51
|
close(): void;
|
|
45
52
|
};
|
|
46
53
|
export type PeerClient<T extends object> = ReturnType<typeof createPeerClient<T>>;
|
|
@@ -7,15 +7,64 @@ const replay_wire_1 = require("../events/replay-wire");
|
|
|
7
7
|
const route_coordinator_1 = require("../events/route-coordinator");
|
|
8
8
|
const route_signal_webrtc_1 = require("../events/route-signal-webrtc");
|
|
9
9
|
function createPeerClient(deps) {
|
|
10
|
-
const { remote, account, initial, rtc, session, accept, policy, peerInitial = () => ({}), history, drain, } = deps;
|
|
10
|
+
const { remote, account, initial, rtc, session, accept, policy, peerInitial = () => ({}), history, drain, journal = 'resume', onPublishError, } = deps;
|
|
11
|
+
const repair = deps.repair ?? 'tail';
|
|
11
12
|
const store = (0, store_1.createStore)(initial, drain !== undefined ? { drain } : {});
|
|
12
13
|
const exposed = (0, store_replay_1.exposeStoreReplay)(store, history !== undefined ? { history } : {});
|
|
14
|
+
let repairing = null;
|
|
15
|
+
function repairEnvelopes(from) {
|
|
16
|
+
const line = exposed.replay;
|
|
17
|
+
if (repair == 'tail' && from >= 0) {
|
|
18
|
+
const tail = line.getSince(from);
|
|
19
|
+
if (tail)
|
|
20
|
+
return tail;
|
|
21
|
+
if (journal == 'sacred') {
|
|
22
|
+
throw new Error('peer publish: local journal evicted seq ' + from + ', sacred relay is unrepairable — raise {history}');
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const kf = exposed.replay.keyframe();
|
|
26
|
+
return kf ? [kf] : [];
|
|
27
|
+
}
|
|
28
|
+
async function runRepair(from) {
|
|
29
|
+
for (const env of repairEnvelopes(from)) {
|
|
30
|
+
const res = await remote.publish(env);
|
|
31
|
+
if (res != null && typeof res == 'object') {
|
|
32
|
+
throw new Error('peer publish: repair rejected at relay seq ' + res.seq);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function queueRepair(from) {
|
|
37
|
+
if (repairing)
|
|
38
|
+
return;
|
|
39
|
+
repairing = runRepair(from)
|
|
40
|
+
.catch(function reportRepairError(e) {
|
|
41
|
+
if (onPublishError)
|
|
42
|
+
onPublishError(e);
|
|
43
|
+
else
|
|
44
|
+
setTimeout(function rethrowRepairError() { throw e; }, 0);
|
|
45
|
+
})
|
|
46
|
+
.finally(() => { repairing = null; });
|
|
47
|
+
}
|
|
48
|
+
function handleVerdict(res) {
|
|
49
|
+
if (res != null && typeof res == 'object' && typeof res.seq == 'number')
|
|
50
|
+
queueRepair(res.seq);
|
|
51
|
+
}
|
|
13
52
|
const offPublish = exposed.replay.line.on(function publishEnvelope(env) {
|
|
14
|
-
|
|
53
|
+
Promise.resolve(remote.publish(env)).then(handleVerdict, function onPublishReject(e) {
|
|
54
|
+
onPublishError?.(e);
|
|
55
|
+
});
|
|
15
56
|
});
|
|
16
57
|
const warmup = exposed.replay.keyframe();
|
|
17
58
|
if (warmup)
|
|
18
|
-
|
|
59
|
+
Promise.resolve(remote.publish(warmup)).then(handleVerdict, e => onPublishError?.(e));
|
|
60
|
+
async function resync() {
|
|
61
|
+
const node = remote.peers[account];
|
|
62
|
+
const relaySeq = Number(await node?.seq?.() ?? -1);
|
|
63
|
+
const localSeq = exposed.replay.keyframe()?.seq ?? -1;
|
|
64
|
+
if (relaySeq >= localSeq)
|
|
65
|
+
return;
|
|
66
|
+
await runRepair(relaySeq);
|
|
67
|
+
}
|
|
19
68
|
const port = {
|
|
20
69
|
send: env => remote.signal.send(env),
|
|
21
70
|
signals: { on: cb => remote.signal.signals.on(cb) },
|
|
@@ -96,6 +145,7 @@ function createPeerClient(deps) {
|
|
|
96
145
|
store,
|
|
97
146
|
peer,
|
|
98
147
|
onRoute: coord.onRoute,
|
|
148
|
+
resync,
|
|
99
149
|
close() {
|
|
100
150
|
for (const view of Array.from(views.values()))
|
|
101
151
|
view.close();
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { StorePatch } from '../Observe/store';
|
|
2
2
|
import { ReplayRemote } from '../events/replay-wire';
|
|
3
3
|
import { SignalEnvelope } from '../events/route-signal-webrtc';
|
|
4
|
-
import { PatchEnvelope } from './peer-relay';
|
|
4
|
+
import { PatchEnvelope, tRelayGap } from './peer-relay';
|
|
5
5
|
export type PeerHostDeps = {
|
|
6
6
|
authorize?: (env: SignalEnvelope) => boolean | Promise<boolean>;
|
|
7
7
|
history?: number;
|
|
8
|
+
gap?: tRelayGap;
|
|
8
9
|
};
|
|
9
10
|
export declare function createPeerHost(deps?: PeerHostDeps): {
|
|
10
11
|
connection: (account: string) => {
|
|
@@ -13,14 +14,17 @@ export declare function createPeerHost(deps?: PeerHostDeps): {
|
|
|
13
14
|
send: (env: SignalEnvelope) => Promise<boolean>;
|
|
14
15
|
signals: import("../..").ListenApi<[SignalEnvelope]>;
|
|
15
16
|
};
|
|
16
|
-
publish: (env: PatchEnvelope) =>
|
|
17
|
+
publish: (env: PatchEnvelope) => import("./peer-relay").RelayPushResult;
|
|
17
18
|
peers: Record<string, ReplayRemote<[StorePatch]>>;
|
|
18
19
|
};
|
|
19
20
|
close: () => void;
|
|
20
21
|
};
|
|
21
22
|
relay: (account: string) => {
|
|
22
|
-
push: (env: PatchEnvelope) =>
|
|
23
|
-
remote: ReplayRemote<[StorePatch]
|
|
23
|
+
push: (env: PatchEnvelope) => import("./peer-relay").RelayPushResult;
|
|
24
|
+
remote: ReplayRemote<[StorePatch]> & {
|
|
25
|
+
seq: () => number;
|
|
26
|
+
};
|
|
27
|
+
gap: tRelayGap;
|
|
24
28
|
seq: () => number;
|
|
25
29
|
snapshot: () => any;
|
|
26
30
|
close: () => void;
|
|
@@ -5,14 +5,14 @@ const rpc_dynamic_1 = require("../rcp/rpc-dynamic");
|
|
|
5
5
|
const route_signal_webrtc_1 = require("../events/route-signal-webrtc");
|
|
6
6
|
const peer_relay_1 = require("./peer-relay");
|
|
7
7
|
function createPeerHost(deps = {}) {
|
|
8
|
-
const { authorize, history } = deps;
|
|
8
|
+
const { authorize, history, gap } = deps;
|
|
9
9
|
const hub = (0, route_signal_webrtc_1.createSignalHub)({ authorize });
|
|
10
10
|
const relays = new Map();
|
|
11
11
|
const peersView = (0, rpc_dynamic_1.noStrict)({});
|
|
12
12
|
function ensureRelay(account) {
|
|
13
13
|
let relay = relays.get(account);
|
|
14
14
|
if (!relay) {
|
|
15
|
-
relay = (0, peer_relay_1.createPatchRelayJournal)({ history });
|
|
15
|
+
relay = (0, peer_relay_1.createPatchRelayJournal)({ history, gap });
|
|
16
16
|
relays.set(account, relay);
|
|
17
17
|
peersView[account] = relay.remote;
|
|
18
18
|
}
|
|
@@ -2,11 +2,19 @@ import { ReplayEvent } from '../events/replay-listen';
|
|
|
2
2
|
import { ReplayRemote } from '../events/replay-wire';
|
|
3
3
|
import { StorePatch } from '../Observe/store';
|
|
4
4
|
export type PatchEnvelope = ReplayEvent<[StorePatch]>;
|
|
5
|
+
export type tRelayGap = 'resume' | 'sacred';
|
|
6
|
+
export type RelayPushResult = boolean | {
|
|
7
|
+
seq: number;
|
|
8
|
+
};
|
|
5
9
|
export declare function createPatchRelayJournal(opts?: {
|
|
6
10
|
history?: number;
|
|
11
|
+
gap?: tRelayGap;
|
|
7
12
|
}): {
|
|
8
|
-
push: (env: PatchEnvelope) =>
|
|
9
|
-
remote: ReplayRemote<[StorePatch]
|
|
13
|
+
push: (env: PatchEnvelope) => RelayPushResult;
|
|
14
|
+
remote: ReplayRemote<[StorePatch]> & {
|
|
15
|
+
seq: () => number;
|
|
16
|
+
};
|
|
17
|
+
gap: tRelayGap;
|
|
10
18
|
seq: () => number;
|
|
11
19
|
snapshot: () => any;
|
|
12
20
|
close: () => void;
|