wenay-common2 1.0.65 → 1.0.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +20 -0
- package/doc/ROADMAP.md +73 -7
- package/doc/changes/1.0.65.md +2 -1
- package/doc/changes/1.0.66.md +8 -0
- package/doc/changes/1.0.67.md +10 -0
- package/doc/wenay-common2-rare.md +73 -3
- package/doc/wenay-common2.md +11 -0
- package/lib/Common/events/replay-index.d.ts +1 -0
- package/lib/Common/events/replay-index.js +1 -0
- package/lib/Common/events/route-coordinator.d.ts +274 -0
- package/lib/Common/events/route-coordinator.js +316 -0
- package/lib/Common/media/media-index.d.ts +1 -0
- package/lib/Common/media/media-index.js +17 -0
- package/lib/Common/media/media-source.d.ts +94 -0
- package/lib/Common/media/media-source.js +537 -0
- package/lib/client.d.ts +2 -0
- package/lib/client.js +3 -1
- package/lib/index.d.ts +2 -0
- package/lib/index.js +3 -1
- package/package.json +4 -2
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MEDIA_FRAME_HEADER_BYTES = exports.MEDIA_FRAME_VERSION = exports.MEDIA_FRAME_MAGIC = void 0;
|
|
4
|
+
exports.encodeMediaFrame = encodeMediaFrame;
|
|
5
|
+
exports.decodeMediaFrame = decodeMediaFrame;
|
|
6
|
+
exports.createAudioSource = createAudioSource;
|
|
7
|
+
exports.createVideoSource = createVideoSource;
|
|
8
|
+
const Listen_1 = require("../events/Listen");
|
|
9
|
+
const replay_listen_1 = require("../events/replay-listen");
|
|
10
|
+
exports.MEDIA_FRAME_MAGIC = 0x57434d32;
|
|
11
|
+
exports.MEDIA_FRAME_VERSION = 1;
|
|
12
|
+
exports.MEDIA_FRAME_HEADER_BYTES = 40;
|
|
13
|
+
const KIND_TO_CODE = {
|
|
14
|
+
'audio-pcm': 1,
|
|
15
|
+
'audio-record': 2,
|
|
16
|
+
'video-frame': 3,
|
|
17
|
+
};
|
|
18
|
+
const CODE_TO_KIND = {
|
|
19
|
+
1: 'audio-pcm',
|
|
20
|
+
2: 'audio-record',
|
|
21
|
+
3: 'video-frame',
|
|
22
|
+
};
|
|
23
|
+
const CODEC_TO_CODE = {
|
|
24
|
+
pcm16: 1,
|
|
25
|
+
float32: 2,
|
|
26
|
+
jpeg: 10,
|
|
27
|
+
png: 11,
|
|
28
|
+
webp: 12,
|
|
29
|
+
'webm-opus': 20,
|
|
30
|
+
};
|
|
31
|
+
const CODE_TO_CODEC = {
|
|
32
|
+
1: 'pcm16',
|
|
33
|
+
2: 'float32',
|
|
34
|
+
10: 'jpeg',
|
|
35
|
+
11: 'png',
|
|
36
|
+
12: 'webp',
|
|
37
|
+
20: 'webm-opus',
|
|
38
|
+
};
|
|
39
|
+
function nowMono() {
|
|
40
|
+
const perf = globalThis.performance;
|
|
41
|
+
return typeof perf?.now == 'function' ? perf.now() : Date.now();
|
|
42
|
+
}
|
|
43
|
+
function toBytes(data) {
|
|
44
|
+
if (ArrayBuffer.isView(data))
|
|
45
|
+
return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
46
|
+
return new Uint8Array(data);
|
|
47
|
+
}
|
|
48
|
+
function clonePayload(data) {
|
|
49
|
+
const src = toBytes(data);
|
|
50
|
+
const out = new Uint8Array(src.byteLength);
|
|
51
|
+
out.set(src);
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
54
|
+
function writeU32(view, offset, value) {
|
|
55
|
+
view.setUint32(offset, Math.max(0, Math.min(0xffffffff, Math.floor(value ?? 0))), true);
|
|
56
|
+
}
|
|
57
|
+
function readPayload(frame, headerBytes) {
|
|
58
|
+
return frame.subarray(headerBytes);
|
|
59
|
+
}
|
|
60
|
+
function encodeMediaFrame(meta, payload) {
|
|
61
|
+
const body = toBytes(payload);
|
|
62
|
+
const out = new Uint8Array(exports.MEDIA_FRAME_HEADER_BYTES + body.byteLength);
|
|
63
|
+
const view = new DataView(out.buffer, out.byteOffset, exports.MEDIA_FRAME_HEADER_BYTES);
|
|
64
|
+
view.setUint32(0, exports.MEDIA_FRAME_MAGIC, true);
|
|
65
|
+
view.setUint8(4, exports.MEDIA_FRAME_VERSION);
|
|
66
|
+
view.setUint8(5, KIND_TO_CODE[meta.kind]);
|
|
67
|
+
view.setUint8(6, CODEC_TO_CODE[meta.codec]);
|
|
68
|
+
view.setUint8(7, 0);
|
|
69
|
+
view.setUint16(8, exports.MEDIA_FRAME_HEADER_BYTES, true);
|
|
70
|
+
view.setUint16(10, 0, true);
|
|
71
|
+
writeU32(view, 12, meta.seq);
|
|
72
|
+
view.setFloat64(16, meta.tMono, true);
|
|
73
|
+
if (meta.kind == 'video-frame') {
|
|
74
|
+
writeU32(view, 24, meta.width);
|
|
75
|
+
writeU32(view, 28, meta.height);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
writeU32(view, 24, meta.sampleRate);
|
|
79
|
+
writeU32(view, 28, meta.channels);
|
|
80
|
+
writeU32(view, 32, meta.nSamples);
|
|
81
|
+
}
|
|
82
|
+
writeU32(view, 36, body.byteLength);
|
|
83
|
+
out.set(body, exports.MEDIA_FRAME_HEADER_BYTES);
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
function decodeMediaFrame(frameLike) {
|
|
87
|
+
const frame = toBytes(frameLike);
|
|
88
|
+
if (frame.byteLength < exports.MEDIA_FRAME_HEADER_BYTES)
|
|
89
|
+
throw new Error('media frame too short');
|
|
90
|
+
const view = new DataView(frame.buffer, frame.byteOffset, exports.MEDIA_FRAME_HEADER_BYTES);
|
|
91
|
+
if (view.getUint32(0, true) != exports.MEDIA_FRAME_MAGIC)
|
|
92
|
+
throw new Error('media frame magic mismatch');
|
|
93
|
+
const version = view.getUint8(4);
|
|
94
|
+
if (version != exports.MEDIA_FRAME_VERSION)
|
|
95
|
+
throw new Error(`media frame version ${version} is not supported`);
|
|
96
|
+
const kind = CODE_TO_KIND[view.getUint8(5)];
|
|
97
|
+
const codec = CODE_TO_CODEC[view.getUint8(6)];
|
|
98
|
+
if (!kind || !codec)
|
|
99
|
+
throw new Error('media frame kind/codec is not supported');
|
|
100
|
+
const headerBytes = view.getUint16(8, true);
|
|
101
|
+
if (headerBytes < exports.MEDIA_FRAME_HEADER_BYTES || headerBytes > frame.byteLength)
|
|
102
|
+
throw new Error('media frame header size is invalid');
|
|
103
|
+
const payloadBytes = view.getUint32(36, true);
|
|
104
|
+
const payload = readPayload(frame, headerBytes);
|
|
105
|
+
if (payload.byteLength != payloadBytes)
|
|
106
|
+
throw new Error('media frame payload size mismatch');
|
|
107
|
+
const base = {
|
|
108
|
+
kind,
|
|
109
|
+
codec,
|
|
110
|
+
seq: view.getUint32(12, true),
|
|
111
|
+
tMono: view.getFloat64(16, true),
|
|
112
|
+
payload,
|
|
113
|
+
};
|
|
114
|
+
if (kind == 'video-frame') {
|
|
115
|
+
return {
|
|
116
|
+
...base,
|
|
117
|
+
width: view.getUint32(24, true),
|
|
118
|
+
height: view.getUint32(28, true),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
...base,
|
|
123
|
+
sampleRate: view.getUint32(24, true),
|
|
124
|
+
channels: view.getUint32(28, true),
|
|
125
|
+
nSamples: view.getUint32(32, true),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function createStats(sourceId, kind) {
|
|
129
|
+
return {
|
|
130
|
+
sourceId,
|
|
131
|
+
kind,
|
|
132
|
+
state: 'idle',
|
|
133
|
+
seq: 0,
|
|
134
|
+
frames: 0,
|
|
135
|
+
bytes: 0,
|
|
136
|
+
dropped: 0,
|
|
137
|
+
startedAt: 0,
|
|
138
|
+
lastAt: 0,
|
|
139
|
+
fps: 0,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
function hasGetUserMedia() {
|
|
143
|
+
const nav = globalThis.navigator;
|
|
144
|
+
return typeof nav?.mediaDevices?.getUserMedia == 'function';
|
|
145
|
+
}
|
|
146
|
+
async function listDevices(kind) {
|
|
147
|
+
const nav = globalThis.navigator;
|
|
148
|
+
if (typeof nav?.mediaDevices?.enumerateDevices != 'function')
|
|
149
|
+
return [];
|
|
150
|
+
const devices = await nav.mediaDevices.enumerateDevices();
|
|
151
|
+
const prefix = kind == 'audio' ? 'audioinput' : 'videoinput';
|
|
152
|
+
return devices
|
|
153
|
+
.filter(d => d?.kind == prefix)
|
|
154
|
+
.map(d => ({ deviceId: String(d.deviceId ?? ''), label: String(d.label ?? ''), kind: String(d.kind ?? '') }));
|
|
155
|
+
}
|
|
156
|
+
function stopTracks(stream) {
|
|
157
|
+
const tracks = typeof stream?.getTracks == 'function' ? stream.getTracks() : [];
|
|
158
|
+
for (const track of tracks)
|
|
159
|
+
track.stop?.();
|
|
160
|
+
}
|
|
161
|
+
function stateFromMediaError(e) {
|
|
162
|
+
const name = String(e?.name ?? '');
|
|
163
|
+
if (name == 'NotAllowedError' || name == 'SecurityError' || name == 'PermissionDeniedError')
|
|
164
|
+
return 'denied';
|
|
165
|
+
if (name == 'NotFoundError' || name == 'DevicesNotFoundError')
|
|
166
|
+
return 'no-device';
|
|
167
|
+
return 'error';
|
|
168
|
+
}
|
|
169
|
+
function defaultReplayOptions(kind) {
|
|
170
|
+
if (kind == 'video') {
|
|
171
|
+
return {
|
|
172
|
+
history: 256,
|
|
173
|
+
current: 'last',
|
|
174
|
+
frame: tail => tail.length ? [tail[tail.length - 1]] : [],
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
return { history: 1024 };
|
|
178
|
+
}
|
|
179
|
+
function createSourceShell(kind, sourceId, transport, replay) {
|
|
180
|
+
const replayOpts = replay == true ? defaultReplayOptions(kind) : replay;
|
|
181
|
+
const [emit, listenApi] = replayOpts ? (0, replay_listen_1.replayListen)(replayOpts) : (0, Listen_1.listen)();
|
|
182
|
+
const stats = createStats(sourceId, kind);
|
|
183
|
+
let state = 'idle';
|
|
184
|
+
let error;
|
|
185
|
+
function setState(next, err) {
|
|
186
|
+
state = next;
|
|
187
|
+
stats.state = next;
|
|
188
|
+
error = err ? String(err?.message ?? err) : undefined;
|
|
189
|
+
stats.error = error;
|
|
190
|
+
}
|
|
191
|
+
function emitFrame(frame, seq) {
|
|
192
|
+
stats.seq = seq;
|
|
193
|
+
stats.frames++;
|
|
194
|
+
stats.bytes += frame.byteLength;
|
|
195
|
+
stats.lastAt = nowMono();
|
|
196
|
+
const age = Math.max(1, stats.lastAt - (stats.startedAt || stats.lastAt));
|
|
197
|
+
stats.fps = Math.round((stats.frames * 10000) / age) / 10;
|
|
198
|
+
emit(frame);
|
|
199
|
+
}
|
|
200
|
+
return { emit, listenApi, stats, get state() { return state; }, get error() { return error; }, setState, emitFrame };
|
|
201
|
+
}
|
|
202
|
+
function attachControl(pair, control) {
|
|
203
|
+
return Object.assign(pair, control);
|
|
204
|
+
}
|
|
205
|
+
function ensureSocketTransport(mode) {
|
|
206
|
+
if (mode == 'webrtc') {
|
|
207
|
+
throw new Error('WebRTC media transport is reserved for a future opt-in SFU/signaling adapter; use transport:"socket" today');
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
function int16PcmFromFloat(input) {
|
|
211
|
+
const out = new Int16Array(input.length);
|
|
212
|
+
for (let i = 0; i < input.length; i++) {
|
|
213
|
+
const x = Math.max(-1, Math.min(1, input[i]));
|
|
214
|
+
out[i] = x < 0 ? x * 0x8000 : x * 0x7fff;
|
|
215
|
+
}
|
|
216
|
+
return new Uint8Array(out.buffer);
|
|
217
|
+
}
|
|
218
|
+
function interleaveChannels(buffers, frames, channels) {
|
|
219
|
+
const out = new Float32Array(frames * channels);
|
|
220
|
+
for (let i = 0; i < frames; i++) {
|
|
221
|
+
for (let ch = 0; ch < channels; ch++)
|
|
222
|
+
out[i * channels + ch] = buffers[ch]?.[i] ?? 0;
|
|
223
|
+
}
|
|
224
|
+
return out;
|
|
225
|
+
}
|
|
226
|
+
function rmsOf(input) {
|
|
227
|
+
if (input.length == 0)
|
|
228
|
+
return 0;
|
|
229
|
+
let sum = 0;
|
|
230
|
+
for (let i = 0; i < input.length; i++)
|
|
231
|
+
sum += input[i] * input[i];
|
|
232
|
+
return Math.sqrt(sum / input.length);
|
|
233
|
+
}
|
|
234
|
+
function audioWorkletCode() {
|
|
235
|
+
return `
|
|
236
|
+
class WenayCommon2PcmProcessor extends AudioWorkletProcessor {
|
|
237
|
+
process(inputs) {
|
|
238
|
+
const input = inputs[0]
|
|
239
|
+
if (!input || !input.length || !input[0]) return true
|
|
240
|
+
const channels = input.length
|
|
241
|
+
const frames = input[0].length
|
|
242
|
+
const out = new Float32Array(frames * channels)
|
|
243
|
+
for (let i = 0; i < frames; i++) {
|
|
244
|
+
for (let ch = 0; ch < channels; ch++) out[i * channels + ch] = input[ch] ? input[ch][i] : 0
|
|
245
|
+
}
|
|
246
|
+
this.port.postMessage({sampleRate, channels, frames, samples: out}, [out.buffer])
|
|
247
|
+
return true
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
registerProcessor('wenay-common2-pcm', WenayCommon2PcmProcessor)
|
|
251
|
+
`;
|
|
252
|
+
}
|
|
253
|
+
function createAudioSource(opts = {}) {
|
|
254
|
+
const sourceId = opts.sourceId ?? 'audio';
|
|
255
|
+
const transport = opts.transport ?? 'socket';
|
|
256
|
+
const shell = createSourceShell('audio', sourceId, transport, opts.replay);
|
|
257
|
+
let deviceId = opts.deviceId;
|
|
258
|
+
let stream = null;
|
|
259
|
+
let audioCtx = null;
|
|
260
|
+
let mediaNode = null;
|
|
261
|
+
let workletNode = null;
|
|
262
|
+
let recorder = null;
|
|
263
|
+
let seq = 0;
|
|
264
|
+
function emitPcm(samples, sampleRate, channels, frames) {
|
|
265
|
+
const codec = opts.format == 'float32' ? 'float32' : 'pcm16';
|
|
266
|
+
const payload = codec == 'float32' ? new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength) : int16PcmFromFloat(samples);
|
|
267
|
+
shell.stats.rms = rmsOf(samples);
|
|
268
|
+
shell.emitFrame(encodeMediaFrame({
|
|
269
|
+
kind: 'audio-pcm',
|
|
270
|
+
codec,
|
|
271
|
+
seq: ++seq,
|
|
272
|
+
tMono: nowMono(),
|
|
273
|
+
sampleRate,
|
|
274
|
+
channels,
|
|
275
|
+
nSamples: frames,
|
|
276
|
+
}, payload), seq);
|
|
277
|
+
}
|
|
278
|
+
async function startPcm(nextStream) {
|
|
279
|
+
const AudioContextCtor = globalThis.AudioContext ?? globalThis.webkitAudioContext;
|
|
280
|
+
if (!AudioContextCtor)
|
|
281
|
+
throw new Error('AudioContext is not available');
|
|
282
|
+
audioCtx = new AudioContextCtor(opts.sampleRate ? { sampleRate: opts.sampleRate } : undefined);
|
|
283
|
+
mediaNode = audioCtx.createMediaStreamSource(nextStream);
|
|
284
|
+
if (opts.worklet != false && audioCtx.audioWorklet && globalThis.Blob && globalThis.URL) {
|
|
285
|
+
const blob = new globalThis.Blob([audioWorkletCode()], { type: 'text/javascript' });
|
|
286
|
+
const url = globalThis.URL.createObjectURL(blob);
|
|
287
|
+
try {
|
|
288
|
+
await audioCtx.audioWorklet.addModule(url);
|
|
289
|
+
}
|
|
290
|
+
finally {
|
|
291
|
+
globalThis.URL.revokeObjectURL(url);
|
|
292
|
+
}
|
|
293
|
+
const AudioWorkletNodeCtor = globalThis.AudioWorkletNode;
|
|
294
|
+
workletNode = new AudioWorkletNodeCtor(audioCtx, 'wenay-common2-pcm', {
|
|
295
|
+
numberOfInputs: 1,
|
|
296
|
+
numberOfOutputs: 0,
|
|
297
|
+
channelCount: opts.channels ?? 1,
|
|
298
|
+
});
|
|
299
|
+
workletNode.port.onmessage = function onWorkletSamples(ev) {
|
|
300
|
+
const data = ev.data ?? {};
|
|
301
|
+
emitPcm(data.samples, data.sampleRate ?? audioCtx.sampleRate, data.channels ?? 1, data.frames ?? 0);
|
|
302
|
+
};
|
|
303
|
+
mediaNode.connect(workletNode);
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
const channels = opts.channels ?? 1;
|
|
307
|
+
const processor = audioCtx.createScriptProcessor(opts.bufferSize ?? 2048, channels, channels);
|
|
308
|
+
processor.onaudioprocess = function onAudioProcess(ev) {
|
|
309
|
+
const input = ev.inputBuffer;
|
|
310
|
+
const frames = input.length;
|
|
311
|
+
const buffers = [];
|
|
312
|
+
for (let ch = 0; ch < channels; ch++)
|
|
313
|
+
buffers.push(input.getChannelData(Math.min(ch, input.numberOfChannels - 1)));
|
|
314
|
+
emitPcm(interleaveChannels(buffers, frames, channels), input.sampleRate ?? audioCtx.sampleRate, channels, frames);
|
|
315
|
+
const out = ev.outputBuffer;
|
|
316
|
+
if (out)
|
|
317
|
+
for (let ch = 0; ch < out.numberOfChannels; ch++)
|
|
318
|
+
out.getChannelData(ch).fill(0);
|
|
319
|
+
};
|
|
320
|
+
mediaNode.connect(processor);
|
|
321
|
+
processor.connect(audioCtx.destination);
|
|
322
|
+
workletNode = processor;
|
|
323
|
+
}
|
|
324
|
+
function startRecord(nextStream) {
|
|
325
|
+
const Recorder = globalThis.MediaRecorder;
|
|
326
|
+
if (!Recorder)
|
|
327
|
+
throw new Error('MediaRecorder is not available');
|
|
328
|
+
const mimeType = opts.recordMimeType ?? 'audio/webm;codecs=opus';
|
|
329
|
+
recorder = new Recorder(nextStream, Recorder.isTypeSupported?.(mimeType) ? { mimeType } : undefined);
|
|
330
|
+
recorder.ondataavailable = async function onRecordChunk(ev) {
|
|
331
|
+
if (!ev.data || ev.data.size == 0)
|
|
332
|
+
return;
|
|
333
|
+
const payload = new Uint8Array(await ev.data.arrayBuffer());
|
|
334
|
+
shell.emitFrame(encodeMediaFrame({
|
|
335
|
+
kind: 'audio-record',
|
|
336
|
+
codec: 'webm-opus',
|
|
337
|
+
seq: ++seq,
|
|
338
|
+
tMono: nowMono(),
|
|
339
|
+
}, payload), seq);
|
|
340
|
+
};
|
|
341
|
+
recorder.start(opts.recordTimesliceMs ?? 1000);
|
|
342
|
+
}
|
|
343
|
+
function stop() {
|
|
344
|
+
recorder?.stop?.();
|
|
345
|
+
recorder = null;
|
|
346
|
+
workletNode?.disconnect?.();
|
|
347
|
+
mediaNode?.disconnect?.();
|
|
348
|
+
audioCtx?.close?.();
|
|
349
|
+
stopTracks(stream);
|
|
350
|
+
stream = null;
|
|
351
|
+
audioCtx = null;
|
|
352
|
+
mediaNode = null;
|
|
353
|
+
workletNode = null;
|
|
354
|
+
shell.setState('idle');
|
|
355
|
+
}
|
|
356
|
+
async function start() {
|
|
357
|
+
try {
|
|
358
|
+
ensureSocketTransport(transport);
|
|
359
|
+
if (!hasGetUserMedia()) {
|
|
360
|
+
shell.setState('no-device');
|
|
361
|
+
return shell.state;
|
|
362
|
+
}
|
|
363
|
+
stop();
|
|
364
|
+
shell.setState('requesting');
|
|
365
|
+
shell.stats.startedAt = nowMono();
|
|
366
|
+
const constraints = { audio: { deviceId: deviceId ? { exact: deviceId } : undefined, channelCount: opts.channels, sampleRate: opts.sampleRate } };
|
|
367
|
+
stream = await globalThis.navigator.mediaDevices.getUserMedia(constraints);
|
|
368
|
+
if (opts.mode == 'record')
|
|
369
|
+
startRecord(stream);
|
|
370
|
+
else
|
|
371
|
+
await startPcm(stream);
|
|
372
|
+
shell.setState('live');
|
|
373
|
+
return shell.state;
|
|
374
|
+
}
|
|
375
|
+
catch (e) {
|
|
376
|
+
stopTracks(stream);
|
|
377
|
+
stream = null;
|
|
378
|
+
shell.setState(stateFromMediaError(e), e);
|
|
379
|
+
return shell.state;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
const control = {
|
|
383
|
+
start,
|
|
384
|
+
stop,
|
|
385
|
+
getStats: () => ({ ...shell.stats }),
|
|
386
|
+
setDevice: async (id) => {
|
|
387
|
+
deviceId = id;
|
|
388
|
+
return shell.state == 'live' ? start() : shell.state;
|
|
389
|
+
},
|
|
390
|
+
listDevices: () => listDevices('audio'),
|
|
391
|
+
get state() { return shell.state; },
|
|
392
|
+
sourceId,
|
|
393
|
+
kind: 'audio',
|
|
394
|
+
transport,
|
|
395
|
+
};
|
|
396
|
+
return attachControl([shell.emit, shell.listenApi], control);
|
|
397
|
+
}
|
|
398
|
+
function mimeForVideo(codec) {
|
|
399
|
+
if (codec == 'png')
|
|
400
|
+
return 'image/png';
|
|
401
|
+
if (codec == 'webp')
|
|
402
|
+
return 'image/webp';
|
|
403
|
+
return 'image/jpeg';
|
|
404
|
+
}
|
|
405
|
+
function canvasToBlob(canvas, mime, quality) {
|
|
406
|
+
if (typeof canvas.convertToBlob == 'function')
|
|
407
|
+
return canvas.convertToBlob({ type: mime, quality });
|
|
408
|
+
return new Promise((resolve, reject) => {
|
|
409
|
+
if (typeof canvas.toBlob != 'function') {
|
|
410
|
+
reject(new Error('canvas.toBlob is not available'));
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
canvas.toBlob((blob) => blob ? resolve(blob) : reject(new Error('canvas.toBlob returned null')), mime, quality);
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
function createVideoSource(opts = {}) {
|
|
417
|
+
const sourceId = opts.sourceId ?? 'video';
|
|
418
|
+
const transport = opts.transport ?? 'socket';
|
|
419
|
+
const shell = createSourceShell('video', sourceId, transport, opts.replay);
|
|
420
|
+
let deviceId = opts.deviceId;
|
|
421
|
+
let stream = null;
|
|
422
|
+
let video = opts.video;
|
|
423
|
+
let canvas = opts.canvas;
|
|
424
|
+
let ctx = null;
|
|
425
|
+
let timer = null;
|
|
426
|
+
let busy = false;
|
|
427
|
+
let seq = 0;
|
|
428
|
+
async function captureOne() {
|
|
429
|
+
if (busy || shell.state != 'live') {
|
|
430
|
+
shell.stats.dropped++;
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
busy = true;
|
|
434
|
+
try {
|
|
435
|
+
const w = opts.width ?? video.videoWidth ?? video.width ?? 0;
|
|
436
|
+
const h = opts.height ?? video.videoHeight ?? video.height ?? 0;
|
|
437
|
+
if (!w || !h)
|
|
438
|
+
return;
|
|
439
|
+
if (canvas.width != w)
|
|
440
|
+
canvas.width = w;
|
|
441
|
+
if (canvas.height != h)
|
|
442
|
+
canvas.height = h;
|
|
443
|
+
ctx.drawImage(video, 0, 0, w, h);
|
|
444
|
+
const codec = opts.codec ?? 'jpeg';
|
|
445
|
+
const blob = await canvasToBlob(canvas, mimeForVideo(codec), opts.quality ?? 0.82);
|
|
446
|
+
const payload = new Uint8Array(await blob.arrayBuffer());
|
|
447
|
+
shell.emitFrame(encodeMediaFrame({
|
|
448
|
+
kind: 'video-frame',
|
|
449
|
+
codec,
|
|
450
|
+
seq: ++seq,
|
|
451
|
+
tMono: nowMono(),
|
|
452
|
+
width: w,
|
|
453
|
+
height: h,
|
|
454
|
+
}, payload), seq);
|
|
455
|
+
}
|
|
456
|
+
catch (e) {
|
|
457
|
+
shell.setState('error', e);
|
|
458
|
+
}
|
|
459
|
+
finally {
|
|
460
|
+
busy = false;
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
function stop() {
|
|
464
|
+
if (timer) {
|
|
465
|
+
clearInterval(timer);
|
|
466
|
+
timer = null;
|
|
467
|
+
}
|
|
468
|
+
stopTracks(stream);
|
|
469
|
+
stream = null;
|
|
470
|
+
if (!opts.video && video) {
|
|
471
|
+
try {
|
|
472
|
+
video.srcObject = null;
|
|
473
|
+
}
|
|
474
|
+
catch { }
|
|
475
|
+
}
|
|
476
|
+
shell.setState('idle');
|
|
477
|
+
}
|
|
478
|
+
async function start() {
|
|
479
|
+
try {
|
|
480
|
+
ensureSocketTransport(transport);
|
|
481
|
+
if (!hasGetUserMedia()) {
|
|
482
|
+
shell.setState('no-device');
|
|
483
|
+
return shell.state;
|
|
484
|
+
}
|
|
485
|
+
const doc = globalThis.document;
|
|
486
|
+
if (!video && !doc?.createElement) {
|
|
487
|
+
shell.setState('error', 'document.createElement is not available');
|
|
488
|
+
return shell.state;
|
|
489
|
+
}
|
|
490
|
+
stop();
|
|
491
|
+
shell.setState('requesting');
|
|
492
|
+
shell.stats.startedAt = nowMono();
|
|
493
|
+
stream = await globalThis.navigator.mediaDevices.getUserMedia({
|
|
494
|
+
video: {
|
|
495
|
+
deviceId: deviceId ? { exact: deviceId } : undefined,
|
|
496
|
+
width: opts.width,
|
|
497
|
+
height: opts.height,
|
|
498
|
+
},
|
|
499
|
+
});
|
|
500
|
+
video = video ?? doc.createElement('video');
|
|
501
|
+
video.muted = true;
|
|
502
|
+
video.playsInline = true;
|
|
503
|
+
video.srcObject = stream;
|
|
504
|
+
await video.play?.();
|
|
505
|
+
canvas = canvas ?? doc.createElement('canvas');
|
|
506
|
+
ctx = canvas.getContext('2d');
|
|
507
|
+
if (!ctx)
|
|
508
|
+
throw new Error('2d canvas context is not available');
|
|
509
|
+
shell.setState('live');
|
|
510
|
+
const ms = Math.max(1, Math.round(1000 / (opts.fps ?? 3)));
|
|
511
|
+
await captureOne();
|
|
512
|
+
timer = setInterval(captureOne, ms);
|
|
513
|
+
return shell.state;
|
|
514
|
+
}
|
|
515
|
+
catch (e) {
|
|
516
|
+
stopTracks(stream);
|
|
517
|
+
stream = null;
|
|
518
|
+
shell.setState(stateFromMediaError(e), e);
|
|
519
|
+
return shell.state;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
const control = {
|
|
523
|
+
start,
|
|
524
|
+
stop,
|
|
525
|
+
getStats: () => ({ ...shell.stats }),
|
|
526
|
+
setDevice: async (id) => {
|
|
527
|
+
deviceId = id;
|
|
528
|
+
return shell.state == 'live' ? start() : shell.state;
|
|
529
|
+
},
|
|
530
|
+
listDevices: () => listDevices('video'),
|
|
531
|
+
get state() { return shell.state; },
|
|
532
|
+
sourceId,
|
|
533
|
+
kind: 'video',
|
|
534
|
+
transport,
|
|
535
|
+
};
|
|
536
|
+
return attachControl([shell.emit, shell.listenApi], control);
|
|
537
|
+
}
|
package/lib/client.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export * from "./Common/events/event";
|
|
|
13
13
|
export * from "./Common/events/SocketBuffer";
|
|
14
14
|
export * from "./Common/events/SocketServerHook";
|
|
15
15
|
export * from "./Common/events/joinListens";
|
|
16
|
+
export * from "./Common/media/media-index";
|
|
16
17
|
export * from "./Common/rcp/rpc-index";
|
|
17
18
|
export * from "./Common/Color";
|
|
18
19
|
export * from "./Common/funcTimeWait";
|
|
@@ -27,3 +28,4 @@ export * as BaseTypes from "./Common/core/BaseTypes";
|
|
|
27
28
|
export * as Params from "./Exchange/CParams";
|
|
28
29
|
export * as Time from "./Common/Time";
|
|
29
30
|
export * as Color from "./Common/Color";
|
|
31
|
+
export * as Media from "./Common/media/media-index";
|
package/lib/client.js
CHANGED
|
@@ -36,7 +36,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
36
36
|
};
|
|
37
37
|
})();
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.Color = exports.Time = exports.Params = exports.BaseTypes = exports.Bars = exports.Math = void 0;
|
|
39
|
+
exports.Media = exports.Color = exports.Time = exports.Params = exports.BaseTypes = exports.Bars = exports.Math = void 0;
|
|
40
40
|
__exportStar(require("./Common/core/Decorator"), exports);
|
|
41
41
|
__exportStar(require("./Common/core/BaseTypes"), exports);
|
|
42
42
|
__exportStar(require("./Common/core/common"), exports);
|
|
@@ -52,6 +52,7 @@ __exportStar(require("./Common/events/event"), exports);
|
|
|
52
52
|
__exportStar(require("./Common/events/SocketBuffer"), exports);
|
|
53
53
|
__exportStar(require("./Common/events/SocketServerHook"), exports);
|
|
54
54
|
__exportStar(require("./Common/events/joinListens"), exports);
|
|
55
|
+
__exportStar(require("./Common/media/media-index"), exports);
|
|
55
56
|
__exportStar(require("./Common/rcp/rpc-index"), exports);
|
|
56
57
|
__exportStar(require("./Common/Color"), exports);
|
|
57
58
|
__exportStar(require("./Common/funcTimeWait"), exports);
|
|
@@ -66,3 +67,4 @@ exports.BaseTypes = __importStar(require("./Common/core/BaseTypes"));
|
|
|
66
67
|
exports.Params = __importStar(require("./Exchange/CParams"));
|
|
67
68
|
exports.Time = __importStar(require("./Common/Time"));
|
|
68
69
|
exports.Color = __importStar(require("./Common/Color"));
|
|
70
|
+
exports.Media = __importStar(require("./Common/media/media-index"));
|
package/lib/index.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ export * from "./Common/events/SocketBuffer";
|
|
|
17
17
|
export * from "./Common/events/SocketServerHook";
|
|
18
18
|
export * from "./Common/events/joinListens";
|
|
19
19
|
export * from "./Common/events/mapListen";
|
|
20
|
+
export * from "./Common/media/media-index";
|
|
20
21
|
export * from "./Common/rcp/rpc-index";
|
|
21
22
|
export * from "./Common/Color";
|
|
22
23
|
export * from "./Common/funcTimeWait";
|
|
@@ -33,3 +34,4 @@ export * as Time from "./Common/Time";
|
|
|
33
34
|
export * as Color from "./Common/Color";
|
|
34
35
|
export * as Observe from "./Common/Observe";
|
|
35
36
|
export * as Replay from "./Common/events/replay-index";
|
|
37
|
+
export * as Media from "./Common/media/media-index";
|
package/lib/index.js
CHANGED
|
@@ -36,7 +36,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
36
36
|
};
|
|
37
37
|
})();
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.Replay = exports.Observe = exports.Color = exports.Time = exports.Params = exports.Bars = exports.Math = void 0;
|
|
39
|
+
exports.Media = exports.Replay = exports.Observe = exports.Color = exports.Time = exports.Params = exports.Bars = exports.Math = void 0;
|
|
40
40
|
__exportStar(require("./Common/node_console"), exports);
|
|
41
41
|
__exportStar(require("./Common/core/Decorator"), exports);
|
|
42
42
|
__exportStar(require("./Common/core/BaseTypes"), exports);
|
|
@@ -56,6 +56,7 @@ __exportStar(require("./Common/events/SocketBuffer"), exports);
|
|
|
56
56
|
__exportStar(require("./Common/events/SocketServerHook"), exports);
|
|
57
57
|
__exportStar(require("./Common/events/joinListens"), exports);
|
|
58
58
|
__exportStar(require("./Common/events/mapListen"), exports);
|
|
59
|
+
__exportStar(require("./Common/media/media-index"), exports);
|
|
59
60
|
__exportStar(require("./Common/rcp/rpc-index"), exports);
|
|
60
61
|
__exportStar(require("./Common/Color"), exports);
|
|
61
62
|
__exportStar(require("./Common/funcTimeWait"), exports);
|
|
@@ -72,3 +73,4 @@ exports.Time = __importStar(require("./Common/Time"));
|
|
|
72
73
|
exports.Color = __importStar(require("./Common/Color"));
|
|
73
74
|
exports.Observe = __importStar(require("./Common/Observe"));
|
|
74
75
|
exports.Replay = __importStar(require("./Common/events/replay-index"));
|
|
76
|
+
exports.Media = __importStar(require("./Common/media/media-index"));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "wenay-common2",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.67",
|
|
4
4
|
"description": "Common library",
|
|
5
5
|
"strict": true,
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
"CLAUDE.md",
|
|
12
12
|
"rpc.md",
|
|
13
13
|
"!doc/target/**/*",
|
|
14
|
+
"!doc/progress/**/*",
|
|
14
15
|
"!**/*.tsbuildinfo"
|
|
15
16
|
],
|
|
16
17
|
"author": "wenay",
|
|
@@ -33,7 +34,8 @@
|
|
|
33
34
|
"./lib/server": "./lib/server.js",
|
|
34
35
|
"./client": "./lib/client.js",
|
|
35
36
|
"./server": "./lib/server.js",
|
|
36
|
-
"./package.json": "./package.json"
|
|
37
|
+
"./package.json": "./package.json",
|
|
38
|
+
"./media": "./lib/Common/media/media-index.js"
|
|
37
39
|
},
|
|
38
40
|
"dependencies": {
|
|
39
41
|
"express": "^5.2.1"
|