converse-framework 0.2.0__py3-none-any.whl

converse_framework/js/browser-voice-client.js

@@ -0,0 +1,248 @@
+/**
+ * browser-voice-client.js — optional composed browser voice client for
+ * converse_framework.
+ *
+ * Combines ``MicFrameSender`` (mic capture) and ``TtsAudioPlayer`` (playback)
+ * into a single helper, with optional ``SpeakerEchoGuard`` integration.
+ *
+ * This module is optional — apps can use ``MicFrameSender`` and
+ * ``TtsAudioPlayer`` independently.  It does **not** modify or depend on
+ * either module's internal API beyond the documented public interface.
+ *
+ * ```html
+ * <script src="tts-audio-player.js"></script>
+ * <script src="mic-frame-sender.js"></script>
+ * <script src="speaker-echo-guard.js"></script>
+ * <script src="browser-voice-client.js"></script>
+ * <script>
+ *   const client = new BrowserVoiceClient({
+ *     webSocket: new WebSocket("ws://localhost:8000/ws"),
+ *   });
+ *   client.start();
+ *
+ *   // Later:
+ *   client.stop();
+ *   client.close();
+ * </script>
+ * ```
+ *
+ * @module
+ */
+
+((root, factory) => {
+	if (typeof define === "function" && define.amd) {
+		define([], factory);
+	} else if (typeof module === "object" && module.exports) {
+		module.exports = factory();
+	} else {
+		root.BrowserVoiceClient = factory();
+	}
+})(this, () => {
+	// -----------------------------------------------------------------------
+	// BrowserVoiceClient
+	// -----------------------------------------------------------------------
+
+	/**
+	 * Create a composed browser voice client.
+	 *
+	 * @param {Object} options
+	 * @param {WebSocket} options.webSocket  - Required: target WebSocket.
+	 * @param {Object}  [options.micOptions]  - Options passed to ``MicFrameSender`` constructor.
+	 * @param {Object}  [options.playerOptions]  - Options passed to ``TtsAudioPlayer`` constructor.
+	 * @param {Object}  [options.guardOptions]  - Options passed to ``SpeakerEchoGuard`` constructor
+	 *     (omit to disable echo guard).
+	 * @param {boolean} [options.autoStart=false]  - If true, call ``start()`` in constructor.
+	 * @param {function(Object):void} [options.onEvent]  - Receive all framework events (from
+	 *     the WebSocket) before they are dispatched.
+	 */
+	function BrowserVoiceClient(options) {
+		if (!options || !options.webSocket) {
+			throw new Error("BrowserVoiceClient requires a webSocket");
+		}
+
+		var ws = options.webSocket;
+		this._ws = ws;
+		this._micOptions = options.micOptions || {};
+		this._playerOptions = options.playerOptions || {};
+		this._guardOptions = options.guardOptions || null;
+		this._onEvent = options.onEvent || null;
+
+		// WebSocket message handler bound so we can removeEventListener
+		this._boundOnMessage = null;
+
+		// Build sub-components
+		this._buildMic();
+		this._buildPlayer();
+		this._buildGuard();
+
+		if (options.autoStart) {
+			this.start();
+		}
+	}
+
+	BrowserVoiceClient.prototype = {
+		constructor: BrowserVoiceClient,
+
+		// -----------------------------------------------------------------------
+		// Public API
+		// -----------------------------------------------------------------------
+
+		/**
+		 * Start mic capture and connect WebSocket event handler.
+		 * @returns {Promise<void>}
+		 */
+		start: async function () {
+			this._startWebSocketHandler();
+			if (this._mic) {
+				try {
+					await this._mic.start();
+				} catch (e) {
+					// Mic start failures are surfaced via the mic's onError callback
+					// and the caller's catch block.
+					throw e;
+				}
+			}
+		},
+
+		/**
+		 * Stop mic capture and disconnect WebSocket handler.
+		 */
+		stop: function () {
+			if (this._mic) {
+				this._mic.stop();
+			}
+			this._stopWebSocketHandler();
+		},
+
+		/**
+		 * Release all resources.
+		 */
+		close: function () {
+			this.stop();
+			if (this._player) {
+				this._player.clear();
+			}
+			if (this._guard) {
+				this._guard.release();
+				this._guard = null;
+			}
+		},
+
+		/**
+		 * Current ``MicFrameSender`` instance.
+		 * @type {Object|null}
+		 */
+		get mic() {
+			return this._mic;
+		},
+
+		/**
+		 * Current ``TtsAudioPlayer`` instance.
+		 * @type {Object|null}
+		 */
+		get player() {
+			return this._player;
+		},
+
+		/**
+		 * Current ``SpeakerEchoGuard`` instance (may be null).
+		 * @type {Object|null}
+		 */
+		get guard() {
+			return this._guard;
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: sub-component construction
+		// -----------------------------------------------------------------------
+
+		_buildMic: function () {
+			var micOpts = Object.assign({}, this._micOptions);
+			if (!micOpts.webSocket) {
+				micOpts.webSocket = this._ws;
+			}
+			// Guard will attach via attachMicSender after construction
+			if (
+				typeof MicFrameSender !== "undefined" ||
+				typeof root.MicFrameSender !== "undefined"
+			) {
+				var Sender = MicFrameSender || root.MicFrameSender;
+				this._mic = new Sender(micOpts);
+			}
+		},
+
+		_buildPlayer: function () {
+			var playerOpts = Object.assign({}, this._playerOptions);
+			if (!playerOpts.webSocket) {
+				playerOpts.webSocket = this._ws;
+			}
+			if (
+				typeof TtsAudioPlayer !== "undefined" ||
+				typeof root.TtsAudioPlayer !== "undefined"
+			) {
+				var Player = TtsAudioPlayer || root.TtsAudioPlayer;
+				this._player = new Player(playerOpts);
+			}
+		},
+
+		_buildGuard: function () {
+			if (
+				!this._guardOptions ||
+				(typeof SpeakerEchoGuard === "undefined" &&
+					typeof root.SpeakerEchoGuard === "undefined")
+			) {
+				return;
+			}
+
+			var Guard = SpeakerEchoGuard || root.SpeakerEchoGuard;
+			this._guard = new Guard(this._guardOptions);
+
+			if (this._mic) {
+				this._guard.attachMicSender(this._mic);
+			}
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: WebSocket event dispatch
+		// -----------------------------------------------------------------------
+
+		_startWebSocketHandler: function () {
+			if (this._boundOnMessage) return;
+
+			this._boundOnMessage = (evt) => {
+				var msg;
+				try {
+					msg = JSON.parse(evt.data);
+				} catch (_) {
+					return; // Not a JSON event
+				}
+
+				// Global event observer
+				if (this._onEvent) {
+					this._onEvent(msg);
+				}
+
+				// Dispatch to player
+				if (this._player && typeof this._player.onEvent === "function") {
+					this._player.onEvent(msg);
+				}
+
+				// Dispatch to guard
+				if (this._guard && typeof this._guard.onTtsEvent === "function") {
+					this._guard.onTtsEvent(msg);
+				}
+			};
+
+			this._ws.addEventListener("message", this._boundOnMessage);
+		},
+
+		_stopWebSocketHandler: function () {
+			if (this._boundOnMessage) {
+				this._ws.removeEventListener("message", this._boundOnMessage);
+				this._boundOnMessage = null;
+			}
+		},
+	};
+
+	return BrowserVoiceClient;
+});

converse_framework/js/mic-frame-sender.js

@@ -0,0 +1,445 @@
+/**
+ * mic-frame-sender.js — browser microphone capture and frame sender for
+ * converse_framework ``audio.frame`` events.
+ *
+ * Captures microphone input via the Web Audio API, downsamples to the
+ * target sample rate, slices into fixed-size PCM s16le frames, and
+ * sends them as base64-encoded payloads over a WebSocket.
+ *
+ * ```html
+ * <script src="mic-frame-sender.js"></script>
+ * <script>
+ *   const ws = new WebSocket("ws://localhost:8000/ws");
+ *   const mic = new MicFrameSender({ webSocket: ws });
+ *   document.getElementById("start-btn").onclick = () => mic.start();
+ *   document.getElementById("stop-btn").onclick = () => mic.stop();
+ * </script>
+ * ```
+ *
+ * Mobile microphone access requires HTTPS, localhost, or a secure tunnel
+ * (Cloudflare Tunnel, ngrok, etc.).  Plain ``http://<lan-ip>`` will be
+ * rejected by ``getUserMedia`` on most mobile browsers.
+ *
+ * @module
+ */
+
+((root, factory) => {
+	if (typeof define === "function" && define.amd) {
+		define([], factory);
+	} else if (typeof module === "object" && module.exports) {
+		module.exports = factory();
+	} else {
+		root.MicFrameSender = factory();
+	}
+})(this, () => {
+	// -----------------------------------------------------------------------
+	// Constants
+	// -----------------------------------------------------------------------
+
+	const ENCODING = "pcm_s16le";
+
+	// -----------------------------------------------------------------------
+	// Utilities
+	// -----------------------------------------------------------------------
+
+	/**
+	 * Downsample an interleaved float32 array from deviceRate to targetRate.
+	 * Simple linear interpolation.
+	 */
+	function downsampleFloat32(samples, deviceRate, targetRate) {
+		if (deviceRate === targetRate) return samples;
+		const ratio = deviceRate / targetRate;
+		const outLen = Math.round(samples.length / ratio);
+		const out = new Float32Array(outLen);
+		for (let i = 0; i < outLen; i++) {
+			const srcIdx = i * ratio;
+			const lo = Math.floor(srcIdx);
+			const hi = Math.min(lo + 1, samples.length - 1);
+			const frac = srcIdx - lo;
+			// Clamp to [-1, 1] to avoid overflow on conversion
+			out[i] = Math.max(
+				-1,
+				Math.min(1, samples[lo] * (1 - frac) + samples[hi] * frac),
+			);
+		}
+		return out;
+	}
+
+	/**
+	 * Convert a Float32Array of samples to a PCM s16le ArrayBuffer.
+	 * Assumes samples in [-1, 1] range.
+	 */
+	function float32ToPcmS16le(samples) {
+		const len = samples.length;
+		const buf = new ArrayBuffer(len * 2);
+		const view = new DataView(buf);
+		for (let i = 0; i < len; i++) {
+			// Clamp to [-1, 1] then scale to int16
+			const s = Math.max(-1, Math.min(1, samples[i]));
+			const val = s < 0 ? s * 0x8000 : s * 0x7fff;
+			view.setInt16(i * 2, Math.round(val), true); // little-endian
+		}
+		return buf;
+	}
+
+	/**
+	 * Convert an ArrayBuffer to a base64-encoded string.
+	 */
+	function arrayBufferToBase64(buf) {
+		const bytes = new Uint8Array(buf);
+		let binary = "";
+		for (let i = 0; i < bytes.length; i++) {
+			binary += String.fromCharCode(bytes[i]);
+		}
+		return btoa(binary);
+	}
+
+	/**
+	 * Number of samples for N ms at a given sample rate.
+	 */
+	function msToSamples(ms, rate) {
+		return Math.round((ms / 1000) * rate);
+	}
+
+	// -----------------------------------------------------------------------
+	// AudioWorklet processor (inline via blob URL)
+	// -----------------------------------------------------------------------
+
+	let _processorUrl = null;
+
+	function getProcessorUrl() {
+		if (_processorUrl) return _processorUrl;
+		const code = [
+			"class CaptureProcessor extends AudioWorkletProcessor {",
+			"  process(inputs, outputs, params) {",
+			"    const input = inputs[0];",
+			"    if (input && input[0] && input[0].length > 0) {",
+			"      this.port.postMessage(input[0]);",
+			"    }",
+			"    return true;",
+			"  }",
+			"}",
+			"registerProcessor('mic-capture-processor', CaptureProcessor);",
+		].join("\n");
+		const blob = new Blob([code], { type: "application/javascript" });
+		_processorUrl = URL.createObjectURL(blob);
+		return _processorUrl;
+	}
+
+	// -----------------------------------------------------------------------
+	// MicFrameSender
+	// -----------------------------------------------------------------------
+
+	/**
+	 * Create a microphone frame sender.
+	 *
+	 * @param {Object} options
+	 * @param {WebSocket} options.webSocket  - target WebSocket.
+	 * @param {number}  [options.sampleRate=16000] - target sample rate.
+	 * @param {number}  [options.channels=1]       - output channel count.
+	 * @param {number}  [options.frameMs=30]       - frame duration in ms.
+	 * @param {string}  [options.mode="chat"]      - conversation mode tag.
+	 * @param {string}  [options.messageType="audio.frame"] - WebSocket message type.
+	 * @param {function(number):void} [options.onLevel] - level callback (0-1).
+	 * @param {function(Error):void}  [options.onError]  - error callback.
+	 * @param {AudioContext} [options.audioContext]  - shared AudioContext (created if omitted).
+	 * @param {function(Object):boolean} [options.shouldSendFrame] - optional gate; called with payload, return false to drop.
+	 */
+	function MicFrameSender(options) {
+		options = options || {};
+		this._ws = options.webSocket || null;
+		this._targetRate = options.sampleRate || 16000;
+		this._channels = options.channels || 1;
+		this._frameMs = options.frameMs || 30;
+		this._mode = options.mode || "chat";
+		this._messageType = options.messageType || "audio.frame";
+		this._onLevel = options.onLevel || null;
+		this._onError = options.onError || null;
+		this._audioContext = options.audioContext || null;
+		this._shouldSendFrame = options.shouldSendFrame || null;
+
+		// Owned AudioContext (if none provided)
+		this._ownedCtx = null;
+
+		// Active stream / nodes / worklet
+		this._stream = null;
+		this._source = null;
+		this._workletNode = null;
+		this._scriptProcessor = null;
+
+		// Frame sequencing
+		this._sequence = 0;
+		this._accumulator = new Float32Array(0);
+		this._frameSamples = msToSamples(this._frameMs, this._targetRate);
+		this._running = false;
+		this._paused = false;
+
+		// Worklet availability
+		this._workletSupported =
+			typeof AudioWorkletNode !== "undefined" &&
+			typeof AudioContext !== "undefined";
+	}
+
+	MicFrameSender.prototype = {
+		constructor: MicFrameSender,
+
+		// -----------------------------------------------------------------------
+		// Public API
+		// -----------------------------------------------------------------------
+
+		/**
+		 * Start capturing and sending mic frames.
+		 * @returns {Promise<void>}
+		 */
+		start: async function () {
+			if (this._running) return;
+			this._paused = false;
+
+			try {
+				const stream = await navigator.mediaDevices.getUserMedia({
+					audio: {
+						sampleRate: { ideal: this._targetRate },
+						channelCount: { ideal: this._channels },
+						echoCancellation: true,
+						noiseSuppression: true,
+					},
+				});
+				this._stream = stream;
+
+				const ctx =
+					this._audioContext ||
+					new (window.AudioContext || window.webkitAudioContext)();
+				if (!this._audioContext) {
+					this._ownedCtx = ctx;
+				}
+
+				// Determine device sample rate from AudioContext
+				const deviceRate = ctx.sampleRate;
+
+				// Ensure context is running (needed after autoplay policy)
+				if (ctx.state === "suspended") {
+					await ctx.resume();
+				}
+
+				const source = ctx.createMediaStreamSource(stream);
+				this._source = source;
+
+				// Try AudioWorklet first, fall back to ScriptProcessorNode
+				try {
+					await this._setupWorkletNode(ctx, source, deviceRate);
+				} catch (_) {
+					this._setupScriptProcessor(ctx, source, deviceRate);
+				}
+
+				this._running = true;
+			} catch (err) {
+				this._safeError(err);
+				throw err;
+			}
+		},
+
+		/**
+		 * Stop capturing and release resources.
+		 */
+		stop: function () {
+			this._running = false;
+			this._paused = false;
+			this._sequence = 0;
+
+			this._teardownNodes();
+			this._teardownStream();
+			this._teardownContext();
+		},
+
+		/**
+		 * Update the conversation mode tag sent with each frame.
+		 * @param {string} mode
+		 */
+		setMode: function (mode) {
+			this._mode = String(mode);
+		},
+
+		/**
+		 * Replace the target WebSocket.  Can be called while running.
+		 * @param {WebSocket|null} ws
+		 */
+		setWebSocket: function (ws) {
+			this._ws = ws;
+		},
+
+		/**
+		 * Release all resources (alias for stop).
+		 */
+		close: function () {
+			this.stop();
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: AudioWorklet branch
+		// -----------------------------------------------------------------------
+
+		_setupWorkletNode: async function (ctx, source, deviceRate) {
+			const url = getProcessorUrl();
+			await ctx.audioWorklet.addModule(url);
+			const workletNode = new AudioWorkletNode(ctx, "mic-capture-processor");
+			this._workletNode = workletNode;
+
+			source.connect(workletNode);
+
+			workletNode.port.onmessage = (evt) => {
+				if (!this._running || this._paused) return;
+				const floatSamples = evt.data;
+				this._processAudio(floatSamples, deviceRate);
+			};
+
+			workletNode.connect(ctx.destination);
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: ScriptProcessorNode fallback
+		// -----------------------------------------------------------------------
+
+		_setupScriptProcessor: function (ctx, source, deviceRate) {
+			// Buffer size = one frame at device rate, rounded to power-of-2
+			const bufSize = this._nextPow2(msToSamples(this._frameMs, deviceRate));
+			const processor = ctx.createScriptProcessor(bufSize, 1, 1);
+			this._scriptProcessor = processor;
+
+			source.connect(processor);
+			processor.connect(ctx.destination);
+
+			processor.onaudioprocess = (evt) => {
+				if (!this._running || this._paused) return;
+				const input = evt.inputBuffer.getChannelData(0);
+				this._processAudio(input, deviceRate);
+			};
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: audio processing pipeline
+		// -----------------------------------------------------------------------
+
+		_processAudio: function (floatSamples, deviceRate) {
+			// Downsample to target rate
+			const downsampled = downsampleFloat32(
+				floatSamples,
+				deviceRate,
+				this._targetRate,
+			);
+
+			// Append to accumulator
+			const acc = this._accumulator;
+			const combined = new Float32Array(acc.length + downsampled.length);
+			combined.set(acc);
+			combined.set(downsampled, acc.length);
+			this._accumulator = combined;
+
+			// Slice and send complete frames
+			const frameSize = this._frameSamples;
+			while (this._accumulator.length >= frameSize) {
+				const frame = this._accumulator.slice(0, frameSize);
+				this._accumulator = this._accumulator.slice(frameSize);
+				this._sendFrame(frame);
+			}
+		},
+
+		_sendFrame: function (frame) {
+			// Report level
+			if (this._onLevel) {
+				let sum = 0;
+				for (let i = 0; i < frame.length; i++) {
+					sum += frame[i] * frame[i];
+				}
+				const rms = Math.sqrt(sum / frame.length);
+				this._onLevel(Math.min(1, rms * 3));
+			}
+
+			const pcmBuf = float32ToPcmS16le(frame);
+			const base64 = arrayBufferToBase64(pcmBuf);
+
+			const payload = {
+				type: this._messageType,
+				payload: {
+					data: base64,
+					encoding: ENCODING,
+					sample_rate: this._targetRate,
+					channels: this._channels,
+					frame_ms: this._frameMs,
+					sequence: this._sequence++,
+					mode: this._mode,
+				},
+			};
+
+			// Optional gate (speaker echo guard)
+			if (this._shouldSendFrame && !this._shouldSendFrame(payload)) {
+				return;
+			}
+
+			if (this._ws && this._ws.readyState === WebSocket.OPEN) {
+				this._ws.send(JSON.stringify(payload));
+			}
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: teardown helpers
+		// -----------------------------------------------------------------------
+
+		_teardownNodes: function () {
+			if (this._workletNode) {
+				this._workletNode.disconnect();
+				this._workletNode = null;
+			}
+			if (this._scriptProcessor) {
+				this._scriptProcessor.disconnect();
+				this._scriptProcessor = null;
+			}
+			if (this._source) {
+				this._source.disconnect();
+				this._source = null;
+			}
+		},
+
+		_teardownStream: function () {
+			if (this._stream) {
+				this._stream.getTracks().forEach((t) => {
+					t.stop();
+				});
+				this._stream = null;
+			}
+		},
+
+		_teardownContext: function () {
+			if (this._ownedCtx) {
+				this._ownedCtx.close();
+				this._ownedCtx = null;
+			}
+		},
+
+		// -----------------------------------------------------------------------
+		// Internal: utilities
+		// -----------------------------------------------------------------------
+
+		_safeError: function (err) {
+			if (this._onError) {
+				this._onError(err instanceof Error ? err : new Error(String(err)));
+			}
+		},
+
+		_nextPow2: (n) => {
+			let v = 1;
+			while (v < n) v <<= 1;
+			return v;
+		},
+	};
+
+	// -----------------------------------------------------------------------
+	// Exports (pure helper functions for testing)
+	// -----------------------------------------------------------------------
+
+	MicFrameSender.downsampleFloat32 = downsampleFloat32;
+	MicFrameSender.float32ToPcmS16le = float32ToPcmS16le;
+	MicFrameSender.arrayBufferToBase64 = arrayBufferToBase64;
+	MicFrameSender.msToSamples = msToSamples;
+
+	return MicFrameSender;
+});