agent-starter-pack 0.0.1b0__py3-none-any.whl

src/frontends/live_api_react/frontend/src/setupTests.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// jest-dom adds custom jest matchers for asserting on DOM nodes.
+// allows you to do things like:
+// expect(element).toHaveTextContent(/react/i)
+// learn more: https://github.com/testing-library/jest-dom
+import "@testing-library/jest-dom";

src/frontends/live_api_react/frontend/src/utils/audio-recorder.ts

@@ -0,0 +1,111 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { audioContext } from "./utils";
+import AudioRecordingWorklet from "./worklets/audio-processing";
+import VolMeterWorket from "./worklets/vol-meter";
+
+import { createWorketFromSrc } from "./audioworklet-registry";
+import EventEmitter from "eventemitter3";
+
+function arrayBufferToBase64(buffer: ArrayBuffer) {
+  var binary = "";
+  var bytes = new Uint8Array(buffer);
+  var len = bytes.byteLength;
+  for (var i = 0; i < len; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return window.btoa(binary);
+}
+
+export class AudioRecorder extends EventEmitter {
+  stream: MediaStream | undefined;
+  audioContext: AudioContext | undefined;
+  source: MediaStreamAudioSourceNode | undefined;
+  recording: boolean = false;
+  recordingWorklet: AudioWorkletNode | undefined;
+  vuWorklet: AudioWorkletNode | undefined;
+
+  private starting: Promise<void> | null = null;
+
+  constructor(public sampleRate = 16000) {
+    super();
+  }
+
+  async start() {
+    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+      throw new Error("Could not request user media");
+    }
+
+    this.starting = new Promise(async (resolve, reject) => {
+      this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      this.audioContext = await audioContext({ sampleRate: this.sampleRate });
+      this.source = this.audioContext.createMediaStreamSource(this.stream);
+
+      const workletName = "audio-recorder-worklet";
+      const src = createWorketFromSrc(workletName, AudioRecordingWorklet);
+
+      await this.audioContext.audioWorklet.addModule(src);
+      this.recordingWorklet = new AudioWorkletNode(
+        this.audioContext,
+        workletName,
+      );
+
+      this.recordingWorklet.port.onmessage = async (ev: MessageEvent) => {
+        // worklet processes recording floats and messages converted buffer
+        const arrayBuffer = ev.data.data.int16arrayBuffer;
+
+        if (arrayBuffer) {
+          const arrayBufferString = arrayBufferToBase64(arrayBuffer);
+          this.emit("data", arrayBufferString);
+        }
+      };
+      this.source.connect(this.recordingWorklet);
+
+      // vu meter worklet
+      const vuWorkletName = "vu-meter";
+      await this.audioContext.audioWorklet.addModule(
+        createWorketFromSrc(vuWorkletName, VolMeterWorket),
+      );
+      this.vuWorklet = new AudioWorkletNode(this.audioContext, vuWorkletName);
+      this.vuWorklet.port.onmessage = (ev: MessageEvent) => {
+        this.emit("volume", ev.data.volume);
+      };
+
+      this.source.connect(this.vuWorklet);
+      this.recording = true;
+      resolve();
+      this.starting = null;
+    });
+  }
+
+  stop() {
+    // its plausible that stop would be called before start completes
+    // such as if the websocket immediately hangs up
+    const handleStop = () => {
+      this.source?.disconnect();
+      this.stream?.getTracks().forEach((track) => track.stop());
+      this.stream = undefined;
+      this.recordingWorklet = undefined;
+      this.vuWorklet = undefined;
+    };
+    if (this.starting) {
+      this.starting.then(handleStop);
+      return;
+    }
+    handleStop();
+  }
+}

src/frontends/live_api_react/frontend/src/utils/audio-streamer.ts

@@ -0,0 +1,270 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  createWorketFromSrc,
+  registeredWorklets,
+} from "./audioworklet-registry";
+
+export class AudioStreamer {
+  public audioQueue: Float32Array[] = [];
+  private isPlaying: boolean = false;
+  private sampleRate: number = 24000;
+  private bufferSize: number = 7680;
+  private processingBuffer: Float32Array = new Float32Array(0);
+  private scheduledTime: number = 0;
+  public gainNode: GainNode;
+  public source: AudioBufferSourceNode;
+  private isStreamComplete: boolean = false;
+  private checkInterval: number | null = null;
+  private initialBufferTime: number = 0.1; //0.1 // 100ms initial buffer
+  private endOfQueueAudioSource: AudioBufferSourceNode | null = null;
+
+  public onComplete = () => {};
+
+  constructor(public context: AudioContext) {
+    this.gainNode = this.context.createGain();
+    this.source = this.context.createBufferSource();
+    this.gainNode.connect(this.context.destination);
+    this.addPCM16 = this.addPCM16.bind(this);
+  }
+
+  async addWorklet<T extends (d: any) => void>(
+    workletName: string,
+    workletSrc: string,
+    handler: T,
+  ): Promise<this> {
+    let workletsRecord = registeredWorklets.get(this.context);
+    if (workletsRecord && workletsRecord[workletName]) {
+      // the worklet already exists on this context
+      // add the new handler to it
+      workletsRecord[workletName].handlers.push(handler);
+      return Promise.resolve(this);
+      //throw new Error(`Worklet ${workletName} already exists on context`);
+    }
+
+    if (!workletsRecord) {
+      registeredWorklets.set(this.context, {});
+      workletsRecord = registeredWorklets.get(this.context)!;
+    }
+
+    // create new record to fill in as becomes available
+    workletsRecord[workletName] = { handlers: [handler] };
+
+    const src = createWorketFromSrc(workletName, workletSrc);
+    await this.context.audioWorklet.addModule(src);
+    const worklet = new AudioWorkletNode(this.context, workletName);
+
+    //add the node into the map
+    workletsRecord[workletName].node = worklet;
+
+    return this;
+  }
+
+  addPCM16(chunk: Uint8Array) {
+    const float32Array = new Float32Array(chunk.length / 2);
+    const dataView = new DataView(chunk.buffer);
+
+    for (let i = 0; i < chunk.length / 2; i++) {
+      try {
+        const int16 = dataView.getInt16(i * 2, true);
+        float32Array[i] = int16 / 32768;
+      } catch (e) {
+        console.error(e);
+        // console.log(
+        //   `dataView.length: ${dataView.byteLength},  i * 2: ${i * 2}`,
+        // );
+      }
+    }
+
+    const newBuffer = new Float32Array(
+      this.processingBuffer.length + float32Array.length,
+    );
+    newBuffer.set(this.processingBuffer);
+    newBuffer.set(float32Array, this.processingBuffer.length);
+    this.processingBuffer = newBuffer;
+
+    while (this.processingBuffer.length >= this.bufferSize) {
+      const buffer = this.processingBuffer.slice(0, this.bufferSize);
+      this.audioQueue.push(buffer);
+      this.processingBuffer = this.processingBuffer.slice(this.bufferSize);
+    }
+
+    if (!this.isPlaying) {
+      this.isPlaying = true;
+      // Initialize scheduledTime only when we start playing
+      this.scheduledTime = this.context.currentTime + this.initialBufferTime;
+      this.scheduleNextBuffer();
+    }
+  }
+
+  private createAudioBuffer(audioData: Float32Array): AudioBuffer {
+    const audioBuffer = this.context.createBuffer(
+      1,
+      audioData.length,
+      this.sampleRate,
+    );
+    audioBuffer.getChannelData(0).set(audioData);
+    return audioBuffer;
+  }
+
+  private scheduleNextBuffer() {
+    const SCHEDULE_AHEAD_TIME = 0.2;
+
+    while (
+      this.audioQueue.length > 0 &&
+      this.scheduledTime < this.context.currentTime + SCHEDULE_AHEAD_TIME
+    ) {
+      const audioData = this.audioQueue.shift()!;
+      const audioBuffer = this.createAudioBuffer(audioData);
+      const source = this.context.createBufferSource();
+
+      if (this.audioQueue.length === 0) {
+        if (this.endOfQueueAudioSource) {
+          this.endOfQueueAudioSource.onended = null;
+        }
+        this.endOfQueueAudioSource = source;
+        source.onended = () => {
+          if (
+            !this.audioQueue.length &&
+            this.endOfQueueAudioSource === source
+          ) {
+            this.endOfQueueAudioSource = null;
+            this.onComplete();
+          }
+        };
+      }
+
+      source.buffer = audioBuffer;
+      source.connect(this.gainNode);
+
+      const worklets = registeredWorklets.get(this.context);
+
+      if (worklets) {
+        Object.entries(worklets).forEach(([workletName, graph]) => {
+          const { node, handlers } = graph;
+          if (node) {
+            source.connect(node);
+            node.port.onmessage = function (ev: MessageEvent) {
+              handlers.forEach((handler) => {
+                handler.call(node.port, ev);
+              });
+            };
+            node.connect(this.context.destination);
+          }
+        });
+      }
+
+      // i added this trying to fix clicks
+      // this.gainNode.gain.setValueAtTime(0, 0);
+      // this.gainNode.gain.linearRampToValueAtTime(1, 1);
+
+      // Ensure we never schedule in the past
+      const startTime = Math.max(this.scheduledTime, this.context.currentTime);
+      source.start(startTime);
+
+      this.scheduledTime = startTime + audioBuffer.duration;
+    }
+
+    if (this.audioQueue.length === 0 && this.processingBuffer.length === 0) {
+      if (this.isStreamComplete) {
+        this.isPlaying = false;
+        if (this.checkInterval) {
+          clearInterval(this.checkInterval);
+          this.checkInterval = null;
+        }
+      } else {
+        if (!this.checkInterval) {
+          this.checkInterval = window.setInterval(() => {
+            if (
+              this.audioQueue.length > 0 ||
+              this.processingBuffer.length >= this.bufferSize
+            ) {
+              this.scheduleNextBuffer();
+            }
+          }, 100) as unknown as number;
+        }
+      }
+    } else {
+      const nextCheckTime =
+        (this.scheduledTime - this.context.currentTime) * 1000;
+      setTimeout(
+        () => this.scheduleNextBuffer(),
+        Math.max(0, nextCheckTime - 50),
+      );
+    }
+  }
+
+  stop() {
+    this.isPlaying = false;
+    this.isStreamComplete = true;
+    this.audioQueue = [];
+    this.processingBuffer = new Float32Array(0);
+    this.scheduledTime = this.context.currentTime;
+
+    if (this.checkInterval) {
+      clearInterval(this.checkInterval);
+      this.checkInterval = null;
+    }
+
+    this.gainNode.gain.linearRampToValueAtTime(
+      0,
+      this.context.currentTime + 0.1,
+    );
+
+    setTimeout(() => {
+      this.gainNode.disconnect();
+      this.gainNode = this.context.createGain();
+      this.gainNode.connect(this.context.destination);
+    }, 200);
+  }
+
+  async resume() {
+    if (this.context.state === "suspended") {
+      await this.context.resume();
+    }
+    this.isStreamComplete = false;
+    this.scheduledTime = this.context.currentTime + this.initialBufferTime;
+    this.gainNode.gain.setValueAtTime(1, this.context.currentTime);
+  }
+
+  complete() {
+    this.isStreamComplete = true;
+    if (this.processingBuffer.length > 0) {
+      this.audioQueue.push(this.processingBuffer);
+      this.processingBuffer = new Float32Array(0);
+      if (this.isPlaying) {
+        this.scheduleNextBuffer();
+      }
+    } else {
+      this.onComplete();
+    }
+  }
+}
+
+// // Usage example:
+// const audioStreamer = new AudioStreamer();
+//
+// // In your streaming code:
+// function handleChunk(chunk: Uint8Array) {
+//   audioStreamer.handleChunk(chunk);
+// }
+//
+// // To start playing (call this in response to a user interaction)
+// await audioStreamer.resume();
+//
+// // To stop playing
+// // audioStreamer.stop();

src/frontends/live_api_react/frontend/src/utils/audioworklet-registry.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A registry to map attached worklets by their audio-context
+ * any module using `audioContext.audioWorklet.addModule(` should register the worklet here
+ */
+export type WorkletGraph = {
+  node?: AudioWorkletNode;
+  handlers: Array<(this: MessagePort, ev: MessageEvent) => any>;
+};
+
+export const registeredWorklets: Map<
+  AudioContext,
+  Record<string, WorkletGraph>
+> = new Map();
+
+export const createWorketFromSrc = (
+  workletName: string,
+  workletSrc: string,
+) => {
+  const script = new Blob(
+    [`registerProcessor("${workletName}", ${workletSrc})`],
+    {
+      type: "application/javascript",
+    },
+  );
+
+  return URL.createObjectURL(script);
+};