agent-starter-pack 0.0.1b0__py3-none-any.whl

src/frontends/live_api_react/frontend/src/utils/multimodal-live-client.ts

@@ -0,0 +1,329 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
+import { EventEmitter } from "eventemitter3";
+import { difference } from "lodash";
+import {
+  ClientContentMessage,
+  isInterrupted,
+  isModelTurn,
+  isServerContenteMessage,
+  isSetupCompleteMessage,
+  isToolCallCancellationMessage,
+  isToolCallMessage,
+  isTurnComplete,
+  LiveIncomingMessage,
+  ModelTurn,
+  RealtimeInputMessage,
+  ServerContent,
+  StreamingLog,
+  ToolCall,
+  ToolCallCancellation,
+  ToolResponseMessage,
+  type LiveConfig,
+} from "../multimodal-live-types";
+import { blobToJSON, base64ToArrayBuffer } from "./utils";
+
+/**
+ * the events that this client will emit
+ */
+interface MultimodalLiveClientEventTypes {
+  open: () => void;
+  log: (log: StreamingLog) => void;
+  close: (event: CloseEvent) => void;
+  audio: (data: ArrayBuffer) => void;
+  content: (data: ServerContent) => void;
+  interrupted: () => void;
+  setupcomplete: () => void;
+  status: (status: string) => void;
+  turncomplete: () => void;
+  toolcall: (toolCall: ToolCall) => void;
+  toolcallcancellation: (toolcallCancellation: ToolCallCancellation) => void;
+}
+
+export type MultimodalLiveAPIClientConnection = {
+  url?: string;
+  runId?: string;
+  userId?: string;
+};
+
+/**
+ * A event-emitting class that manages the connection to the websocket and emits
+ * events to the rest of the application.
+ * If you dont want to use react you can still use this.
+ */
+export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEventTypes> {
+  public ws: WebSocket | null = null;
+  protected config: LiveConfig | null = null;
+  public url: string = "";
+  private runId: string;
+  private userId?: string;
+  constructor({ url, userId, runId }: MultimodalLiveAPIClientConnection) {
+    super();
+    url = url || `ws://localhost:8000/ws`;
+    this.url = new URL("ws", url).href;
+    this.userId = userId;
+    this.runId = runId || crypto.randomUUID(); // Ensure runId is always a string by providing default
+    this.send = this.send.bind(this);
+  }
+
+  log(type: string, message: StreamingLog["message"]) {
+    const log: StreamingLog = {
+      date: new Date(),
+      type,
+      message,
+    };
+    this.emit("log", log);
+  }
+
+  connect(newRunId?: string): Promise<boolean> {
+    const ws = new WebSocket(this.url);
+
+    // Update runId if provided
+    if (newRunId) {
+      this.runId = newRunId;
+    }
+
+    ws.addEventListener("message", async (evt: MessageEvent) => {
+      if (evt.data instanceof Blob) {
+        this.receive(evt.data);
+      } else if (typeof evt.data === "string") {
+        try {
+          const jsonData = JSON.parse(evt.data);
+          if (jsonData.status) {
+            this.log("server.status", jsonData.status);
+            console.log("Status:", jsonData.status); // This will show in console
+          }
+        } catch (error) {
+          console.error("Error parsing message:", error);
+        }
+      } else {
+        console.log("Unhandled message type:", evt);
+      }
+    });
+
+    return new Promise((resolve, reject) => {
+      const onError = (ev: Event) => {
+        this.disconnect(ws);
+        const message = `Could not connect to "${this.url}"`;
+        this.log(`server.${ev.type}`, message);
+        reject(new Error(message));
+      };
+      ws.addEventListener("error", onError);
+      ws.addEventListener("open", (ev: Event) => {
+        this.log(`client.${ev.type}`, `connected to socket`);
+        this.emit("open");
+
+        this.ws = ws;
+        // Send initial setup message with runId
+        const setupMessage = {
+          setup: {
+            run_id: this.runId,
+            user_id: this.userId,
+          },
+        };
+        this._sendDirect(setupMessage);
+        ws.removeEventListener("error", onError);
+        ws.addEventListener("close", (ev: CloseEvent) => {
+          console.log(ev);
+          this.disconnect(ws);
+          let reason = ev.reason || "";
+          if (reason.toLowerCase().includes("error")) {
+            const prelude = "ERROR]";
+            const preludeIndex = reason.indexOf(prelude);
+            if (preludeIndex > 0) {
+              reason = reason.slice(
+                preludeIndex + prelude.length + 1,
+                Infinity,
+              );
+            }
+          }
+          this.log(
+            `server.${ev.type}`,
+            `disconnected ${reason ? `with reason: ${reason}` : ``}`,
+          );
+          this.emit("close", ev);
+        });
+        resolve(true);
+      });
+    });
+  }
+
+  disconnect(ws?: WebSocket) {
+    // could be that this is an old websocket and there's already a new instance
+    // only close it if its still the correct reference
+    if ((!ws || this.ws === ws) && this.ws) {
+      this.ws.close();
+      this.ws = null;
+      this.log("client.close", `Disconnected`);
+      return true;
+    }
+    return false;
+  }
+  protected async receive(blob: Blob) {
+    const response = (await blobToJSON(blob)) as LiveIncomingMessage;
+    console.log("Parsed response:", response);
+
+    if (isToolCallMessage(response)) {
+      this.log("server.toolCall", response);
+      this.emit("toolcall", response.toolCall);
+      return;
+    }
+    if (isToolCallCancellationMessage(response)) {
+      this.log("receive.toolCallCancellation", response);
+      this.emit("toolcallcancellation", response.toolCallCancellation);
+      return;
+    }
+
+    if (isSetupCompleteMessage(response)) {
+      this.log("server.send", "setupComplete");
+      this.emit("setupcomplete");
+      return;
+    }
+
+    // this json also might be `contentUpdate { interrupted: true }`
+    // or contentUpdate { end_of_turn: true }
+    if (isServerContenteMessage(response)) {
+      const { serverContent } = response;
+      if (isInterrupted(serverContent)) {
+        this.log("receive.serverContent", "interrupted");
+        this.emit("interrupted");
+        return;
+      }
+      if (isTurnComplete(serverContent)) {
+        this.log("server.send", "turnComplete");
+        this.emit("turncomplete");
+        //plausible there's more to the message, continue
+      }
+
+      if (isModelTurn(serverContent)) {
+        let parts: Part[] = serverContent.modelTurn.parts;
+
+        // when its audio that is returned for modelTurn
+        const audioParts = parts.filter(
+          (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
+        );
+        const base64s = audioParts.map((p) => p.inlineData?.data);
+
+        // strip the audio parts out of the modelTurn
+        const otherParts = difference(parts, audioParts);
+        // console.log("otherParts", otherParts);
+
+        base64s.forEach((b64) => {
+          if (b64) {
+            const data = base64ToArrayBuffer(b64);
+            this.emit("audio", data);
+            this.log(`server.audio`, `buffer (${data.byteLength})`);
+          }
+        });
+        if (!otherParts.length) {
+          return;
+        }
+
+        parts = otherParts;
+
+        const content: ModelTurn = { modelTurn: { parts } };
+        this.emit("content", content);
+        this.log(`server.content`, response);
+      }
+    } else {
+      console.log("received unmatched message", response);
+      this.log("received unmatched message", response);
+    }
+  }
+
+  /**
+   * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
+   */
+  sendRealtimeInput(chunks: GenerativeContentBlob[]) {
+    let hasAudio = false;
+    let hasVideo = false;
+    for (let i = 0; i < chunks.length; i++) {
+      const ch = chunks[i];
+      if (ch.mimeType.includes("audio")) {
+        hasAudio = true;
+      }
+      if (ch.mimeType.includes("image")) {
+        hasVideo = true;
+      }
+      if (hasAudio && hasVideo) {
+        break;
+      }
+    }
+    const message =
+      hasAudio && hasVideo
+        ? "audio + video"
+        : hasAudio
+          ? "audio"
+          : hasVideo
+            ? "video"
+            : "unknown";
+
+    const data: RealtimeInputMessage = {
+      realtimeInput: {
+        mediaChunks: chunks,
+      },
+    };
+    this._sendDirect(data);
+    this.log(`client.realtimeInput`, message);
+  }
+
+  /**
+   *  send a response to a function call and provide the id of the functions you are responding to
+   */
+  sendToolResponse(toolResponse: ToolResponseMessage["toolResponse"]) {
+    const message: ToolResponseMessage = {
+      toolResponse,
+    };
+
+    this._sendDirect(message);
+    this.log(`client.toolResponse`, message);
+  }
+
+  /**
+   * send normal content parts such as { text }
+   */
+  send(parts: Part | Part[], turnComplete: boolean = true) {
+    parts = Array.isArray(parts) ? parts : [parts];
+    const content: Content = {
+      role: "user",
+      parts,
+    };
+
+    const clientContentRequest: ClientContentMessage = {
+      clientContent: {
+        turns: [content],
+        turnComplete,
+      },
+    };
+
+    this._sendDirect(clientContentRequest);
+    this.log(`client.send`, clientContentRequest);
+  }
+
+  /**
+   *  used internally to send all messages
+   *  don't use directly unless trying to send an unsupported message type
+   */
+  _sendDirect(request: object) {
+    if (!this.ws) {
+      throw new Error("WebSocket is not connected");
+    }
+    const str = JSON.stringify(request);
+    this.ws.send(str);
+  }
+}

src/frontends/live_api_react/frontend/src/utils/store-logger.ts

@@ -0,0 +1,64 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { create } from "zustand";
+import { StreamingLog } from "../multimodal-live-types";
+
+interface StoreLoggerState {
+  maxLogs: number;
+  logs: StreamingLog[];
+  log: (streamingLog: StreamingLog) => void;
+  clearLogs: () => void;
+}
+
+export const useLoggerStore = create<StoreLoggerState>((set, get) => ({
+  maxLogs: 500,
+  logs: [], //mockLogs,
+  log: ({ date, type, message }: StreamingLog) => {
+    set((state) => {
+      const prevLog = state.logs.at(-1);
+      if (prevLog && prevLog.type === type && prevLog.message === message) {
+        return {
+          logs: [
+            ...state.logs.slice(0, -1),
+            {
+              date,
+              type,
+              message,
+              count: prevLog.count ? prevLog.count + 1 : 1,
+            } as StreamingLog,
+          ],
+        };
+      }
+      return {
+        logs: [
+          ...state.logs.slice(-(get().maxLogs - 1)),
+          {
+            date,
+            type,
+            message,
+          } as StreamingLog,
+        ],
+      };
+    });
+  },
+
+  clearLogs: () => {
+    console.log("clear log");
+    set({ logs: [] });
+  },
+  setMaxLogs: (n: number) => set({ maxLogs: n }),
+}));

src/frontends/live_api_react/frontend/src/utils/utils.ts

@@ -0,0 +1,86 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type GetAudioContextOptions = AudioContextOptions & {
+  id?: string;
+};
+
+const map: Map<string, AudioContext> = new Map();
+
+export const audioContext: (
+  options?: GetAudioContextOptions,
+) => Promise<AudioContext> = (() => {
+  const didInteract = new Promise((res) => {
+    window.addEventListener("pointerdown", res, { once: true });
+    window.addEventListener("keydown", res, { once: true });
+  });
+
+  return async (options?: GetAudioContextOptions) => {
+    try {
+      const a = new Audio();
+      a.src =
+        "data:audio/wav;base64,UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA";
+      await a.play();
+      if (options?.id && map.has(options.id)) {
+        const ctx = map.get(options.id);
+        if (ctx) {
+          return ctx;
+        }
+      }
+      const ctx = new AudioContext(options);
+      if (options?.id) {
+        map.set(options.id, ctx);
+      }
+      return ctx;
+    } catch (e) {
+      await didInteract;
+      if (options?.id && map.has(options.id)) {
+        const ctx = map.get(options.id);
+        if (ctx) {
+          return ctx;
+        }
+      }
+      const ctx = new AudioContext(options);
+      if (options?.id) {
+        map.set(options.id, ctx);
+      }
+      return ctx;
+    }
+  };
+})();
+
+export const blobToJSON = (blob: Blob) =>
+  new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => {
+      if (reader.result) {
+        const json = JSON.parse(reader.result as string);
+        resolve(json);
+      } else {
+        reject("oops");
+      }
+    };
+    reader.readAsText(blob);
+  });
+
+export function base64ToArrayBuffer(base64: string) {
+  var binaryString = atob(base64);
+  var bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes.buffer;
+}

src/frontends/live_api_react/frontend/src/utils/worklets/audio-processing.ts

@@ -0,0 +1,73 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AudioRecordingWorklet = `
+class AudioProcessingWorklet extends AudioWorkletProcessor {
+
+  // send and clear buffer every 2048 samples, 
+  // which at 16khz is about 8 times a second
+  buffer = new Int16Array(2048);
+
+  // current write index
+  bufferWriteIndex = 0;
+
+  constructor() {
+    super();
+    this.hasAudio = false;
+  }
+
+  /**
+   * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0]
+   * @param outputs Float32Array[][]
+   */
+  process(inputs) {
+    if (inputs[0].length) {
+      const channel0 = inputs[0][0];
+      this.processChunk(channel0);
+    }
+    return true;
+  }
+
+  sendAndClearBuffer(){
+    this.port.postMessage({
+      event: "chunk",
+      data: {
+        int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
+      },
+    });
+    this.bufferWriteIndex = 0;
+  }
+
+  processChunk(float32Array) {
+    const l = float32Array.length;
+    
+    for (let i = 0; i < l; i++) {
+      // convert float32 -1 to 1 to int16 -32768 to 32767
+      const int16Value = float32Array[i] * 32768;
+      this.buffer[this.bufferWriteIndex++] = int16Value;
+      if(this.bufferWriteIndex >= this.buffer.length) {
+        this.sendAndClearBuffer();
+      }
+    }
+
+    if(this.bufferWriteIndex >= this.buffer.length) {
+      this.sendAndClearBuffer();
+    }
+  }
+}
+`;
+
+export default AudioRecordingWorklet;

src/frontends/live_api_react/frontend/src/utils/worklets/vol-meter.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const VolMeterWorket = `
+  class VolMeter extends AudioWorkletProcessor {
+    volume
+    updateIntervalInMS
+    nextUpdateFrame
+
+    constructor() {
+      super()
+      this.volume = 0
+      this.updateIntervalInMS = 25
+      this.nextUpdateFrame = this.updateIntervalInMS
+      this.port.onmessage = event => {
+        if (event.data.updateIntervalInMS) {
+          this.updateIntervalInMS = event.data.updateIntervalInMS
+        }
+      }
+    }
+
+    get intervalInFrames() {
+      return (this.updateIntervalInMS / 1000) * sampleRate
+    }
+
+    process(inputs) {
+      const input = inputs[0]
+
+      if (input.length > 0) {
+        const samples = input[0]
+        let sum = 0
+        let rms = 0
+
+        for (let i = 0; i < samples.length; ++i) {
+          sum += samples[i] * samples[i]
+        }
+
+        rms = Math.sqrt(sum / samples.length)
+        this.volume = Math.max(rms, this.volume * 0.7)
+
+        this.nextUpdateFrame -= samples.length
+        if (this.nextUpdateFrame < 0) {
+          this.nextUpdateFrame += this.intervalInFrames
+          this.port.postMessage({volume: this.volume})
+        }
+      }
+
+      return true
+    }
+  }`;
+
+export default VolMeterWorket;

src/frontends/live_api_react/frontend/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noFallthroughCasesInSwitch": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "preserve"
+  },
+  "include": ["src", "src/**/*"],
+  "ts-node": {
+    "compilerOptions": {
+      "module": "commonjs"
+    }
+  }
+}