agent-starter-pack 0.3.3__py3-none-any.whl → 0.21.0__py3-none-any.whl

agent_starter_pack/frontends/adk_live_react/frontend/src/components/transcription-preview/transcription-preview.scss

@@ -0,0 +1,150 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.transcription-preview {
+  width: 0;
+  height: 100%;
+  background: var(--Neutral-00);
+  border-left: 1px solid var(--Neutral-20);
+  display: flex;
+  flex-direction: column;
+  gap: 0;
+  overflow: hidden;
+  transition: width 0.2s ease-in-out;
+  flex-shrink: 0;
+
+  &.open {
+    width: 300px;
+  }
+
+  .transcription-section {
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    overflow: hidden;
+    opacity: 0;
+    transition: opacity 0.2s ease-in-out 0.1s;
+
+    &:first-child {
+      border-bottom: 1px solid var(--Neutral-20);
+    }
+
+    .transcription-header {
+      display: flex;
+      align-items: center;
+      gap: 8px;
+      padding: 12px 20px;
+      background: var(--Neutral-05);
+      border-bottom: 1px solid var(--Neutral-20);
+      flex-shrink: 0;
+
+      .material-symbols-outlined {
+        font-size: 20px;
+        color: var(--Neutral-70);
+      }
+
+      h3 {
+        margin: 0;
+        font-size: 12px;
+        font-weight: 600;
+        color: var(--Neutral-90);
+        text-transform: uppercase;
+        letter-spacing: 0.5px;
+      }
+    }
+
+    .transcription-content {
+      flex: 1;
+      padding: 12px 16px;
+      overflow-y: auto;
+      overflow-x: hidden;
+      word-wrap: break-word;
+
+      p {
+        margin: 0 0 12px 0;
+        color: var(--Neutral-90);
+        font-size: 13px;
+        line-height: 1.5;
+        white-space: pre-wrap;
+        word-break: break-word;
+        font-family: var(--font-family);
+
+        &.placeholder {
+          color: var(--Neutral-50);
+          font-style: italic;
+        }
+
+        &.previous {
+          color: var(--Neutral-60);
+          opacity: 0.7;
+        }
+
+        &.current {
+          color: var(--Neutral-90);
+          font-weight: 500;
+        }
+
+        &:last-child {
+          margin-bottom: 0;
+        }
+      }
+    }
+  }
+
+  &.open .transcription-section {
+    opacity: 1;
+  }
+
+  .input-section {
+    .transcription-header {
+      .material-symbols-outlined {
+        color: var(--Green-500);
+      }
+
+      h3 {
+        color: var(--Green-500);
+      }
+    }
+
+    .transcription-content {
+      background: linear-gradient(
+        to bottom,
+        rgba(0, 200, 100, 0.02),
+        transparent
+      );
+    }
+  }
+
+  .output-section {
+    .transcription-header {
+      .material-symbols-outlined {
+        color: var(--Blue-500);
+      }
+
+      h3 {
+        color: var(--Blue-500);
+      }
+    }
+
+    .transcription-content {
+      background: linear-gradient(
+        to bottom,
+        rgba(0, 100, 255, 0.02),
+        transparent
+      );
+    }
+  }
+}

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/hooks/use-live-api.ts

@@ -76,6 +76,10 @@ export function useLiveAPI({
       setConnected(false);
     };
 
+    const onSetupComplete = () => {
+      setConnected(true);
+    };
+
     const stopAudioStreamer = () => audioStreamerRef.current?.stop();
 
     const onAudio = (data: ArrayBuffer) =>
@@ -83,12 +87,14 @@ export function useLiveAPI({
 
     client
       .on("close", onClose)
+      .on("setupcomplete", onSetupComplete)
       .on("interrupted", stopAudioStreamer)
       .on("audio", onAudio);
 
     return () => {
       client
         .off("close", onClose)
+        .off("setupcomplete", onSetupComplete)
         .off("interrupted", stopAudioStreamer)
         .off("audio", onAudio);
     };
@@ -97,8 +103,8 @@ export function useLiveAPI({
   const connect = useCallback(async () => {
     client.disconnect();
     await client.connect();
-    setConnected(true);
-  }, [client, setConnected]);
+    // Don't set connected here - wait for setupcomplete event
+  }, [client]);
 
   const disconnect = useCallback(async () => {
     client.disconnect();

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/multimodal-live-types.ts

@@ -94,7 +94,8 @@ export type LiveIncomingMessage =
   | ToolCallMessage
   | ToolCallCancellationMessage
   | SetupCompleteMessage
-  | ServerContentMessage;
+  | ServerContentMessage
+  | AdkEvent;
 
 export type SetupCompleteMessage = { setupComplete: {} };
 
@@ -148,7 +149,7 @@ export type StreamingLog = {
 
 // Type-Guards
 
-const prop = (a: any, prop: string, kind: string = "object") =>
+const prop = (a: any, prop: string) =>
   typeof a === "object" && typeof a[prop] === "object";
 
 // outgoing messages
@@ -240,3 +241,40 @@ export const isToolCallCancellation = (
   a: unknown,
 ): a is ToolCallCancellationMessage["toolCallCancellation"] =>
   typeof a === "object" && Array.isArray((a as any).ids);
+
+// ADK Event types
+export interface AdkEvent {
+  user_id: string;
+  session_id: string;
+  author: string;
+  actions: {
+    state_delta: any;
+    artifact_delta: any;
+    requested_auth_configs: any;
+    requested_tool_confirmations: any;
+  };
+  id: string;
+  timestamp: number;
+  // Optional ADK fields
+  input_transcription?: { text: string };
+  output_transcription?: { text: string };
+  content?: any;
+  interrupted?: boolean;
+  turn_complete?: boolean;
+  partial?: boolean;
+}
+
+// ADK Event type guards
+export const isAdkEvent = (a: unknown): a is AdkEvent =>
+  typeof a === "object" &&
+  a !== null &&
+  typeof (a as any).user_id === "string" &&
+  typeof (a as any).session_id === "string" &&
+  typeof (a as any).author === "string" &&
+  typeof (a as any).actions === "object";
+
+export const isInputTranscription = (a: unknown): a is AdkEvent =>
+  isAdkEvent(a) && (a as AdkEvent).input_transcription !== undefined;
+
+export const isOutputTranscription = (a: unknown): a is AdkEvent =>
+  isAdkEvent(a) && (a as AdkEvent).output_transcription !== undefined;

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/audio-recorder.ts

@@ -50,7 +50,7 @@ export class AudioRecorder extends EventEmitter {
       throw new Error("Could not request user media");
     }
 
-    this.starting = new Promise(async (resolve, reject) => {
+    this.starting = new Promise(async (resolve) => {
       this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
       this.audioContext = await audioContext({ sampleRate: this.sampleRate });
       this.source = this.audioContext.createMediaStreamSource(this.stream);

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/audio-streamer.ts

@@ -154,7 +154,7 @@ export class AudioStreamer {
       const worklets = registeredWorklets.get(this.context);
 
       if (worklets) {
-        Object.entries(worklets).forEach(([workletName, graph]) => {
+        Object.entries(worklets).forEach(([, graph]) => {
           const { node, handlers } = graph;
           if (node) {
             source.connect(node);

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/multimodal-live-client.ts

@@ -18,7 +18,6 @@ import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
 import { EventEmitter } from "eventemitter3";
 import { difference } from "lodash";
 import {
-  ClientContentMessage,
   isInterrupted,
   isModelTurn,
   isServerContenteMessage,
@@ -26,15 +25,18 @@ import {
   isToolCallCancellationMessage,
   isToolCallMessage,
   isTurnComplete,
+  isAdkEvent,
+  isInputTranscription,
+  isOutputTranscription,
   LiveIncomingMessage,
   ModelTurn,
-  RealtimeInputMessage,
   ServerContent,
   StreamingLog,
   ToolCall,
   ToolCallCancellation,
   ToolResponseMessage,
   type LiveConfig,
+  type AdkEvent,
 } from "../multimodal-live-types";
 import { blobToJSON, base64ToArrayBuffer } from "./utils";
 
@@ -53,6 +55,10 @@ interface MultimodalLiveClientEventTypes {
   turncomplete: () => void;
   toolcall: (toolCall: ToolCall) => void;
   toolcallcancellation: (toolcallCancellation: ToolCallCancellation) => void;
+  // ADK events
+  inputtranscription: (text: string) => void;
+  outputtranscription: (text: string) => void;
+  adkevent: (event: AdkEvent) => void;
 }
 
 export type MultimodalLiveAPIClientConnection = {
@@ -72,15 +78,27 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   public url: string = "";
   private runId: string;
   private userId?: string;
+  private firstContentSent: boolean = false;
+  private audioChunksSent: number = 0;
+  private lastAudioSendTime: number = 0;
+  private readonly INITIAL_SEND_INTERVAL_MS = 300; // Start slow: 300ms between chunks
+  private readonly NORMAL_SEND_INTERVAL_MS = 125; // Normal rate: 125ms (8 chunks/sec)
+  private readonly RAMPUP_CHUNKS = 10; // Number of chunks to send at reduced rate
+
   constructor({ url, userId, runId }: MultimodalLiveAPIClientConnection) {
     super();
-    url = url || `ws://localhost:8000/ws`;
+    const defaultWsUrl = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
+    url = url || defaultWsUrl;
     this.url = new URL("ws", url).href;
     this.userId = userId;
     this.runId = runId || crypto.randomUUID(); // Ensure runId is always a string by providing default
     this.send = this.send.bind(this);
   }
 
+  get currentRunId(): string {
+    return this.runId;
+  }
+
   log(type: string, message: StreamingLog["message"]) {
     const log: StreamingLog = {
       date: new Date(),
@@ -98,15 +116,37 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       this.runId = newRunId;
     }
 
+    // Reset connection state
+    this.firstContentSent = false;
+    this.audioChunksSent = 0;
+    this.lastAudioSendTime = 0;
+
     ws.addEventListener("message", async (evt: MessageEvent) => {
       if (evt.data instanceof Blob) {
         this.receive(evt.data);
       } else if (typeof evt.data === "string") {
         try {
           const jsonData = JSON.parse(evt.data);
-          if (jsonData.status) {
+          
+          // Handle different message types from backend
+          if (jsonData.setupComplete) {
+            this.emit("setupcomplete");
+            this.log("server.setupComplete", "Session ready");
+          } else if (jsonData.serverContent) {
+            // Handle serverContent messages
+            this.receive(new Blob([JSON.stringify(jsonData)], {type: 'application/json'}));
+          } else if (jsonData.toolCall) {
+            // Handle tool calls
+            this.receive(new Blob([JSON.stringify(jsonData)], {type: 'application/json'}));
+          } else if (jsonData.status) {
             this.log("server.status", jsonData.status);
-            console.log("Status:", jsonData.status); // This will show in console
+            console.log("Status:", jsonData.status);
+          } else if (jsonData.error) {
+            this.log("server.error", jsonData.error);
+            console.error("Server error:", jsonData.error);
+          } else {
+            // Try to process as a regular message
+            this.receive(new Blob([JSON.stringify(jsonData)], {type: 'application/json'}));
           }
         } catch (error) {
           console.error("Error parsing message:", error);
@@ -129,11 +169,12 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         this.emit("open");
 
         this.ws = ws;
-        // Send initial setup message with runId
+        // Send initial setup message with user_id for backend
         const setupMessage = {
+          user_id: this.userId || "default_user",
           setup: {
             run_id: this.runId,
-            user_id: this.userId,
+            user_id: this.userId || "default_user",
           },
         };
         this._sendDirect(setupMessage);
@@ -213,11 +254,18 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       if (isModelTurn(serverContent)) {
         let parts: Part[] = serverContent.modelTurn.parts;
 
-        // when its audio that is returned for modelTurn
+        // when its audio that is returned for modelTurn (check both camelCase and snake_case)
         const audioParts = parts.filter(
-          (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
+          (p: any) => {
+            const inlineData = p.inlineData || p.inline_data;
+            const mimeType = inlineData?.mimeType || inlineData?.mime_type;
+            return inlineData && mimeType && mimeType.startsWith("audio/pcm");
+          }
         );
-        const base64s = audioParts.map((p) => p.inlineData?.data);
+        const base64s = audioParts.map((p: any) => {
+          const inlineData = p.inlineData || p.inline_data;
+          return inlineData?.data;
+        });
 
         // strip the audio parts out of the modelTurn
         const otherParts = difference(parts, audioParts);
@@ -240,6 +288,98 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         this.emit("content", content);
         this.log(`server.content`, response);
       }
+    } else if (isAdkEvent(response)) {
+      // Handle ADK events
+      this.emit("adkevent", response);
+
+      // Handle specific ADK event types
+      if (isInputTranscription(response)) {
+        this.emit("inputtranscription", response.input_transcription!.text);
+      }
+
+      if (isOutputTranscription(response)) {
+        this.emit("outputtranscription", response.output_transcription!.text);
+      }
+
+      // Handle ADK content (text responses from agent)
+      if (response.content && response.content.parts) {
+        const parts = response.content.parts;
+
+        // Extract function calls for tool call logging
+        const functionCallParts = parts.filter((p: any) => p.function_call);
+
+        // Log function calls as tool calls for the console
+        if (functionCallParts.length > 0) {
+          const functionCalls = functionCallParts.map((p: any) => ({
+            id: p.function_call.id,
+            name: p.function_call.name,
+            args: p.function_call.args || {}
+          }));
+
+          const toolCallMessage = {
+            toolCall: {
+              functionCalls: functionCalls
+            }
+          };
+
+          this.log("server.toolCall", toolCallMessage);
+          this.emit("toolcall", toolCallMessage.toolCall);
+        }
+        
+        // Extract audio parts for playing (check both camelCase and snake_case)
+        const audioParts = parts.filter(
+          (p: any) => {
+            const inlineData = p.inlineData || p.inline_data;
+            const mimeType = inlineData?.mimeType || inlineData?.mime_type;
+            return inlineData && mimeType && mimeType.startsWith("audio/");
+          }
+        );
+        
+        // Play audio if present
+        audioParts.forEach((audioPart: any) => {
+          const inlineData = audioPart.inlineData || audioPart.inline_data;
+          if (inlineData && inlineData.data) {
+            const audioData = base64ToArrayBuffer(inlineData.data);
+            
+            // Only emit audio if we have a valid buffer with data
+            if (audioData.byteLength > 0) {
+              this.emit("audio", audioData);
+              this.log(`server.audio`, `buffer (${audioData.byteLength}) - ${inlineData.mime_type || inlineData.mimeType}`);
+            } else {
+              this.log(`server.audio`, `invalid audio buffer - skipped`);
+            }
+          }
+        });
+        
+        // Send content for other parts (text, etc.) - exclude function calls and audio
+        const nonAudioNonFunctionParts = parts.filter(
+          (p: any) => {
+            const inlineData = p.inlineData || p.inline_data;
+            const mimeType = inlineData?.mimeType || inlineData?.mime_type;
+            const hasAudio = inlineData && mimeType && mimeType.startsWith("audio/");
+            const hasFunctionCall = p.function_call;
+            return !hasAudio && !hasFunctionCall;
+          }
+        );
+        
+        if (nonAudioNonFunctionParts.length > 0) {
+          const content: ModelTurn = { modelTurn: { parts: nonAudioNonFunctionParts } };
+          this.emit("content", content);
+          this.log("server.content", `content with ${nonAudioNonFunctionParts.length} non-audio, non-function parts`);
+        }
+      }
+      
+      // Handle turn complete
+      if (response.turn_complete) {
+        this.emit("turncomplete");
+        this.log("server.turncomplete", "ADK turn complete");
+      }
+      
+      // Handle interruption
+      if (response.interrupted) {
+        this.emit("interrupted");
+        this.log("server.interrupted", "ADK interrupted");
+      }
     } else {
       console.log("received unmatched message", response);
       this.log("received unmatched message", response);
@@ -250,6 +390,12 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
    * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
    */
   sendRealtimeInput(chunks: GenerativeContentBlob[]) {
+    // Don't send if WebSocket is not open - this prevents flooding the queue
+    // during connection setup
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+
     let hasAudio = false;
     let hasVideo = false;
     for (let i = 0; i < chunks.length; i++) {
@@ -264,6 +410,25 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         break;
       }
     }
+
+    // Throttle audio chunks during initial connection phase
+    if (hasAudio && !hasVideo) {
+      const now = Date.now();
+
+      // Calculate required interval based on how many chunks we've sent
+      const requiredInterval = this.audioChunksSent < this.RAMPUP_CHUNKS
+        ? this.INITIAL_SEND_INTERVAL_MS
+        : this.NORMAL_SEND_INTERVAL_MS;
+
+      // If not enough time has passed since last send, drop this chunk
+      if (this.lastAudioSendTime > 0 && (now - this.lastAudioSendTime) < requiredInterval) {
+        return;
+      }
+
+      this.lastAudioSendTime = now;
+      this.audioChunksSent++;
+    }
+
     const message =
       hasAudio && hasVideo
         ? "audio + video"
@@ -273,12 +438,26 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
             ? "video"
             : "unknown";
 
-    const data: RealtimeInputMessage = {
-      realtimeInput: {
-        mediaChunks: chunks,
-      },
-    };
-    this._sendDirect(data);
+    // Convert to LiveRequest format for backend
+    for (const chunk of chunks) {
+      let data: any = {
+        blob: {
+          mimeType: chunk.mimeType,
+          data: chunk.data,
+        },
+      };
+
+      // For remote mode: wrap first content in {user_id, live_request} format
+      if (!this.firstContentSent) {
+        data = {
+          user_id: this.userId || "default_user",
+          live_request: data,
+        };
+        this.firstContentSent = true;
+      }
+
+      this._sendDirect(data);
+    }
     this.log(`client.realtimeInput`, message);
   }
 
@@ -297,22 +476,29 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   /**
    * send normal content parts such as { text }
    */
-  send(parts: Part | Part[], turnComplete: boolean = true) {
+  send(parts: Part | Part[], _turnComplete: boolean = true) {
     parts = Array.isArray(parts) ? parts : [parts];
     const content: Content = {
       role: "user",
       parts,
     };
 
-    const clientContentRequest: ClientContentMessage = {
-      clientContent: {
-        turns: [content],
-        turnComplete,
-      },
+    // Convert to LiveRequest format for backend
+    let data: any = {
+      content: content,
     };
 
-    this._sendDirect(clientContentRequest);
-    this.log(`client.send`, clientContentRequest);
+    // For remote mode: wrap first content in {user_id, live_request} format
+    if (!this.firstContentSent) {
+      data = {
+        user_id: this.userId || "default_user",
+        live_request: data,
+      };
+      this.firstContentSent = true;
+    }
+
+    this._sendDirect(data);
+    this.log(`client.send`, `content with ${parts.length} parts`);
   }
 
   /**

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/utils.ts

@@ -76,11 +76,33 @@ export const blobToJSON = (blob: Blob) =>
     reader.readAsText(blob);
   });
 
+function cleanBase64String(base64: string): string {
+  // Convert URL-safe base64 to standard base64
+  let cleaned = base64
+    .replace(/-/g, '+')  // Replace - with +
+    .replace(/_/g, '/')  // Replace _ with /
+    .replace(/[^A-Za-z0-9+/=]/g, ''); // Remove any other invalid characters
+  
+  // Ensure proper padding (base64 strings must be multiples of 4)
+  return cleaned + '='.repeat((4 - cleaned.length % 4) % 4);
+}
+
 export function base64ToArrayBuffer(base64: string) {
-  var binaryString = atob(base64);
-  var bytes = new Uint8Array(binaryString.length);
-  for (let i = 0; i < binaryString.length; i++) {
-    bytes[i] = binaryString.charCodeAt(i);
+  const cleanedBase64 = cleanBase64String(base64);
+  
+  try {
+    var binaryString = atob(cleanedBase64);
+    var bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes.buffer;
+  } catch (error) {
+    console.error('Failed to decode base64 audio data:', error);
+    console.error('Original base64 length:', base64.length);
+    console.error('Cleaned base64 length:', cleanedBase64.length);
+    console.error('First 100 chars:', base64.substring(0, 100));
+    // Return empty buffer on error
+    return new ArrayBuffer(0);
   }
-  return bytes.buffer;
 }