agent-starter-pack 0.15.7__py3-none-any.whl → 0.17.0__py3-none-any.whl

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/hooks/use-live-api.ts

@@ -76,6 +76,10 @@ export function useLiveAPI({
       setConnected(false);
     };
 
+    const onSetupComplete = () => {
+      setConnected(true);
+    };
+
     const stopAudioStreamer = () => audioStreamerRef.current?.stop();
 
     const onAudio = (data: ArrayBuffer) =>
@@ -83,12 +87,14 @@ export function useLiveAPI({
 
     client
       .on("close", onClose)
+      .on("setupcomplete", onSetupComplete)
       .on("interrupted", stopAudioStreamer)
       .on("audio", onAudio);
 
     return () => {
       client
         .off("close", onClose)
+        .off("setupcomplete", onSetupComplete)
         .off("interrupted", stopAudioStreamer)
         .off("audio", onAudio);
     };
@@ -97,8 +103,8 @@ export function useLiveAPI({
   const connect = useCallback(async () => {
     client.disconnect();
     await client.connect();
-    setConnected(true);
-  }, [client, setConnected]);
+    // Don't set connected here - wait for setupcomplete event
+  }, [client]);
 
   const disconnect = useCallback(async () => {
     client.disconnect();

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/multimodal-live-types.ts

@@ -94,7 +94,8 @@ export type LiveIncomingMessage =
   | ToolCallMessage
   | ToolCallCancellationMessage
   | SetupCompleteMessage
-  | ServerContentMessage;
+  | ServerContentMessage
+  | AdkEvent;
 
 export type SetupCompleteMessage = { setupComplete: {} };
 
@@ -148,7 +149,7 @@ export type StreamingLog = {
 
 // Type-Guards
 
-const prop = (a: any, prop: string, kind: string = "object") =>
+const prop = (a: any, prop: string) =>
   typeof a === "object" && typeof a[prop] === "object";
 
 // outgoing messages
@@ -240,3 +241,38 @@ export const isToolCallCancellation = (
   a: unknown,
 ): a is ToolCallCancellationMessage["toolCallCancellation"] =>
   typeof a === "object" && Array.isArray((a as any).ids);
+
+// ADK Event types
+export interface AdkEvent {
+  invocation_id: string;
+  author: string;
+  actions: {
+    state_delta: any;
+    artifact_delta: any;
+    requested_auth_configs: any;
+    requested_tool_confirmations: any;
+  };
+  id: string;
+  timestamp: number;
+  // Optional ADK fields
+  input_transcription?: { text: string };
+  output_transcription?: { text: string };
+  content?: any;
+  interrupted?: boolean;
+  turn_complete?: boolean;
+  partial?: boolean;
+}
+
+// ADK Event type guards
+export const isAdkEvent = (a: unknown): a is AdkEvent =>
+  typeof a === "object" && 
+  a !== null &&
+  typeof (a as any).invocation_id === "string" &&
+  typeof (a as any).author === "string" &&
+  typeof (a as any).actions === "object";
+
+export const isInputTranscription = (a: unknown): a is AdkEvent =>
+  isAdkEvent(a) && (a as AdkEvent).input_transcription !== undefined;
+
+export const isOutputTranscription = (a: unknown): a is AdkEvent =>
+  isAdkEvent(a) && (a as AdkEvent).output_transcription !== undefined;

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/audio-recorder.ts

@@ -50,7 +50,7 @@ export class AudioRecorder extends EventEmitter {
       throw new Error("Could not request user media");
     }
 
-    this.starting = new Promise(async (resolve, reject) => {
+    this.starting = new Promise(async (resolve) => {
       this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
       this.audioContext = await audioContext({ sampleRate: this.sampleRate });
       this.source = this.audioContext.createMediaStreamSource(this.stream);

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/audio-streamer.ts

@@ -154,7 +154,7 @@ export class AudioStreamer {
       const worklets = registeredWorklets.get(this.context);
 
       if (worklets) {
-        Object.entries(worklets).forEach(([workletName, graph]) => {
+        Object.entries(worklets).forEach(([, graph]) => {
           const { node, handlers } = graph;
           if (node) {
             source.connect(node);

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/multimodal-live-client.ts

@@ -18,7 +18,6 @@ import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
 import { EventEmitter } from "eventemitter3";
 import { difference } from "lodash";
 import {
-  ClientContentMessage,
   isInterrupted,
   isModelTurn,
   isServerContenteMessage,
@@ -26,15 +25,18 @@ import {
   isToolCallCancellationMessage,
   isToolCallMessage,
   isTurnComplete,
+  isAdkEvent,
+  isInputTranscription,
+  isOutputTranscription,
   LiveIncomingMessage,
   ModelTurn,
-  RealtimeInputMessage,
   ServerContent,
   StreamingLog,
   ToolCall,
   ToolCallCancellation,
   ToolResponseMessage,
   type LiveConfig,
+  type AdkEvent,
 } from "../multimodal-live-types";
 import { blobToJSON, base64ToArrayBuffer } from "./utils";
 
@@ -53,6 +55,10 @@ interface MultimodalLiveClientEventTypes {
   turncomplete: () => void;
   toolcall: (toolCall: ToolCall) => void;
   toolcallcancellation: (toolcallCancellation: ToolCallCancellation) => void;
+  // ADK events
+  inputtranscription: (text: string) => void;
+  outputtranscription: (text: string) => void;
+  adkevent: (event: AdkEvent) => void;
 }
 
 export type MultimodalLiveAPIClientConnection = {
@@ -72,6 +78,13 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   public url: string = "";
   private runId: string;
   private userId?: string;
+  private firstContentSent: boolean = false;
+  private audioChunksSent: number = 0;
+  private lastAudioSendTime: number = 0;
+  private readonly INITIAL_SEND_INTERVAL_MS = 300; // Start slow: 300ms between chunks
+  private readonly NORMAL_SEND_INTERVAL_MS = 125; // Normal rate: 125ms (8 chunks/sec)
+  private readonly RAMPUP_CHUNKS = 10; // Number of chunks to send at reduced rate
+
   constructor({ url, userId, runId }: MultimodalLiveAPIClientConnection) {
     super();
     const defaultWsUrl = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
@@ -103,15 +116,37 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       this.runId = newRunId;
     }
 
+    // Reset connection state
+    this.firstContentSent = false;
+    this.audioChunksSent = 0;
+    this.lastAudioSendTime = 0;
+
     ws.addEventListener("message", async (evt: MessageEvent) => {
       if (evt.data instanceof Blob) {
         this.receive(evt.data);
       } else if (typeof evt.data === "string") {
         try {
           const jsonData = JSON.parse(evt.data);
-          if (jsonData.status) {
+          
+          // Handle different message types from backend
+          if (jsonData.setupComplete) {
+            this.emit("setupcomplete");
+            this.log("server.setupComplete", "Session ready");
+          } else if (jsonData.serverContent) {
+            // Handle serverContent messages
+            this.receive(new Blob([JSON.stringify(jsonData)], {type: 'application/json'}));
+          } else if (jsonData.toolCall) {
+            // Handle tool calls
+            this.receive(new Blob([JSON.stringify(jsonData)], {type: 'application/json'}));
+          } else if (jsonData.status) {
             this.log("server.status", jsonData.status);
-            console.log("Status:", jsonData.status); // This will show in console
+            console.log("Status:", jsonData.status);
+          } else if (jsonData.error) {
+            this.log("server.error", jsonData.error);
+            console.error("Server error:", jsonData.error);
+          } else {
+            // Try to process as a regular message
+            this.receive(new Blob([JSON.stringify(jsonData)], {type: 'application/json'}));
           }
         } catch (error) {
           console.error("Error parsing message:", error);
@@ -134,11 +169,12 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         this.emit("open");
 
         this.ws = ws;
-        // Send initial setup message with runId
+        // Send initial setup message with user_id for backend
         const setupMessage = {
+          user_id: this.userId || "default_user",
           setup: {
             run_id: this.runId,
-            user_id: this.userId,
+            user_id: this.userId || "default_user",
           },
         };
         this._sendDirect(setupMessage);
@@ -218,11 +254,18 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       if (isModelTurn(serverContent)) {
         let parts: Part[] = serverContent.modelTurn.parts;
 
-        // when its audio that is returned for modelTurn
+        // when its audio that is returned for modelTurn (check both camelCase and snake_case)
         const audioParts = parts.filter(
-          (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
+          (p: any) => {
+            const inlineData = p.inlineData || p.inline_data;
+            const mimeType = inlineData?.mimeType || inlineData?.mime_type;
+            return inlineData && mimeType && mimeType.startsWith("audio/pcm");
+          }
         );
-        const base64s = audioParts.map((p) => p.inlineData?.data);
+        const base64s = audioParts.map((p: any) => {
+          const inlineData = p.inlineData || p.inline_data;
+          return inlineData?.data;
+        });
 
         // strip the audio parts out of the modelTurn
         const otherParts = difference(parts, audioParts);
@@ -245,6 +288,98 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         this.emit("content", content);
         this.log(`server.content`, response);
       }
+    } else if (isAdkEvent(response)) {
+      // Handle ADK events
+      this.emit("adkevent", response);
+
+      // Handle specific ADK event types
+      if (isInputTranscription(response)) {
+        this.emit("inputtranscription", response.input_transcription!.text);
+      }
+
+      if (isOutputTranscription(response)) {
+        this.emit("outputtranscription", response.output_transcription!.text);
+      }
+
+      // Handle ADK content (text responses from agent)
+      if (response.content && response.content.parts) {
+        const parts = response.content.parts;
+
+        // Extract function calls for tool call logging
+        const functionCallParts = parts.filter((p: any) => p.function_call);
+
+        // Log function calls as tool calls for the console
+        if (functionCallParts.length > 0) {
+          const functionCalls = functionCallParts.map((p: any) => ({
+            id: p.function_call.id,
+            name: p.function_call.name,
+            args: p.function_call.args || {}
+          }));
+
+          const toolCallMessage = {
+            toolCall: {
+              functionCalls: functionCalls
+            }
+          };
+
+          this.log("server.toolCall", toolCallMessage);
+          this.emit("toolcall", toolCallMessage.toolCall);
+        }
+        
+        // Extract audio parts for playing (check both camelCase and snake_case)
+        const audioParts = parts.filter(
+          (p: any) => {
+            const inlineData = p.inlineData || p.inline_data;
+            const mimeType = inlineData?.mimeType || inlineData?.mime_type;
+            return inlineData && mimeType && mimeType.startsWith("audio/");
+          }
+        );
+        
+        // Play audio if present
+        audioParts.forEach((audioPart: any) => {
+          const inlineData = audioPart.inlineData || audioPart.inline_data;
+          if (inlineData && inlineData.data) {
+            const audioData = base64ToArrayBuffer(inlineData.data);
+            
+            // Only emit audio if we have a valid buffer with data
+            if (audioData.byteLength > 0) {
+              this.emit("audio", audioData);
+              this.log(`server.audio`, `buffer (${audioData.byteLength}) - ${inlineData.mime_type || inlineData.mimeType}`);
+            } else {
+              this.log(`server.audio`, `invalid audio buffer - skipped`);
+            }
+          }
+        });
+        
+        // Send content for other parts (text, etc.) - exclude function calls and audio
+        const nonAudioNonFunctionParts = parts.filter(
+          (p: any) => {
+            const inlineData = p.inlineData || p.inline_data;
+            const mimeType = inlineData?.mimeType || inlineData?.mime_type;
+            const hasAudio = inlineData && mimeType && mimeType.startsWith("audio/");
+            const hasFunctionCall = p.function_call;
+            return !hasAudio && !hasFunctionCall;
+          }
+        );
+        
+        if (nonAudioNonFunctionParts.length > 0) {
+          const content: ModelTurn = { modelTurn: { parts: nonAudioNonFunctionParts } };
+          this.emit("content", content);
+          this.log("server.content", `content with ${nonAudioNonFunctionParts.length} non-audio, non-function parts`);
+        }
+      }
+      
+      // Handle turn complete
+      if (response.turn_complete) {
+        this.emit("turncomplete");
+        this.log("server.turncomplete", "ADK turn complete");
+      }
+      
+      // Handle interruption
+      if (response.interrupted) {
+        this.emit("interrupted");
+        this.log("server.interrupted", "ADK interrupted");
+      }
     } else {
       console.log("received unmatched message", response);
       this.log("received unmatched message", response);
@@ -255,6 +390,12 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
    * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
    */
   sendRealtimeInput(chunks: GenerativeContentBlob[]) {
+    // Don't send if WebSocket is not open - this prevents flooding the queue
+    // during connection setup
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+
     let hasAudio = false;
     let hasVideo = false;
     for (let i = 0; i < chunks.length; i++) {
@@ -269,6 +410,25 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         break;
       }
     }
+
+    // Throttle audio chunks during initial connection phase
+    if (hasAudio && !hasVideo) {
+      const now = Date.now();
+
+      // Calculate required interval based on how many chunks we've sent
+      const requiredInterval = this.audioChunksSent < this.RAMPUP_CHUNKS
+        ? this.INITIAL_SEND_INTERVAL_MS
+        : this.NORMAL_SEND_INTERVAL_MS;
+
+      // If not enough time has passed since last send, drop this chunk
+      if (this.lastAudioSendTime > 0 && (now - this.lastAudioSendTime) < requiredInterval) {
+        return;
+      }
+
+      this.lastAudioSendTime = now;
+      this.audioChunksSent++;
+    }
+
     const message =
       hasAudio && hasVideo
         ? "audio + video"
@@ -278,12 +438,26 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
             ? "video"
             : "unknown";
 
-    const data: RealtimeInputMessage = {
-      realtimeInput: {
-        mediaChunks: chunks,
-      },
-    };
-    this._sendDirect(data);
+    // Convert to LiveRequest format for backend
+    for (const chunk of chunks) {
+      let data: any = {
+        blob: {
+          mimeType: chunk.mimeType,
+          data: chunk.data,
+        },
+      };
+
+      // For remote mode: wrap first content in {user_id, live_request} format
+      if (!this.firstContentSent) {
+        data = {
+          user_id: this.userId || "default_user",
+          live_request: data,
+        };
+        this.firstContentSent = true;
+      }
+
+      this._sendDirect(data);
+    }
     this.log(`client.realtimeInput`, message);
   }
 
@@ -302,22 +476,29 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   /**
    * send normal content parts such as { text }
    */
-  send(parts: Part | Part[], turnComplete: boolean = true) {
+  send(parts: Part | Part[], _turnComplete: boolean = true) {
     parts = Array.isArray(parts) ? parts : [parts];
     const content: Content = {
       role: "user",
       parts,
     };
 
-    const clientContentRequest: ClientContentMessage = {
-      clientContent: {
-        turns: [content],
-        turnComplete,
-      },
+    // Convert to LiveRequest format for backend
+    let data: any = {
+      content: content,
     };
 
-    this._sendDirect(clientContentRequest);
-    this.log(`client.send`, clientContentRequest);
+    // For remote mode: wrap first content in {user_id, live_request} format
+    if (!this.firstContentSent) {
+      data = {
+        user_id: this.userId || "default_user",
+        live_request: data,
+      };
+      this.firstContentSent = true;
+    }
+
+    this._sendDirect(data);
+    this.log(`client.send`, `content with ${parts.length} parts`);
   }
 
   /**

{src/frontends/live_api_react → agent_starter_pack/frontends/adk_live_react}/frontend/src/utils/utils.ts

@@ -76,11 +76,33 @@ export const blobToJSON = (blob: Blob) =>
     reader.readAsText(blob);
   });
 
+function cleanBase64String(base64: string): string {
+  // Convert URL-safe base64 to standard base64
+  let cleaned = base64
+    .replace(/-/g, '+')  // Replace - with +
+    .replace(/_/g, '/')  // Replace _ with /
+    .replace(/[^A-Za-z0-9+/=]/g, ''); // Remove any other invalid characters
+  
+  // Ensure proper padding (base64 strings must be multiples of 4)
+  return cleaned + '='.repeat((4 - cleaned.length % 4) % 4);
+}
+
 export function base64ToArrayBuffer(base64: string) {
-  var binaryString = atob(base64);
-  var bytes = new Uint8Array(binaryString.length);
-  for (let i = 0; i < binaryString.length; i++) {
-    bytes[i] = binaryString.charCodeAt(i);
+  const cleanedBase64 = cleanBase64String(base64);
+  
+  try {
+    var binaryString = atob(cleanedBase64);
+    var bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+      bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes.buffer;
+  } catch (error) {
+    console.error('Failed to decode base64 audio data:', error);
+    console.error('Original base64 length:', base64.length);
+    console.error('Cleaned base64 length:', cleanedBase64.length);
+    console.error('First 100 chars:', base64.substring(0, 100));
+    // Return empty buffer on error
+    return new ArrayBuffer(0);
   }
-  return bytes.buffer;
 }

{src → agent_starter_pack}/frontends/streamlit/frontend/utils/local_chat_history.py

@@ -109,6 +109,8 @@ class LocalChatMessageHistory(BaseChatMessageHistory):
                 for msg in messages
                 if msg["type"] in ("ai", "human") and isinstance(msg["content"], str)
             ]
+            # Convert messages to the format expected by chain_title.invoke
+            messages = {"messages": messages}
 
             response = chain_title.invoke(messages)
             title = (

{src → agent_starter_pack}/resources/docs/adk-cheatsheet.md

@@ -1464,8 +1464,8 @@ if __name__ == "__main__":
 
 For detailed specifications of all classes, methods, and commands, refer to the official reference documentation.
 
-*   [Python API Reference](./api-reference/python/index.html)
-*   [Java API Reference](./api-reference/java/index.html)
-*   [CLI Reference](./api-reference/cli/index.html)
-*   [REST API Reference](./api-reference/rest/index.md)
-*   [Agent Config YAML Reference](./api-reference/agentconfig/index.html)
+*   [Python API Reference](https://github.com/google/adk-docs/tree/main/docs/api-reference/python)
+*   [Java API Reference](https://github.com/google/adk-docs/tree/main/docs/api-reference/java)
+*   [CLI Reference](https://github.com/google/adk-docs/tree/main/docs/api-reference/cli)
+*   [REST API Reference](https://github.com/google/adk-docs/tree/main/docs/api-reference/rest)
+*   [Agent Config YAML Reference](https://github.com/google/adk-docs/tree/main/docs/api-reference/agentconfig)

agent_starter_pack/resources/idx/.idx/dev.nix

@@ -0,0 +1,64 @@
+
+# To learn more about how to use Nix to configure your environment
+# see: https://firebase.google.com/docs/studio/customize-workspace
+{ pkgs, ... }: {
+  # Which nixpkgs channel to use.
+  channel = "stable-24.11"; # or "unstable"
+
+  # Use https://search.nixos.org/packages to find packages
+  packages = [
+    pkgs.uv
+    pkgs.gnumake
+    pkgs.terraform
+    pkgs.gh
+  ];
+  # Sets environment variables in the workspace
+  env = {};
+  idx = {
+    # Search for the extensions you want on https://open-vsx.org/ and use "publisher.id"
+    extensions = [
+    ];
+    workspace = {
+      # Runs when a workspace is first created with this `dev.nix` file
+      onCreate = {
+        create-venv = ''
+        # Load environment variables from .env file if it exists
+        source .env
+
+        # Beautiful prints for gcloud setup
+        echo ""
+        echo "╔════════════════════════════════════════════════════════════╗"
+        echo "║                  🔐 GCLOUD SETUP REQUIRED                  ║"
+        echo "╚════════════════════════════════════════════════════════════╝"
+        echo ""
+        echo "📝 Before proceeding, please ensure:"
+        echo "   1️⃣  You are logged in to gcloud"
+        echo "   2️⃣  You have selected the correct project"
+        echo ""
+
+        auth_status=$(gcloud auth list --quiet 2>&1)
+
+        echo ""
+        echo "⚙️  We will now set the project you want to use..."
+        gcloud config get project
+
+        echo ""
+        echo "💡 Need to setup? Run these commands:"
+        echo "   → gcloud auth login"
+        echo "   → gcloud config set project YOUR_PROJECT_ID"
+        echo ""
+
+        echo "Running agent starter pack creation..."
+        uvx agent-starter-pack create $WS_NAME
+        code ~/$WS_NAME/$WS_NAME/README.md
+        exec bash
+        '';
+        # Open editors for the following files by default, if they exist:
+        default.openFiles = [];
+      };
+      # To run something each time the workspace is (re)started, use the `onStart` hook
+    };
+    # Enable previews and customize configuration
+    previews = {};
+  };
+}

agent_starter_pack/resources/idx/idx-template.json

@@ -0,0 +1,6 @@
+{
+  "name": "Agent Starter Pack",
+  "description": "Production-ready Gen AI Agent templates for Google Cloud. Addressing common challenges (Deployment & Operations, Evaluation, Customization, Observability) in building and deploying GenAI agents.",
+  "icon": "https://github.com/GoogleCloudPlatform/agent-starter-pack/blob/main/docs/images/icon.png?raw=true",
+  "params": []
+}

{src → agent_starter_pack}/resources/idx/idx-template.nix

@@ -1,3 +1,4 @@
+
 # No user-configurable parameters
 # Accept additional arguments to this template corresponding to template
 # parameter IDs
@@ -14,8 +15,6 @@
 
     # Create .env file with the parameter values
     cat > "$out/.env" << EOF
-    AGENT_NAME=${agent_name}
-    GOOGLE_CLOUD_PROJECT=${google_cloud_project_id}
     WS_NAME=$WS_NAME
     EOF
 
@@ -23,4 +22,4 @@
     # Git repository
     rm -rf "$out/.git" "$out/idx-template".{nix,json}
   '';
-}
+}