agent-starter-pack 0.0.1b0__py3-none-any.whl

src/frontends/live_api_react/frontend/src/contexts/LiveAPIContext.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createContext, FC, ReactNode, useContext } from "react";
+import { useLiveAPI, UseLiveAPIResults } from "../hooks/use-live-api";
+
+const LiveAPIContext = createContext<UseLiveAPIResults | undefined>(undefined);
+
+export type LiveAPIProviderProps = {
+  children: ReactNode;
+  url?: string;
+  userId?: string;
+};
+
+export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
+  url,
+  userId,
+  children,
+}) => {
+  const liveAPI = useLiveAPI({ url, userId });
+
+  return (
+    <LiveAPIContext.Provider value={liveAPI}>
+      {children}
+    </LiveAPIContext.Provider>
+  );
+};
+
+export const useLiveAPIContext = () => {
+  const context = useContext(LiveAPIContext);
+  if (!context) {
+    throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
+  }
+  return context;
+};

src/frontends/live_api_react/frontend/src/hooks/use-live-api.ts

@@ -0,0 +1,115 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+  Dispatch,
+  SetStateAction,
+} from "react";
+import { MultimodalLiveClient } from "../utils/multimodal-live-client";
+import { AudioStreamer } from "../utils/audio-streamer";
+import { audioContext } from "../utils/utils";
+import VolMeterWorket from "../utils/worklets/vol-meter";
+
+export type UseLiveAPIResults = {
+  client: MultimodalLiveClient;
+  connected: boolean;
+  connect: () => Promise<void>;
+  disconnect: () => Promise<void>;
+  volume: number;
+};
+
+export type UseLiveAPIProps = {
+  url?: string;
+  userId?: string;
+  onRunIdChange?: Dispatch<SetStateAction<string>>;
+};
+
+export function useLiveAPI({
+  url,
+  userId,
+}: UseLiveAPIProps): UseLiveAPIResults {
+  const client = useMemo(
+    () => new MultimodalLiveClient({ url, userId }),
+    [url, userId],
+  );
+  const audioStreamerRef = useRef<AudioStreamer | null>(null);
+
+  const [connected, setConnected] = useState(false);
+  const [volume, setVolume] = useState(0);
+
+  // register audio for streaming server -> speakers
+  useEffect(() => {
+    if (!audioStreamerRef.current) {
+      audioContext({ id: "audio-out" }).then((audioCtx: AudioContext) => {
+        audioStreamerRef.current = new AudioStreamer(audioCtx);
+        audioStreamerRef.current
+          .addWorklet<any>("vumeter-out", VolMeterWorket, (ev: any) => {
+            setVolume(ev.data.volume);
+          })
+          .then(() => {
+            // Successfully added worklet
+          });
+      });
+    }
+  }, [audioStreamerRef]);
+
+  useEffect(() => {
+    const onClose = () => {
+      setConnected(false);
+    };
+
+    const stopAudioStreamer = () => audioStreamerRef.current?.stop();
+
+    const onAudio = (data: ArrayBuffer) =>
+      audioStreamerRef.current?.addPCM16(new Uint8Array(data));
+
+    client
+      .on("close", onClose)
+      .on("interrupted", stopAudioStreamer)
+      .on("audio", onAudio);
+
+    return () => {
+      client
+        .off("close", onClose)
+        .off("interrupted", stopAudioStreamer)
+        .off("audio", onAudio);
+    };
+  }, [client]);
+
+  const connect = useCallback(async () => {
+    client.disconnect();
+    await client.connect();
+    setConnected(true);
+  }, [client, setConnected]);
+
+  const disconnect = useCallback(async () => {
+    client.disconnect();
+    setConnected(false);
+  }, [setConnected, client]);
+
+  return {
+    client,
+    connected,
+    connect,
+    disconnect,
+    volume,
+  };
+}

src/frontends/live_api_react/frontend/src/hooks/use-media-stream-mux.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type UseMediaStreamResult = {
+  type: "webcam" | "screen";
+  start: () => Promise<MediaStream>;
+  stop: () => void;
+  isStreaming: boolean;
+  stream: MediaStream | null;
+};

src/frontends/live_api_react/frontend/src/hooks/use-screen-capture.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useState, useEffect } from "react";
+import { UseMediaStreamResult } from "./use-media-stream-mux";
+
+export function useScreenCapture(): UseMediaStreamResult {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [isStreaming, setIsStreaming] = useState(false);
+
+  useEffect(() => {
+    const handleStreamEnded = () => {
+      setIsStreaming(false);
+      setStream(null);
+    };
+    if (stream) {
+      stream
+        .getTracks()
+        .forEach((track) => track.addEventListener("ended", handleStreamEnded));
+      return () => {
+        stream
+          .getTracks()
+          .forEach((track) =>
+            track.removeEventListener("ended", handleStreamEnded),
+          );
+      };
+    }
+  }, [stream]);
+
+  const start = async () => {
+    // const controller = new CaptureController();
+    // controller.setFocusBehavior("no-focus-change");
+    const mediaStream = await navigator.mediaDevices.getDisplayMedia({
+      video: true,
+      // controller
+    });
+    setStream(mediaStream);
+    setIsStreaming(true);
+    return mediaStream;
+  };
+
+  const stop = () => {
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+      setIsStreaming(false);
+    }
+  };
+
+  const result: UseMediaStreamResult = {
+    type: "screen",
+    start,
+    stop,
+    isStreaming,
+    stream,
+  };
+
+  return result;
+}

src/frontends/live_api_react/frontend/src/hooks/use-webcam.ts

@@ -0,0 +1,69 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useState, useEffect } from "react";
+import { UseMediaStreamResult } from "./use-media-stream-mux";
+
+export function useWebcam(): UseMediaStreamResult {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [isStreaming, setIsStreaming] = useState(false);
+
+  useEffect(() => {
+    const handleStreamEnded = () => {
+      setIsStreaming(false);
+      setStream(null);
+    };
+    if (stream) {
+      stream
+        .getTracks()
+        .forEach((track) => track.addEventListener("ended", handleStreamEnded));
+      return () => {
+        stream
+          .getTracks()
+          .forEach((track) =>
+            track.removeEventListener("ended", handleStreamEnded),
+          );
+      };
+    }
+  }, [stream]);
+
+  const start = async () => {
+    const mediaStream = await navigator.mediaDevices.getUserMedia({
+      video: true,
+    });
+    setStream(mediaStream);
+    setIsStreaming(true);
+    return mediaStream;
+  };
+
+  const stop = () => {
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+      setIsStreaming(false);
+    }
+  };
+
+  const result: UseMediaStreamResult = {
+    type: "webcam",
+    start,
+    stop,
+    isStreaming,
+    stream,
+  };
+
+  return result;
+}

src/frontends/live_api_react/frontend/src/index.css

@@ -0,0 +1,28 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+body {
+  margin: 0;
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen,
+    Ubuntu, Cantarell, "Fira Sans", "Droid Sans", "Helvetica Neue", sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+code {
+  font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
+    monospace;
+}

src/frontends/live_api_react/frontend/src/index.tsx

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import './index.css';
+import App from './App';
+import reportWebVitals from './reportWebVitals';
+
+const root = ReactDOM.createRoot(
+  document.getElementById('root') as HTMLElement
+);
+root.render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);
+
+// If you want to start measuring performance in your app, pass a function
+// to log results (for example: reportWebVitals(console.log))
+// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
+reportWebVitals();

src/frontends/live_api_react/frontend/src/multimodal-live-types.ts

@@ -0,0 +1,242 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type {
+  Content,
+  FunctionCall,
+  GenerationConfig,
+  GenerativeContentBlob,
+  Part,
+  Tool,
+} from "@google/generative-ai";
+
+/**
+ * this module contains type-definitions and Type-Guards
+ */
+
+// Type-definitions
+
+/* outgoing types */
+
+/**
+ * the config to initiate the session
+ */
+export type LiveConfig = {
+  model: string;
+  systemInstruction?: { parts: Part[] };
+  generationConfig?: Partial<LiveGenerationConfig>;
+  tools?: Array<Tool | { googleSearch: {} } | { codeExecution: {} }>;
+};
+
+export type LiveGenerationConfig = GenerationConfig & {
+  responseModalities: "text" | "audio" | "image";
+  speechConfig?: {
+    voiceConfig?: {
+      prebuiltVoiceConfig?: {
+        voiceName: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede" | string;
+      };
+    };
+  };
+};
+
+export type LiveOutgoingMessage =
+  | SetupMessage
+  | ClientContentMessage
+  | RealtimeInputMessage
+  | ToolResponseMessage;
+
+export type SetupMessage = {
+  setup: LiveConfig;
+};
+
+export type ClientContentMessage = {
+  clientContent: {
+    turns: Content[];
+    turnComplete: boolean;
+  };
+};
+
+export type RealtimeInputMessage = {
+  realtimeInput: {
+    mediaChunks: GenerativeContentBlob[];
+  };
+};
+
+export type ToolResponseMessage = {
+  toolResponse: {
+    functionResponses: LiveFunctionResponse[];
+  };
+};
+
+export type ToolResponse = ToolResponseMessage["toolResponse"];
+
+export type LiveFunctionResponse = {
+  response: object;
+  id: string;
+};
+
+/** Incoming types */
+
+export type LiveIncomingMessage =
+  | ToolCallMessage
+  | ToolCallCancellationMessage
+  | SetupCompleteMessage
+  | ServerContentMessage;
+
+export type SetupCompleteMessage = { setupComplete: {} };
+
+export type ServerContentMessage = {
+  serverContent: ServerContent;
+};
+
+export type ServerContent = ModelTurn | TurnComplete | Interrupted;
+
+export type ModelTurn = {
+  modelTurn: {
+    parts: Part[];
+  };
+};
+
+export type TurnComplete = { turnComplete: boolean };
+
+export type Interrupted = { interrupted: true };
+
+export type ToolCallCancellationMessage = {
+  toolCallCancellation: {
+    ids: string[];
+  };
+};
+
+export type ToolCallCancellation =
+  ToolCallCancellationMessage["toolCallCancellation"];
+
+export type ToolCallMessage = {
+  toolCall: ToolCall;
+};
+
+export type LiveFunctionCall = FunctionCall & {
+  id: string;
+};
+
+/**
+ * A `toolCall` message
+ */
+export type ToolCall = {
+  functionCalls: LiveFunctionCall[];
+};
+
+/** log types */
+export type StreamingLog = {
+  date: Date;
+  type: string;
+  count?: number;
+  message: string | LiveOutgoingMessage | LiveIncomingMessage;
+};
+
+// Type-Guards
+
+const prop = (a: any, prop: string, kind: string = "object") =>
+  typeof a === "object" && typeof a[prop] === "object";
+
+// outgoing messages
+export const isSetupMessage = (a: unknown): a is SetupMessage =>
+  prop(a, "setup");
+
+export const isClientContentMessage = (a: unknown): a is ClientContentMessage =>
+  prop(a, "clientContent");
+
+export const isRealtimeInputMessage = (a: unknown): a is RealtimeInputMessage =>
+  prop(a, "realtimeInput");
+
+export const isToolResponseMessage = (a: unknown): a is ToolResponseMessage =>
+  prop(a, "toolResponse");
+
+// incoming messages
+export const isSetupCompleteMessage = (a: unknown): a is SetupCompleteMessage =>
+  prop(a, "setupComplete");
+
+export const isServerContenteMessage = (a: any): a is ServerContentMessage =>
+  prop(a, "serverContent");
+
+export const isToolCallMessage = (a: any): a is ToolCallMessage =>
+  prop(a, "toolCall");
+
+export const isToolCallCancellationMessage = (
+  a: unknown,
+): a is ToolCallCancellationMessage =>
+  prop(a, "toolCallCancellation") &&
+  isToolCallCancellation((a as any).toolCallCancellation);
+
+export const isModelTurn = (a: any): a is ModelTurn =>
+  typeof (a as ModelTurn).modelTurn === "object";
+
+export const isTurnComplete = (a: any): a is TurnComplete =>
+  typeof (a as TurnComplete).turnComplete === "boolean";
+
+export const isInterrupted = (a: any): a is Interrupted =>
+  (a as Interrupted).interrupted;
+
+export function isToolCall(value: unknown): value is ToolCall {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    Array.isArray(candidate.functionCalls) &&
+    candidate.functionCalls.every((call) => isLiveFunctionCall(call))
+  );
+}
+
+export function isToolResponse(value: unknown): value is ToolResponse {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    Array.isArray(candidate.functionResponses) &&
+    candidate.functionResponses.every((resp) => isLiveFunctionResponse(resp))
+  );
+}
+
+export function isLiveFunctionCall(value: unknown): value is LiveFunctionCall {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    typeof candidate.name === "string" &&
+    typeof candidate.id === "string" &&
+    typeof candidate.args === "object" &&
+    candidate.args !== null
+  );
+}
+
+export function isLiveFunctionResponse(
+  value: unknown,
+): value is LiveFunctionResponse {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    typeof candidate.response === "object" && typeof candidate.id === "string"
+  );
+}
+
+export const isToolCallCancellation = (
+  a: unknown,
+): a is ToolCallCancellationMessage["toolCallCancellation"] =>
+  typeof a === "object" && Array.isArray((a as any).ids);

src/frontends/live_api_react/frontend/src/react-app-env.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/// <reference types="react-scripts" />

src/frontends/live_api_react/frontend/src/reportWebVitals.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ReportHandler } from "web-vitals";
+
+const reportWebVitals = (onPerfEntry?: ReportHandler) => {
+  if (onPerfEntry && onPerfEntry instanceof Function) {
+    import("web-vitals").then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
+      getCLS(onPerfEntry);
+      getFID(onPerfEntry);
+      getFCP(onPerfEntry);
+      getLCP(onPerfEntry);
+      getTTFB(onPerfEntry);
+    });
+  }
+};
+
+export default reportWebVitals;