agent-starter-pack 0.0.1b0__py3-none-any.whl

src/frontends/live_api_react/frontend/src/components/control-tray/ControlTray.tsx

@@ -0,0 +1,217 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import cn from "classnames";
+
+import { memo, ReactNode, RefObject, useEffect, useRef, useState } from "react";
+import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
+import { UseMediaStreamResult } from "../../hooks/use-media-stream-mux";
+import { useScreenCapture } from "../../hooks/use-screen-capture";
+import { useWebcam } from "../../hooks/use-webcam";
+import { AudioRecorder } from "../../utils/audio-recorder";
+import AudioPulse from "../audio-pulse/AudioPulse";
+import "./control-tray.scss";
+
+export type ControlTrayProps = {
+  videoRef: RefObject<HTMLVideoElement>;
+  children?: ReactNode;
+  supportsVideo: boolean;
+  onVideoStreamChange?: (stream: MediaStream | null) => void;
+};
+
+type MediaStreamButtonProps = {
+  isStreaming: boolean;
+  onIcon: string;
+  offIcon: string;
+  start: () => Promise<any>;
+  stop: () => any;
+};
+
+/**
+ * button used for triggering webcam or screen-capture
+ */
+const MediaStreamButton = memo(
+  ({ isStreaming, onIcon, offIcon, start, stop }: MediaStreamButtonProps) =>
+    isStreaming ? (
+      <button className="action-button" onClick={stop}>
+        <span className="material-symbols-outlined">{onIcon}</span>
+      </button>
+    ) : (
+      <button className="action-button" onClick={start}>
+        <span className="material-symbols-outlined">{offIcon}</span>
+      </button>
+    ),
+);
+
+function ControlTray({
+  videoRef,
+  children,
+  onVideoStreamChange = () => {},
+  supportsVideo,
+}: ControlTrayProps) {
+  const videoStreams = [useWebcam(), useScreenCapture()];
+  const [activeVideoStream, setActiveVideoStream] =
+    useState<MediaStream | null>(null);
+  const [webcam, screenCapture] = videoStreams;
+  const [inVolume, setInVolume] = useState(0);
+  const [audioRecorder] = useState(() => new AudioRecorder());
+  const [muted, setMuted] = useState(false);
+  const renderCanvasRef = useRef<HTMLCanvasElement>(null);
+  const connectButtonRef = useRef<HTMLButtonElement>(null);
+
+  const { client, connected, connect, disconnect, volume } =
+    useLiveAPIContext();
+
+  useEffect(() => {
+    if (!connected && connectButtonRef.current) {
+      connectButtonRef.current.focus();
+    }
+  }, [connected]);
+  useEffect(() => {
+    document.documentElement.style.setProperty(
+      "--volume",
+      `${Math.max(5, Math.min(inVolume * 200, 8))}px`,
+    );
+  }, [inVolume]);
+
+  useEffect(() => {
+    const onData = (base64: string) => {
+      client.sendRealtimeInput([
+        {
+          mimeType: "audio/pcm;rate=16000",
+          data: base64,
+        },
+      ]);
+    };
+    if (connected && !muted && audioRecorder) {
+      audioRecorder.on("data", onData).on("volume", setInVolume).start();
+    } else {
+      audioRecorder.stop();
+    }
+    return () => {
+      audioRecorder.off("data", onData).off("volume", setInVolume);
+    };
+  }, [connected, client, muted, audioRecorder]);
+
+  useEffect(() => {
+    if (videoRef.current) {
+      videoRef.current.srcObject = activeVideoStream;
+    }
+
+    let timeoutId = -1;
+
+    function sendVideoFrame() {
+      const video = videoRef.current;
+      const canvas = renderCanvasRef.current;
+
+      if (!video || !canvas) {
+        return;
+      }
+
+      const ctx = canvas.getContext("2d")!;
+      canvas.width = video.videoWidth * 0.25;
+      canvas.height = video.videoHeight * 0.25;
+      if (canvas.width + canvas.height > 0) {
+        ctx.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height);
+        const base64 = canvas.toDataURL("image/jpeg", 1.0);
+        const data = base64.slice(base64.indexOf(",") + 1, Infinity);
+        client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]);
+      }
+      if (connected) {
+        timeoutId = window.setTimeout(sendVideoFrame, 1000 / 0.5);
+      }
+    }
+    if (connected && activeVideoStream !== null) {
+      requestAnimationFrame(sendVideoFrame);
+    }
+    return () => {
+      clearTimeout(timeoutId);
+    };
+  }, [connected, activeVideoStream, client, videoRef]);
+
+  //handler for swapping from one video-stream to the next
+  const changeStreams = (next?: UseMediaStreamResult) => async () => {
+    if (next) {
+      const mediaStream = await next.start();
+      setActiveVideoStream(mediaStream);
+      onVideoStreamChange(mediaStream);
+    } else {
+      setActiveVideoStream(null);
+      onVideoStreamChange(null);
+    }
+
+    videoStreams.filter((msr) => msr !== next).forEach((msr) => msr.stop());
+  };
+
+  return (
+    <section className="control-tray">
+      <canvas style={{ display: "none" }} ref={renderCanvasRef} />
+      <nav className={cn("actions-nav", { disabled: !connected })}>
+        <button
+          className={cn("action-button mic-button")}
+          onClick={() => setMuted(!muted)}
+        >
+          {!muted ? (
+            <span className="material-symbols-outlined filled">mic</span>
+          ) : (
+            <span className="material-symbols-outlined filled">mic_off</span>
+          )}
+        </button>
+
+        <div className="action-button no-action outlined">
+          <AudioPulse volume={volume} active={connected} hover={false} />
+        </div>
+
+        {supportsVideo && (
+          <>
+            <MediaStreamButton
+              isStreaming={screenCapture.isStreaming}
+              start={changeStreams(screenCapture)}
+              stop={changeStreams()}
+              onIcon="cancel_presentation"
+              offIcon="present_to_all"
+            />
+            <MediaStreamButton
+              isStreaming={webcam.isStreaming}
+              start={changeStreams(webcam)}
+              stop={changeStreams()}
+              onIcon="videocam_off"
+              offIcon="videocam"
+            />
+          </>
+        )}
+        {children}
+      </nav>
+
+      <div className={cn("connection-container", { connected })}>
+        <div className="connection-button-container">
+          <button
+            ref={connectButtonRef}
+            className={cn("action-button connect-toggle", { connected })}
+            onClick={connected ? disconnect : connect}
+          >
+            <span className="material-symbols-outlined filled">
+              {connected ? "pause" : "play_arrow"}
+            </span>
+          </button>
+        </div>
+        <span className="text-indicator">Streaming</span>
+      </div>
+    </section>
+  );
+}
+
+export default memo(ControlTray);

src/frontends/live_api_react/frontend/src/components/control-tray/control-tray.scss

@@ -0,0 +1,201 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* stylelint-disable */
+.action-button {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  background: var(--Neutral-20);
+  color: var(--Neutral-60);
+  font-size: 1.25rem;
+  line-height: 1.75rem;
+  text-transform: lowercase;
+  cursor: pointer;
+  animation: opacity-pulse 3s ease-in infinite;
+  transition: all 0.2s ease-in-out;
+  width: 48px;
+  height: 48px;
+  border-radius: 18px;
+  border: 1px solid rgba(0, 0, 0, 0);
+  user-select: none;
+  cursor: pointer;
+
+  &:focus {
+    border: 2px solid var(--Neutral-20);
+    outline: 2px solid var(--Neutral-80);
+  }
+
+  &.outlined {
+    background: var(--Neutral-2);
+    border: 1px solid var(--Neutral-20);
+  }
+
+  .no-action {
+    pointer-events: none;
+  }
+
+  &:hover {
+    background: rgba(0, 0, 0, 0);
+    border: 1px solid var(--Neutral-20);
+  }
+
+  &.connected {
+    background: var(--Blue-800);
+    color: var(--Blue-500);
+
+    &:hover {
+      border: 1px solid var(--Blue-500);
+    }
+  }
+}
+
+@property --volume {
+  syntax: "length";
+  inherit: false;
+  initial-value: 0px;
+}
+
+.disabled .mic-button,
+.mic-button.disabled {
+  &:before {
+    background: rgba(0, 0, 0, 0);
+  }
+}
+
+.mic-button {
+  position: relative;
+  background-color: var(--accent-red);
+  z-index: 1;
+  color: black;
+  transition: all 0.2s ease-in;
+
+  &:focus {
+    border: 2px solid var(--Neutral-20);
+    outline: 2px solid var(--Red-500);
+  }
+
+  &:hover {
+    background-color: var(--Red-400);
+  }
+
+  &:before {
+    position: absolute;
+    z-index: -1;
+    top: calc(var(--volume) * -1);
+    left: calc(var(--volume) * -1);
+    display: block;
+    content: "";
+    opacity: 0.35;
+    background-color: var(--Red-500);
+    width: calc(100% + var(--volume) * 2);
+    height: calc(100% + var(--volume) * 2);
+    border-radius: 24px;
+    transition: all 0.02s ease-in-out;
+  }
+}
+
+.connect-toggle {
+  &:focus {
+    border: 2px solid var(--Neutral-20);
+    outline: 2px solid var(--Neutral-80);
+  }
+
+  &:not(.connected) {
+    background-color: var(--Blue-500);
+    color: var(--Neutral-5);
+  }
+}
+
+.control-tray {
+  position: absolute;
+  bottom: 0;
+  left: 50%;
+  transform: translate(-50%, 0);
+  display: inline-flex;
+  justify-content: center;
+  align-items: flex-start;
+  gap: 8px;
+  padding-bottom: 18px;
+
+  .disabled .action-button,
+  .action-button.disabled {
+    background: rgba(0, 0, 0, 0);
+    border: 1px solid var(--Neutral-30, #404547);
+    color: var(--Neutral-30);
+  }
+
+  .connection-container {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+    gap: 4px;
+
+    .connection-button-container {
+      border-radius: 27px;
+      border: 1px solid var(--Neutral-30);
+      background: var(--Neutral-5);
+      padding: 10px;
+    }
+
+    .text-indicator {
+      font-size: 11px;
+      color: var(--Blue-500);
+      user-select: none;
+    }
+
+    &:not(.connected) {
+      .text-indicator {
+        opacity: 0;
+      }
+    }
+  }
+}
+
+.actions-nav {
+  background: var(--Neutral-5);
+  border: 1px solid var(--Neutral-30);
+  border-radius: 27px;
+  display: inline-flex;
+  gap: 12px;
+  align-items: center;
+  overflow: clip;
+  padding: 10px;
+
+  transition: all 0.6s ease-in;
+
+  & > * {
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    gap: 1rem;
+  }
+}
+
+@keyframes opacity-pulse {
+  0% {
+    opacity: 0.9;
+  }
+
+  50% {
+    opacity: 1;
+  }
+
+  100% {
+    opacity: 0.9;
+  }
+}

src/frontends/live_api_react/frontend/src/components/logger/Logger.tsx

@@ -0,0 +1,241 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "./logger.scss";
+
+import { Part } from "@google/generative-ai";
+import cn from "classnames";
+import { ReactNode } from "react";
+import { useLoggerStore } from "../../utils/store-logger";
+import {
+  ClientContentMessage,
+  isClientContentMessage,
+  isInterrupted,
+  isModelTurn,
+  isServerContenteMessage,
+  isToolCallCancellationMessage,
+  isToolCallMessage,
+  isToolResponseMessage,
+  isTurnComplete,
+  ModelTurn,
+  ServerContentMessage,
+  StreamingLog,
+  ToolCallCancellationMessage,
+  ToolCallMessage,
+  ToolResponseMessage,
+} from "../../multimodal-live-types";
+
+const formatTime = (d: Date) => d.toLocaleTimeString().slice(0, -3);
+
+const LogEntry = ({
+  log,
+  MessageComponent,
+}: {
+  log: StreamingLog;
+  MessageComponent: ({
+    message,
+  }: {
+    message: StreamingLog["message"];
+  }) => ReactNode;
+}): JSX.Element => (
+  <li
+    className={cn(
+      `plain-log`,
+      `source-${log.type.slice(0, log.type.indexOf("."))}`,
+      {
+        receive: log.type.includes("receive"),
+        send: log.type.includes("send"),
+      },
+    )}
+  >
+    <span className="timestamp">{formatTime(log.date)}</span>
+    <span className="source">{log.type}</span>
+    <span className="message">
+      <MessageComponent message={log.message} />
+    </span>
+    {log.count && <span className="count">{log.count}</span>}
+  </li>
+);
+
+const PlainTextMessage = ({
+  message,
+}: {
+  message: StreamingLog["message"];
+}) => <span>{message as string}</span>;
+
+type Message = { message: StreamingLog["message"] };
+
+const AnyMessage = ({ message }: Message) => (
+  <pre>{JSON.stringify(message, null, "  ")}</pre>
+);
+
+const RenderPart = ({ part }: { part: Part }) =>
+  part.text && part.text.length ? (
+    <p className="part part-text">{part.text}</p>
+  ) : (
+    <div className="part part-inlinedata">
+      <h5>Inline Data: {part.inlineData?.mimeType}</h5>
+    </div>
+  );
+
+const ClientContentLog = ({ message }: Message) => {
+  const { turns, turnComplete } = (message as ClientContentMessage)
+    .clientContent;
+  return (
+    <div className="rich-log client-content user">
+      <h4 className="roler-user">User</h4>
+      {turns.map((turn, i) => (
+        <div key={`message-turn-${i}`}>
+          {turn.parts
+            .filter((part) => !(part.text && part.text === "\n"))
+            .map((part, j) => (
+              <RenderPart part={part} key={`message-turh-${i}-part-${j}`} />
+            ))}
+        </div>
+      ))}
+      {!turnComplete ? <span>turnComplete: false</span> : ""}
+    </div>
+  );
+};
+
+const ToolCallLog = ({ message }: Message) => {
+  const { toolCall } = message as ToolCallMessage;
+  return (
+    <div className={cn("rich-log tool-call")}>
+      {toolCall.functionCalls.map((fc, i) => (
+        <div key={fc.id} className="part part-functioncall">
+          <h5>Function call: {fc.name}</h5>
+          <pre>{JSON.stringify(fc, null, "  ")}</pre>
+        </div>
+      ))}
+    </div>
+  );
+};
+
+const ToolCallCancellationLog = ({ message }: Message): JSX.Element => (
+  <div className={cn("rich-log tool-call-cancellation")}>
+    <span>
+      {" "}
+      ids:{" "}
+      {(message as ToolCallCancellationMessage).toolCallCancellation.ids.map(
+        (id) => (
+          <span className="inline-code" key={`cancel-${id}`}>
+            "{id}"
+          </span>
+        ),
+      )}
+    </span>
+  </div>
+);
+
+const ToolResponseLog = ({ message }: Message): JSX.Element => (
+  <div className={cn("rich-log tool-response")}>
+    {(message as ToolResponseMessage).toolResponse.functionResponses.map(
+      (fc) => (
+        <div key={`tool-response-${fc.id}`} className="part">
+          <h5>Function Response: {fc.id}</h5>
+          <pre>{JSON.stringify(fc.response, null, "  ")}</pre>
+        </div>
+      ),
+    )}
+  </div>
+);
+
+const ModelTurnLog = ({ message }: Message): JSX.Element => {
+  const serverContent = (message as ServerContentMessage).serverContent;
+  const { modelTurn } = serverContent as ModelTurn;
+  const { parts } = modelTurn;
+
+  return (
+    <div className="rich-log model-turn model">
+      <h4 className="role-model">Model</h4>
+      {parts
+        .filter((part) => !(part.text && part.text === "\n"))
+        .map((part, j) => (
+          <RenderPart part={part} key={`model-turn-part-${j}`} />
+        ))}
+    </div>
+  );
+};
+
+const CustomPlainTextLog = (msg: string) => () => (
+  <PlainTextMessage message={msg} />
+);
+
+export type LoggerFilterType = "conversations" | "tools" | "none";
+
+export type LoggerProps = {
+  filter: LoggerFilterType;
+};
+
+const filters: Record<LoggerFilterType, (log: StreamingLog) => boolean> = {
+  tools: (log: StreamingLog) =>
+    isToolCallMessage(log.message) ||
+    isToolResponseMessage(log.message) ||
+    isToolCallCancellationMessage(log.message),
+  conversations: (log: StreamingLog) =>
+    isClientContentMessage(log.message) || isServerContenteMessage(log.message),
+  none: () => true,
+};
+
+const component = (log: StreamingLog) => {
+  if (typeof log.message === "string") {
+    return PlainTextMessage;
+  }
+  if (isClientContentMessage(log.message)) {
+    return ClientContentLog;
+  }
+  if (isToolCallMessage(log.message)) {
+    return ToolCallLog;
+  }
+  if (isToolCallCancellationMessage(log.message)) {
+    return ToolCallCancellationLog;
+  }
+  if (isToolResponseMessage(log.message)) {
+    return ToolResponseLog;
+  }
+  if (isServerContenteMessage(log.message)) {
+    const { serverContent } = log.message;
+    if (isInterrupted(serverContent)) {
+      return CustomPlainTextLog("interrupted");
+    }
+    if (isTurnComplete(serverContent)) {
+      return CustomPlainTextLog("turnComplete");
+    }
+    if (isModelTurn(serverContent)) {
+      return ModelTurnLog;
+    }
+  }
+  return AnyMessage;
+};
+
+export default function Logger({ filter = "none" }: LoggerProps) {
+  const { logs } = useLoggerStore();
+
+  const filterFn = filters[filter];
+
+  return (
+    <div className="logger">
+      <ul className="logger-list">
+        {logs.filter(filterFn).map((log, key) => {
+          return (
+            <LogEntry MessageComponent={component(log)} log={log} key={key} />
+          );
+        })}
+      </ul>
+    </div>
+  );
+}