agent-starter-pack 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

src/frontends/live_api_react/frontend/package-lock.json

@@ -5914,9 +5914,9 @@
       "license": "ISC"
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.12",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
-      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0",
@@ -9051,9 +9051,9 @@
       }
     },
     "node_modules/filelist/node_modules/brace-expansion": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
-      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"
@@ -16819,9 +16819,9 @@
       }
     },
     "node_modules/sucrase/node_modules/brace-expansion": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
-      "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
       "license": "MIT",
       "dependencies": {
         "balanced-match": "^1.0.0"

src/frontends/live_api_react/frontend/src/App.tsx

@@ -18,62 +18,32 @@ import { useRef, useState } from "react";
 import "./App.scss";
 import { LiveAPIProvider } from "./contexts/LiveAPIContext";
 import SidePanel from "./components/side-panel/SidePanel";
-import ControlTray from "./components/control-tray/ControlTray";
 import cn from "classnames";
 
-// Use relative URLs that work with integrated setup and deployments
-const defaultHost = window.location.host;
+// In development mode (frontend on :8501), connect to backend on :8000
+const isDevelopment = window.location.port === '8501';
+const defaultHost = isDevelopment ? `${window.location.hostname}:8000` : window.location.host;
 const defaultUri = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${defaultHost}/`;
 
 function App() {
   const videoRef = useRef<HTMLVideoElement>(null);
   const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
   const [serverUrl, setServerUrl] = useState<string>(defaultUri);
-  const [runId] = useState<string>(crypto.randomUUID());
   const [userId, setUserId] = useState<string>("user1");
 
-  // Feedback state
-  const [feedbackScore, setFeedbackScore] = useState<number>(10);
-  const [feedbackText, setFeedbackText] = useState<string>("");
-  const [sendFeedback, setShowFeedback] = useState(false);
-
-  const submitFeedback = async () => {
-    const feedbackUrl = new URL('feedback', serverUrl.replace('ws', 'http')).href;
-    
-    try {
-      const response = await fetch(feedbackUrl, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json'
-        },
-        body: JSON.stringify({
-          score: feedbackScore,
-          text: feedbackText,
-          run_id: runId,
-          user_id: userId,
-          log_type: "feedback"
-        })
-      });
-      if (!response.ok) {
-        throw new Error(`Failed to submit feedback: Server returned status ${response.status} ${response.statusText}`);
-      }
-
-      // Clear feedback after successful submission
-      setFeedbackScore(10);
-      setFeedbackText("");
-      setShowFeedback(false);
-      alert("Feedback submitted successfully!");
-    } catch (error) {
-      console.error('Error submitting feedback:', error);
-      alert(`Failed to submit feedback:  ${error}`);
-    }
-  };
-
   return (
     <div className="App">
       <LiveAPIProvider url={serverUrl} userId={userId}>
         <div className="streaming-console">
-          <SidePanel />
+          <SidePanel 
+            videoRef={videoRef}
+            supportsVideo={true}
+            onVideoStreamChange={setVideoStream}
+            serverUrl={serverUrl}
+            userId={userId}
+            onServerUrlChange={setServerUrl}
+            onUserIdChange={setUserId}
+          />
           <main>
             <div className="main-app-area">
               <video
@@ -85,117 +55,6 @@ function App() {
                 playsInline
               />
             </div>
-            <ControlTray
-              videoRef={videoRef}
-              supportsVideo={true}
-              onVideoStreamChange={setVideoStream}
-            >
-            </ControlTray>
-            <div className="url-setup" style={{position: 'absolute', top: 0, left: 0, right: 0, pointerEvents: 'auto', zIndex: 1000, padding: '2px', marginBottom: '2px', display: 'flex', justifyContent: 'space-between', alignItems: 'center', background: 'rgba(255, 255, 255, 0.9)'}}>
-              <div>
-                <label htmlFor="server-url">Server URL:</label>
-                <input
-                  id="server-url"
-                  type="text"
-                  value={serverUrl}
-                  onChange={(e) => setServerUrl(e.target.value)}
-                  placeholder="Enter server URL"
-                  style={{
-                    cursor: 'text',
-                    padding: '4px',
-                    margin: '0 4px', 
-                    borderRadius: '2px',
-                    border: '1px solid #ccc',
-                    fontSize: '14px',
-                    fontFamily: 'system-ui, -apple-system, sans-serif',
-                    width: '200px'
-                  }}
-                />
-                <label htmlFor="user-id">User ID:</label>
-                <input
-                  id="user-id"
-                  type="text"
-                  value={userId}
-                  onChange={(e) => setUserId(e.target.value)}
-                  placeholder="Enter user ID"
-                  style={{
-                    cursor: 'text',
-                    padding: '4px',
-                    margin: '0 4px', 
-                    borderRadius: '2px',
-                    border: '1px solid #ccc',
-                    fontSize: '14px',
-                    fontFamily: 'system-ui, -apple-system, sans-serif',
-                    width: '100px'
-                  }}
-                />
-              </div>
-
-              {/* Feedback Button */}
-              <button 
-                onClick={() => setShowFeedback(!sendFeedback)}
-                style={{
-                  padding: '5px 10px',
-                  margin: '10px',
-                  cursor: 'pointer'
-                }}
-              >
-                {sendFeedback ? 'Hide Feedback' : 'Send Feedback'}
-              </button>
-            </div>
-
-            {/* Feedback Overlay Section */}
-            {sendFeedback && (
-              <div className="feedback-section" style={{
-                position: 'absolute',
-                top: '50%',
-                left: '50%',
-                transform: 'translate(-50%, -50%)',
-                padding: '20px',
-                background: 'rgba(255, 255, 255, 0.95)',
-                boxShadow: '0 0 10px rgba(0,0,0,0.2)',
-                borderRadius: '8px',
-                zIndex: 1001,
-                minWidth: '300px'
-              }}>
-                <h3>Provide Feedback</h3>
-                <div>
-                  <label htmlFor="feedback-score">Score (0-10): </label>
-                  <input
-                    id="feedback-score"
-                    type="number"
-                    min="0"
-                    max="10"
-                    value={feedbackScore}
-                    onChange={(e) => setFeedbackScore(Number(e.target.value))}
-                    style={{margin: '0 10px'}}
-                  />
-                </div>
-                <div style={{marginTop: '10px'}}>
-                  <label htmlFor="feedback-text">Comments: </label>
-                  <textarea
-                    id="feedback-text"
-                    value={feedbackText}
-                    onChange={(e) => setFeedbackText(e.target.value)}
-                    style={{
-                      width: '100%',
-                      height: '60px',
-                      margin: '5px 0'
-                    }}
-                  />
-                </div>
-                <button
-                  onClick={submitFeedback}
-                  style={{
-                    padding: '5px 10px',
-                    marginTop: '5px',
-                    cursor: 'pointer'
-                  }}
-                >
-                  Submit Feedback
-                </button>
-              </div>
-            )}
           </main>
         </div>
       </LiveAPIProvider>

src/frontends/live_api_react/frontend/src/components/side-panel/SidePanel.tsx

@@ -15,11 +15,16 @@
  */
 
 import cn from "classnames";
-import { useEffect, useRef, useState } from "react";
+import { memo, ReactNode, RefObject, useEffect, useRef, useState } from "react";
 import { RiSidebarFoldLine, RiSidebarUnfoldLine } from "react-icons/ri";
 import Select from "react-select";
 import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
 import { useLoggerStore } from "../../utils/store-logger";
+import { UseMediaStreamResult } from "../../hooks/use-media-stream-mux";
+import { useScreenCapture } from "../../hooks/use-screen-capture";
+import { useWebcam } from "../../hooks/use-webcam";
+import { AudioRecorder } from "../../utils/audio-recorder";
+import AudioPulse from "../audio-pulse/AudioPulse";
 import Logger, { LoggerFilterType } from "../logger/Logger";
 import "./side-panel.scss";
 
@@ -29,9 +34,58 @@ const filterOptions = [
   { value: "none", label: "All" },
 ];
 
-export default function SidePanel() {
-  const { connected, client } = useLiveAPIContext();
+export type SidePanelProps = {
+  videoRef?: RefObject<HTMLVideoElement>;
+  children?: ReactNode;
+  supportsVideo?: boolean;
+  onVideoStreamChange?: (stream: MediaStream | null) => void;
+  serverUrl?: string;
+  userId?: string;
+  onServerUrlChange?: (url: string) => void;
+  onUserIdChange?: (userId: string) => void;
+};
+
+type MediaStreamButtonProps = {
+  isStreaming: boolean;
+  onIcon: string;
+  offIcon: string;
+  start: () => Promise<any>;
+  stop: () => any;
+};
+
+const MediaStreamButton = memo(
+  ({ isStreaming, onIcon, offIcon, start, stop }: MediaStreamButtonProps) =>
+    isStreaming ? (
+      <button className={cn("action-button", { active: isStreaming })} onClick={stop}>
+        <span className="material-symbols-outlined">{onIcon}</span>
+      </button>
+    ) : (
+      <button className="action-button" onClick={start}>
+        <span className="material-symbols-outlined">{offIcon}</span>
+      </button>
+    ),
+);
+
+function SidePanel({
+  videoRef,
+  children,
+  onVideoStreamChange = () => {},
+  supportsVideo = true,
+  serverUrl = "ws://localhost:8000/",
+  userId = "user1",
+  onServerUrlChange = () => {},
+  onUserIdChange = () => {},
+}: SidePanelProps) {
+  const { connected, client, connect, disconnect, volume } = useLiveAPIContext();
   const [open, setOpen] = useState(true);
+  const [connectionExpanded, setConnectionExpanded] = useState(false);
+
+  // Auto-collapse connection settings when panel is closed
+  useEffect(() => {
+    if (!open && connectionExpanded) {
+      setConnectionExpanded(false);
+    }
+  }, [open, connectionExpanded]);
   const loggerRef = useRef<HTMLDivElement>(null);
   const loggerLastHeightRef = useRef<number>(-1);
   const { log, logs } = useLoggerStore();
@@ -42,6 +96,135 @@ export default function SidePanel() {
     label: string;
   } | null>(null);
   const inputRef = useRef<HTMLTextAreaElement>(null);
+  
+  // Control states
+  const videoStreams = [useWebcam(), useScreenCapture()];
+  const [activeVideoStream, setActiveVideoStream] = useState<MediaStream | null>(null);
+  const [webcam, screenCapture] = videoStreams;
+  const [inVolume, setInVolume] = useState(0);
+  const [audioRecorder] = useState(() => new AudioRecorder());
+  const [muted, setMuted] = useState(false);
+  const renderCanvasRef = useRef<HTMLCanvasElement>(null);
+  const connectButtonRef = useRef<HTMLButtonElement>(null);
+  
+  // Feedback state (local to SidePanel)
+  const [feedbackScore, setFeedbackScore] = useState<number>(10);
+  const [feedbackText, setFeedbackText] = useState<string>("");
+  const [sendFeedback, setShowFeedback] = useState(false);
+
+  useEffect(() => {
+    if (!connected && connectButtonRef.current) {
+      connectButtonRef.current.focus();
+    }
+  }, [connected]);
+  
+  useEffect(() => {
+    document.documentElement.style.setProperty(
+      "--volume",
+      `${Math.max(5, Math.min(inVolume * 200, 8))}px`,
+    );
+  }, [inVolume]);
+
+  useEffect(() => {
+    const onData = (base64: string) => {
+      client.sendRealtimeInput([
+        {
+          mimeType: "audio/pcm;rate=16000",
+          data: base64,
+        },
+      ]);
+    };
+    if (connected && !muted && audioRecorder) {
+      audioRecorder.on("data", onData).on("volume", setInVolume).start();
+    } else {
+      audioRecorder.stop();
+    }
+    return () => {
+      audioRecorder.off("data", onData).off("volume", setInVolume);
+    };
+  }, [connected, client, muted, audioRecorder]);
+
+  useEffect(() => {
+    if (videoRef && videoRef.current) {
+      videoRef.current.srcObject = activeVideoStream;
+    }
+
+    let timeoutId = -1;
+
+    function sendVideoFrame() {
+      const video = videoRef && videoRef.current;
+      const canvas = renderCanvasRef.current;
+
+      if (!video || !canvas) {
+        return;
+      }
+
+      const ctx = canvas.getContext("2d")!;
+      canvas.width = video.videoWidth * 0.25;
+      canvas.height = video.videoHeight * 0.25;
+      if (canvas.width + canvas.height > 0) {
+        ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+        const base64 = canvas.toDataURL("image/jpeg", 1.0);
+        const data = base64.slice(base64.indexOf(",") + 1, Infinity);
+        client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]);
+      }
+      if (connected) {
+        timeoutId = window.setTimeout(sendVideoFrame, 1000 / 0.5);
+      }
+    }
+    if (connected && activeVideoStream !== null) {
+      requestAnimationFrame(sendVideoFrame);
+    }
+    return () => {
+      clearTimeout(timeoutId);
+    };
+  }, [connected, activeVideoStream, client, videoRef]);
+
+  //handler for swapping from one video-stream to the next
+  const changeStreams = (next?: UseMediaStreamResult) => async () => {
+    if (next) {
+      const mediaStream = await next.start();
+      setActiveVideoStream(mediaStream);
+      onVideoStreamChange(mediaStream);
+    } else {
+      setActiveVideoStream(null);
+      onVideoStreamChange(null);
+    }
+
+    videoStreams.filter((msr) => msr !== next).forEach((msr) => msr.stop());
+  };
+
+  const submitFeedback = async () => {
+    const feedbackUrl = new URL('feedback', serverUrl.replace('ws', 'http')).href;
+    
+    try {
+      const response = await fetch(feedbackUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          score: feedbackScore,
+          text: feedbackText,
+          run_id: client.currentRunId,
+          user_id: userId,
+          log_type: "feedback"
+        })
+      });
+      if (!response.ok) {
+        throw new Error(`Failed to submit feedback: Server returned status ${response.status} ${response.statusText}`);
+      }
+
+      // Clear feedback after successful submission
+      setFeedbackScore(10);
+      setFeedbackText("");
+      setShowFeedback(false);
+      alert("Feedback submitted successfully!");
+    } catch (error) {
+      console.error('Error submitting feedback:', error);
+      alert(`Failed to submit feedback:  ${error}`);
+    }
+  };
 
   //scroll the log to the bottom when new logs come in
   useEffect(() => {
@@ -74,6 +257,7 @@ export default function SidePanel() {
 
   return (
     <div className={`side-panel ${open ? "open" : ""}`}>
+      <canvas style={{ display: "none" }} ref={renderCanvasRef} />
       <header className="top">
         <h2>Console</h2>
         {open ? (
@@ -86,6 +270,106 @@ export default function SidePanel() {
           </button>
         )}
       </header>
+      
+      {/* Connection Settings Section */}
+      <section className="connection-settings">
+        <button 
+          className="connection-expander"
+          onClick={() => setConnectionExpanded(!connectionExpanded)}
+        >
+          Connection Settings
+          <span>{connectionExpanded ? '▼' : '▶'}</span>
+        </button>
+        {connectionExpanded && open && (
+          <div className="connection-content">
+            <div className="setting-group">
+              <label htmlFor="server-url">Server URL</label>
+              <input
+                id="server-url"
+                type="text"
+                value={serverUrl}
+                onChange={(e) => onServerUrlChange(e.target.value)}
+                placeholder="ws://localhost:8000/"
+                className="setting-input"
+              />
+            </div>
+            <div className="setting-group">
+              <label htmlFor="user-id">User ID</label>
+              <input
+                id="user-id"
+                type="text"
+                value={userId}
+                onChange={(e) => onUserIdChange(e.target.value)}
+                placeholder="user123"
+                className="setting-input"
+              />
+            </div>
+            <button 
+              onClick={() => setShowFeedback(!sendFeedback)}
+              className="feedback-button"
+            >
+              {sendFeedback ? 'Hide Feedback' : 'Send Feedback'}
+            </button>
+          </div>
+        )}
+      </section>
+
+      {/* Control Tray - All buttons moved here */}
+      <section className="control-tray">
+        <nav className={cn("actions-nav", { disabled: !connected })}>
+          <button
+            ref={connectButtonRef}
+            className={cn("action-button connect-toggle", { connected })}
+            onClick={connected ? disconnect : connect}
+          >
+            <span className="material-symbols-outlined filled">
+              {connected ? "pause" : "play_arrow"}
+            </span>
+          </button>
+
+          <button
+            className={cn("action-button mic-button", { active: !muted })}
+            onClick={() => setMuted(!muted)}
+          >
+            {!muted ? (
+              <span className="material-symbols-outlined filled">mic</span>
+            ) : (
+              <span className="material-symbols-outlined filled">mic_off</span>
+            )}
+          </button>
+
+          <div className="action-button no-action outlined">
+            <AudioPulse volume={volume} active={connected} hover={false} />
+          </div>
+
+          {supportsVideo && (
+            <>
+              <MediaStreamButton
+                isStreaming={screenCapture.isStreaming}
+                start={changeStreams(screenCapture)}
+                stop={changeStreams()}
+                onIcon="cancel_presentation"
+                offIcon="present_to_all"
+              />
+              <MediaStreamButton
+                isStreaming={webcam.isStreaming}
+                start={changeStreams(webcam)}
+                stop={changeStreams()}
+                onIcon="videocam_off"
+                offIcon="videocam"
+              />
+            </>
+          )}
+          {children}
+        </nav>
+
+        <div className="connection-status">
+          <span className={cn("text-indicator", { connected })}>
+            {connected ? "Streaming" : "Disconnected"}
+          </span>
+        </div>
+      </section>
+
       <section className="indicators">
         <Select
           className="react-select"
@@ -156,6 +440,71 @@ export default function SidePanel() {
           </button>
         </div>
       </div>
+      
+      {/* Audio Pulse Bottom Section */}
+      <section className="audio-pulse-bottom">
+        <div className="pulse-container">
+          <AudioPulse volume={volume} active={connected} hover={false} />
+          <span className="pulse-label">
+            {connected ? (volume > 0 ? "AI Speaking..." : "AI Ready") : "Not connected"}
+          </span>
+        </div>
+      </section>
+
+      {/* Feedback Overlay Section */}
+      {sendFeedback && (
+        <div className="feedback-section" style={{
+          position: 'absolute',
+          top: '50%',
+          left: '50%',
+          transform: 'translate(-50%, -50%)',
+          padding: '20px',
+          background: 'rgba(255, 255, 255, 0.95)',
+          boxShadow: '0 0 10px rgba(0,0,0,0.2)',
+          borderRadius: '8px',
+          zIndex: 1001,
+          minWidth: '300px'
+        }}>
+          <h3>Provide Feedback</h3>
+          <div>
+            <label htmlFor="feedback-score">Score (0-10): </label>
+            <input
+              id="feedback-score"
+              type="number"
+              min="0"
+              max="10"
+              value={feedbackScore}
+              onChange={(e) => setFeedbackScore(Number(e.target.value))}
+              style={{margin: '0 10px'}}
+            />
+          </div>
+          <div style={{marginTop: '10px'}}>
+            <label htmlFor="feedback-text">Comments: </label>
+            <textarea
+              id="feedback-text"
+              value={feedbackText}
+              onChange={(e) => setFeedbackText(e.target.value)}
+              style={{
+                width: '100%',
+                height: '60px',
+                margin: '5px 0'
+              }}
+            />
+          </div>
+          <button
+            onClick={submitFeedback}
+            style={{
+              padding: '5px 10px',
+              marginTop: '5px',
+              cursor: 'pointer'
+            }}
+          >
+            Submit Feedback
+          </button>
+        </div>
+      )}
     </div>
   );
 }
+
+export default memo(SidePanel);