sparkecoder 0.1.64 → 0.1.66

package/web/.next/standalone/web/src/components/chat-interface.tsx

@@ -112,6 +112,11 @@ import {
   type Checkpoint,
   type RunAgentAttachment,
   type VersionInfo,
+  getSessionFiles,
+  getBrowserStreamStatus,
+  getBrowserRecordings,
+  type SessionFile as ApiSessionFile,
+  type BrowserRecordingInfo,
 } from '@/lib/api';
 import { TodoPanel } from '@/components/ai-elements/todo-panel';
 import { getConfig, type AppConfig } from '@/lib/config';
@@ -124,7 +129,7 @@ import {
   SelectTrigger,
   SelectValue,
 } from '@/components/ui/select';
-import { MessageSquare, Copy, RefreshCw, AlertTriangle, Terminal as TerminalIcon, FileCode, Radio, Pencil, Check, Settings, RotateCcw, FolderOpen, PanelLeft, FileIcon, Download, X, ChevronDown, ChevronUp, Play, ArrowUp, Trash2, Monitor, ListChecks } from 'lucide-react';
+import { MessageSquare, Copy, RefreshCw, AlertTriangle, Terminal as TerminalIcon, FileCode, Radio, Pencil, Check, Settings, RotateCcw, FolderOpen, PanelLeft, FileIcon, Download, X, ChevronDown, ChevronUp, Play, ArrowUp, Trash2, Monitor, ListChecks, MoreVertical, Globe, Video } from 'lucide-react';
 import { useSidebar } from '@/components/ui/sidebar';
 import {
   Dialog,
@@ -136,6 +141,13 @@ import {
 import { Switch } from '@/components/ui/switch';
 import { Label } from '@/components/ui/label';
 import { Input } from '@/components/ui/input';
+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuSeparator,
+  DropdownMenuTrigger,
+} from '@/components/ui/dropdown-menu';
 import { WriteFileTool, type WriteFileInput, type WriteFileOutput } from '@/components/ai-elements/write-file-tool';
 import { ReadFileTool, type ReadFileInput, type ReadFileOutput } from '@/components/ai-elements/read-file-tool';
 import { BashTool, type BashInput, type BashOutput } from '@/components/ai-elements/bash-tool';
@@ -146,6 +158,8 @@ import { LinterTool, type LinterInput, type LinterOutput } from '@/components/ai
 import { CodeGraphTool, type CodeGraphInput, type CodeGraphOutput } from '@/components/ai-elements/code-graph-tool';
 import { CompleteTaskTool } from '@/components/ai-elements/complete-task-tool';
 import { SpeechInput } from '@/components/ai-elements/speech-input';
+import { BrowserPreview } from '@/components/browser-preview';
+import { BrowserRecording } from '@/components/ai-elements/browser-recording';
 import {
   MentionInputProvider,
   MentionTextarea,
@@ -424,6 +438,25 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
   const [updateBannerDismissed, setUpdateBannerDismissed] = useState(false);
   const [isCreatingUpdateSession, setIsCreatingUpdateSession] = useState(false);
 
+  // Browser PIP preview state
+  const [browserPreviewActive, setBrowserPreviewActive] = useState(false);
+  const [browserPreviewFrame, setBrowserPreviewFrame] = useState<string | null>(null);
+  const [browserPreviewMetadata, setBrowserPreviewMetadata] = useState<{
+    deviceWidth: number;
+    deviceHeight: number;
+    pageScaleFactor: number;
+    offsetTop: number;
+    scrollOffsetX: number;
+    scrollOffsetY: number;
+  } | null>(null);
+  const [browserPreviewDismissed, setBrowserPreviewDismissed] = useState(false);
+  const browserFrameTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  // Session files (GCS-backed recordings, uploads, etc.)
+  const [sessionFiles, setSessionFiles] = useState<ApiSessionFile[]>([]);
+  const [browserRecordingsOpen, setBrowserRecordingsOpen] = useState(false);
+  const [browserRecordings, setBrowserRecordings] = useState<BrowserRecordingInfo[]>([]);
+
   // Load config for available models
   useEffect(() => {
     getConfig().then(setConfig);
@@ -947,6 +980,36 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
       return;
     }
 
+    // Browser PIP preview events
+    if (event.type === 'browser-frame') {
+      setBrowserPreviewFrame(event.data);
+      setBrowserPreviewMetadata(event.metadata || null);
+      if (!browserPreviewDismissed) {
+        setBrowserPreviewActive(true);
+      }
+      if (browserFrameTimeoutRef.current) {
+        clearTimeout(browserFrameTimeoutRef.current);
+      }
+      browserFrameTimeoutRef.current = setTimeout(() => {
+        setBrowserPreviewActive(false);
+      }, 30000);
+      return;
+    }
+
+    if (event.type === 'browser-status') {
+      if (event.connected === false) {
+        if (browserFrameTimeoutRef.current) {
+          clearTimeout(browserFrameTimeoutRef.current);
+        }
+        browserFrameTimeoutRef.current = setTimeout(() => {
+          setBrowserPreviewActive(false);
+          setBrowserPreviewFrame(null);
+          setBrowserPreviewDismissed(false);
+        }, 5000);
+      }
+      return;
+    }
+
     switch (event.type) {
       case 'start':
         // Message started - could use messageId for tracking
@@ -1325,7 +1388,7 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
         setIsWatching(false);
         playDing();
         
-        // Only refresh checkpoints (for revert buttons) - NOT chatItems.
+        // Refresh checkpoints and session files (recordings may have been uploaded)
         {
           const finishSessionId = session.id;
           getSessionCheckpoints(finishSessionId)
@@ -1334,6 +1397,16 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
               setCheckpoints(checkpointsData.checkpoints || []);
             })
             .catch(() => {});
+          // Delay file refresh slightly to allow recording upload to complete
+          setTimeout(() => {
+            if (sessionIdRef.current !== finishSessionId) return;
+            getSessionFiles(finishSessionId)
+              .then((files) => {
+                if (sessionIdRef.current !== finishSessionId) return;
+                setSessionFiles(files);
+              })
+              .catch(() => {});
+          }, 3000);
         }
         
         // Send next queued message directly from the finish event.
@@ -1414,12 +1487,17 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
       currentTextRef.current = '';
       currentReasoningRef.current = '';
       toolCallsRef.current = [];
+      setBrowserPreviewActive(false);
+      setBrowserPreviewFrame(null);
+      setBrowserPreviewDismissed(false);
+      setSessionFiles([]);
       
       try {
-        // Load existing messages and checkpoints in parallel
-        const [apiMessages, checkpointsData] = await Promise.all([
+        // Load existing messages, checkpoints, and session files in parallel
+        const [apiMessages, checkpointsData, filesData] = await Promise.all([
           getSessionMessages(session.id),
           getSessionCheckpoints(session.id).catch(() => ({ checkpoints: [] })),
+          getSessionFiles(session.id).catch(() => []),
         ]);
         
         // Don't update state if session changed during async work
@@ -1431,6 +1509,7 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
         const converted = convertApiMessages(sorted);
         setChatItems(converted);
         setCheckpoints(checkpointsData.checkpoints || []);
+        setSessionFiles(filesData);
 
         // Check if there's an active stream to watch
         const streamInfo = await getActiveStream(session.id);
@@ -2404,6 +2483,16 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
           </TooltipProvider>
         </div>
         <div className="flex items-center gap-2 shrink-0">
+          {/* Browser recordings */}
+          {sessionFiles.filter(f => f.category === 'browser-recording' && f.downloadUrl).map((file) => (
+            <BrowserRecording
+              key={file.id}
+              fileName={file.fileName}
+              downloadUrl={file.downloadUrl!}
+              sizeBytes={file.sizeBytes}
+              createdAt={file.createdAt}
+            />
+          ))}
           {isWatching && (
             <TooltipProvider>
               <Tooltip>
@@ -2451,6 +2540,44 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
               <TooltipContent>Session Settings</TooltipContent>
             </Tooltip>
           </TooltipProvider>
+
+          {/* Browser / More actions menu */}
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button size="icon" variant="ghost" className="size-7 hover:bg-accent transition-colors">
+                <MoreVertical className="size-4" />
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent align="end" className="w-52">
+              <DropdownMenuItem
+                onClick={() => {
+                  setBrowserPreviewDismissed(false);
+                  // Check if there's an active browser stream
+                  getBrowserStreamStatus(session.id).then((status) => {
+                    if (status?.active) {
+                      setBrowserPreviewActive(true);
+                      console.log('[browser] Reconnecting PIP — stream active');
+                    } else {
+                      console.log('[browser] No active browser stream');
+                    }
+                  });
+                }}
+              >
+                <Globe className="size-4 mr-2" />
+                Open Browser PIP
+              </DropdownMenuItem>
+              <DropdownMenuItem
+                onClick={async () => {
+                  const recs = await getBrowserRecordings(session.id);
+                  setBrowserRecordings(recs);
+                  setBrowserRecordingsOpen(true);
+                }}
+              >
+                <Video className="size-4 mr-2" />
+                Browser Recordings
+              </DropdownMenuItem>
+            </DropdownMenuContent>
+          </DropdownMenu>
         </div>
       </div>
       )}
@@ -2531,6 +2658,51 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
         </DialogContent>
       </Dialog>
 
+      {/* Browser Recordings Dialog */}
+      <Dialog open={browserRecordingsOpen} onOpenChange={setBrowserRecordingsOpen}>
+        <DialogContent className="max-w-2xl w-full p-0 overflow-hidden rounded-xl">
+          <DialogHeader className="px-5 pt-5 pb-3">
+            <DialogTitle className="flex items-center gap-2 text-sm font-medium">
+              <div className="flex items-center justify-center size-6 rounded-lg bg-violet-500/10 dark:bg-violet-400/10">
+                <Video className="size-3.5 text-violet-600 dark:text-violet-400" />
+              </div>
+              Browser Recordings
+            </DialogTitle>
+          </DialogHeader>
+          <div className="px-5 pb-5 space-y-3">
+            {browserRecordings.length === 0 ? (
+              <p className="text-sm text-muted-foreground py-4 text-center">No browser recordings for this session</p>
+            ) : (
+              browserRecordings.map((rec) => (
+                <div key={rec.id} className="flex items-center gap-3 p-3 rounded-lg border bg-muted/30 hover:bg-muted/50 transition-colors">
+                  <div className="flex items-center justify-center size-10 rounded-lg bg-violet-500/10 dark:bg-violet-400/10 shrink-0">
+                    <Play className="size-4 text-violet-600 dark:text-violet-400 ml-0.5" />
+                  </div>
+                  <div className="min-w-0 flex-1">
+                    <p className="text-sm font-medium truncate">{rec.fileName}</p>
+                    <p className="text-xs text-muted-foreground">
+                      {rec.createdAt && new Date(rec.createdAt).toLocaleString()}
+                      {rec.sizeBytes != null && ` · ${rec.sizeBytes > 1024 * 1024 ? `${(rec.sizeBytes / (1024 * 1024)).toFixed(1)} MB` : `${(rec.sizeBytes / 1024).toFixed(0)} KB`}`}
+                    </p>
+                  </div>
+                  {rec.downloadUrl && (
+                    <Button
+                      size="sm"
+                      variant="outline"
+                      className="shrink-0"
+                      onClick={() => window.open(rec.downloadUrl!, '_blank')}
+                    >
+                      <Play className="size-3 mr-1" />
+                      Watch
+                    </Button>
+                  )}
+                </div>
+              ))
+            )}
+          </div>
+        </DialogContent>
+      </Dialog>
+
       {/* Update Available Banner - hidden in embed mode */}
       {!isEmbed && versionInfo?.updateAvailable && !updateBannerDismissed && (
         <div className="px-4 py-2 border-b border-amber-500/30 bg-amber-500/10 flex items-center justify-between gap-3">
@@ -3431,6 +3603,18 @@ export function ChatInterface({ session, isEmbed = false }: ChatInterfaceProps)
           </MentionInputProvider>
         </div>
       </div>
+
+      {/* Browser PIP preview */}
+      <BrowserPreview
+        sessionId={session.id}
+        frame={browserPreviewFrame}
+        metadata={browserPreviewMetadata}
+        active={browserPreviewActive && !browserPreviewDismissed}
+        onDismiss={() => {
+          setBrowserPreviewDismissed(true);
+          setBrowserPreviewActive(false);
+        }}
+      />
     </div>
   );
 }

package/web/.next/standalone/web/src/lib/api.ts

@@ -486,11 +486,22 @@ export type SSEEvent =
   | { type: 'data-stream-id'; streamId: string }
   | { type: 'data-session'; data: { id: string; name: string; workingDirectory: string; model: string } }
   | { type: 'data-user-message'; data: { id: string; content: string } }
+  | { type: 'browser-frame'; data: string; metadata: BrowserFrameMetadata }
+  | { type: 'browser-status'; connected: boolean; screencasting: boolean; viewportWidth?: number; viewportHeight?: number }
   | { type: 'finish-step' }
   | { type: 'finish'; finishReason?: string }
   | { type: 'abort' }
   | { type: 'error'; errorText: string };
 
+export interface BrowserFrameMetadata {
+  deviceWidth: number;
+  deviceHeight: number;
+  pageScaleFactor: number;
+  offsetTop: number;
+  scrollOffsetX: number;
+  scrollOffsetY: number;
+}
+
 // Terminal stream events
 export interface TerminalStreamEvent {
   type: 'status' | 'stdout' | 'exit';
@@ -848,3 +859,111 @@ export async function getWorkspaceFiles(
   
   return res.json();
 }
+
+// ============================================
+// Browser Input API (pair-browsing)
+// ============================================
+
+export interface BrowserInputEvent {
+  type: 'input_mouse' | 'input_keyboard' | 'input_touch';
+  eventType: string;
+  x?: number;
+  y?: number;
+  button?: string;
+  clickCount?: number;
+  deltaX?: number;
+  deltaY?: number;
+  key?: string;
+  code?: string;
+  text?: string;
+  modifiers?: number;
+  touchPoints?: Array<{ x: number; y: number; id?: number }>;
+}
+
+export async function sendBrowserInput(
+  sessionId: string,
+  event: BrowserInputEvent
+): Promise<void> {
+  await fetch(`${getApiBase()}/agents/${sessionId}/browser-input`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(event),
+  });
+}
+
+// ============================================
+// Session Files API (GCS-backed)
+// ============================================
+
+export interface SessionFile {
+  id: string;
+  fileName: string;
+  contentType: string;
+  sizeBytes: number | null;
+  category: string;
+  createdAt: string;
+  downloadUrl: string | null;
+  downloadUrlExpiresAt: string | null;
+}
+
+export async function getSessionFiles(sessionId: string): Promise<SessionFile[]> {
+  const res = await fetch(`${getApiBase()}/sessions/${sessionId}/session-files`);
+  if (!res.ok) {
+    if (res.status === 404) return [];
+    throw new Error(`Failed to get session files: ${res.statusText}`);
+  }
+  const data = await res.json();
+  return data.files || [];
+}
+
+export async function getFileDownloadUrl(fileId: string): Promise<{ downloadUrl: string; expiresAt: string }> {
+  const res = await fetch(`${getApiBase()}/sessions/files/${fileId}/download`);
+  if (!res.ok) {
+    throw new Error(`Failed to get download URL: ${res.statusText}`);
+  }
+  return res.json();
+}
+
+// ============================================
+// Browser Recording API
+// ============================================
+
+export interface BrowserRecordingInfo {
+  id: string;
+  fileName: string;
+  sizeBytes: number | null;
+  createdAt: string;
+  downloadUrl: string | null;
+  expiresAt: string | null;
+}
+
+export async function getBrowserRecordings(sessionId: string): Promise<BrowserRecordingInfo[]> {
+  const res = await fetch(`${getApiBase()}/sessions/${sessionId}/browser-recording`);
+  if (!res.ok) return [];
+  const data = await res.json();
+  return data.recordings || [];
+}
+
+// ============================================
+// Browser Stream Status API
+// ============================================
+
+export interface BrowserStreamStatus {
+  sessionId: string;
+  active: boolean;
+  hasProxy: boolean;
+  latestFrame: {
+    metadata: { deviceWidth: number; deviceHeight: number };
+    timestamp: number;
+  } | null;
+}
+
+export async function getBrowserStreamStatus(sessionId: string): Promise<BrowserStreamStatus | null> {
+  try {
+    const res = await fetch(`${getApiBase()}/agents/${sessionId}/browser-stream`);
+    if (!res.ok) return null;
+    return res.json();
+  } catch {
+    return null;
+  }
+}