serve-emul 0.0.4

package/src/ui/components/session-panel.tsx

@@ -0,0 +1,127 @@
+import { useEffect, useState } from "react";
+
+type SessionEvent = {
+  id: number;
+  at: string;
+  delayMs: number;
+  source: string;
+  kind: "gesture" | "location";
+  gesture?: { type: string };
+  location?: { latitude: number; longitude: number };
+};
+
+type SessionSnapshot = {
+  events: SessionEvent[];
+  recording: boolean;
+  replaying: boolean;
+  lastError: string | null;
+};
+
+function labelForEvent(event: SessionEvent): string {
+  if (event.kind === "gesture") return `${event.gesture?.type ?? "gesture"} • ${event.source}`;
+  const lat = event.location?.latitude.toFixed(5) ?? "?";
+  const lng = event.location?.longitude.toFixed(5) ?? "?";
+  return `location ${lat}, ${lng}`;
+}
+
+export function SessionPanel() {
+  const [session, setSession] = useState<SessionSnapshot | null>(null);
+  const [multiplier, setMultiplier] = useState("1");
+  const [status, setStatus] = useState("Ready");
+
+  const refresh = () => {
+    fetch("/api/session")
+      .then((r) => r.json() as Promise<SessionSnapshot>)
+      .then(setSession)
+      .catch(() => setStatus("Session unavailable"));
+  };
+
+  useEffect(() => {
+    refresh();
+    const timer = setInterval(refresh, 1000);
+    return () => clearInterval(timer);
+  }, []);
+
+  const replay = async () => {
+    const rate = Number(multiplier);
+    if (!Number.isFinite(rate) || rate <= 0) {
+      setStatus("Rate must be positive");
+      return;
+    }
+    const res = await fetch("/api/session/replay", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ multiplier: rate }),
+    });
+    const data = await res.json() as { ok?: boolean; error?: string; session?: SessionSnapshot };
+    if (!res.ok || !data.ok) {
+      setStatus(data.error ?? "Replay failed");
+      return;
+    }
+    setSession(data.session ?? null);
+    setStatus("Replaying");
+  };
+
+  const stopReplay = async () => {
+    const res = await fetch("/api/session/replay/stop", { method: "POST" });
+    const data = await res.json() as { session?: SessionSnapshot };
+    setSession(data.session ?? null);
+    setStatus("Replay stopped");
+  };
+
+  const clear = async () => {
+    const res = await fetch("/api/session", { method: "DELETE" });
+    const data = await res.json() as { session?: SessionSnapshot };
+    setSession(data.session ?? null);
+    setStatus("Cleared");
+  };
+
+  const copy = async () => {
+    await navigator.clipboard.writeText(JSON.stringify(session?.events ?? [], null, 2));
+    setStatus("Copied");
+  };
+
+  const recent = session?.events.slice(-6).reverse() ?? [];
+
+  return (
+    <section className="tool-panel session-panel">
+      <div className="panel-heading">
+        <h2>Session</h2>
+        <div className="location-status">
+          {session?.replaying ? "Replaying" : status} • {session?.events.length ?? 0}
+        </div>
+      </div>
+      <div className="coordinate-grid">
+        <label>
+          Rate
+          <input
+            inputMode="decimal"
+            onChange={(e) => setMultiplier(e.currentTarget.value)}
+            value={multiplier}
+          />
+        </label>
+        <label>
+          Mode
+          <input readOnly value={session?.recording ? "Recording" : "Paused"} />
+        </label>
+      </div>
+      <div className="panel-actions">
+        <button onClick={() => void replay()}>Replay</button>
+        <button onClick={() => void stopReplay()}>Stop</button>
+        <button onClick={() => void clear()}>Clear</button>
+        <button onClick={() => void copy()}>Copy</button>
+      </div>
+      <div className="session-list">
+        {recent.length
+          ? recent.map((event) => (
+              <div key={event.id}>
+                <span>+{Math.round(event.delayMs)}ms</span>
+                {labelForEvent(event)}
+              </div>
+            ))
+          : <div>No recorded events</div>}
+      </div>
+      {session?.lastError && <div className="route-meta">{session.lastError}</div>}
+    </section>
+  );
+}

package/src/ui/components/status-bar.tsx

@@ -0,0 +1,19 @@
+import type { DeviceSize } from "../lib/use-stream";
+
+type Props = {
+  status: string;
+  deviceSize: DeviceSize | null;
+  fps: number;
+};
+
+export function StatusBar({ status, deviceSize, fps }: Props) {
+  const meta =
+    status +
+    (deviceSize ? ` • ${deviceSize.width}×${deviceSize.height} • ${fps} fps` : "");
+  return (
+    <header>
+      <h1>serve-emul</h1>
+      <div className="meta">{meta}</div>
+    </header>
+  );
+}

package/src/ui/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>serve-emul</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/main.tsx"></script>
+  </body>
+</html>

package/src/ui/lib/h264.ts

@@ -0,0 +1,35 @@
+export function buildCodecString(spsHeaderAndPayload: Uint8Array): string {
+  const profile = spsHeaderAndPayload[1].toString(16).padStart(2, "0");
+  const constraints = spsHeaderAndPayload[2].toString(16).padStart(2, "0");
+  const level = spsHeaderAndPayload[3].toString(16).padStart(2, "0");
+  return `avc1.${profile}${constraints}${level}`;
+}
+
+export type ScanResult = {
+  isKey: boolean;
+  spsBytes: Uint8Array | null;
+};
+
+export function scanAU(buf: Uint8Array): ScanResult {
+  let isKey = false;
+  let spsBytes: Uint8Array | null = null;
+  const len = buf.length;
+  let i = 0;
+  while (i + 2 < len) {
+    if (buf[i] === 0 && buf[i + 1] === 0) {
+      let codeLen = 0;
+      if (buf[i + 2] === 1) codeLen = 3;
+      else if (i + 3 < len && buf[i + 2] === 0 && buf[i + 3] === 1) codeLen = 4;
+      if (codeLen) {
+        const headerByte = buf[i + codeLen];
+        const nalType = headerByte & 0x1f;
+        if (nalType === 7 && !spsBytes) spsBytes = buf.subarray(i + codeLen);
+        if (nalType === 5) isKey = true;
+        i += codeLen + 1;
+        continue;
+      }
+    }
+    i++;
+  }
+  return { isKey, spsBytes };
+}

package/src/ui/lib/use-stream.ts

@@ -0,0 +1,368 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import type { RefObject } from "react";
+import { buildCodecString, scanAU } from "./h264";
+
+export type DeviceSize = { width: number; height: number };
+
+export type StreamState = {
+  status: string;
+  fps: number;
+  deviceSize: DeviceSize | null;
+};
+
+export type Sender = (msg: Record<string, unknown>, ack?: boolean) => void;
+
+type ApiInfo = {
+  size: DeviceSize;
+  status?: "streaming" | "stopped" | "error";
+  lastFrameAt?: string | null;
+  lastError?: string | null;
+};
+
+const SOFT_DECODE_QUEUE_SIZE = 4;
+const DECODER_RECOVERY_COOLDOWN_MS = 1500;
+const KEYFRAME_REQUEST_COOLDOWN_MS = 1500;
+const FRAME_QUEUE_SIZE = 2;
+const FRAME_META_MAGIC = 0x53454d55; // "SEMU"
+const FRAME_META_VERSION = 1;
+const FRAME_META_HEADER_BYTES = 16;
+const FRAME_FLAG_KEY = 1 << 0;
+
+type FramePacket = {
+  data: Uint8Array;
+  isKey: boolean | null;
+  timestamp: number | null;
+};
+
+function parseFramePacket(raw: ArrayBuffer | Uint8Array): FramePacket {
+  const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
+  if (bytes.byteLength > FRAME_META_HEADER_BYTES) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, FRAME_META_HEADER_BYTES);
+    if (view.getUint32(0, false) === FRAME_META_MAGIC && view.getUint8(4) === FRAME_META_VERSION) {
+      const pts = view.getBigUint64(8, false);
+      return {
+        data: bytes.subarray(FRAME_META_HEADER_BYTES),
+        isKey: (view.getUint8(5) & FRAME_FLAG_KEY) !== 0,
+        timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null,
+      };
+    }
+  }
+  return { data: bytes, isKey: null, timestamp: null };
+}
+
+export function useStream(canvasRef: RefObject<HTMLCanvasElement>) {
+  const [state, setState] = useState<StreamState>({
+    status: "connecting…",
+    fps: 0,
+    deviceSize: null,
+  });
+  const wsRef = useRef<WebSocket | null>(null);
+
+  const send = useCallback<Sender>((msg, ack = true) => {
+    const ws = wsRef.current;
+    if (!ws || ws.readyState !== WebSocket.OPEN) return;
+    ws.send(JSON.stringify(ack ? msg : { ...msg, ack: false }));
+  }, []);
+
+  useEffect(() => {
+    const canDecode = "VideoDecoder" in globalThis && "EncodedVideoChunk" in globalThis;
+    if (!canDecode) {
+      setState((s) => ({ ...s, status: "WebCodecs unsupported" }));
+      return;
+    }
+
+    let cancelled = false;
+    let reconnectDelay = 500;
+    let retryTimer: ReturnType<typeof setTimeout> | null = null;
+    let decoder: VideoDecoder | null = null;
+    let sawKeyframe = false;
+    let frameIdx = 0;
+    let fpsCount = 0;
+    let fpsTimer = performance.now();
+    let frameQueue: (VideoFrame | null)[] = new Array(FRAME_QUEUE_SIZE).fill(null);
+    let frameQueueHead = 0;
+    let frameQueueCount = 0;
+    let renderRaf = 0;
+    let lastDecoderRecoveryAt = 0;
+    let lastKeyframeRequestAt = 0;
+    let droppingUntilKeyframe = false;
+    let healthTimer: ReturnType<typeof setInterval> | null = null;
+
+    const setStatus = (s: string) =>
+      setState((prev) => (prev.status === s ? prev : { ...prev, status: s }));
+
+    const clearFrameQueue = () => {
+      if (renderRaf) {
+        cancelAnimationFrame(renderRaf);
+        renderRaf = 0;
+      }
+      for (let i = 0; i < FRAME_QUEUE_SIZE; i++) {
+        frameQueue[i]?.close();
+        frameQueue[i] = null;
+      }
+      frameQueueHead = 0;
+      frameQueueCount = 0;
+    };
+
+    const closeDecoder = () => {
+      if (!decoder) return;
+      try {
+        if (decoder.state !== "closed") decoder.close();
+      } catch {}
+      decoder = null;
+    };
+
+    const beginDecoderRecovery = () => {
+      const now = performance.now();
+      if (now - lastDecoderRecoveryAt < DECODER_RECOVERY_COOLDOWN_MS && droppingUntilKeyframe) return;
+      lastDecoderRecoveryAt = now;
+      closeDecoder();
+      clearFrameQueue();
+      sawKeyframe = false;
+      frameIdx = 0;
+      droppingUntilKeyframe = true;
+      requestKeyframe();
+      setStatus("recovering video");
+    };
+
+    const requestKeyframe = () => {
+      const ws = wsRef.current;
+      const now = performance.now();
+      if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      if (now - lastKeyframeRequestAt < KEYFRAME_REQUEST_COOLDOWN_MS) return;
+      lastKeyframeRequestAt = now;
+      ws.send(JSON.stringify({ type: "reset-video", ack: false }));
+    };
+
+    const renderFromQueue = (canvas: HTMLCanvasElement, ctx: CanvasRenderingContext2D) => {
+      renderRaf = 0;
+      if (frameQueueCount === 0) return;
+
+      const tail = (frameQueueHead - frameQueueCount + FRAME_QUEUE_SIZE) % FRAME_QUEUE_SIZE;
+      const frame = frameQueue[tail]!;
+      frameQueue[tail] = null;
+      frameQueueCount--;
+
+      if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+        canvas.width = frame.displayWidth;
+        canvas.height = frame.displayHeight;
+      }
+      ctx.drawImage(frame, 0, 0);
+      frame.close();
+
+      fpsCount++;
+      const now = performance.now();
+      if (now - fpsTimer >= 1000) {
+        const fps = Math.round((fpsCount * 1000) / (now - fpsTimer));
+        fpsCount = 0;
+        fpsTimer = now;
+        setState((s) => (s.fps === fps ? s : { ...s, fps }));
+      }
+
+      if (frameQueueCount > 0) {
+        renderRaf = requestAnimationFrame(() => renderFromQueue(canvas, ctx));
+      }
+    };
+
+    const ensureDecoder = (spsBytes: Uint8Array): boolean => {
+      if (decoder?.state === "configured") return true;
+      closeDecoder();
+      const canvas = canvasRef.current;
+      const ctx = canvas?.getContext("2d", { alpha: false, desynchronized: true });
+      if (!canvas || !ctx) return false;
+      const codec = buildCodecString(spsBytes);
+      let dec: VideoDecoder;
+      dec = new VideoDecoder({
+        output: (frame) => {
+          if (decoder !== dec) {
+            frame.close();
+            return;
+          }
+          if (frameQueueCount >= FRAME_QUEUE_SIZE) {
+            const tail = (frameQueueHead - frameQueueCount + FRAME_QUEUE_SIZE) % FRAME_QUEUE_SIZE;
+            frameQueue[tail]?.close();
+            frameQueue[tail] = null;
+            frameQueueCount--;
+          }
+          frameQueue[frameQueueHead] = frame;
+          frameQueueHead = (frameQueueHead + 1) % FRAME_QUEUE_SIZE;
+          frameQueueCount++;
+          if (!renderRaf) {
+            renderRaf = requestAnimationFrame(() => renderFromQueue(canvas, ctx));
+          }
+        },
+        error: (e) => {
+          console.error("VideoDecoder error", e);
+          setStatus("decoder error");
+          if (decoder === dec) beginDecoderRecovery();
+        },
+      });
+      try {
+        dec.configure({ codec, optimizeForLatency: true });
+        decoder = dec;
+        console.log("VideoDecoder configured:", codec);
+        return true;
+      } catch (e) {
+        console.error("VideoDecoder configure failed", e);
+        try {
+          dec.close();
+        } catch {}
+        setStatus("decoder config failed");
+        requestKeyframe();
+        return false;
+      }
+    };
+
+    const feedFrame = (raw: ArrayBuffer | Uint8Array) => {
+      const packet = parseFramePacket(raw);
+      const needsScan =
+        packet.isKey === null ||
+        (packet.isKey && (!decoder || decoder.state !== "configured" || droppingUntilKeyframe));
+      const scanned = needsScan ? scanAU(packet.data) : null;
+      const isKey = packet.isKey ?? scanned?.isKey ?? false;
+      const spsBytes = scanned?.spsBytes ?? null;
+      if (spsBytes && !ensureDecoder(spsBytes)) return;
+
+      if (droppingUntilKeyframe) {
+        if (!isKey) return;
+        if (!decoder || decoder.state !== "configured") {
+          requestKeyframe();
+          return;
+        }
+        droppingUntilKeyframe = false;
+      }
+
+      if (!decoder || decoder.state !== "configured") {
+        if (!isKey) requestKeyframe();
+        return;
+      }
+
+      if (decoder.decodeQueueSize > SOFT_DECODE_QUEUE_SIZE) {
+        beginDecoderRecovery();
+        return;
+      }
+
+      if (!sawKeyframe) {
+        if (!isKey) {
+          requestKeyframe();
+          return;
+        }
+        sawKeyframe = true;
+        setStatus("streaming");
+      }
+      try {
+        decoder.decode(
+          new EncodedVideoChunk({
+            type: isKey ? "key" : "delta",
+            timestamp: packet.timestamp ?? Math.round((frameIdx * 1_000_000) / 60),
+            data: packet.data,
+          }),
+        );
+        frameIdx++;
+      } catch (e) {
+        console.error("decode failed", e);
+        beginDecoderRecovery();
+      }
+    };
+
+    const connect = () => {
+      if (cancelled) return;
+      const proto = location.protocol === "https:" ? "wss:" : "ws:";
+      const ws = new WebSocket(`${proto}//${location.host}/ws?frame-meta=1`);
+      ws.binaryType = "arraybuffer";
+      wsRef.current = ws;
+      ws.onopen = () => {
+        reconnectDelay = 500;
+        setStatus("streaming");
+      };
+      ws.onerror = () => setStatus("connection error");
+      ws.onclose = () => {
+        if (cancelled) return;
+        const retryIn = reconnectDelay;
+        setStatus(`disconnected — retrying in ${Math.round(retryIn / 1000)}s`);
+        try {
+          decoder?.close();
+        } catch {}
+        decoder = null;
+        frameIdx = 0;
+        sawKeyframe = false;
+        reconnectDelay = Math.min(Math.round(reconnectDelay * 1.6), 5000);
+        retryTimer = setTimeout(connect, retryIn);
+      };
+      ws.onmessage = (e) => {
+        if (typeof e.data === "string") {
+          try {
+            const msg = JSON.parse(e.data) as { type?: string; size?: DeviceSize };
+            if (
+              msg.type === "video-session" &&
+              msg.size &&
+              Number.isFinite(msg.size.width) &&
+              Number.isFinite(msg.size.height)
+            ) {
+              closeDecoder();
+              clearFrameQueue();
+              frameIdx = 0;
+              sawKeyframe = false;
+              droppingUntilKeyframe = true;
+              setState((s) => ({ ...s, deviceSize: msg.size! }));
+              requestKeyframe();
+            }
+          } catch {}
+          return;
+        }
+        feedFrame(e.data);
+      };
+    };
+
+    connect();
+
+    const applyServerStatus = (d: ApiInfo) => {
+      const lastFrameAgeMs = d.lastFrameAt ? Date.now() - Date.parse(d.lastFrameAt) : Infinity;
+      setState((s) => ({
+        ...s,
+        deviceSize: d.size,
+        status:
+          d.status && d.status !== "streaming"
+            ? d.lastError || d.status
+            : lastFrameAgeMs > 3000
+              ? "stream stalled"
+              : s.status,
+      }));
+    };
+
+    fetch("/health")
+      .then((r) => r.json() as Promise<ApiInfo>)
+      .then((d) => {
+        if (cancelled) return;
+        applyServerStatus(d);
+      })
+      .catch(() => {
+        if (!cancelled) setStatus("metadata unavailable");
+      });
+
+    healthTimer = setInterval(() => {
+      fetch("/health")
+        .then((r) => r.json() as Promise<ApiInfo>)
+        .then((d) => {
+          if (!cancelled) applyServerStatus(d);
+        })
+        .catch(() => {
+          if (!cancelled) setStatus("metadata unavailable");
+        });
+    }, 1500);
+
+    return () => {
+      cancelled = true;
+      if (retryTimer) clearTimeout(retryTimer);
+      if (healthTimer) clearInterval(healthTimer);
+      try {
+        wsRef.current?.close();
+      } catch {}
+      clearFrameQueue();
+      closeDecoder();
+      wsRef.current = null;
+    };
+  }, [canvasRef]);
+
+  return { state, send };
+}

package/src/ui/main.tsx

@@ -0,0 +1,7 @@
+import { createRoot } from "react-dom/client";
+import { App } from "./app";
+import "./styles.css";
+
+const root = document.getElementById("root");
+if (!root) throw new Error("missing #root");
+createRoot(root).render(<App />);