rnxsim 0.1.313 → 0.1.314

package/cli/commands/film.ts

@@ -0,0 +1,379 @@
+// sootsim film — record a flow on the flat canvas, then re-play that
+// recording on the 3d device stage under a camera cinematic and capture the
+// stage as video.
+//
+// running a flow inside live 3d mode is too slow to look good (the guest
+// app, the shell, and the WebGL stage all compete for the same cores), so
+// filming is two-phase: the flow records at full speed on the flat canvas
+// via the engine recording store, then the 3d recording editor plays the
+// webm back on the phone screen while only the stage renders. the stage
+// canvas is captured with MediaRecorder (WebGL content only — the editor's
+// DOM dock never appears in the capture).
+
+import { spawnSync } from 'node:child_process'
+import { existsSync, mkdirSync, unlinkSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { dirname, extname, join, resolve } from 'node:path'
+import {
+  callInBridgeWrite,
+  createBridgeFromParsed,
+  parseBridgeCliArgs,
+  type WsBridge,
+} from '../ws-bridge'
+import { buildConfigureOptions, waitForThreeModeRuntimeReadyOnBridge } from './three-mode'
+
+interface FilmOptions {
+  port?: number
+  verbose?: boolean
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
+
+// flags consumed by film itself; everything else that buildConfigureOptions
+// understands (--background, --colorway, --env-*, --room, --focus*, ...)
+// passes through to the stage configure call.
+const FILM_VALUE_FLAGS = ['--out', '--fps', '--script', '--slow']
+const FILM_BOOLEAN_FLAGS = ['--keep-three']
+
+const CONFIGURE_VALUE_FLAGS = [
+  '--background',
+  '--gradient',
+  '--gradient-angle',
+  '--colorway',
+  '--environment',
+  '--env-shape',
+  '--env-spread',
+  '--env-size',
+  '--env-color',
+  '--env-glow',
+  '--env-plastic',
+  '--room',
+  '--focus',
+  '--focus-amount',
+  '--focus-range',
+]
+
+const MAX_FILM_MS = 120_000
+const STAGE_BITRATE = 16_000_000
+
+function flagValue(args: string[], flag: string): string | undefined {
+  const i = args.indexOf(flag)
+  return i >= 0 ? args[i + 1] : undefined
+}
+
+async function evalInPage<T>(bridge: WsBridge, code: string, timeoutMs = 30_000) {
+  return (await bridge.send({ type: 'evaluate', code }, { timeoutMs })) as T
+}
+
+export async function runFilm(args: string[], opts: FilmOptions): Promise<number> {
+  if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
+    console.log(`
+rnxsim film — record a flow flat, replay it on the 3d stage, save a clip
+
+usage:
+  rnxsim film <flow.yaml> [options]
+
+how it works:
+  1. the flow runs at full speed on the flat canvas while the engine
+     recording store captures the device screen
+  2. 3d mode + the recording editor come up and play that capture back on
+     the phone screen while a camera script animates
+  3. the WebGL stage canvas is recorded and saved
+
+options:
+  --out <path>           output file (.webm, or .mp4 when ffmpeg is
+                         installed). default: ~/Desktop/sootsim-films/<flow>.webm
+  --script <id>          camera script to animate during playback
+                         (default: hero-arc)
+  --fps <n>              stage capture framerate (default: 30)
+  --slow <ms>            per-step delay forwarded to the flow runner
+  --keep-three           leave 3d mode + the recording editor on afterwards
+  --sim <id>             target a specific connected sim
+
+stage decor (same flags as \`rnxsim mode three configure\`):
+  --background <id> · --gradient <stops> · --gradient-angle <deg>
+  --colorway <id> · --room on|off · --focus on|off · --focus-amount <id>
+  --focus-range <id> · --environment on|off · --env-shape/spread/size/color
+  --env-glow on|off · --env-plastic on|off
+
+examples:
+  rnxsim film flows/feed-scroll.yaml
+  rnxsim film flows/swap.yaml --script hero-arc --background gradient-tide --out ~/Desktop/swap-3d.mp4
+  rnxsim film flows/login.yaml --colorway deep-blue --environment on --env-shape pane --room on
+`)
+    return 0
+  }
+
+  const flowPath = args.find((a) => !a.startsWith('--') && /\.ya?ml$/.test(a))
+  if (!flowPath || !existsSync(resolve(flowPath))) {
+    console.error(`  film: flow file not found: ${flowPath ?? '(none given)'}`)
+    return 1
+  }
+
+  const fps = Number.parseInt(flagValue(args, '--fps') ?? '30', 10)
+  const scriptId = flagValue(args, '--script') ?? 'hero-arc'
+  const keepThree = args.includes('--keep-three')
+  const flowName = flowPath.replace(/^.*\//, '').replace(/\.ya?ml$/, '')
+  const outArg = flagValue(args, '--out')
+  const outPath = resolve(
+    outArg ?? join(homedir(), 'Desktop', 'sootsim-films', `${flowName}.webm`),
+  )
+  const wantsMp4 = extname(outPath).toLowerCase() === '.mp4'
+  if (wantsMp4 && spawnSync('ffmpeg', ['-version']).status !== 0) {
+    console.error('  film: --out ends in .mp4 but ffmpeg is not installed')
+    console.error('  install ffmpeg or use a .webm output path')
+    return 1
+  }
+
+  const parsed = parseBridgeCliArgs(args, {
+    port: opts.port,
+    stripValueFlags: [...FILM_VALUE_FLAGS, ...CONFIGURE_VALUE_FLAGS],
+    stripBooleanFlags: FILM_BOOLEAN_FLAGS,
+  })
+  const bridge = createBridgeFromParsed({ ...parsed, commandTimeoutMs: 60_000 })
+
+  try {
+    // remember the shell state we mutate so a plain run is side-effect free
+    const before = await evalInPage<{ threeMode?: boolean; editor?: boolean }>(
+      bridge,
+      `(() => { const s = window.SootSim?.bridges?.settings?.get?.() || {}
+         return { threeMode: s.threeMode === true, editor: s.threeRecordingEditor === true } })()`,
+    )
+
+    // -- phase 1: record the flow on the flat canvas ----------------------
+    console.log(`  film: recording flow ${flowName} on the flat canvas`)
+    const started = await evalInPage<boolean>(
+      bridge,
+      `(async () => !!(await window.SootSim?.bridges?.startRecording?.('video', undefined, { skipEntitlement: true })))()`,
+    )
+    if (!started) {
+      console.error(
+        '  film: engine recording store did not start (is @sootsim/plugin-recording installed?)',
+      )
+      return 1
+    }
+    const t0 = Date.now()
+    await sleep(500)
+
+    const { runFlowPlayback } = await import('./flow')
+    const playbackArgs = [flowPath]
+    const simFlag = flagValue(args, '--sim')
+    if (simFlag) playbackArgs.push('--sim', simFlag)
+    const slowFlag = flagValue(args, '--slow')
+    if (slowFlag) playbackArgs.push('--slow', slowFlag)
+    const flowCode = await runFlowPlayback(playbackArgs)
+    await sleep(400)
+    const durationMs = Math.min(Date.now() - t0, MAX_FILM_MS)
+    await evalInPage(bridge, `window.SootSim?.bridges?.stopRecording?.()`)
+    if (flowCode !== 0) {
+      console.error(`  film: flow failed (exit ${flowCode}) — not filming a broken take`)
+      return flowCode
+    }
+
+    let hasBlob = false
+    for (let i = 0; i < 50; i++) {
+      await sleep(400)
+      const st = await evalInPage<{ state?: string; hasBlob?: boolean }>(
+        bridge,
+        `(() => { const s = window.SootSim?.bridges?.getRecordingState?.()
+           return { state: s?.state, hasBlob: !!s?.lastVideoBlob } })()`,
+      )
+      if (st?.state === 'idle' && st.hasBlob) {
+        hasBlob = true
+        break
+      }
+    }
+    if (!hasBlob) {
+      console.error('  film: flow recording never produced a video blob')
+      return 1
+    }
+    console.log(`  film: flow captured (${(durationMs / 1000).toFixed(1)}s)`)
+
+    // -- phase 2: bring up the 3d stage + recording editor -----------------
+    await evalInPage(
+      bridge,
+      `window.SootSim?.bridges?.settings?.set?.('threeMode', true)`,
+    )
+    await waitForThreeModeRuntimeReadyOnBridge(bridge)
+    await sleep(1400)
+    await evalInPage(
+      bridge,
+      `window.SootSim?.bridges?.settings?.set?.('threeRecordingEditor', true)`,
+    )
+    let replaySeeded = false
+    for (let i = 0; i < 60; i++) {
+      await sleep(250)
+      const seeded = await evalInPage<boolean>(
+        bridge,
+        `(() => { const rs = window.SootSim?.replayStore
+           return rs?.value?.preview?.surface === 'video' && !!rs?.value?.preview?.flowVideoUrl })()`,
+      )
+      if (seeded) {
+        replaySeeded = true
+        break
+      }
+    }
+    if (!replaySeeded) {
+      console.error('  film: recording editor did not seed the replay store')
+      return 1
+    }
+
+    const configure = buildConfigureOptions(args)
+    delete configure.script
+    configure.resetPose = true
+    await callInBridgeWrite(bridge, 'SootSim.bridges.threeMode.configure', configure)
+    await sleep(2400)
+
+    // -- phase 3: capture the stage while the replay + camera script run ---
+    console.log(`  film: filming the 3d stage (script: ${scriptId})`)
+    const rec = await evalInPage<{ ok: boolean; error?: string }>(
+      bridge,
+      `(() => {
+        const c = window.SootSim?.bridges?.threeMode?.getCanvas?.()
+        if (!c || !c.captureStream) return { ok: false, error: 'no stage canvas' }
+        let mime = 'video/webm;codecs=vp9'
+        if (!('MediaRecorder' in window) || !MediaRecorder.isTypeSupported(mime)) mime = 'video/webm'
+        const stream = c.captureStream(${fps})
+        const mr = new MediaRecorder(stream, { mimeType: mime, videoBitsPerSecond: ${STAGE_BITRATE} })
+        const st = { mr, chunks: [], b64: null }
+        mr.ondataavailable = (e) => { if (e.data && e.data.size) st.chunks.push(e.data) }
+        mr.onstop = () => {
+          const blob = new Blob(st.chunks, { type: 'video/webm' })
+          const fr = new FileReader()
+          fr.onload = () => { st.b64 = String(fr.result).split(',')[1] || '' }
+          fr.readAsDataURL(blob)
+        }
+        window.__sootsimFilm = st
+        mr.start(200)
+        return { ok: true }
+      })()`,
+    )
+    if (!rec?.ok) {
+      console.error(`  film: stage capture failed: ${rec?.error ?? 'unknown'}`)
+      return 1
+    }
+
+    await evalInPage(
+      bridge,
+      `((dur) => {
+        const rs = window.SootSim?.replayStore
+        if (!rs) return false
+        const probed = rs.value?.playback?.durationMs
+        const total = probed && probed > 500 ? probed : dur
+        const start = performance.now()
+        const tick = () => {
+          const t = performance.now() - start
+          rs.emit({ ...rs.value, playback: { ...rs.value.playback, playing: true, scrubbing: false, currentTimeMs: Math.min(t, total) } })
+          if (t < total) requestAnimationFrame(tick)
+        }
+        requestAnimationFrame(tick)
+        return true
+      })(${durationMs})`,
+    )
+    await callInBridgeWrite(bridge, 'SootSim.bridges.threeMode.configure', {
+      script: { id: scriptId, progress: 1, animate: true, slow: true },
+    })
+    await sleep(durationMs + 700)
+
+    await evalInPage(
+      bridge,
+      `(() => { const s = window.__sootsimFilm; if (s?.mr?.state === 'recording') s.mr.stop(); return true })()`,
+    )
+    let b64Size = 0
+    for (let i = 0; i < 60; i++) {
+      await sleep(500)
+      const probe = await evalInPage<{ ready: boolean; size: number }>(
+        bridge,
+        `(() => { const s = window.__sootsimFilm; return { ready: !!(s && s.b64), size: s && s.b64 ? s.b64.length : 0 } })()`,
+      )
+      if (probe?.ready && probe.size > 0) {
+        b64Size = probe.size
+        break
+      }
+    }
+    if (!b64Size) {
+      console.error('  film: stage recording never finished encoding')
+      return 1
+    }
+
+    // stream the base64 back in chunks (bridge messages cap out on very
+    // large single payloads)
+    const CHUNK = 2_400_000
+    let b64 = ''
+    for (let offset = 0; offset < b64Size; offset += CHUNK) {
+      const part = await evalInPage<{ data: string }>(
+        bridge,
+        `(() => ({ data: window.__sootsimFilm.b64.substr(${offset}, ${CHUNK}) }))()`,
+      )
+      b64 += part.data
+    }
+    await evalInPage(
+      bridge,
+      `(() => { try { delete window.__sootsimFilm } catch {} return true })()`,
+    )
+
+    const webmBytes = Buffer.from(b64, 'base64')
+    mkdirSync(dirname(outPath), { recursive: true })
+    if (wantsMp4) {
+      const tmpWebm = `${outPath}.tmp.webm`
+      writeFileSync(tmpWebm, webmBytes)
+      const ff = spawnSync(
+        'ffmpeg',
+        [
+          '-y',
+          '-i',
+          tmpWebm,
+          '-c:v',
+          'libx264',
+          // libx264 + yuv420p requires even dimensions; the stage canvas is
+          // whatever the window layout produced (e.g. 750x987)
+          '-vf',
+          'scale=trunc(iw/2)*2:trunc(ih/2)*2',
+          '-pix_fmt',
+          'yuv420p',
+          '-movflags',
+          '+faststart',
+          outPath,
+        ],
+        { stdio: ['ignore', 'ignore', 'pipe'] },
+      )
+      if (ff.status !== 0) {
+        // keep the source webm so the capture isn't lost
+        const tail = String(ff.stderr ?? '')
+          .split('\n')
+          .filter(Boolean)
+          .slice(-4)
+          .join('\n  ')
+        console.error(`  film: ffmpeg transcode failed:\n  ${tail}`)
+        console.error(`  film: source kept at ${tmpWebm}`)
+        return 1
+      }
+      unlinkSync(tmpWebm)
+    } else {
+      writeFileSync(outPath, webmBytes)
+    }
+
+    // -- restore shell state ----------------------------------------------
+    if (!keepThree) {
+      if (!before?.editor) {
+        await evalInPage(
+          bridge,
+          `window.SootSim?.bridges?.settings?.set?.('threeRecordingEditor', false)`,
+        )
+      }
+      if (!before?.threeMode) {
+        await evalInPage(
+          bridge,
+          `window.SootSim?.bridges?.settings?.set?.('threeMode', false)`,
+        )
+      }
+    }
+
+    console.log(
+      `  film: saved ${outPath} (${(webmBytes.length / 1e6).toFixed(1)}MB source)`,
+    )
+    return 0
+  } finally {
+    bridge.close()
+  }
+}