storyboard-bridge 0.6.0 → 0.7.0

package/index.mjs

@@ -40,6 +40,8 @@ const VERSION = createRequire(import.meta.url)('./package.json').version;
 // in-flight Higgsfield render children, keyed by job id — so a `higgsfield-cancel` can kill the right one
 // (the backend sends it when the user cancels, freeing the single render slot instead of leaving it running).
 const hfChildren = new Map();
+// in-flight tryinfer.com render children, keyed by job id — so a `tryinfer-cancel` kills the right one.
+const tfChildren = new Map();
 // in-flight official-CLI render children (the `higgsfield-cli` credit-billed path — separate from the CDP one).
 const hfCliChildren = new Map();
 
@@ -60,6 +62,8 @@ const IS_WIN = process.platform === 'win32';
 // debug tab + python + the Higgsfield login). Opt in with --higgsfield; the backend routes everyone's
 // Higgsfield jobs to a bridge that advertises this.
 const HIGGSFIELD = process.argv.includes('--higgsfield') || process.env.STORYBOARD_HIGGSFIELD === '1';
+// --infer → this machine is the project's tryinfer.com Seedance 2.0 host (its logged-in Chrome debug tab).
+const INFER = process.argv.includes('--infer') || process.env.STORYBOARD_INFER === '1';
 // Auto-confirm Higgsfield's copyright/likeness "rights verification" gate on a rendered clip (otherwise a
 // gated clip stalls until timeout). ON by default — it asserts you own the rights to YOUR generated content.
 // Opt out with --no-own-rights or STORYBOARD_OWN_RIGHTS=0.
@@ -69,6 +73,7 @@ const PYTHON_BIN = arg('python-bin', process.env.PYTHON_BIN || (IS_WIN ? 'python
 const HF_CLI = arg('higgsfield-cli-bin', process.env.HIGGSFIELD_CLI_BIN || 'higgsfield');
 const CDP_HTTP = arg('cdp', process.env.CDP_HTTP || 'http://localhost:9222'); // the debug Chrome endpoint
 const HF_SCRIPT = join(dirname(fileURLToPath(import.meta.url)), 'higgsfield_client.py');
+const TF_SCRIPT = join(dirname(fileURLToPath(import.meta.url)), 'tryinfer_client.py');
 // claude supports `--output-format json` (wraps as {result}); gemini's CLI (<=0.1.x) does NOT — it
 // prints raw text, and passing the flag makes it dump --help. Newer gemini supports it (wraps as
 // {response}) — opt in with GEMINI_JSON=1. extractText() handles wrapped OR raw either way.
@@ -193,15 +198,19 @@ const YTDLP_VIDEO_EXTS = ['mp4', 'webm', 'mov', 'm4v', 'mkv', 'ogv'];
 // cap what we ship back over the socket — base64 inflates ~1.37×, and the ws frame limit is 100 MiB, so
 // 60 MB of video → ~82 MB frame, safely under. Short social clips are far smaller; big files → download+upload.
 const YTDLP_MAX_BYTES = 60 * 1024 * 1024;
-async function runYtdlp(url) {
+// Cap RESOLUTION, not filesize. X/HLS only report size after probing, so `--max-filesize` ABORTS m3u8
+// streams ("File is larger than max-filesize") before downloading at all. Picking best mp4 <=1080p keeps
+// the file small enough to relay (a clip's 1440p variant can be ~100 MB, its 720p ~17 MB); the post-download
+// byte guard below is the real backstop. Overridable per job via `format`.
+const YTDLP_FMT = 'b[ext=mp4][height<=1080]/b[height<=1080]/bv*[height<=1080][ext=mp4]+ba/b[ext=mp4]/b';
+async function runYtdlp(url, format) {
   if (!/^https?:\/\//i.test(String(url || ''))) throw new Error('provide an http(s) URL');
   const dir = await mkdtemp(join(tmpdir(), 'sb-ytdlp-'));
   try {
     const args = [
       '--no-playlist', '--no-progress', '--no-warnings', '--restrict-filenames',
-      '-f', 'b[ext=mp4]/bv*[ext=mp4]+ba/b', '--merge-output-format', 'mp4',
-      // keep clips reasonable for the socket round-trip
-      '--max-filesize', String(YTDLP_MAX_BYTES),
+      '-f', (typeof format === 'string' && format.trim()) ? format.trim() : YTDLP_FMT,
+      '--merge-output-format', 'mp4',
       '-o', join(dir, '%(title).80B.%(ext)s'), String(url),
     ];
     await new Promise((res, rej) => {
@@ -302,6 +311,50 @@ async function handleHiggsfield(msg, ws) {
   }
 }
 
+// ---- tryinfer.com i2v: run the local Python client (CDP → logged-in Chrome debug tab) and stream its
+// NDJSON progress/done/error events back. Reference images arrive as PUBLIC URLs (tryinfer fetches them),
+// so — unlike Higgsfield — there's no image upload / temp files: we just pass --image-url per ref. ----
+function handleTryinfer(msg, ws) {
+  const id = msg.id;
+  const p = msg.params || {};
+  try {
+    const args = ['-u', TF_SCRIPT, '--emit-json'];
+    if (p.resumeTaskId) {
+      args.push('--resume-task', String(p.resumeTaskId)); // poll an existing task, no submit
+    } else {
+      args.push('--prompt', String(p.prompt ?? ''),
+        '--duration', String(Math.max(1, Math.round(p.durationSec || 5))),
+        '--aspect', String(p.aspectRatio || '1:1'),
+        '--resolution', String(p.resolution || '1080p'));
+      if (p.audio === false) args.push('--no-audio');
+      for (const u of (Array.isArray(p.imageUrls) ? p.imageUrls : [])) args.push('--image-url', String(u));
+    }
+    const child = spawn(PYTHON_BIN, args, { env: { ...process.env, CDP_HTTP }, stdio: ['ignore', 'pipe', 'pipe'] });
+    tfChildren.set(id, child);
+    let buf = '', sawDone = false, stderrTail = '';
+    child.stdout.on('data', (d) => {
+      buf += d.toString();
+      let nl;
+      while ((nl = buf.indexOf('\n')) >= 0) {
+        const line = buf.slice(0, nl).trim();
+        buf = buf.slice(nl + 1);
+        if (!line) continue;
+        let ev; try { ev = JSON.parse(line); } catch { continue; }
+        if (ev.event === 'done') sawDone = true;
+        ws.send(JSON.stringify({ type: 'tryinferEvent', id, ...ev }));
+      }
+    });
+    child.stderr.on('data', (d) => { stderrTail = (stderrTail + d.toString()).slice(-8000); });
+    child.on('error', (e) => { tfChildren.delete(id); ws.send(JSON.stringify({ type: 'tryinferEvent', id, event: 'error', reason: 'spawn', detail: `${PYTHON_BIN}: ${e.message}` })); });
+    child.on('close', (code) => {
+      tfChildren.delete(id);
+      if (!sawDone && code !== 0) ws.send(JSON.stringify({ type: 'tryinferEvent', id, event: 'error', reason: 'exit', detail: `python exited ${code}: ${stderrTail}` }));
+    });
+  } catch (e) {
+    ws.send(JSON.stringify({ type: 'tryinferEvent', id, event: 'error', reason: 'bridge', detail: String(e?.message ?? e) }));
+  }
+}
+
 // Pull the result video URL out of the CLI's `--json` output. The exact schema isn't publicly documented,
 // so scan defensively: each line (the CLI may print logs then a final JSON object), then known fields, then
 // any media URL. On a miss the caller returns the raw tail in the error so we can lock the field fast.
@@ -420,7 +473,7 @@ function connect() {
       gemini: await cliAvailable(BIN_FOR.gemini),
     };
     const hfCli = await cliAvailable(HF_CLI); // official `higgsfield` CLI present + runnable → can serve credit-billed renders
-    ws.send(JSON.stringify({ type: 'hello', version: VERSION, provider: PROVIDER, providers, files: !!FILES_ROOT, higgsfield: HIGGSFIELD, higgsfieldCli: hfCli, inflight: [...inflightJobs] }));
+    ws.send(JSON.stringify({ type: 'hello', version: VERSION, provider: PROVIDER, providers, files: !!FILES_ROOT, higgsfield: HIGGSFIELD, higgsfieldCli: hfCli, infer: INFER, inflight: [...inflightJobs] }));
     log(`connected to ${url} — serving jobs with: ${PROVIDER} (${BIN})${FILES_ROOT ? `; files → ${FILES_ROOT}` : ''}${HIGGSFIELD ? `; HIGGSFIELD HOST (python: ${PYTHON_BIN}, cdp: ${CDP_HTTP})` : ''}${hfCli ? `; HIGGSFIELD CLI (${HF_CLI})` : ''}`);
     if (!providers[PROVIDER]) {
       log(`WARNING: '${BIN}' was not found / not runnable. Install it and log in, or pass --provider/--*_BIN. Jobs will fail until then.`);
@@ -467,7 +520,7 @@ function connect() {
       }
     } else if (msg.type === 'ytdlp' && msg.id) {
       log(`ytdlp ${msg.id} → ${String(msg.url || '').slice(0, 80)}`);
-      runYtdlp(msg.url).then(
+      runYtdlp(msg.url, msg.format).then(
         (r) => { deliver({ type: 'ytdlpResult', id: msg.id, data: r.data, ext: r.ext, title: r.title }); log(`ytdlp ${msg.id} ✓ ${r.ext} (${Math.round(r.data.length * 0.75 / 1024)}KB)`); },
         (e) => { deliver({ type: 'ytdlpError', id: msg.id, message: String(e?.message ?? e) }); log(`ytdlp ${msg.id} ✗ ${e?.message ?? e}`); },
       );
@@ -483,6 +536,12 @@ function connect() {
     } else if (msg.type === 'higgsfield-cli-cancel' && msg.id) {
       const child = hfCliChildren.get(msg.id);
       if (child) { log(`higgsfield-cli ${msg.id} → cancel`); try { child.kill('SIGTERM'); } catch { /* already gone */ } hfCliChildren.delete(msg.id); }
+    } else if (msg.type === 'tryinfer' && msg.id) {
+      if (!INFER) ws.send(JSON.stringify({ type: 'tryinferEvent', id: msg.id, event: 'error', reason: 'disabled', detail: 'this bridge was not started with --infer' }));
+      else { log(`tryinfer ${msg.id} → running`); handleTryinfer(msg, ws); }
+    } else if (msg.type === 'tryinfer-cancel' && msg.id) {
+      const child = tfChildren.get(msg.id);
+      if (child) { log(`tryinfer ${msg.id} → cancel`); try { child.kill('SIGTERM'); } catch { /* already gone */ } tfChildren.delete(msg.id); }
     }
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "storyboard-bridge",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Desktop bridge that powers a hosted Storyboard AI webapp with your own local Claude Code / Gemini CLI login (no API key).",
   "type": "module",
   "bin": {

package/tryinfer_client.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+"""
+tryinfer.com (Infer) Seedance 2.0 reference-to-video, driven through your ALREADY-LOGGED-IN Chrome tab
+over the DevTools Protocol — the same in-page-fetch trick as higgsfield_client.py. Every /api call runs
+as fetch() INSIDE the real tab, so tryinfer's Clerk SESSION COOKIE rides along automatically (its API is
+cookie-authed — no bearer, no token minting).
+
+We pass reference_image_urls as ARBITRARY PUBLIC URLs (verified: tryinfer fetches them), so there is NO
+upload step — the backend publishes each board to a public R2 URL and hands us the URL.
+
+Flow (from bridge/tryinfer_capture.json):
+  POST /api/create/generation-tasks         {args:{model,capability,input:{reference_image_urls,prompt,
+                                             duration_seconds,aspect_ratio,resolution,audio}}, group:{…}}
+                                             -> {id, status:"PENDING"}
+  GET  /api/create/generation-tasks/{id}     -> status PENDING|RUNNING|SUCCEEDED|FAILED
+  GET  /api/create/generation-tasks/{id}/result -> {output:{video_url, width, height, …}}
+
+Two run modes:
+  • --emit-json : machine mode for the desktop bridge — NDJSON events on stdout, human logs on stderr.
+  • (default)   : standalone test — human logs on stderr, final JSON on stdout.
+
+Deps (same Windows Python as the Higgsfield bridge): websocket-client
+Usage (standalone test):
+  python tryinfer_client.py --prompt "a slow cinematic push-in" [--image-url URL] [--duration 5] \
+      [--aspect 1:1] [--resolution 1080p] [--no-audio]
+"""
+import argparse
+import json
+import sys
+import time
+import urllib.request
+import uuid
+
+try:
+    import websocket  # websocket-client
+except ImportError:
+    sys.exit("Missing dep. Install with:  pip install websocket-client")
+
+
+def log(*a):
+    print(*a, file=sys.stderr, flush=True)
+
+
+# NDJSON progress protocol (only active in --emit-json). The backend's TryinferProvider parses these to
+# drive the node's live status: {"event":"progress","phase":…}, {"event":"done","taskId","videoUrl"}, error.
+_JSON_OUT = None
+
+
+def event(**fields):
+    if _JSON_OUT is not None:
+        _JSON_OUT.write(json.dumps(fields) + "\n")
+        _JSON_OUT.flush()
+
+
+def http_json(url):
+    with urllib.request.urlopen(url, timeout=10) as r:
+        return json.loads(r.read().decode())
+
+
+# ---- minimal synchronous CDP client (same shape as higgsfield_client.py) ----
+class CDP:
+    def __init__(self, ws_url):
+        self.ws = websocket.create_connection(ws_url, max_size=None, timeout=30)
+        self._id = 0
+
+    def send(self, method, params=None, session_id=None):
+        self._id += 1
+        msg = {"id": self._id, "method": method, "params": params or {}}
+        if session_id:
+            msg["sessionId"] = session_id
+        self.ws.send(json.dumps(msg))
+        return self._id
+
+    def wait_for(self, msg_id, timeout=30):
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            self.ws.settimeout(max(0.1, deadline - time.time()))
+            try:
+                msg = json.loads(self.ws.recv())
+            except websocket.WebSocketTimeoutException:
+                break
+            if msg.get("id") == msg_id:
+                if "error" in msg:
+                    raise RuntimeError(f"CDP error for {msg_id}: {msg['error']}")
+                return msg.get("result", {})
+        raise TimeoutError(f"Timed out waiting for CDP response id={msg_id}")
+
+    def call(self, method, params=None, session_id=None, timeout=30):
+        return self.wait_for(self.send(method, params, session_id), timeout)
+
+    def close(self):
+        try:
+            self.ws.close()
+        except Exception:
+            pass
+
+
+class PageResponse:
+    def __init__(self, status, text):
+        self.status_code = status
+        self.text = text
+
+    def json(self):
+        return json.loads(self.text)
+
+
+def _fetch_expr(method, url, body):
+    """fetch() in the page context. credentials:'include' → the Clerk session cookie rides along
+    (tryinfer's API is cookie-authed, so no Authorization bearer is needed, unlike Higgsfield)."""
+    parts = [
+        "(async()=>{try{",
+        f"const o={{method:{json.dumps(method)},headers:{{}},credentials:'include'}};",
+    ]
+    if body is not None:
+        parts.append("o.headers['content-type']='application/json';")
+        parts.append(f"o.body=JSON.stringify({json.dumps(body)});")
+    parts += [
+        f"const r=await fetch({json.dumps(url)},o);",
+        "const x=await r.text();",
+        "return JSON.stringify({status:r.status,body:x});",
+        "}catch(e){return JSON.stringify({status:0,error:String((e&&e.message)||e)});}})()",
+    ]
+    return "".join(parts)
+
+
+class BrowserSession:
+    def __init__(self, cdp_http, match):
+        ver = http_json(f"{cdp_http}/json/version")
+        log(f"Connected to {ver.get('Browser')}")
+        tid = url = None
+        for t in http_json(f"{cdp_http}/json/list"):
+            if t.get("type") == "page" and match in (t.get("url") or ""):
+                tid, url = t["id"], t["url"]
+                break
+        if not tid:
+            raise RuntimeError(f"No open tab whose URL contains '{match}'. Open tryinfer.com and log in.")
+        log(f"Routing API calls through tab: {url}")
+        self.cdp = CDP(ver["webSocketDebuggerUrl"])
+        self.session_id = self.cdp.call("Target.attachToTarget", {"targetId": tid, "flatten": True})["sessionId"]
+        self.cdp.call("Runtime.enable", session_id=self.session_id)
+
+    def request(self, method, url, body=None, timeout=120, retries=3):
+        last = None
+        for attempt in range(retries):
+            res = self.cdp.call(
+                "Runtime.evaluate",
+                {"expression": _fetch_expr(method, url, body), "awaitPromise": True, "returnByValue": True},
+                session_id=self.session_id, timeout=timeout)
+            val = res.get("result", {}).get("value")
+            if val is None:
+                last = f"evaluate failed: {res.get('exceptionDetails')}"
+            else:
+                data = json.loads(val)
+                if data.get("status") == 0:
+                    last = f"in-page fetch error: {data.get('error')}"
+                else:
+                    return PageResponse(data["status"], data["body"])
+            if attempt < retries - 1:
+                log(f"   transient request error, retrying… ({last})")
+                time.sleep(3)
+        raise RuntimeError(last)
+
+    def close(self):
+        try:
+            self.cdp.call("Target.detachFromTarget", {"sessionId": self.session_id}, timeout=5)
+        except Exception:
+            pass
+        self.cdp.close()
+
+
+API = "https://tryinfer.com/api"
+RATIO_WORD = {"1:1": "square", "16:9": "landscape", "9:16": "portrait", "4:3": "landscape", "3:4": "portrait"}
+DONE = {"SUCCEEDED", "COMPLETED"}
+FAILED = {"FAILED", "ERROR", "CANCELLED", "CANCELED"}
+
+
+def submit(session, prompt, image_urls, duration, aspect, resolution, audio):
+    """POST the generation task; reference_image_urls = the public URLs we pass. Returns the task id."""
+    body = {
+        "args": {
+            "model": "seedance-2.0-pro",
+            "capability": "reference-to-video",
+            "input": {
+                "reference_image_urls": image_urls,
+                "prompt": prompt,
+                "duration_seconds": duration,
+                "aspect_ratio": aspect,
+                "resolution": resolution,
+                "audio": audio,
+            },
+        },
+        "group": {
+            "groupId": str(uuid.uuid4()),
+            "position": 0,
+            "meta": {
+                "prompt": prompt, "medium": "video", "ratio": RATIO_WORD.get(aspect, "square"),
+                "kind": "reference", "referenceImageUrls": image_urls, "referenceRequestIds": [],
+            },
+        },
+    }
+    r = session.request("POST", f"{API}/create/generation-tasks", body=body)
+    if r.status_code not in (200, 201, 202):
+        raise RuntimeError(f"submit failed ({r.status_code}): {r.text[:800]}")
+    tid = r.json().get("id")
+    if not tid:
+        raise RuntimeError(f"submit returned no task id: {r.text[:400]}")
+    return tid
+
+
+def poll(session, task_id, on_status=None, timeout=1800, interval=3):
+    """Poll the task to a terminal state. Fires on_status(status) on each change. Returns final task JSON."""
+    deadline = time.time() + timeout
+    last = None
+    while time.time() < deadline:
+        j = session.request("GET", f"{API}/create/generation-tasks/{task_id}").json()
+        st = j.get("status")
+        if st != last:
+            last = st
+            if on_status:
+                on_status(st)
+        if st in DONE:
+            return j
+        if st in FAILED:
+            err = j.get("error")
+            raise RuntimeError(f"task {st}: {json.dumps(err) if err else 'no detail'}")
+        time.sleep(interval)
+    raise TimeoutError(f"task {task_id} did not finish within {timeout}s (last status {last})")
+
+
+def get_result(session, task_id):
+    """GET the result; return (video_url, output_dict). Raises on moderation block / missing url."""
+    j = session.request("GET", f"{API}/create/generation-tasks/{task_id}/result").json()
+    mod = j.get("moderation_status")
+    if mod and mod != "allowed":
+        raise RuntimeError(f"moderation blocked: {mod}")
+    out = j.get("output") or {}
+    url = out.get("video_url")
+    if not url:
+        raise RuntimeError(f"no video_url in result: {json.dumps(j)[:800]}")
+    return url, out
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--prompt", default="")
+    ap.add_argument("--image-url", action="append", default=[],
+                    help="public reference image URL; repeat for ordered refs. Default (test) is gstatic's sample JPEG.")
+    ap.add_argument("--duration", type=int, default=5)
+    ap.add_argument("--aspect", default="1:1")
+    ap.add_argument("--resolution", default="1080p")
+    ap.add_argument("--no-audio", action="store_true")
+    ap.add_argument("--resume-task", default=None, help="skip submit; just poll+result an existing task id")
+    ap.add_argument("--emit-json", action="store_true", help="NDJSON events on stdout, human logs on stderr")
+    ap.add_argument("--match", default="tryinfer", help="substring of the logged-in tab's URL")
+    ap.add_argument("--host", default="localhost")
+    ap.add_argument("--port", type=int, default=9222)
+    args = ap.parse_args()
+
+    if args.emit_json:
+        global _JSON_OUT
+        _JSON_OUT = sys.stdout
+        sys.stdout = sys.stderr  # any stray print() can't corrupt the event stream
+
+    image_urls = args.image_url or ["https://www.gstatic.com/webp/gallery/1.jpg"]  # default = permissive test image
+    sess = BrowserSession(f"http://{args.host}:{args.port}", args.match)
+    try:
+        if args.resume_task:
+            task_id = args.resume_task
+            log(f"resuming task {task_id}")
+            event(event="progress", phase="rendering", taskId=task_id)
+        else:
+            task_id = submit(sess, args.prompt, image_urls, args.duration, args.aspect, args.resolution, not args.no_audio)
+            log(f"submitted task {task_id}")
+            event(event="progress", phase="submitted", taskId=task_id)
+        poll(sess, task_id, on_status=lambda st: (log(f"   status={st}"), event(event="progress", phase=st.lower(), taskId=task_id)))
+        video_url, out = get_result(sess, task_id)
+    except Exception as e:
+        event(event="error", reason="tryinfer", detail=str(e))
+        log(f"\n❌ {e}")
+        sys.exit(1)
+    finally:
+        sess.close()
+
+    event(event="done", taskId=task_id, videoUrl=video_url,
+          width=out.get("width"), height=out.get("height"), duration=out.get("duration_seconds"))
+    log(f"\n✅ {out.get('width')}x{out.get('height')}  {out.get('duration_seconds')}s")
+    if not args.emit_json:
+        print(json.dumps({"ok": True, "video_url": video_url, "width": out.get("width"),
+                          "height": out.get("height"), "duration": out.get("duration_seconds")}))
+
+
+if __name__ == "__main__":
+    main()