retake-dev 0.4.0

package/src/core.js

@@ -0,0 +1,265 @@
+// What every way of serving Retake shares: the runtime (as source, or as the
+// script tag a server injects into a frame's page), the dock page, and an HTML
+// stream transform that puts the runtime first in <head>. The Vite plugin, the
+// front server (server/front.js) and the site's deployed build (apps/site) all use these.
+import crypto from "node:crypto"
+import fs from "node:fs"
+import path from "node:path"
+import zlib from "node:zlib"
+import { Transform } from "node:stream"
+import { fileURLToPath } from "node:url"
+
+const SRC = path.dirname(fileURLToPath(import.meta.url))
+const read = (...p) => fs.readFileSync(path.join(SRC, ...p), "utf8")
+
+// How the runtime is set up for the server it runs behind (`RT` in the
+// runtime). The defaults are the Vite plugin's: a frame marked by `?__wb=app`.
+//   marker:      "url" (the frame's URL carries `__wb=app`) or "header" (the front
+//                server knows the dock's frames by Sec-Fetch-Dest; no URL marker)
+//   bootAt:      "dcl" (the clock starts at DOMContentLoaded) or "load" (F47)
+//   exemptUrls:  the dev server's own traffic, as regular expressions (F54)
+//   holdScripts: scripts added later are held to their recorded moment (F48)
+//   next:        the Next.js major version, for its debug channel (F49)
+//   docId, docStored: this page's kept copy on the front server, and whether
+//                this is it (F56)
+// The front server's setup is frontRuntime() in server/detect.js.
+export const RT_DEFAULTS = Object.freeze({ marker: "url", bootAt: "dcl", exemptUrls: [], holdScripts: false, next: null, docId: null, docStored: false })
+
+// The files of a concatenated set ("runtime" or "shell"), in the order they
+// run: later files may use earlier files' top-level bindings. (The type check,
+// scripts/typecheck.mjs, concatenates them the same way.)
+export const setFiles = (set) => fs.readdirSync(path.join(SRC, set)).filter((f) => f.endsWith(".js")).sort()
+
+// Read on every page load, so edits to the tool apply on refresh.
+/** @param {import("../types/index.js").RuntimeConfig} [rt] */
+export function runtimeSource(rt = {}) {
+  const files = setFiles("runtime")
+  const body = files.map((f) => `// ---- ${f}\n${read("runtime", f)}`).join("\n")
+  const config = JSON.stringify({ ...RT_DEFAULTS, ...rt })
+  return `;(function () {\n"use strict";\nif (window.__retake) return;\nconst RT = Object.freeze(${config});\n${body}\n})();`
+}
+
+/** @param {import("../types/index.js").ShellOptions} [options] */
+export function shellHtml(options = {}) {
+  const config = {
+    codeBranches: !!options.codeBranches,
+    features: ["continue", "segments", "serialize"],
+    ...(options.appPage ? { appPage: options.appPage } : {}),
+    ...(options.marker && options.marker !== "url" ? { marker: options.marker } : {}),
+    // Behind the front server: rebuilds ask for the page as it was recorded (F56).
+    ...(options.docs ? { docs: true } : {}),
+  }
+  return read("shell", "shell.html")
+    .replace(
+      "/*CONFIG*/",
+      () => `window.__retakeConfig = ${JSON.stringify(config)};` + (options.token ? `window.__RETAKE_TOKEN = ${JSON.stringify(options.token)};` : ""),
+    )
+    .replace("/*CSS*/", () => read("shell", "shell.css"))
+    .replace("/*JS*/", () => {
+      const files = setFiles("shell")
+      return `;(function () {\n"use strict";\n${files.map((f) => read("shell", f)).join("\n")}\n})();`
+    })
+}
+
+// The script a server injects into a page it can't tell apart from the app's
+// own frames (any iframe document). It takes itself out of the DOM first (so an
+// app hydrating the whole document never sees it), then runs the runtime only
+// in a frame the dock made (`isAppFrame`, or, with the URL marker, a URL with
+// `__wb=app`). Anywhere else `__retake` is `{ inert: true }` and the runtime
+// returns at its first line.
+/** @param {import("../types/index.js").RuntimeConfig} [rt] */
+export function runtimeScript(rt = {}) {
+  const marker = rt.marker || RT_DEFAULTS.marker
+  const guard =
+    `;(function(){try{var d=document.currentScript;d&&d.remove()}catch(e){}` +
+    `var ok=false;try{var s=window.parent!==window&&window.parent.__retakeShell;` +
+    `ok=!!s&&((typeof s.isAppFrame==="function"&&s.isAppFrame(window.frameElement))` +
+    (marker === "url" ? `||/[?&]__wb=app(&|$)/.test(location.search)` : "") +
+    `)}catch(e){}if(!ok)window.__retake={inert:true}})();`
+  return `${guard}\n${runtimeSource(rt)}`
+}
+
+/** @param {import("../types/index.js").RuntimeTagOptions} [options] */
+export function runtimeTag({ rt = {}, nonce = null, script = runtimeScript(rt) } = {}) {
+  return `<script data-retake${nonce ? ` nonce="${String(nonce).replace(/"/g, "&quot;")}"` : ""}>${script}</script>`
+}
+
+// For a CSP without 'unsafe-inline': the hash source that allows this script.
+export const scriptHash = (script) => `'sha256-${crypto.createHash("sha256").update(script, "utf8").digest("base64")}'`
+
+const HOLD_MAX = 64 * 1024
+
+// Where the tag goes in the HTML seen so far, or -1 to wait for more. `s` is
+// the bytes as latin1 (one char per byte), so indices are byte offsets and a
+// multi-byte character is never split.
+function insertAt(raw, final) {
+  // Comments are blanked out (same length) first: a `<head>` written inside one
+  // (before the real one) isn't the head. One still open at the end hides the rest.
+  const s = raw.replace(/<!--[\s\S]*?(?:-->|$)/g, (c) => " ".repeat(c.length))
+  const head = /<head(?=[\s>/])[^>]*>/i.exec(s)
+  if (head) {
+    let at = head.index + head[0].length
+    // A <meta charset> right after it stays first (it must be in the first 1024 bytes).
+    const rest = s.slice(at)
+    const meta = /<meta\b[^>]*\bcharset\s*=[^>]*>/i.exec(rest)
+    if (meta && !/<script\b/i.test(rest.slice(0, meta.index))) at += meta.index + meta[0].length
+    return at
+  }
+  // An opening <head that hasn't closed yet: wait for it.
+  if (!final && /<head(?:[\s/][^>]*)?$/i.test(s)) return -1
+  const first = s.search(/<(body|script|link|meta|style|title)\b/i)
+  if (first >= 0) return first
+  if (!final && s.length < HOLD_MAX) return -1
+  // Nothing to go by (the hold cap, or the page ended): after the doctype,
+  // not before it (that would put the page in quirks mode).
+  const doc = /^\s*(?:<!--[\s\S]*?-->\s*)*<!doctype[^>]*>/i.exec(raw)
+  return doc ? doc[0].length : 0
+}
+
+const decompressor = (encoding) => {
+  const e = String(encoding || "").trim().toLowerCase()
+  if (!e || e === "identity") return null
+  if (e === "gzip" || e === "x-gzip") return zlib.createGunzip()
+  if (e === "br") return zlib.createBrotliDecompress()
+  if (e === "deflate") return zlib.createInflate()
+  if (e === "zstd" && zlib.createZstdDecompress) return zlib.createZstdDecompress()
+  throw new Error(`can't decode content-encoding ${encoding}`)
+}
+
+/**
+ * A byte stream transform that inserts `tag` into an HTML document as it
+ * streams: after the `<head …>` opening tag (and a `<meta charset>` right
+ * after it), else before the first <body>/<script>/<link>/<meta>. It holds at
+ * most 64 KB back; past that, or if the page ends first, the tag goes after
+ * the doctype. Everything after the insertion point passes straight through.
+ * @param {string} tag
+ * @param {{ encoding?: string }} [options] the body's content-encoding (gzip,
+ *   br, deflate): it's decoded first, and the output is plain.
+ */
+export function injectHtml(tag, { encoding } = {}) {
+  const tagBytes = Buffer.from(tag, "utf8")
+  let held = []
+  let heldLen = 0
+  let done = false
+  const feed = (chunk, final) => {
+    if (done) return chunk
+    if (chunk && chunk.length) {
+      held.push(chunk)
+      heldLen += chunk.length
+    }
+    const buf = Buffer.concat(held, heldLen)
+    const at = insertAt(buf.toString("latin1"), final)
+    if (at < 0) {
+      held = [buf]
+      return null
+    }
+    done = true
+    held = []
+    return Buffer.concat([buf.subarray(0, at), tagBytes, buf.subarray(at)])
+  }
+  const unzip = decompressor(encoding)
+  if (!unzip) {
+    return new Transform({
+      transform(chunk, _, cb) {
+        cb(null, feed(chunk, false) || undefined)
+      },
+      flush(cb) {
+        cb(null, feed(null, true) || undefined)
+      },
+    })
+  }
+  /** @type {import("node:stream").TransformCallback | null} */
+  let flushed = null
+  const t = new Transform({
+    transform(chunk, _, cb) {
+      unzip.write(chunk) ? cb() : unzip.once("drain", cb)
+    },
+    flush(cb) {
+      flushed = cb
+      unzip.end()
+    },
+  })
+  unzip.on("data", (d) => {
+    const out = feed(d, false)
+    if (out && out.length) t.push(out)
+  })
+  unzip.on("end", () => {
+    const out = feed(null, true)
+    if (out && out.length) t.push(out)
+    if (flushed) flushed()
+  })
+  unzip.on("error", (err) => t.destroy(err))
+  return t
+}
+
+/**
+ * Runs an HTML response that a server writes in its own way (res.writeHead,
+ * write, end, or a stream piped into res) through injectHtml. Whether it's
+ * HTML is decided at the first write, from its Content-Type; anything else
+ * passes through untouched.
+ * @param {import("node:http").ServerResponse} res
+ * @param {string} tag
+ */
+export function injectResponse(res, tag) {
+  const write = res.write.bind(res)
+  const end = res.end.bind(res)
+  const writeHead = res.writeHead.bind(res)
+  /** @type {Transform | null} */
+  let t = null
+  let decided = false
+  const decide = () => {
+    if (decided) return
+    decided = true
+    if (!/^\s*text\/html\b/i.test(String(res.getHeader("content-type") || "")) || res.statusCode === 204 || res.statusCode === 304 || res.req?.method === "HEAD") return
+    const encoding = String(res.getHeader("content-encoding") || "")
+    for (const h of ["content-length", "content-encoding", "etag"]) res.removeHeader(h)
+    res.setHeader("cache-control", "no-store")
+    let out
+    try {
+      out = t = injectHtml(tag, { encoding })
+    } catch {
+      return
+    }
+    out.on("data", (c) => {
+      if (!write(c)) {
+        out.pause()
+        res.once("drain", () => out.resume())
+      }
+    })
+    out.on("end", () => end())
+    out.on("error", () => res.destroy())
+  }
+  res.writeHead = /** @type {any} */ (function (status, message, headers) {
+    if (typeof message !== "string") {
+      headers = message
+      message = undefined
+    }
+    // Headers given here are folded in first, so they can be changed.
+    if (Array.isArray(headers)) for (let i = 0; i < headers.length; i += 2) res.setHeader(headers[i], headers[i + 1])
+    else if (headers) for (const [k, v] of Object.entries(headers)) res.setHeader(k, v)
+    res.statusCode = status
+    if (message) res.statusMessage = message
+    decide()
+    return writeHead(res.statusCode)
+  })
+  const args = (chunk, encoding, cb) => {
+    if (typeof chunk === "function") return [null, null, chunk]
+    if (typeof encoding === "function") return [chunk, null, encoding]
+    return [chunk, encoding, cb]
+  }
+  res.write = /** @type {any} */ (function (chunk, encoding, cb) {
+    decide()
+    if (!t) return write(chunk, encoding, cb)
+    ;[chunk, encoding, cb] = args(chunk, encoding, cb)
+    return t.write(typeof chunk === "string" ? Buffer.from(chunk, encoding || "utf8") : chunk, cb)
+  })
+  res.end = /** @type {any} */ (function (chunk, encoding, cb) {
+    decide()
+    if (!t) return end(chunk, encoding, cb)
+    ;[chunk, encoding, cb] = args(chunk, encoding, cb)
+    if (chunk != null && chunk.length) t.write(typeof chunk === "string" ? Buffer.from(chunk, encoding || "utf8") : chunk)
+    if (cb) res.once("finish", cb)
+    t.end()
+    return res
+  })
+}

package/src/plugin.js

@@ -0,0 +1,137 @@
+// Vite plugin. A page request gets the dock: a small shell page with the
+// timeline docked at the bottom (like DevTools) and the prototype in a frame
+// above it. The frame's own request (`?__wb=app`) gets the real page with the
+// time runtime injected as its first script, before any app code. Dev only.
+import { AsyncLocalStorage } from "node:async_hooks"
+import crypto from "node:crypto"
+import fs from "node:fs"
+import path from "node:path"
+import { codeVersions } from "./code-versions.js"
+import { injectResponse, runtimeSource, runtimeTag, shellHtml } from "./core.js"
+import { createBus, sessionApi } from "./server/api.js"
+import { frontRuntime } from "./server/detect.js"
+
+// The site's deployed build (apps/site/app/retake-dock, retake-runtime) and the tests import these from here.
+export { runtimeSource, shellHtml, runtimeTag, injectHtml } from "./core.js"
+
+const NESTED_DEST = new Set(["iframe", "frame", "embed", "object"])
+const pageKind = new AsyncLocalStorage()
+
+/**
+ * @param {import("../types/index.js").RetakeOptions} [options]
+ *   `codeBranches`: each timeline keeps its own version of the code; stepping
+ *   into a timeline checks its code out on disk (see code-versions.js).
+ *   `root`: where .retake/ goes (default: Vite's root).
+ *   Public types: types/index.d.ts (keep them in step).
+ * @returns {import("vite").Plugin[]}
+ */
+export function retake(options = {}) {
+  // Mutating /__retake/ requests must carry this (see CONTRACT.md).
+  options = { ...options, token: options.token || crypto.randomBytes(16).toString("hex") }
+  const bus = createBus()
+  let ssr = false
+  // The dock for a top-level page load, the runtime injected into the dock's
+  // frame (Sec-Fetch-Dest: iframe; isAppFrame() tells it from an app's own
+  // iframe), and anything else left alone. Which one a request is stays known
+  // while the framework renders it (pageKind), for frameworks that call
+  // server.transformIndexHtml themselves.
+  const ssrPages = (req, res, next) => {
+    const url = new URL(req.url || "/", "http://x")
+    const mode = req.headers["sec-fetch-mode"]
+    const dest = req.headers["sec-fetch-dest"]
+    const optOut = url.searchParams.get("retake") === "0"
+    if (url.pathname.startsWith("/__retake/") || mode !== "navigate" || optOut) return pageKind.run("plain", next)
+    if (req.method === "GET" && dest === "document" && req.headers["sec-fetch-site"] !== "cross-site") {
+      res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" })
+      return res.end(shellHtml({ ...options, marker: "header" }))
+    }
+    if (dest !== "iframe" && dest !== "frame") return pageKind.run("plain", next)
+    delete req.headers["accept-encoding"]
+    // Set up like the front server's frames (clock at load, scripts held: see frontRuntime).
+    injectResponse(res, runtimeTag({ rt: { ...frontRuntime("vite"), marker: "header" } }))
+    pageKind.run("frame", next)
+  }
+  /** @type {import("vite").Plugin} */
+  const main = {
+    name: "retake",
+    apply: "serve",
+    // CSS source maps in dev, so cssSourceFor() can point at the original file.
+    config(cfg) {
+      if (options.enabled === false) return
+      if (!cfg.css || cfg.css.devSourcemap === undefined) return { css: { devSourcemap: true } }
+    },
+    configureServer(server) {
+      if (options.enabled === false) return
+      // No index.html: a framework renders the pages itself (React Router,
+      // SvelteKit, Astro, TanStack Start...), and transformIndexHtml never sees
+      // them. Then pages are told apart by their Sec-Fetch-* headers, like the
+      // front server does (header marker), and the frame's HTML is injected as
+      // it's written. The URL is never touched (a `__wb` would leak into the app).
+      ssr = !fs.existsSync(path.join(server.config.root, "index.html"))
+      if (ssr) server.middlewares.use(ssrPages)
+      // Only a top-level page load gets the dock. An iframe the app itself
+      // embeds (or a frame navigating to another page of a multi-page app)
+      // gets its plain page. The dock's own frame asks for `?__wb=app`.
+      else server.middlewares.use((req, res, next) => {
+        const dest = req.headers["sec-fetch-dest"]
+        if (dest && NESTED_DEST.has(dest) && req.url && !/[?&]__wb=/.test(req.url)) {
+          const add = (u) => u + (u.includes("?") ? "&" : "?") + "__wb=plain"
+          req.url = add(req.url)
+          if (req.originalUrl) req.originalUrl = add(req.originalUrl)
+        }
+        next()
+      })
+      sessionApi(server, { token: options.token, bus, root: options.root })
+      if (options.codeBranches) codeVersions(server, { token: options.token, bus })
+      const httpServer = server.httpServer
+      if (httpServer) {
+        httpServer.once("listening", () => {
+          const a = httpServer.address()
+          const port = a && typeof a === "object" ? a.port : server.config.server.port
+          const proto = server.config.server.https ? "https" : "http"
+          const base = server.config.base || "/"
+          if (options.banner !== false) console.log(`\n  \x1b[1mRetake\x1b[0m  timeline docked at ${proto}://localhost:${port}${base}${options.codeBranches ? "  (code branches on)" : ""}${ssr ? "  (server-rendered pages)" : ""}\n`)
+        })
+      }
+    },
+    // With code branches the dock decides when the frame reloads (it replays
+    // up to the current moment on the new code), so Vite's HMR stands down.
+    handleHotUpdate() {
+      if (options.codeBranches) return []
+    },
+    transformIndexHtml: {
+      order: "pre",
+      handler(html, ctx) {
+        if (options.enabled === false) return
+        // A server-rendered page (see ssrPages): the response is injected, if at all.
+        if (pageKind.getStore()) return
+        const params = new URL(ctx.originalUrl || ctx.path, "http://x").searchParams
+        // `?retake=0` opts a page load out entirely.
+        if (params.get("retake") === "0") return
+        if (params.get("__wb") === "plain") return
+        if (params.get("__wb") !== "app") return shellHtml(options)
+        return [{ tag: "script", attrs: { "data-retake": "" }, children: runtimeSource(), injectTo: "head-prepend" }]
+      },
+    },
+  }
+  // The dock page must not run Vite's client: a full reload (an edit HMR
+  // can't apply) would reload the whole dock and drop the session. Only the
+  // app frame keeps the client, so only the app reloads. Other plugins' module
+  // scripts go too (plugin-react's refresh preamble imports the client); the
+  // dock's own scripts are classic ones.
+  /** @type {import("vite").Plugin} */
+  const stripClient = {
+    name: "retake:shell-without-vite-client",
+    apply: "serve",
+    transformIndexHtml: {
+      order: "post",
+      handler(html) {
+        if (options.enabled === false || !html.includes('id="wb-dock"')) return
+        return html.replace(/<script\b[^>]*\btype=["']?module["']?[^>]*>[\s\S]*?<\/script>\s*/gi, "")
+      },
+    },
+  }
+  return [main, stripClient]
+}
+
+export default retake