rvlib-shipkit 2.1.1 → 2.3.0

package/src/serve/hub-runtime.ts

@@ -58,6 +58,7 @@ import { hubHandler, type CheckTap, type HubDeps, type HubHandler } from './serv
 import type { projectGutter } from './project-gutter-route.ts'
 import { defaultHubPort } from './hub-port.ts'
 import { LAYOUT } from '../core/layout.ts'
+import { storeDir, storeStamp } from '../review/store.ts'
 export { defaultHubPort }
 
 export type ServeIo = { print: (l: string) => void; printErr: (l: string) => void }
@@ -347,9 +348,10 @@ export async function resolveHubDeps(root: string, boot: RepoConfig, io: ServeIo
         graphs,
         studio,
         refresh,
-        // the config and the run record, by their stats: what the warm rows
-        // and the runs page read besides the tracked files the watch sees
-        stamp: () => `${configStamp(root)}|${record === '' ? '' : recordStamp(record)}`,
+        // the config, the run record and the review store (gitignored, its
+        // own watch counts its writes): what the warm rows and the runs page
+        // read besides the tracked files the watch sees
+        stamp: () => `${configStamp(root)}|${record === '' ? '' : recordStamp(record)}|${storeStamp(storeDir(root))}`,
         runCli,
         // the run happens IN THIS PROCESS, so every check result reaches the
         // page as it finishes. A shelled run could only ever deliver one wall

package/src/serve/same-origin.ts

@@ -10,5 +10,9 @@ export function sameOrigin(req: Request): boolean {
     if (site != null && site !== 'same-origin' && site !== 'none') return false
     const origin = req.headers.get('origin')
     if (origin == null) return true
-    try { return new URL(origin).host === new URL(req.url).host } catch { return false }
+    // behind a proxy that frames the page under its own origin (pkm-multi's admin), the host the browser asked is the
+    // proxy's, which it passes on. A browser cannot forge the header: a page of another site that sets it is held at
+    // its preflight, which this server never answers
+    const host = req.headers.get('x-forwarded-host') ?? new URL(req.url).host
+    try { return new URL(origin).host === host } catch { return false }
 }

package/src/serve/serve-hub.ts

@@ -32,6 +32,8 @@ import { sameOrigin } from './same-origin.ts'
 import { LAYOUT } from '../core/layout.ts'
 import { WarmCell } from './warm-cell.ts'
 import { readmeRoutes } from './readme-route.ts'
+import { reviewRoutes } from '../review/routes.ts'
+import { embedRoute } from './embed.ts'
 
 export { sameOrigin }
 
@@ -338,6 +340,12 @@ export function hubHandler(d: HubDeps, o: { base?: string; projects?: Gutter } =
             // the empty page shows the README, its images served from the repo
             const readme = await readmeRoutes(d.root, path)
             if (readme != null) return readme
+            // the review or trailer page alone, for another tool's frame (embed.ts)
+            const embed = await embedRoute(path)
+            if (embed != null) return embed
+            // the one feedback store and the trailer view (review/routes.ts)
+            const review = await reviewRoutes(d.root, req, path)
+            if (review != null) return review
             if (path === '/api/status.json') {
                 const p = join(d.root, LAYOUT.status)
                 return Response.json({ statusMd: existsSync(p) ? await Bun.file(p).text() : null })

package/src/shipkit-files.ts

@@ -6,13 +6,21 @@
 //                         supervises it, the home page switches it
 //   widget({...})         html the daemon renders every few seconds, mounted
 //                         on the home page
+//   trailer({...})        what `sk trailer` cannot read from a file: the
+//                         sounds the repo resolves by id (trailer/adapter.ts)
+//   review({...})         the repo's parts of the review store: its subject
+//                         kinds, topics, sources, auto rules, the dispatcher
+//                         (review/unit.ts)
 //
 // the one constraint a reader breaks first: every kind is IMPORTED by the
 // process that reads it (discovery, pm's config child, the daemon), so a file
 // keeps its top level cheap and puts heavy work inside a function.
 import { check, type CheckDef } from './check.ts'
+import { review, type AutoRule, type KindDef, type ReviewDef, type ReviewSource, type ReviewUnit } from './review/unit.ts'
+import { trailer, type TrailerDef, type TrailerUnit } from './trailer/adapter.ts'
 
-export { check }
+export { check, review, trailer }
+export type { AutoRule, KindDef, ReviewDef, ReviewSource, ReviewUnit, TrailerDef, TrailerUnit }
 
 /** one button beside a global process: a command, or a function of the file */
 export type ProcessAction = {
@@ -69,7 +77,7 @@ export type WidgetDef = {
 export type GlobalProcessUnit = GlobalProcessDef & { kind: 'globalProcess' }
 export type WidgetUnit = WidgetDef & { kind: 'widget' }
 
-export const UNIT_KINDS = ['globalProcess', 'widget'] as const
+export const UNIT_KINDS = ['globalProcess', 'widget', 'trailer', 'review'] as const
 
 const KEBAB = /^[a-z0-9][a-z0-9-]*$/
 const words = (v: unknown): boolean => Array.isArray(v) && v.length > 0 && v.every((w) => typeof w === 'string')
@@ -123,9 +131,9 @@ export function widget(d: WidgetDef): WidgetUnit {
 }
 
 /** the kind a default export says it is: a unit, else a check */
-export function unitKind(d: unknown): 'globalProcess' | 'widget' | 'check' {
+export function unitKind(d: unknown): (typeof UNIT_KINDS)[number] | 'check' {
     const k = (d as { kind?: unknown } | null)?.kind
-    return k === 'globalProcess' || k === 'widget' ? k : 'check'
+    return UNIT_KINDS.includes(k as (typeof UNIT_KINDS)[number]) ? (k as (typeof UNIT_KINDS)[number]) : 'check'
 }
 
 export type { CheckDef }

package/src/shot/client.ts

@@ -0,0 +1,67 @@
+// the one way code asks for a shot or a clip: `shotUrl` and `clipUrl` post to
+// the warm shot service (serve-shot.ts), start it through pm when it is down,
+// and shoot in process only when the machine has no pm (CI, SHIPKIT_NO_HOME).
+// `sk shot` and the trailer's `url` film both call these.
+import { DEFAULT_FPS, DEFAULT_VIEWPORT, DEFAULT_WAIT_MS, type ClipAsk, type ShotAsk, shotUrlBase, type Viewport } from './shot-ask.ts'
+import type { ShotDone } from './shooter.ts'
+
+/** the global process id pm knows the service by (shot.shipkit.ts) */
+export const SHOT_PROCESS = 'shot'
+
+export type ShotInput = { url: string; out: string; viewport?: Viewport; dpr?: number; reload?: boolean; waitMs?: number }
+export type ClipInput = ShotInput & { seconds: number; fps?: number; dom?: boolean }
+export type ShotResult = ShotDone & { via: 'warm' | 'started' | 'local' }
+
+const shotAsk = (i: ShotInput): ShotAsk => ({
+    url: i.url,
+    out: i.out,
+    viewport: i.viewport ?? DEFAULT_VIEWPORT,
+    dpr: i.dpr ?? 1,
+    reload: i.reload ?? false,
+    waitMs: i.waitMs ?? DEFAULT_WAIT_MS,
+})
+
+/** a png of `url` at `out` (absolute), through the warm browser */
+export const shotUrl = (i: ShotInput, log: (l: string) => void = () => {}): Promise<ShotResult> => ask('/shot', shotAsk(i), log)
+
+/** an mp4 of `seconds` of `url` at `out` (absolute): a page that is a canvas through its WebCodecs, any other page (or
+ *  `dom`) as chromium paints it */
+export const clipUrl = (i: ClipInput, log: (l: string) => void = () => {}): Promise<ShotResult> =>
+    ask('/clip', { ...shotAsk(i), seconds: i.seconds, fps: i.fps ?? DEFAULT_FPS, dom: i.dom ?? false } satisfies ClipAsk, log)
+
+/** null when nothing listens (start it), else the service's answer; a refusal throws its reason */
+async function post(route: string, a: ShotAsk): Promise<ShotDone | null> {
+    let r: Response
+    try {
+        r = await fetch(`${shotUrlBase()}${route}`, { method: 'POST', body: JSON.stringify(a), headers: { 'content-type': 'application/json' } })
+    } catch {
+        return null
+    }
+    if (!r.ok) throw new Error(await r.text())
+    return (await r.json()) as ShotDone
+}
+
+async function ask(route: '/shot' | '/clip', a: ShotAsk | ClipAsk, log: (l: string) => void): Promise<ShotResult> {
+    const warm = await post(route, a)
+    if (warm != null) return { ...warm, via: 'warm' }
+    const { pmUp, pmUrl } = await import('../home/home-paths.ts')
+    if (process.env['CI'] == null && process.env['SHIPKIT_NO_HOME'] == null && (await pmUp())) {
+        log(`shot: starting the shot service (sk gp start ${SHOT_PROCESS})`)
+        const { pmSwitchDeps, switchProcess } = await import('../cli/cmd_GlobalProcesses.ts')
+        const code = await switchProcess(SHOT_PROCESS, 'start', pmSwitchDeps(log, pmUrl()), 60_000)
+        if (code !== 0) throw new Error(`the shot service did not start through pm: sk gp shows where it stands`)
+        const started = await post(route, a)
+        if (started == null) throw new Error(`pm says the shot service is up and nothing answers on ${shotUrlBase()}`)
+        return { ...started, via: 'started' }
+    }
+    log('shot: no pm on this machine, shooting in process (a cold browser)')
+    const { Shooter } = await import('./shooter.ts')
+    const { dirname } = await import('path')
+    const s = new Shooter([process.cwd(), dirname(import.meta.dir)])
+    try {
+        const done = route === '/clip' ? await s.clip(a as ClipAsk) : await s.shot(a)
+        return { ...done, via: 'local' }
+    } finally {
+        await s.close()
+    }
+}

package/src/shot/clip-encoder.ts

@@ -0,0 +1,146 @@
+// a clip filmed by the page itself: each frame of its largest canvas goes to
+// the page's WebCodecs encoder (H.264, the media engine) on the page's own
+// animation frames, in real time, and the stream comes back in takes of at
+// most TAKE bytes, never whole (one answer of a long clip's base64 can close
+// the browser). ffmpeg then muxes it into an mp4 with no second encode.
+//
+// the constraint a reader breaks first: `installClip`, `takeClip` and
+// `CLIP_INIT` run IN the page, so they import nothing and read only their
+// argument and the document. A WebGL canvas keeps no drawing buffer after it
+// presents, so the clip's page is opened with CLIP_INIT, which asks every
+// WebGL context for `preserveDrawingBuffer`.
+
+/** what the page films: seconds of its largest canvas at fps, H.264 at a fixed quantizer */
+export type PageClipAsk = { seconds: number; fps: number; quantizer: number; keyEvery: number }
+
+/** what the page says once the film ran */
+export type PageClipDone = { frames: number; repeated: number; width: number; height: number; canvasOfViewport: string }
+
+/** IN THE PAGE, as an init script: every WebGL context keeps its drawing buffer, so a frame reads back after it presents */
+export const CLIP_INIT = `(() => {
+    const get = HTMLCanvasElement.prototype.getContext
+    HTMLCanvasElement.prototype.getContext = function (type, attrs) {
+        if (type === 'webgl' || type === 'webgl2' || type === 'experimental-webgl') attrs = Object.assign({}, attrs, { preserveDrawingBuffer: true })
+        return get.call(this, type, attrs)
+    }
+})()`
+
+type ClipState = {
+    chunks: Uint8Array[]
+    next: number
+    failed: string | null
+    done: Promise<PageClipDone>
+}
+
+/** IN THE PAGE: starts filming the largest visible canvas; null once it films, else why it cannot */
+export async function installClip(a: PageClipAsk): Promise<string | null> {
+    const w = window as unknown as { __skClip?: ClipState }
+    if (typeof VideoEncoder === 'undefined') return 'this page has no WebCodecs (VideoEncoder)'
+    const canvases = [...document.querySelectorAll('canvas')].filter((c) => c.width > 0 && c.height > 0 && c.getBoundingClientRect().width > 0)
+    const canvas = canvases.sort((x, y) => y.width * y.height - x.width * x.height)[0]
+    if (canvas == null) return 'the page draws no visible <canvas>: a clip films the page\'s canvas through WebCodecs'
+    const even = (v: number): number => Math.max(2, 2 * Math.floor(v / 2))
+    const width = even(canvas.width)
+    const height = even(canvas.height)
+    const blocks = Math.ceil(width / 16) * Math.ceil(height / 16)
+    const level = blocks <= 8192 ? '28' : blocks <= 36864 ? '33' : '34'
+    const config: VideoEncoderConfig = {
+        codec: `avc1.6400${level}`,
+        width,
+        height,
+        framerate: a.fps,
+        latencyMode: 'quality',
+        bitrateMode: 'quantizer',
+        hardwareAcceleration: 'prefer-hardware',
+        avc: { format: 'annexb' },
+    }
+    if (!(await VideoEncoder.isConfigSupported(config)).supported) return `the page's H.264 encoder takes no ${width}x${height} at a fixed quantizer`
+    const box = canvas.getBoundingClientRect()
+    const canvasOfViewport = `${Math.round(box.width)}x${Math.round(box.height)} css px of ${innerWidth}x${innerHeight}`
+    const st: ClipState = { chunks: [], next: 0, failed: null, done: Promise.resolve({ frames: 0, repeated: 0, width, height, canvasOfViewport }) }
+    const enc = new VideoEncoder({
+        output: (chunk) => {
+            const b = new Uint8Array(chunk.byteLength)
+            chunk.copyTo(b)
+            st.chunks.push(b)
+        },
+        error: (e) => (st.failed = `the page's encoder failed: ${e.message}`),
+    })
+    enc.configure(config)
+    const total = Math.round(a.seconds * a.fps)
+    const step = 1000 / a.fps
+    const rect = { x: 0, y: 0, width, height }
+    st.done = new Promise<PageClipDone>((ok, fail) => {
+        let n = 0
+        let repeated = 0
+        let t0 = -1
+        const tick = (t: number): void => {
+            try {
+                if (st.failed) throw new Error(st.failed)
+                if (t0 < 0) t0 = t
+                // a frame for every step of real time passed: a slow page repeats its last drawing, so the clip keeps its length
+                let made = 0
+                while (n < total && n * step <= t - t0) {
+                    const frame = new VideoFrame(canvas, { timestamp: Math.round(n * step * 1000), duration: Math.round(step * 1000), visibleRect: rect })
+                    enc.encode(frame, { keyFrame: n % a.keyEvery === 0, avc: { quantizer: a.quantizer } } as VideoEncoderEncodeOptions)
+                    frame.close()
+                    if (made++ > 0) repeated++
+                    n++
+                }
+                if (n < total) requestAnimationFrame(tick)
+                else
+                    void enc
+                        .flush()
+                        .then(() => {
+                            enc.close()
+                            if (st.failed) throw new Error(st.failed)
+                            ok({ frames: n, repeated, width, height, canvasOfViewport })
+                        })
+                        .catch(fail)
+            } catch (e) {
+                if (enc.state !== 'closed') enc.close()
+                fail(e)
+            }
+        }
+        requestAnimationFrame(tick)
+    })
+    // a rejection is read by `clipDone`, never left unhandled in the page
+    st.done.catch(() => undefined)
+    w.__skClip = st
+    return null
+}
+
+/** IN THE PAGE: the stream made since the last take, at most `max` bytes (whole chunks, one at least), base64; '' when
+ *  none waits */
+export async function takeClip(max: number): Promise<string> {
+    const st = (window as unknown as { __skClip: ClipState }).__skClip
+    if (st.failed) throw new Error(st.failed)
+    const from = st.next
+    let bytes = 0
+    while (st.next < st.chunks.length && (st.next === from || bytes + st.chunks[st.next]!.byteLength <= max)) bytes += st.chunks[st.next++]!.byteLength
+    if (st.next === from) return ''
+    const part = st.chunks.slice(from, st.next)
+    for (let i = from; i < st.next; i++) st.chunks[i] = new Uint8Array(0)
+    const url = await new Promise<string>((ok, fail) => {
+        const r = new FileReader()
+        r.onload = () => ok(String(r.result))
+        r.onerror = () => fail(new Error('the stream did not read back'))
+        r.readAsDataURL(new Blob(part as BlobPart[]))
+    })
+    return url.slice(url.indexOf(',') + 1)
+}
+
+/** IN THE PAGE: resolves once the film ran and the encoder flushed */
+export const clipDone = (): Promise<PageClipDone> => (window as unknown as { __skClip: ClipState }).__skClip.done
+
+/** the most stream one take carries, bytes: its base64 is a third more */
+export const TAKE = 8 << 20
+
+/** H.264's quantizer, 0 to 51, lower is better: 24 films a UI or a 3D scene sharp at a few MB a minute */
+export const CLIP_QUANTIZER = 24
+
+/** a key frame a second at 30 fps: a cut into the footage seeks no further back than that */
+export const KEY_EVERY = 30
+
+/** ffmpeg's arguments to mux a raw H.264 stream (stdin) into an mp4, no second encode */
+export const muxArgs = (fps: number, out: string): string[] => ['-hide_banner', '-nostdin', '-y', '-loglevel', 'error', '-fflags', '+genpts', '-r', String(fps), '-f', 'h264', '-i', 'pipe:0', '-c:v', 'copy', '-movflags', '+faststart', out]

package/src/shot/dom-clip.ts

@@ -0,0 +1,124 @@
+// a clip of the page itself, for a page the canvas path cannot film: chromium
+// sends each painted frame of the page (CDP `Page.startScreencast`, jpeg),
+// `FrameTimeline` lays them on the clip's fixed frame grid (a frame holds
+// until the next one paints, so a still page still fills its seconds), and
+// ffmpeg encodes the grid to H.264 as the frames arrive.
+//
+// measured against Playwright's `recordVideo` on a 3 s animated page: 60
+// painted frames a second and 90 of 90 distinct frames in the mp4, against
+// 65 of 89 for recordVideo (25 fps vp8, then a second encode to trim it).
+//
+// the constraint a reader breaks first: the screencast must take EVERY frame
+// (`everyNthFrame: 1`). Every second frame drops 6 to 21 distinct frames of
+// 90, because its arrival jitter beats against the clip's grid.
+import type { Viewport } from './shot-ask.ts'
+
+export type ClipPath = 'canvas' | 'dom'
+
+/** css box of the page's largest visible canvas, the one the canvas path would film */
+export type CanvasBox = { width: number; height: number }
+
+/** a canvas this share of the viewport or more is the page: the canvas path films it. Under it, the page is DOM with
+ *  a canvas in it (a chart, a sparkline), and filming the canvas alone would drop the page */
+export const CANVAS_SHARE = 0.5
+
+export function clipPathOf(o: { dom: boolean; canvas: CanvasBox | null; viewport: Viewport }): ClipPath {
+    if (o.dom || o.canvas == null) return 'dom'
+    return o.canvas.width * o.canvas.height >= CANVAS_SHARE * o.viewport.width * o.viewport.height ? 'canvas' : 'dom'
+}
+
+/** IN THE PAGE: the css box of the largest visible canvas by drawing buffer (installClip's pick), null when none */
+export const largestCanvas = (): CanvasBox | null => {
+    const c = [...document.querySelectorAll('canvas')].filter((x) => x.width > 0 && x.height > 0 && x.getBoundingClientRect().width > 0).sort((x, y) => y.width * y.height - x.width * x.height)[0]
+    if (c == null) return null
+    const r = c.getBoundingClientRect()
+    return { width: r.width, height: r.height }
+}
+
+/** the screencast request: every frame, jpeg. Its frames are css pixels whatever the device pixel ratio */
+export const screencastParams = (viewport: Viewport): Record<string, unknown> => ({
+    format: 'jpeg',
+    quality: 90,
+    maxWidth: viewport.width,
+    maxHeight: viewport.height,
+    everyNthFrame: 1,
+})
+
+/** the painted frames on the clip's fixed grid of `fps` × `seconds` slots. Slot i shows the last frame painted at or
+ *  before i / fps, timed from the first frame */
+export class FrameTimeline {
+    readonly total: number
+    #filled = 0
+    #t0: number | null = null
+    #shown = 0
+
+    constructor(
+        readonly fps: number,
+        readonly seconds: number,
+    ) {
+        this.total = Math.round(seconds * fps)
+    }
+
+    /** a frame painted at `t` seconds (any clock, the same for every frame): how many slots the PREVIOUS frame fills,
+     *  and whether this frame lands past the clip's end (it is then dropped) */
+    arrive(t: number): { previous: number; past: boolean } {
+        this.#t0 ??= t
+        const rel = t - this.#t0
+        const upTo = Math.min(this.total, Math.ceil(rel * this.fps))
+        const previous = this.#shown === 0 ? 0 : Math.max(0, upTo - this.#filled)
+        this.#filled += previous
+        const past = rel >= this.seconds
+        if (!past) this.#shown++
+        return { previous, past }
+    }
+
+    /** the slots the last frame fills once filming stops, so the clip is exactly `seconds` long */
+    end(): number {
+        if (this.#shown === 0) return 0
+        const rest = this.total - this.#filled
+        this.#filled = this.total
+        return rest
+    }
+
+    /** distinct frames the page painted inside the clip */
+    get shown(): number {
+        return this.#shown
+    }
+
+    /** the page's own frame rate over the clip */
+    get realFps(): number {
+        return this.#shown / this.seconds
+    }
+}
+
+/** ffmpeg's arguments: a stream of jpegs (stdin) at `fps` into an H.264 mp4 of exactly `frames` frames */
+export const domEncodeArgs = (fps: number, frames: number, out: string): string[] => [
+    '-hide_banner',
+    '-nostdin',
+    '-y',
+    '-loglevel',
+    'error',
+    '-f',
+    'image2pipe',
+    '-framerate',
+    String(fps),
+    '-c:v',
+    'mjpeg',
+    '-i',
+    'pipe:0',
+    '-vf',
+    'scale=trunc(iw/2)*2:trunc(ih/2)*2',
+    '-frames:v',
+    String(frames),
+    '-c:v',
+    'libx264',
+    '-preset',
+    'veryfast',
+    '-crf',
+    '20',
+    '-pix_fmt',
+    'yuv420p',
+    '-movflags',
+    '+faststart',
+    out,
+]

package/src/shot/serve-shot.ts

@@ -0,0 +1,54 @@
+// the warm shot service, the `shot` global process pm keeps up: an http
+// door on 127.0.0.1 over one Shooter. `sk shot` and `shotUrl` post an ask
+// and get back where the file landed. The browser launches on the first
+// ask and closes after BROWSER_IDLE_MS; this process itself stays, idle.
+import { dirname } from 'path'
+import { LIB_VERSION } from '../core/lib-version.ts'
+import { type ClipAsk, isClip, type ShotAsk, shotPort } from './shot-ask.ts'
+import { Shooter } from './shooter.ts'
+
+export const SHOT_SERVICE = 'shipkit-shot'
+
+export function serveShots(port = shotPort(), roots = [process.cwd(), dirname(import.meta.dir)]): { stop(): Promise<void> } {
+    const shooter = new Shooter(roots)
+    const server = Bun.serve({
+        hostname: '127.0.0.1',
+        port,
+        idleTimeout: 0,
+        async fetch(req) {
+            const path = new URL(req.url).pathname
+            if (req.method === 'GET' && path === '/health') return Response.json({ service: SHOT_SERVICE, version: LIB_VERSION, pid: process.pid, ...shooter.warm })
+            if (req.method === 'POST' && (path === '/shot' || path === '/clip')) {
+                let a: ShotAsk | ClipAsk
+                try {
+                    a = (await req.json()) as ShotAsk | ClipAsk
+                } catch {
+                    return new Response('the ask is not json', { status: 400 })
+                }
+                if (typeof a?.url !== 'string' || typeof a.out !== 'string' || !a.out.startsWith('/')) return new Response('an ask needs url and an absolute out', { status: 400 })
+                try {
+                    const done = path === '/clip' ? (isClip(a) ? await shooter.clip(a) : null) : await shooter.shot(a)
+                    if (done == null) return new Response('a clip ask needs seconds and fps', { status: 400 })
+                    return Response.json(done)
+                } catch (e) {
+                    return new Response((e as Error).message ?? String(e), { status: 500 })
+                }
+            }
+            return new Response('not found: GET /health, POST /shot, POST /clip', { status: 404 })
+        },
+    })
+    return {
+        async stop() {
+            await shooter.close()
+            void server.stop(true)
+        },
+    }
+}
+
+if (import.meta.main) {
+    const s = serveShots()
+    console.log(`ok shots on 127.0.0.1:${shotPort()}`)
+    const bye = (): void => void s.stop().finally(() => process.exit(0))
+    process.on('SIGTERM', bye)
+    process.on('SIGINT', bye)
+}