rvlib-shipkit 2.3.0 → 2.5.0

package/src/serve/serve-hub.ts

@@ -7,6 +7,7 @@
 import { existsSync } from 'fs'
 import { join } from 'path'
 import { positionsFile } from 'rvlib-shipkit/src/wiki/graph-positions.ts'
+import { pageOf, searchWiki } from 'rvlib-shipkit/src/wiki/graph-pages.ts'
 import { graphRoutes } from 'rvlib-shipkit/src/wiki/serve-graph.ts'
 import type { StudioRoutes } from '../assets/server/studio-server.ts'
 import { comfyBoxUp, type ComfyBox, type ComfyPanelUp } from '../comfy/comfy-panel.ts'
@@ -30,6 +31,7 @@ import { RUN_EXIT } from '../home/home-words.ts'
 import { projectGutter } from './project-gutter-route.ts'
 import { sameOrigin } from './same-origin.ts'
 import { LAYOUT } from '../core/layout.ts'
+import { heldNow, heldText } from '../core/last-good.ts'
 import { WarmCell } from './warm-cell.ts'
 import { readmeRoutes } from './readme-route.ts'
 import { reviewRoutes } from '../review/routes.ts'
@@ -245,7 +247,7 @@ export function hubHandler(d: HubDeps, o: { base?: string; projects?: Gutter } =
                     collections: r.collections,
                     resources: r.resources,
                     planned: d.planned?.() ?? null,
-                    hub: d.hubProblems?.list() ?? [],
+                    hub: [...(d.hubProblems?.list() ?? []), ...heldProblem()],
                 })
             }
             // the previous address, answered while pages opened before the
@@ -411,6 +413,12 @@ export function hubHandler(d: HubDeps, o: { base?: string; projects?: Gutter } =
                 if (path === `/graph/${g.id}`) return page(routes.page())
                 if (path === `/graph/${g.id}/graph.json`) return Response.json(held)
                 if (path === `/graph/${g.id}/positions`) return routes.save(req)
+                // the wikis page's search bar and its preview panel: only a node of the graph is read
+                if (path === `/graph/${g.id}/search`) return g.dir == null ? new Response('this wiki names no folder to read', { status: 404 }) : Response.json(searchWiki(held, g.dir, new URL(req.url).searchParams.get('q') ?? ''))
+                if (path === `/graph/${g.id}/page`) {
+                    const p = g.dir == null ? null : pageOf(held, g.dir, new URL(req.url).searchParams.get('node') ?? '')
+                    return p == null ? new Response('not a page of this wiki', { status: 404 }) : Response.json(p)
+                }
             }
             // the media API namespace — the studio route table, ONE prefix strip
             if (d.studio != null && path.startsWith('/media/')) return d.studio.handle(req, path.slice('/media'.length))
@@ -420,6 +428,12 @@ export function hubHandler(d: HubDeps, o: { base?: string; projects?: Gutter } =
     }
 }
 
+/** the machine runs older code than its tree: every hub's header says so */
+const heldProblem = (): { id: string; text: string; severity: 'yellow' }[] => {
+    const h = heldNow()
+    return h == null ? [] : [{ id: 'last-good', text: heldText(h), severity: 'yellow' }]
+}
+
 export function serveHub(d: HubDeps, h: HubHandler = hubHandler(d)): Hub {
     const server: Server = Bun.serve({
         hostname: '127.0.0.1',

package/src/serve/stale-restart.ts

@@ -7,10 +7,14 @@
 // watch fds, and a full table fails every spawn with EBADF. Either way it
 // re-executes in place (core/reexec.ts) once no run is in flight, which is a
 // clean table too. Only a restart that throws reaches a human, once, through
-// `say`.
+// `say`. With `loads`, it restarts only onto code that loads: while the tree
+// fails, it keeps the running image and tells `hold` why.
 import { assertLeakHealable, FD_HEALED_ENV, fdTableLeak } from '../core/fd-table.ts'
 
-export type StaleTick = 'fresh' | 'busy' | 'restarted' | 'impossible'
+export type StaleTick = 'fresh' | 'busy' | 'probing' | 'held' | 'restarted' | 'impossible'
+
+/** does the code a restart would run load: `pending` while a probe runs */
+export type LoadVerdict = 'ok' | 'pending' | { file: string; error: string }
 
 export type StaleRestartOpts = {
     /** the version this process booted with */
@@ -30,6 +34,10 @@ export type StaleRestartOpts = {
     /** the hub's status line: a text when the restart is impossible, null
      *  once there is nothing to say */
     say: (text: string | null) => void
+    /** asked once a restart is due, never before (core/load-probe.ts) */
+    loads?: () => LoadVerdict
+    /** why the due restart waits: a file of the tree fails to load. Null once it loads */
+    hold?: (why: { file: string; error: string } | null) => void
     everyMs?: number
 }
 
@@ -47,6 +55,15 @@ export class StaleRestart {
         }
         if (this.failed != null) return 'impossible'
         if (this.o.busy()) return 'busy'
+        if (this.o.loads != null) {
+            const v = this.o.loads()
+            if (v === 'pending') return 'probing'
+            if (v !== 'ok') {
+                this.o.hold?.(v)
+                return 'held'
+            }
+            this.o.hold?.(null)
+        }
         try {
             if (stale.leak) {
                 assertLeakHealable()

package/src/shot/client.ts

@@ -8,7 +8,7 @@ import type { ShotDone } from './shooter.ts'
 /** the global process id pm knows the service by (shot.shipkit.ts) */
 export const SHOT_PROCESS = 'shot'
 
-export type ShotInput = { url: string; out: string; viewport?: Viewport; dpr?: number; reload?: boolean; waitMs?: number }
+export type ShotInput = { url: string; out: string; viewport?: Viewport; dpr?: number; reload?: boolean; waitMs?: number; step?: string; at?: number }
 export type ClipInput = ShotInput & { seconds: number; fps?: number; dom?: boolean }
 export type ShotResult = ShotDone & { via: 'warm' | 'started' | 'local' }
 
@@ -21,13 +21,25 @@ const shotAsk = (i: ShotInput): ShotAsk => ({
     waitMs: i.waitMs ?? DEFAULT_WAIT_MS,
 })
 
-/** a png of `url` at `out` (absolute), through the warm browser */
-export const shotUrl = (i: ShotInput, log: (l: string) => void = () => {}): Promise<ShotResult> => ask('/shot', shotAsk(i), log)
+/** a service of an older shipkit takes a stepped ask, ignores the step and shoots the page as it loads: never kept */
+function stepped(i: { step?: string }, r: ShotResult): void {
+    if (i.step && r.stepped !== true) throw new Error(`the shot service on ${shotUrlBase()} did not step the page (it says: ${r.note}); it runs an older shipkit: sk gp restart ${SHOT_PROCESS}`)
+}
+
+/** a png of `url` at `out` (absolute), through the warm browser; with `step`, the page drawn at `at` first */
+export async function shotUrl(i: ShotInput, log: (l: string) => void = () => {}): Promise<ShotResult> {
+    const r = await ask('/shot', { ...shotAsk(i), ...(i.step ? { step: i.step, at: i.at ?? 0 } : {}) }, log)
+    stepped(i, r)
+    return r
+}
 
 /** an mp4 of `seconds` of `url` at `out` (absolute): a page that is a canvas through its WebCodecs, any other page (or
  *  `dom`) as chromium paints it */
-export const clipUrl = (i: ClipInput, log: (l: string) => void = () => {}): Promise<ShotResult> =>
-    ask('/clip', { ...shotAsk(i), seconds: i.seconds, fps: i.fps ?? DEFAULT_FPS, dom: i.dom ?? false } satisfies ClipAsk, log)
+export async function clipUrl(i: ClipInput, log: (l: string) => void = () => {}): Promise<ShotResult> {
+    const r = await ask('/clip', { ...shotAsk(i), seconds: i.seconds, fps: i.fps ?? DEFAULT_FPS, dom: i.dom ?? false, ...(i.step ? { step: i.step } : {}) } satisfies ClipAsk, log)
+    stepped(i, r)
+    return r
+}
 
 /** null when nothing listens (start it), else the service's answer; a refusal throws its reason */
 async function post(route: string, a: ShotAsk): Promise<ShotDone | null> {

package/src/shot/shooter.ts

@@ -59,7 +59,7 @@ export const SHOT_NICE = 10
 /** a page that paints nothing once the screencast starts is filmed from one still taken after this */
 const FIRST_FRAME_MS = 1000
 
-export type ShotDone = { out: string; ms: number; phases: Record<string, number>; errors: string[]; note: string }
+export type ShotDone = { out: string; ms: number; phases: Record<string, number>; errors: string[]; note: string; stepped?: true }
 
 type Slot = { key: string; url: string; ctx: Context; page: Page; cdp: Cdp; errors: string[]; frozen: boolean }
 
@@ -169,6 +169,8 @@ export class Shooter {
         const phases: Record<string, number> = {}
         const t0 = performance.now()
         let move = pageMove(this.#slot == null || this.#slot.page.isClosed() ? null : { url: this.#slot.url, key: this.#slot.key }, a)
+        // a stepped still draws its moment on a fresh load, never on a page some other shot left at another time
+        if (a.step && move === 'none') move = 'reload'
         if (move === 'boot') {
             const old = this.#slot
             this.#slot = null
@@ -200,13 +202,22 @@ export class Shooter {
         let t = performance.now()
         const how = await waitReady(() => slot.page.evaluate(readMark), move === 'none' ? 0 : a.waitMs, { now: () => performance.now(), sleep: (n) => slot.page.waitForTimeout(n) })
         phases['ready'] = ms(t)
+        if (a.step) {
+            t = performance.now()
+            const { stepTo } = await import('./stepped-clip.ts')
+            if (!(await slot.page.evaluate(stepTo, [a.step, a.at ?? 0] as [string, number]))) throw new Error(`the page has no window.${a.step}(t): a stepped page draws the time it is given there`)
+            await waitReady(() => slot.page.evaluate(readMark), 0, { now: () => performance.now(), sleep: (n) => slot.page.waitForTimeout(n) })
+            // the warm page now shows that moment: the next plain shot of this url loads it again
+            slot.url = ''
+            phases['step'] = ms(t)
+        }
         t = performance.now()
         const r = await slot.cdp.send('Page.captureScreenshot', { format: 'png', optimizeForSpeed: true })
         phases['capture'] = ms(t)
         mkdirSync(dirname(a.out), { recursive: true })
         await Bun.write(a.out, Buffer.from(String(r['data']), 'base64'))
-        const note = `${move === 'none' ? 'warm page' : thawed ? 'frozen page reloaded' : move}, ${how === 'ready' ? 'ready mark' : move === 'none' ? 'no wait' : `${a.waitMs} ms after load`}`
-        return { out: a.out, ms: ms(t0), phases, errors: [...slot.errors], note }
+        const note = `${move === 'none' ? 'warm page' : thawed ? 'frozen page reloaded' : move}, ${how === 'ready' ? 'ready mark' : move === 'none' ? 'no wait' : `${a.waitMs} ms after load`}${a.step ? `, drawn at ${a.at ?? 0}s through window.${a.step}(t)` : ''}`
+        return { out: a.out, ms: ms(t0), phases, errors: [...slot.errors], note, ...(a.step ? { stepped: true as const } : {}) }
     }
 
     async #clip(a: ClipAsk): Promise<ShotDone> {
@@ -223,6 +234,11 @@ export class Shooter {
             await waitReady(() => page.evaluate(readMark), a.waitMs, { now: () => performance.now(), sleep: (n) => page.waitForTimeout(n) })
             phases['load'] = ms(t)
             t = performance.now()
+            if (a.step) {
+                const note = await (await import('./stepped-clip.ts')).stepPage(await ctx.newCDPSession(page), page, a as ClipAsk & { step: string })
+                phases['film'] = ms(t)
+                return { out: a.out, ms: ms(t0), phases, errors, note, stepped: true }
+            }
             const path: ClipPath = clipPathOf({ dom: a.dom === true, canvas: await page.evaluate(largestCanvas), viewport: a.viewport })
             const why = path === 'canvas' ? await page.evaluate(installClip, { seconds: a.seconds, fps: a.fps, quantizer: CLIP_QUANTIZER, keyEvery: KEY_EVERY } satisfies PageClipAsk) : null
             if (path === 'dom' || why != null) {

package/src/shot/shot-ask.ts

@@ -17,6 +17,9 @@ export type ShotAsk = {
     reload: boolean
     /** a page with no ready mark is shot this long after its load (a warm page at once) */
     waitMs: number
+    /** a still of a stepped page: `window[step](at)` on a fresh load, then its ready mark, then the shot */
+    step?: string
+    at?: number
 }
 
 export type ClipAsk = ShotAsk & {
@@ -24,6 +27,9 @@ export type ClipAsk = ShotAsk & {
     fps: number
     /** film the page itself even when a canvas fills it (dom-clip.ts); a page with no big canvas is filmed so anyway */
     dom?: boolean
+    /** step the page frame by frame (stepped-clip.ts): before each frame `window[step](t)`, t its time in seconds, then
+     *  its ready mark; a slow page films as smooth as a fast one */
+    step?: string
 }
 
 export const DEFAULT_SHOT_PORT = 4796
@@ -113,7 +119,9 @@ export const isClip = (a: ShotAsk | ClipAsk): a is ClipAsk => 'seconds' in a
 
 /** a request held past this (a clip: past its own seconds) fails loud and frees the turn */
 export const REQUEST_MAX_MS = 120_000
-export const capMsOf = (a: ShotAsk | ClipAsk): number => REQUEST_MAX_MS + (isClip(a) ? a.seconds * 1000 : 0)
+export const capMsOf = (a: ShotAsk | ClipAsk): number => REQUEST_MAX_MS + (isClip(a) ? (a.step ? a.seconds * a.fps * STEP_FRAME_MS : a.seconds * 1000) : 0)
+/** a stepped frame's budget in the request's cap: a page that draws a frame slower than this is no page to step */
+export const STEP_FRAME_MS = 1000
 
 // ── the ready mark ──────────────────────────────────────────────────────────
 // A page that knows when it is drawn says so: `<meta name="shot-ready"

package/src/shot/stepped-clip.ts

@@ -0,0 +1,53 @@
+// a clip filmed frame by frame (wiki build/trailer.md, film page): before each frame the page is asked to draw the
+// frame's time, `window[step](t)`, then the shot service waits for its ready mark and two painted frames, takes one
+// screenshot, and ffmpeg encodes the frames in order. The page's own speed never shows: a frame that takes a second to
+// draw sits beside one that took a millisecond. The page owns its clock while it is stepped
+import { mkdirSync } from 'fs'
+import { dirname } from 'path'
+import { domEncodeArgs } from './dom-clip.ts'
+import { type ClipAsk, readMark, waitReady } from './shot-ask.ts'
+
+type Cdp = { send(method: string, params?: Record<string, unknown>): Promise<Record<string, unknown>> }
+type Page = { evaluate<R, A>(fn: (a: A) => R | Promise<R>, arg: A): Promise<R>; waitForTimeout(ms: number): Promise<void> }
+
+/** the frames a stepped clip holds, and each one's time */
+export const stepTimes = (seconds: number, fps: number): number[] => Array.from({ length: Math.max(1, Math.round(seconds * fps)) }, (_, i) => +(i / fps).toFixed(6))
+
+/** IN THE PAGE: the hook asked to draw time t; false when the page has no such hook */
+export const stepTo = async ([hook, t]: [string, number]): Promise<boolean> => {
+    const f = (window as unknown as Record<string, unknown>)[hook]
+    if (typeof f !== 'function') return false
+    await (f as (t: number) => unknown)(t)
+    await new Promise<void>((ok) => requestAnimationFrame(() => requestAnimationFrame(() => ok())))
+    return true
+}
+
+export async function stepPage(cdp: Cdp, page: Page, a: ClipAsk & { step: string }): Promise<string> {
+    const ffmpeg = Bun.which('ffmpeg') ?? (process.platform === 'darwin' ? '/opt/homebrew/bin/ffmpeg' : null)
+    if (ffmpeg == null) throw new Error('ffmpeg is not installed: brew install ffmpeg')
+    mkdirSync(dirname(a.out), { recursive: true })
+    const times = stepTimes(a.seconds, a.fps)
+    const p = Bun.spawn([ffmpeg, ...domEncodeArgs(a.fps, times.length, a.out)], { stdin: 'pipe', stdout: 'ignore', stderr: 'pipe' })
+    const stderr = new Response(p.stderr).text()
+    const t0 = performance.now()
+    let px = ''
+    try {
+        for (const t of times) {
+            if (!(await page.evaluate(stepTo, [a.step, t] as [string, number]))) throw new Error(`the page has no window.${a.step}(t): a stepped page draws the time it is given there`)
+            await waitReady(() => page.evaluate(readMark, undefined), 0, { now: () => performance.now(), sleep: (n) => page.waitForTimeout(n) })
+            const shot = await cdp.send('Page.captureScreenshot', { format: 'jpeg', quality: 92 })
+            const jpeg = Buffer.from(String(shot['data']), 'base64')
+            // the frame's pixels, from the jpeg's start of frame
+            const sof = jpeg.indexOf(Buffer.from([0xff, 0xc0]))
+            if (px === '' && sof >= 0) px = `${jpeg.readUInt16BE(sof + 7)}x${jpeg.readUInt16BE(sof + 5)}`
+            p.stdin.write(jpeg)
+            await p.stdin.flush()
+        }
+    } finally {
+        await Promise.resolve(p.stdin.end()).catch(() => undefined)
+    }
+    const err = await stderr
+    if ((await p.exited) !== 0) throw new Error(`ffmpeg encode → ${a.out}: ${err.trim().split('\n').slice(-6).join('\n') || `exit ${p.exitCode}`}`)
+    const s = (performance.now() - t0) / 1000
+    return `${times.length} frames ${px} at ${a.fps} fps, each stepped through window.${a.step}(t) and its ready mark (${(s / times.length * 1000).toFixed(0)} ms a frame), H.264 by ffmpeg`
+}

package/src/trailer/adapter.ts

@@ -3,15 +3,38 @@
 // resolves the sounds that are not files, names the paths a take's world is, checks the content the language cannot
 // know, refines one film line's take, and turns on the guards that read the repo's own words. Discovered like every
 // unit file (load-unit.ts)
-import type { TakeLength } from './grammar.ts'
+import type { SoundRun, TakeLength } from './grammar.ts'
+import { type Words, wordsProblem } from './words.ts'
 
+/** the repo's sounds by id; null: not the repo's, the house makes it (src/trailer/sound/, design call 9) */
 export interface TrailerSounds {
     /** a music id to an audio file of at least `seconds`, cached by the repo */
-    music?(id: string, o: { take?: string; seconds: number }): Promise<string>
+    music?(id: string, o: { take?: string; seconds: number }): Promise<string | null>
     /** an sfx id to an audio file */
-    sfx?(id: string): Promise<string>
+    sfx?(id: string): Promise<string | null>
     /** a voice line, found by its text, to an audio file */
-    voice?(text: string): Promise<string>
+    voice?(text: string): Promise<string | null>
+}
+
+/** a sound line the house makes (`resolveSound` and `soundStates` of src/trailer/sound/index.ts, the shape it reads):
+ *  the line read, the prompt, or a `.mid` file's path (`text`), how it is made (`seed`, `by`, `for`, `hit`), what a music
+ *  cue covers (`seconds`), or the line's own `run` command */
+export interface SoundRequest {
+    kind: 'music' | 'sfx' | 'voice'
+    text: string
+    seed?: number
+    by?: string
+    for?: number
+    hit?: boolean
+    seconds?: number
+    line: number
+    run?: SoundRun
+    /** the cut's folder: a `reads` file is found beside it first, then from the repo root */
+    dir: string
+    /** the text is a quoted prompt, never an id */
+    prompt?: boolean
+    /** music: the take of a song id */
+    take?: string
 }
 
 /** a film line of the project CLI as the unit sees it: its words as written, the text of each `@<file>` word */
@@ -66,6 +89,8 @@ export interface TrailerDef {
     states?: { ignore?: string[]; underWay?: string[] }
     /** a clip take that drops its picture for a few frames is filmed again once, then refused (holes.ts) */
     holes?: boolean
+    /** the words a `film page` of this repo takes (words.ts): each checked at its line, its query made plain */
+    page?: { words: Words }
 }
 
 export type TrailerUnit = TrailerDef & { kind: 'trailer' }
@@ -73,6 +98,10 @@ export type TrailerUnit = TrailerDef & { kind: 'trailer' }
 export function trailer(d: TrailerDef): TrailerUnit {
     for (const [k, f] of Object.entries(d.sound ?? {})) if (typeof f !== 'function') throw new Error(`trailer: sound.${k} is a function returning an audio file`)
     for (const k of ['check', 'take'] as const) if (d[k] !== undefined && typeof d[k] !== 'function') throw new Error(`trailer: ${k} is a function`)
+    if (d.page !== undefined) {
+        const why = wordsProblem(d.page.words) ?? Object.entries(d.page.words).flatMap(([k, w]) => (w.arg !== undefined ? [`page word ${k} declares arg: a page word has a key (it is the url's query)`] : []))[0] ?? null
+        if (why !== null) throw new Error(`trailer: ${why}`)
+    }
     if (d.stamp !== undefined && (!Array.isArray(d.stamp) || d.stamp.some((p) => typeof p !== 'string' || p === '' || p.startsWith('/')))) throw new Error('trailer: stamp is a list of repo relative paths')
     return { ...d, kind: 'trailer' }
 }