pixi-effects 0.37.0 → 0.38.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +6 -5
- package/ai/chat-template.html +3 -3
- package/ai/tools/lookup.mjs +6 -3
- package/ai/tools/media-page.html +122 -0
- package/ai/tools/media.mjs +748 -0
- package/ai/tools/mediaBrowser.mjs +47 -0
- package/ai/tools/pixi-effects.mjs +11 -2
- package/dist/{Base-BlA6SeqJ.d.cts → Base-BKKwxavW.d.cts} +1 -1
- package/dist/{Base-YqM0Ljvp.d.ts → Base-ioh_AjLB.d.ts} +1 -1
- package/dist/Composition-23KGIJX4.cjs +16 -0
- package/dist/{Composition-MHDI7UNZ.cjs.map → Composition-23KGIJX4.cjs.map} +1 -1
- package/dist/Composition-5SQXSZNA.js +7 -0
- package/dist/{Composition-5KZETM4J.js.map → Composition-5SQXSZNA.js.map} +1 -1
- package/dist/Controller.d.cts +3 -2
- package/dist/Controller.d.ts +3 -2
- package/dist/{Movie-CfHaGmUP.d.ts → Movie-BtwFI2ne.d.ts} +3 -161
- package/dist/{Movie-CJN_gvLB.d.cts → Movie-CSWUTnMd.d.cts} +3 -161
- package/dist/{Spans-FSSO2HGX.js → Spans-4IBQASGK.js} +4 -4
- package/dist/{Spans-FSSO2HGX.js.map → Spans-4IBQASGK.js.map} +1 -1
- package/dist/{Spans-WVJH77BW.cjs → Spans-WJFPHMSQ.cjs} +9 -9
- package/dist/{Spans-WVJH77BW.cjs.map → Spans-WJFPHMSQ.cjs.map} +1 -1
- package/dist/{chunk-IPQNT65V.js → chunk-4CSL5QSL.js} +3 -3
- package/dist/{chunk-IPQNT65V.js.map → chunk-4CSL5QSL.js.map} +1 -1
- package/dist/{chunk-QHIOOK5S.cjs → chunk-7NXKBRT7.cjs} +4 -4
- package/dist/{chunk-QHIOOK5S.cjs.map → chunk-7NXKBRT7.cjs.map} +1 -1
- package/dist/{chunk-MJMOPWMK.cjs → chunk-FE42PKHP.cjs} +62 -62
- package/dist/{chunk-MJMOPWMK.cjs.map → chunk-FE42PKHP.cjs.map} +1 -1
- package/dist/{chunk-E26M4NTL.cjs → chunk-KNMDDUA3.cjs} +48 -453
- package/dist/chunk-KNMDDUA3.cjs.map +1 -0
- package/dist/chunk-LQDDDIC4.cjs +431 -0
- package/dist/chunk-LQDDDIC4.cjs.map +1 -0
- package/dist/{chunk-SBR75G73.js → chunk-UDTLYYGY.js} +4 -4
- package/dist/{chunk-SBR75G73.js.map → chunk-UDTLYYGY.js.map} +1 -1
- package/dist/chunk-UHNSL2KJ.js +425 -0
- package/dist/chunk-UHNSL2KJ.js.map +1 -0
- package/dist/{chunk-KNDEKWQO.js → chunk-XOCRS5MW.js} +9 -414
- package/dist/chunk-XOCRS5MW.js.map +1 -0
- package/dist/index.cjs +38 -33
- package/dist/index.d.cts +6 -5
- package/dist/index.d.ts +6 -5
- package/dist/index.js +4 -3
- package/dist/loader.css +1 -1
- package/dist/present.cjs +4 -3
- package/dist/present.cjs.map +1 -1
- package/dist/present.d.cts +3 -2
- package/dist/present.d.ts +3 -2
- package/dist/present.js +4 -3
- package/dist/present.js.map +1 -1
- package/dist/sound.cjs +12 -0
- package/dist/sound.cjs.map +1 -0
- package/dist/sound.d.cts +1 -0
- package/dist/sound.d.ts +1 -0
- package/dist/sound.js +3 -0
- package/dist/sound.js.map +1 -0
- package/dist/soundEntry-DT13DNt8.d.cts +191 -0
- package/dist/soundEntry-DT13DNt8.d.ts +191 -0
- package/dist/three.cjs +5 -5
- package/dist/three.d.cts +2 -2
- package/dist/three.d.ts +2 -2
- package/dist/three.js +1 -1
- package/dist/{types-DvnXXm33.d.cts → types-BXi-0iZS.d.cts} +1 -1
- package/dist/{types-DvnXXm33.d.ts → types-BXi-0iZS.d.ts} +1 -1
- package/docs/api.md +19 -0
- package/docs/media.md +136 -0
- package/llms-full.txt +170 -3
- package/llms.txt +1 -0
- package/package.json +2 -1
- package/skills/pixi-effects/SKILL.md +10 -2
- package/skills/pixi-effects/template.html +6 -6
- package/dist/Composition-5KZETM4J.js +0 -7
- package/dist/Composition-MHDI7UNZ.cjs +0 -16
- package/dist/chunk-E26M4NTL.cjs.map +0 -1
- package/dist/chunk-KNDEKWQO.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.38.1
|
|
4
|
+
|
|
5
|
+
**Docs, and 21 pieces leave the gallery (64 → 43)**
|
|
6
|
+
|
|
7
|
+
- **Gallery: 21 pieces removed** at the owner's choice (AURA ONE, Orbits, KAGEOKURI, Escapement, Hanabi Night, hanbun, Nengajo: Hatsuhi, Now Showing, Paper Planes, Tadaima, Hikari no oto, Same Grey, Nenrin, KOYU, Key and Lock, Pendulum Wave, Yakou ressha, Hanon Trio Night, Binary Counted, Made of Seconds, Kanreki): their pages, posters, notes, cards and counts are gone (`scripts/gallery-swap.mjs --remove`); the measurements in the guide and FACTS that named Hanabi Night now say "a firework piece".
|
|
8
|
+
|
|
9
|
+
- **Docs: what an agent does when its shell is a browser chat's sandbox** (AGENTS.md, SKILL.md). An agent in claude.ai's code execution (a small root container with a weak CPU and a network limited to package managers: starting Chrome and rendering there is very slow) stalled for minutes on `check` (Chrome refuses to start as root without `--no-sandbox`; with the CDNs unreachable `check` waits its whole 240 s and says only "timed out") and on rewiring the page to local libraries, and was stopped by the person twice. The docs now say: there, follow the chat way (the template from CHAT.md, no `npx`, no `npm i`, no `check` / `render` / `view`), hand over the page, do not render in the sandbox (the person exports with the player's download button on their own computer, warnings come back from the red box), stop after two tries at anything that stalls, say what was not checked. Not changed yet: `check` still hides Chrome's own error and the reason a page did not load (proposals in the project notes).
|
|
10
|
+
|
|
11
|
+
## 0.38.0
|
|
12
|
+
|
|
13
|
+
**Media tools for AI agents: look at, listen to, cut and join footage with no ffmpeg**
|
|
14
|
+
|
|
15
|
+
- **New: `pixi-effects probe | trim | frames | sound | concat | audio`** (docs: `docs/media.md`, `help media`). `concat a b … -o out` joins clips by copy (refused, naming what differs, when the video or audio codec, shown size, rotation, sample rate, channels or the presence of sound or picture differ; each clip starts where the longest track of the one before it ends, so picture and sound do not drift apart over many joins; the report says where each clip starts), and `audio file -o out.m4a` copies the sound out (AAC → `.m4a`, Opus → `.webm`, any → `.mka`). Both were asked for by the trial below (an agent making a highlights reel could only plan it), run in Node with no browser, are made aside and moved into place, and were played in Chrome (the colours before and after the seam) and run on the iPhone clip. A fresh review of them found, and this release fixes: clips of the same codec with different decoder parameters (profile, level, parameter sets) were joined into a file whose later clips decode as garbage (now refused, naming it; a `trim --exact` cut is such a case), a sound that began later than its picture lost that offset in every clip (one start per clip, not one per track), AAC's encoder-delay packet was copied into the middle of the join as a gap of silence, an output that was the input under another spelling (`A.MP4`) overwrote it on a disk that ignores case, hundreds of inputs held a file each and ran into the descriptor limit, a refused command left an empty new folder, a second sound track was dropped without a word, and `audio` lost the sound's lead-in offset without saying so. Description of the first four: For an agent that is handed a video or a sound: `probe <file|url>` (container, duration, size, per track codec / size as shown / frame rate and whether it varies / rotation / bitrate, key packet times with `--keyframes`, and notes: `no audio`, `rotated 90°`, `variable frame rate`), `frames <file> --at 1,50%,f120,end,"every 2s" [--size 480] [--sheet]` (PNGs of the frames, named by time, and a contact sheet with the times under each frame), `sound <file> [--waveform]` (integrated loudness, true peak, clipping, silences, level over time and a waveform: the numbers `check` gives for a movie, from the same code), `trim <file> <start> <end> -o <out> [--exact]` (a cut by copy, with the real times). Every command: a short report or `--json`; exit 0 done / 1 a problem / 2 usage; a bad file is one sentence naming it, never a stack trace.
|
|
16
|
+
- **Honest about what a cut is.** A copy can only start on a key frame, so `trim` starts on the next one and says it (`asked 3.3–4.9 s, got 4–4.9 s`); a range with no key frame inside is refused with the key frames around it; `--exact` re-encodes in the browser for a start on the frame asked. mediabunny's own default (start exactly, with hidden earlier frames and negative timestamps) plays right in Chrome and ffmpeg (measured by playing the cut and reading the colour of every frame, after a first attempt that seeked a paused `<video>` gave a wrong answer) but other players were not measured, so the cut starts ON the key frame and the report is read back from the written file. A fresh reviewer found that the first version of this left the picture starting 0.7 s late with the sound from 0 (the Chrome test had been blind to it: it read the first frame for every time before the first frame); the test now reads the file's tracks back and fails on a late start. The output is made aside and moved into place, so a refused copy leaves nothing and never empties an existing file.
|
|
17
|
+
- **No ffmpeg anywhere, offline.** `probe` and `trim` by copy run in Node with mediabunny's file source and target, no browser (about 0.1 s). `frames`, `sound` and `trim --exact` decode or re-encode, so they use the headless Chrome that `check` and `render` use; mediabunny comes from this package's own bundle through a local server (Range reads in, PUT out: a long video never goes through CDP). A test runs the four commands with only `node` on `PATH`. If ffmpeg is installed and the job is a long transcode, the docs say to use ffmpeg: measured on one 3.9 MB file on one Mac, a lossless 5.6 s cut took 0.05–0.15 s in ffmpeg and 0.10–0.15 s in mediabunny under Node, and a 5.6 s re-encode 0.6–0.75 s in ffmpeg (the browser re-encode was not timed).
|
|
18
|
+
- **New export: `inspectSound(pcm, { window? })`**, the measurement of `movie.inspectAudio()` for the sound of any file (loudness BS.1770, true peak, clipped samples, silences, level rows; a last window that is only a sliver is not a row). It is also built on its own as `dist/sound.js` with no PixiJS in it, which the media page loads so it works offline (a test follows every import of that file). On a real export the command read the −14.8 LUFS that `check` had said (true peak −3.6 against −3.7: the file is AAC); on a very quiet one −30 against −29.7.
|
|
19
|
+
- Found while building it: mediabunny's `UrlSource` retries a dead URL for ever by default and its delay is in seconds (the command gives up after two retries and says so); its own console lines are silenced so a failure is one line; `getDisplayWidth()` is already the width after a rotation. The staged site no longer carries the tools' browser page (`ai/tools/media-page.html`).
|
|
20
|
+
- **A trial by three fresh agents** (four clips: two of our renders, one with a rotation flag, one with no sound; only these commands and their docs; findings in `pixi-effects-opus/M1..M3`) found, and this release fixes: `trim --exact` files were 70–79 ms longer than reported (now only the picture is re-encoded, the sound is copied byte for byte, and `actual` is read back from the file); an audio bitrate that read 157 kb/s on a file and 55 on its cut (a declared figure against a measured one: both are measured now); a rotated clip's text did not say the stored size or the direction (`180×320 shown (stored 320×180, turned 90° clockwise)`; note that a flag on a landscape picture shows it sideways, which `frames` shows too); `--at every 2s` needed quotes (`every:2s` works, and the error says to quote); `sound` gave whole-dB rows with no times, no loudest stretch, no headroom and no verdict (now: loudest and quietest rows with times, `--window`, and a `reference` line with how many LU under −14 and the headroom to −1 dBTP, whether raising it would need a limiter; the library's own notes start at −24 and −9 LUFS, which the docs now say); `help media` opened with a line of the DSL (an exact heading now crowds out files that only mention the word). Then on one real iPhone clip (HEVC, 1080×1920 stored sideways and turned, variable frame rate ~59.94 fps): `probe`, `frames` (upright), `sound` and the copy cuts (0.2 s, HEVC kept) were right, four cuts matched the source frame at their first and last time, and `--exact` (8 s) turned the HEVC into H.264 without saying so, which it now does (`the picture was re-encoded as avc (the source is hevc)`). Not built: an option to show a rotated video unrotated, re-encoding clips so that different ones can be joined, `convert`, `scenes`, tempo.
|
|
21
|
+
- Spec and plan: `docs/superpowers/specs/2026-10-14-media-tools-design.md`, `docs/superpowers/plans/2026-10-14-media-tools.md`. Not claimed anywhere yet (landing page, FACTS): the trial above used our own renders, not footage from a phone or a camera.
|
|
22
|
+
|
|
3
23
|
## 0.37.0
|
|
4
24
|
|
|
5
25
|
**The `canvas` layer: draw it yourself, and what eight trial pieces taught**
|
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# pixi-effects
|
|
2
2
|
|
|
3
|
-
> **Status**: experimental — current release `0.
|
|
3
|
+
> **Status**: experimental — current release `0.38.1`. The API may still change between minor versions; see the [CHANGELOG](./CHANGELOG.md) for what each release changed.
|
|
4
4
|
|
|
5
5
|
**[Guide →](https://yjmtmtk.github.io/pixi-effects/guide/)** · **[Gallery →](https://yjmtmtk.github.io/pixi-effects/examples/gallery/)** · 52 portfolio pieces written as plain data by AI models · **[Examples →](https://yjmtmtk.github.io/pixi-effects/examples/)** · 15 numbered examples, a music lab + an in-browser playground.
|
|
6
6
|
|
|
@@ -49,8 +49,8 @@ Drop the imports into an [importmap](https://developer.mozilla.org/docs/Web/HTML
|
|
|
49
49
|
"imports": {
|
|
50
50
|
"pixi.js": "https://esm.sh/pixi.js@8.22.0?bundle-deps",
|
|
51
51
|
"mediabunny": "https://esm.sh/mediabunny",
|
|
52
|
-
"pixi-effects": "https://cdn.jsdelivr.net/npm/pixi-effects@0.
|
|
53
|
-
"pixi-effects/controller": "https://cdn.jsdelivr.net/npm/pixi-effects@0.
|
|
52
|
+
"pixi-effects": "https://cdn.jsdelivr.net/npm/pixi-effects@0.38.1/dist/index.js",
|
|
53
|
+
"pixi-effects/controller": "https://cdn.jsdelivr.net/npm/pixi-effects@0.38.1/dist/Controller.js"
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
</script>
|
|
@@ -61,7 +61,7 @@ Drop the imports into an [importmap](https://developer.mozilla.org/docs/Web/HTML
|
|
|
61
61
|
</script>
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
Load the `dist/` files **as they are** (jsDelivr's `/npm/…/dist/…`, or unpkg's `https://unpkg.com/pixi-effects@0.
|
|
64
|
+
Load the `dist/` files **as they are** (jsDelivr's `/npm/…/dist/…`, or unpkg's `https://unpkg.com/pixi-effects@0.38.1/dist/index.js`) rather than a CDN-rebundled build such as `esm.sh/pixi-effects` or jsDelivr's `+esm`: the entries (`pixi-effects`, `…/controller`, `…/three`) share internal chunks, which only works when each file is served untouched.
|
|
65
65
|
|
|
66
66
|
> **Using three.js?** Add two more entries (`three` and `pixi-effects/three`) to this importmap — see [Adding three.js](#adding-threejs-optional) below.
|
|
67
67
|
>
|
|
@@ -77,7 +77,7 @@ Add two more entries to the importmap above: three.js itself, and the `pixi-effe
|
|
|
77
77
|
"imports": {
|
|
78
78
|
"...": "(everything from the importmap above)",
|
|
79
79
|
"three": "https://esm.sh/three@0.178.0",
|
|
80
|
-
"pixi-effects/three": "https://cdn.jsdelivr.net/npm/pixi-effects@0.
|
|
80
|
+
"pixi-effects/three": "https://cdn.jsdelivr.net/npm/pixi-effects@0.38.1/dist/three.js"
|
|
81
81
|
}
|
|
82
82
|
}
|
|
83
83
|
</script>
|
|
@@ -176,6 +176,7 @@ This library is designed to be written by AI: a video is plain data, and every m
|
|
|
176
176
|
|
|
177
177
|
- **Start a page, and read only what you need:** `npx pixi-effects new my-video` writes `my-video.html` from the template; `npx pixi-effects recipe [id]` lists the tested recipes and prints one with its code; `npx pixi-effects help <word>` prints the sections of the docs about a word (`mask`, `threeD`, `keyframes`, `music` …). An AI starts from the 2 KB [quick start](./skills/pixi-effects/reference/quickstart.md), not from the cheatsheet.
|
|
178
178
|
- **Look at a page with its timeline, and scrub one with the other:** `npx pixi-effects view my-video.html` (it also watches the folder: when the work changes it shows "Updated hh:mm:ss — reload (R)" and never switches by itself; reload comes back to the same frame) (or `node ai/tools/view.mjs …`) opens your browser on a viewer: the page on top, the timeline of every layer under it with a playhead that follows the movie. Click or drag the timeline to seek, click a layer's name to jump to where it starts, Space plays, ← / → step a frame, + / − (or Ctrl/⌘ + wheel) zoom, the names stay in place. For the human who wants to see what the AI made.
|
|
179
|
+
- **Look at, listen to and cut footage you were handed, with no ffmpeg:** `npx pixi-effects probe clip.mp4` (codecs, size, rotation, sound), `frames clip.mp4 --sheet` (a contact sheet of any video), `sound clip.mp4` (loudness, peak, silences), `trim clip.mp4 3 8 -o cut.mp4` (by copy, with the real times), `concat a.mp4 b.mp4 -o both.mp4` (join clips by copy) and `audio clip.mp4 -o sound.m4a` (the sound alone). They use mediabunny, so there is nothing else to install (`frames`, `sound` and `trim --exact` use Chrome). If ffmpeg is installed and the job is a long transcode, use ffmpeg. See [`docs/media.md`](./docs/media.md).
|
|
179
180
|
- **Render a page to a video file, headless:** `npx pixi-effects render my-video.html -o my-video.mp4` (or `node ai/tools/render.mjs …`; Node ≥ 22 and Chrome installed, no dependencies). The container follows the extension (`mp4 webm mov mkv`); `--quality very-low…very-high`, `--query lang=ja` (added to the page URL), `--strict` (exit 1 when the page logged a warning), `--quiet`. It waits for `window.__ready`, runs `movie.render()` and streams the file to disk; exit 0 = written, 1 = the page or the render failed (no file). For scripts, CI and batches; `pixi-effects check` below is the review.
|
|
180
181
|
- **A talk as a PDF:** `npx pixi-effects render my-talk.html -o my-talk.pdf` (one page per page of a `deck()` / `stops` movie; `--all-stops` makes a page of every step). The review tool writes `stops.png` for such a page, one picture per stop.
|
|
181
182
|
- **One-command review for the AI that wrote the page:** `npx pixi-effects check my-video.html` (or `node ai/tools/check.mjs my-video.html`; Node ≥ 22 and Chrome installed, no dependencies). It opens the page in its own headless Chrome and reports warnings, layout (`movie.inspect` over the whole timeline), the soundtrack (`movie.inspectAudio`) and a real export decoded again, and writes a contact sheet PNG and a `timeline.html` (every layer as a bar on a time axis) to look at. Exit code 0 / 1.
|
package/ai/chat-template.html
CHANGED
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"imports": {
|
|
60
60
|
"pixi.js": "https://cdn.jsdelivr.net/npm/pixi.js@8.22.0/dist/pixi.min.mjs",
|
|
61
61
|
"mediabunny": "https://cdn.jsdelivr.net/npm/mediabunny@1.61.3/+esm",
|
|
62
|
-
"pixi-effects": "https://cdn.jsdelivr.net/npm/pixi-effects@0.
|
|
63
|
-
"pixi-effects/controller": "https://cdn.jsdelivr.net/npm/pixi-effects@0.
|
|
64
|
-
"pixi-effects/present": "https://cdn.jsdelivr.net/npm/pixi-effects@0.
|
|
62
|
+
"pixi-effects": "https://cdn.jsdelivr.net/npm/pixi-effects@0.38.1/dist/index.js",
|
|
63
|
+
"pixi-effects/controller": "https://cdn.jsdelivr.net/npm/pixi-effects@0.38.1/dist/Controller.js",
|
|
64
|
+
"pixi-effects/present": "https://cdn.jsdelivr.net/npm/pixi-effects@0.38.1/dist/present.js"
|
|
65
65
|
}
|
|
66
66
|
}
|
|
67
67
|
</script>
|
package/ai/tools/lookup.mjs
CHANGED
|
@@ -15,7 +15,7 @@ const RECIPES = 'skills/pixi-effects/reference/recipes.md';
|
|
|
15
15
|
/** What `help` reads, most useful first (the number is a small bonus in the ranking). */
|
|
16
16
|
const CORPUS = [
|
|
17
17
|
['docs/dsl.md', 10], ['skills/pixi-effects/reference/cheatsheet.md', 8], ['skills/pixi-effects/reference/pitfalls.md', 6],
|
|
18
|
-
['docs/api.md', 4], ['skills/pixi-effects/reference/direction.md', 2],
|
|
18
|
+
['docs/api.md', 4], ['docs/media.md', 7], ['skills/pixi-effects/reference/direction.md', 2],
|
|
19
19
|
];
|
|
20
20
|
const MAX_HITS = 5, MAX_LINES_PER_HIT = 30, MAX_OUTPUT_LINES = 170;
|
|
21
21
|
|
|
@@ -118,11 +118,14 @@ export function searchDocs(root, word) {
|
|
|
118
118
|
};
|
|
119
119
|
const call = tokens.some(calls);
|
|
120
120
|
const exact = tokens.length === 1 && flat(heading) === flat(tokens[0]); // the heading IS the word
|
|
121
|
-
scored.push({ s, score: (exact ? 1500 : 0) + (inHeading ? 1000 : 0) + (row ? 1100 : 0) + (call ? 300 : 0) + Math.min(count, 50) + bonus });
|
|
121
|
+
scored.push({ s, exact, strong: exact || inHeading || row || call, score: (exact ? 1500 : 0) + (inHeading ? 1000 : 0) + (row ? 1100 : 0) + (call ? 300 : 0) + Math.min(count, 50) + bonus });
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
124
|
scored.sort((a, b) => b.score - a.score);
|
|
125
|
-
|
|
125
|
+
// a section that IS the word (an exact heading) is the answer: other documents that only mention the word are noise next to it (`help media` is the media tools, not a line of the DSL that says "media")
|
|
126
|
+
const top = scored[0];
|
|
127
|
+
const kept = top && top.exact ? scored.filter(h => h.s.file === top.s.file || h.strong) : scored;
|
|
128
|
+
return kept.slice(0, MAX_HITS).map(({ s }) => {
|
|
126
129
|
const body = s.lines.filter((l, i) => i === 0 || l.trim() !== '' || true);
|
|
127
130
|
const shown = body.slice(0, MAX_LINES_PER_HIT);
|
|
128
131
|
return { file: s.file, line: s.line, heading: s.heading, lines: shown, more: Math.max(0, body.length - shown.length) };
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8" />
|
|
5
|
+
<title>pixi-effects media</title>
|
|
6
|
+
</head>
|
|
7
|
+
<body>
|
|
8
|
+
<!--
|
|
9
|
+
The browser side of `pixi-effects frames | sound | trim --exact` (ai/tools/media.mjs opens it in the headless Chrome the other tools use).
|
|
10
|
+
It reads the input by Range requests from /__in/<id>, decodes with mediabunny's own bundle (/__mb/, so it works offline), and hands every file it makes back with
|
|
11
|
+
PUT /__out/<name>: a long video never goes through CDP. `window.media.<name>(args)` returns a plain object (JSON) or throws a sentence.
|
|
12
|
+
-->
|
|
13
|
+
<script type="module">
|
|
14
|
+
import * as mb from '/__mb/mediabunny.min.mjs';
|
|
15
|
+
import { inspectSound } from '/__lib/sound.js'; // the loudness code `check` uses, built with no PixiJS in it
|
|
16
|
+
mb.Logging.level = mb.LogLevel.Silent;
|
|
17
|
+
|
|
18
|
+
const put = async (name, blob) => {
|
|
19
|
+
const res = await fetch('/__out/' + name, { method: 'PUT', body: blob });
|
|
20
|
+
if (!res.ok) throw new Error('could not hand ' + name + ' to the command (' + res.status + ')');
|
|
21
|
+
};
|
|
22
|
+
const open = (id) => new mb.Input({ source: new mb.UrlSource('/__in/' + id, { getRetryDelay: (n) => (n < 3 ? 0.3 * n : null) }), formats: mb.ALL_FORMATS });
|
|
23
|
+
const toBlob = (canvas) => (canvas.convertToBlob ? canvas.convertToBlob({ type: 'image/png' }) : new Promise((ok) => canvas.toBlob(ok, 'image/png')));
|
|
24
|
+
const pad = (n) => String(n).padStart(2, '0');
|
|
25
|
+
|
|
26
|
+
window.media = {
|
|
27
|
+
/** PNGs of the frames at `times` ([{ t, label }]), `size` px wide at most (a small video is not enlarged), and with `sheet` one contact sheet. */
|
|
28
|
+
async frames({ id, times, size, sheet }) {
|
|
29
|
+
const input = await open(id);
|
|
30
|
+
const track = await input.getPrimaryVideoTrack();
|
|
31
|
+
if (!track) throw new Error('no video track');
|
|
32
|
+
if (!(await track.canDecode())) throw new Error('this browser cannot decode ' + track.codec + ' (the video codec of the file)');
|
|
33
|
+
const width = Math.min(size || 480, await track.getDisplayWidth()); // the width as it is shown (after the turn of a rotated video); a small video is not enlarged
|
|
34
|
+
const sink = new mb.CanvasSink(track, { width });
|
|
35
|
+
const firstFrame = await track.getFirstTimestamp();
|
|
36
|
+
const frames = [], kept = [];
|
|
37
|
+
for (let i = 0; i < times.length; i++) {
|
|
38
|
+
const wrapped = await sink.getCanvas(Math.max(times[i].t, firstFrame)); // a time before the first frame is the first frame
|
|
39
|
+
if (!wrapped) throw new Error('there is no frame at ' + times[i].t + ' s');
|
|
40
|
+
const file = pad(i + 1) + '-' + times[i].label + '.png';
|
|
41
|
+
await put(file, await toBlob(wrapped.canvas));
|
|
42
|
+
frames.push({ file, requested: times[i].t, actual: wrapped.timestamp, width: wrapped.canvas.width, height: wrapped.canvas.height });
|
|
43
|
+
if (sheet) kept.push(wrapped.canvas);
|
|
44
|
+
}
|
|
45
|
+
let sheetFile = null;
|
|
46
|
+
if (sheet && kept.length) {
|
|
47
|
+
const cols = Math.min(kept.length, 3), rows = Math.ceil(kept.length / cols), cw = kept[0].width, ch = kept[0].height, strip = 22;
|
|
48
|
+
const c = new OffscreenCanvas(cols * cw, rows * (ch + strip)), g = c.getContext('2d');
|
|
49
|
+
g.fillStyle = '#10141d'; g.fillRect(0, 0, c.width, c.height);
|
|
50
|
+
g.font = '14px ui-monospace, Menlo, monospace'; g.textBaseline = 'middle';
|
|
51
|
+
kept.forEach((k, i) => {
|
|
52
|
+
const x = (i % cols) * cw, y = Math.floor(i / cols) * (ch + strip);
|
|
53
|
+
g.drawImage(k, x, y);
|
|
54
|
+
g.fillStyle = '#e9edf5'; g.fillText(frames[i].actual.toFixed(2) + ' s', x + 8, y + ch + strip / 2);
|
|
55
|
+
});
|
|
56
|
+
sheetFile = 'sheet.png';
|
|
57
|
+
await put(sheetFile, await c.convertToBlob({ type: 'image/png' }));
|
|
58
|
+
}
|
|
59
|
+
return { frames, sheet: sheetFile };
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
/** Loudness, peak, clipping, silences and level over time of the first audio track (all of it is decoded into memory: the command refuses very long files), and with `waveform` a picture. */
|
|
63
|
+
window.media.sound = async ({ id, waveform, window: windowSeconds }) => {
|
|
64
|
+
const input = await open(id);
|
|
65
|
+
const track = await input.getPrimaryAudioTrack();
|
|
66
|
+
if (!track) throw new Error('no audio track');
|
|
67
|
+
if (!(await track.canDecode())) throw new Error('this browser cannot decode ' + track.codec + ' (the audio codec of the file)');
|
|
68
|
+
const duration = await track.computeDuration(); // the sound's own length: a video file is often a little longer than its audio
|
|
69
|
+
const sink = new mb.AudioBufferSink(track);
|
|
70
|
+
const parts = [];
|
|
71
|
+
let rate = 0, channels = 0;
|
|
72
|
+
for await (const { buffer, timestamp } of sink.buffers()) {
|
|
73
|
+
rate = buffer.sampleRate; channels = buffer.numberOfChannels;
|
|
74
|
+
parts.push({ at: Math.round(timestamp * rate), data: Array.from({ length: channels }, (_, c) => buffer.getChannelData(c).slice()) });
|
|
75
|
+
}
|
|
76
|
+
if (!parts.length) throw new Error('the audio track has no samples');
|
|
77
|
+
const total = Math.max(parts.reduce((m, p) => Math.max(m, p.at + p.data[0].length), 0), Math.round(duration * rate));
|
|
78
|
+
const data = Array.from({ length: channels }, () => new Float32Array(total));
|
|
79
|
+
for (const p of parts) p.data.forEach((d, c) => data[c].set(d, Math.max(0, p.at)));
|
|
80
|
+
const pcm = { numberOfChannels: channels, sampleRate: rate, length: total, getChannelData: (c) => data[c] };
|
|
81
|
+
const report = inspectSound(pcm, { window: windowSeconds });
|
|
82
|
+
let waveformFile = null;
|
|
83
|
+
if (waveform) {
|
|
84
|
+
const W = 1200, H = 240, c = new OffscreenCanvas(W, H), g = c.getContext('2d');
|
|
85
|
+
g.fillStyle = '#10141d'; g.fillRect(0, 0, W, H);
|
|
86
|
+
g.fillStyle = '#7fb4ff';
|
|
87
|
+
const per = Math.max(1, Math.floor(total / W));
|
|
88
|
+
for (let x = 0; x < W; x++) {
|
|
89
|
+
let lo = 0, hi = 0;
|
|
90
|
+
for (let c = 0; c < channels; c++) for (let i = x * per; i < Math.min(total, (x + 1) * per); i++) { const v = data[c][i]; if (v < lo) lo = v; if (v > hi) hi = v; } // every channel: a sound only on one side is not an empty picture
|
|
91
|
+
g.fillRect(x, H / 2 - hi * (H / 2 - 4), 1, Math.max(1, (hi - lo) * (H / 2 - 4)));
|
|
92
|
+
}
|
|
93
|
+
g.fillStyle = '#e9edf5'; g.font = '13px ui-monospace, Menlo, monospace'; g.fillText('0 s', 6, H - 6); g.fillText(report.duration.toFixed(1) + ' s', W - 60, H - 6);
|
|
94
|
+
waveformFile = 'waveform.png';
|
|
95
|
+
await put(waveformFile, await c.convertToBlob({ type: 'image/png' }));
|
|
96
|
+
}
|
|
97
|
+
return { ...report, waveform: waveformFile };
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
const FORMATS = { '.mp4': mb.Mp4OutputFormat, '.m4v': mb.Mp4OutputFormat, '.m4a': mb.Mp4OutputFormat, '.mov': mb.MovOutputFormat, '.webm': mb.WebMOutputFormat, '.mkv': mb.MkvOutputFormat, '.mka': mb.MkvOutputFormat };
|
|
101
|
+
|
|
102
|
+
/** [start, end] re-encoded (every track), so the cut starts on the frame asked for; the file is held in memory until it is handed back. */
|
|
103
|
+
window.media.trimExact = async ({ id, start, end, ext, outName }) => {
|
|
104
|
+
const input = await open(id);
|
|
105
|
+
const Format = FORMATS[ext];
|
|
106
|
+
if (!Format) throw new Error('cannot write ' + ext);
|
|
107
|
+
const target = new mb.BufferTarget();
|
|
108
|
+
const output = new mb.Output({ format: new Format(), target });
|
|
109
|
+
// only the PICTURE is re-encoded (that is what makes the start exact); the sound is copied: re-encoding it squeezed a 157 kb/s track to 55 kb/s and left it running 70-80 ms past the picture
|
|
110
|
+
const conversion = await mb.Conversion.init({ input, output, trim: { start, end }, video: { forceTranscode: true }, showWarnings: false });
|
|
111
|
+
if (!conversion.isValid || conversion.discardedTracks.length) {
|
|
112
|
+
throw new Error('this browser cannot re-encode it into ' + ext + ' (' + conversion.discardedTracks.map((d) => d.track.type + ': ' + d.reason).join(', ') + ')');
|
|
113
|
+
}
|
|
114
|
+
await conversion.execute();
|
|
115
|
+
await put(outName, new Blob([target.buffer]));
|
|
116
|
+
return { bytes: target.buffer.byteLength };
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
window.__mediaReady = true;
|
|
120
|
+
</script>
|
|
121
|
+
</body>
|
|
122
|
+
</html>
|