vizmatic 0.8.0 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PROMPT.md +10 -9
- package/README.md +167 -61
- package/ROADMAP.md +5 -4
- package/SCIENCE.md +1 -2
- package/assets/THIRD_PARTY_LICENSES.md +3 -3
- package/assets/fonts/SourceSerif4-24pt-SemiBold.ttf +0 -0
- package/assets/licenses/raster-codecs-MIT.txt +2 -2
- package/dist/{chunk-ZB3HK7RM.js → chunk-4MXGLMBG.js} +274 -24
- package/dist/chunk-GGFDBJ25.js +88 -0
- package/dist/{chunk-USDVNKO7.js → chunk-I745F4QK.js} +2323 -357
- package/dist/{chunk-DZHX76RS.js → chunk-JZPMGVBP.js} +32 -3
- package/dist/cli.js +443 -104
- package/dist/index.cjs +1055 -98
- package/dist/index.d.cts +216 -4
- package/dist/index.d.ts +216 -4
- package/dist/index.js +734 -63
- package/dist/scene-browser-chunks/chunk-RQNLYYJN.js +70 -0
- package/dist/scene-browser-chunks/{scene-browser-layout-M3XSXMV4.js → scene-browser-layout-RC3ECYRK.js} +1 -1
- package/dist/scene-browser.js +16 -16
- package/dist/scene-cli.js +1 -1
- package/dist/scene-runtime.cjs +24 -3
- package/dist/scene-runtime.js +1 -1
- package/dist/{server-2FYHZ4UO.js → server-Q4COUZHV.js} +3 -3
- package/dist/stage-browser/browser.d.ts +8240 -315
- package/dist/stage-browser/stage.js +22 -5
- package/dist/stage-export.cjs +3640 -298
- package/dist/stage-export.d.cts +56 -3
- package/dist/stage-export.d.ts +56 -3
- package/dist/stage-export.js +677 -149
- package/dist/stage.cjs +33026 -7554
- package/dist/stage.d.cts +7809 -286
- package/dist/stage.d.ts +7809 -286
- package/dist/stage.js +26066 -2599
- package/dist/{svg-import-C-RG1t7s.d.cts → svg-import-B5BGbC1t.d.cts} +455 -53
- package/dist/{svg-import-C-RG1t7s.d.ts → svg-import-B5BGbC1t.d.ts} +455 -53
- package/dist/{video-BSwNJZkn.d.ts → video-CiN1MDCS.d.cts} +27 -2
- package/dist/{video-BSwNJZkn.d.cts → video-CiN1MDCS.d.ts} +27 -2
- package/examples/README.md +1 -1
- package/examples/_teaching/layer-stack-connector.ts +36 -0
- package/examples/_teaching/layer-stack-recurrence.ts +29 -0
- package/examples/_teaching/layer-stack-tiny-gpt.ts +37 -0
- package/examples/_teaching/layer-stack-transformer.ts +44 -0
- package/examples/catalog-convolution-kernel.tsx +6 -7
- package/examples/{aurora-incident.tsx → checkout-incident.tsx} +1 -1
- package/examples/films/beam-search.ts +52 -0
- package/examples/films/cache-miss.ts +4 -3
- package/examples/films/catalog.json +30 -0
- package/examples/films/forecast.ts +354 -0
- package/examples/films/passage/chunking.ts +40 -0
- package/examples/films/passage/evidence.ts +44 -0
- package/examples/films/passage/prompt.ts +34 -0
- package/examples/films/paxos.ts +596 -0
- package/examples/films/taylor-series.ts +1 -1
- package/examples/narrated-cues/README.md +32 -0
- package/examples/narrated-cues/data/numbers.json +1 -0
- package/examples/narrated-cues/film.json +9 -0
- package/examples/narrated-cues/film.ts +78 -0
- package/examples/narrated-cues/package.json +1 -0
- package/examples/narrated-cues/script.json +57 -0
- package/examples/narrated-cues/timing.json +1 -0
- package/examples/narrated-cues/tsconfig.json +1 -0
- package/examples/proofs/decoding-bars.ts +67 -0
- package/examples/proofs/pipeline.ts +27 -0
- package/examples/proofs/table/evaluation.ts +41 -0
- package/examples/proofs/table/ledger.ts +47 -0
- package/examples/proofs/table/routing.ts +46 -0
- package/package.json +9 -4
- package/templates/narrated/README.md +61 -0
- package/templates/narrated/data/numbers.json +5 -0
- package/templates/narrated/film.json +12 -0
- package/templates/narrated/film.ts +14 -0
- package/templates/narrated/gitignore +8 -0
- package/templates/narrated/package.json +1 -0
- package/templates/narrated/scenes/answer.ts +25 -0
- package/templates/narrated/scenes/kit.ts +39 -0
- package/templates/narrated/scenes/question.ts +31 -0
- package/templates/narrated/script.json +26 -0
- package/templates/narrated/sources.md +22 -0
- package/templates/narrated/timing.json +1 -0
- package/templates/narrated/tsconfig.json +1 -0
- package/tools/narration/README.md +123 -0
- package/tools/narration/beats.py +43 -0
- package/tools/narration/checks.py +587 -0
- package/tools/narration/draft.py +73 -0
- package/tools/narration/film.py +341 -0
- package/tools/narration/lag.py +58 -0
- package/tools/narration/mix.py +265 -0
- package/tools/narration/probe.py +99 -0
- package/tools/narration/record.py +33 -0
- package/tools/narration/resound.py +170 -0
- package/tools/narration/soundbed.py +404 -0
- package/tools/narration/sync.py +93 -0
- package/tools/narration/transcribe.py +84 -0
- package/tools/narration/verify.py +59 -0
- package/tools/narration/voice.py +113 -0
- package/dist/chunk-6N6P2GJG.js +0 -88
- package/dist/scene-browser-chunks/chunk-5MVLYGXM.js +0 -70
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
#!/usr/bin/env -S uv run --script
|
|
2
|
+
# /// script
|
|
3
|
+
# requires-python = ">=3.12,<3.13"
|
|
4
|
+
# dependencies = ["numpy", "scipy", "soundfile"]
|
|
5
|
+
# ///
|
|
6
|
+
"""Lay the voice on the timeline of the film, and add a quiet pad and selected sound effects.
|
|
7
|
+
|
|
8
|
+
The voice stays on top: the pad from soundbed.py stays well under it, and rises only in a long pause
|
|
9
|
+
and where the film asks for a swell. The film gives its seed, its levels, its swells and its effects
|
|
10
|
+
in film.json, under "mix". The README of this folder describes each key.
|
|
11
|
+
|
|
12
|
+
It writes narration.wav for the render. An agent cannot hear the result, so it prints the levels and
|
|
13
|
+
writes the voice alone, the pad alone, and the mix before the loudness pass under stills/.
|
|
14
|
+
|
|
15
|
+
Usage: mix.py <folder>"""
|
|
16
|
+
import hashlib
|
|
17
|
+
import io
|
|
18
|
+
import subprocess
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
import soundfile as sf
|
|
23
|
+
from scipy.signal import resample_poly
|
|
24
|
+
|
|
25
|
+
import checks
|
|
26
|
+
import film
|
|
27
|
+
import soundbed
|
|
28
|
+
|
|
29
|
+
RATE = 48000
|
|
30
|
+
BELL = (659.25, 987.77, 1318.51) # E5, B5, E6.
|
|
31
|
+
LOW_BELL = (329.63, 493.88, 659.25) # The same bell, one octave lower.
|
|
32
|
+
CLEAR_VOICE = 18.0 # Decibels that the pad must stay under the voice while the voice speaks.
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def word_start(beat, word, last, where):
|
|
36
|
+
"""When the voice starts a word of one beat, in seconds from the start of the beat."""
|
|
37
|
+
hits = [spoken["start"] for spoken in beat.take["words"] if spoken["text"].strip(".,:?").lower() == word.lower()]
|
|
38
|
+
if not hits:
|
|
39
|
+
said = " ".join(spoken["text"] for spoken in beat.take["words"])
|
|
40
|
+
film.fail(f'{where}.word: the voice does not say "{word}" in beat {beat.index} of scene "{beat.scene}". It says: {said}')
|
|
41
|
+
return hits[-1] if last else hits[0]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def anchor(spec, line, where):
|
|
45
|
+
"""A time in the film, in seconds, from one of three references: the film ("at", and a negative
|
|
46
|
+
value counts from the end), the title card ("intro", a share of it), or a beat ("scene" and "beat",
|
|
47
|
+
with "edge" for the end of its voice, or "word" for a word of it). "offset" moves the time."""
|
|
48
|
+
if "at" in spec:
|
|
49
|
+
base = spec["at"] if spec["at"] >= 0 else line.total + spec["at"]
|
|
50
|
+
elif "intro" in spec:
|
|
51
|
+
base = line.intro * spec["intro"]
|
|
52
|
+
else:
|
|
53
|
+
beat = line.beat(spec["scene"], spec.get("beat", 0))
|
|
54
|
+
if "word" in spec:
|
|
55
|
+
base = beat.start + word_start(beat, spec["word"], spec.get("last", False), where)
|
|
56
|
+
else:
|
|
57
|
+
base = beat.voice_end if spec.get("edge", "start") == "end" else beat.start
|
|
58
|
+
return base + spec.get("offset", 0.0)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def lay_voice(root, line, samples):
|
|
62
|
+
"""The takes at their places, as one track with its peak at 0.89."""
|
|
63
|
+
voice = np.zeros(samples, dtype=np.float32)
|
|
64
|
+
for beat in line.beats:
|
|
65
|
+
data, rate = sf.read(root / "audio" / beat.take["wav"], dtype="float32")
|
|
66
|
+
if rate != RATE:
|
|
67
|
+
data = resample_poly(data, RATE, rate).astype(np.float32)
|
|
68
|
+
start = int(beat.start * RATE)
|
|
69
|
+
voice[start:start + len(data)] += data[: max(0, samples - start)]
|
|
70
|
+
return voice / np.abs(voice).max() * 0.89
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def swell(seconds, center, width, db):
|
|
74
|
+
"""A rise of the pad around one moment: a bell curve that is `db` decibels under the voice at its top."""
|
|
75
|
+
return 10 ** (db / 20) * np.exp(-((seconds - center) / width) ** 2)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def chime(samples, at, notes=BELL, gain=0.11, until=None):
|
|
79
|
+
"""A soft bell of three notes. With `until`, it fades over half a second and is silent from that time on."""
|
|
80
|
+
out = np.zeros(samples, dtype=np.float32)
|
|
81
|
+
start = int(at * RATE)
|
|
82
|
+
length = min(samples - start, int(5 * RATE))
|
|
83
|
+
t = np.arange(length) / RATE
|
|
84
|
+
out[start:start + length] = soundbed.bell_samples(t, notes, gain)
|
|
85
|
+
if until is not None:
|
|
86
|
+
out[start:start + length] *= np.clip((until - (at + t)) / 0.5, 0, 1).astype(np.float32)
|
|
87
|
+
return out
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def default_swells(line, card):
|
|
91
|
+
"""Without a list in film.json: a swell under the title card, when the film has one, and under the last card.
|
|
92
|
+
The last one starts a quarter of a second after the last word, so that the pad does not rise under the voice."""
|
|
93
|
+
# The rise under the title card is as wide as the card allows, so that it is over when the first word starts.
|
|
94
|
+
title = [{"intro": 0.5, "width": min(1.6, line.intro * 0.57)}] if line.intro > 0 else []
|
|
95
|
+
last_word = {"scene": -1, "beat": -1, "edge": "end", "offset": 0.25}
|
|
96
|
+
return title + [{"at": -2.4, "width": 1.6, "db": card + 1, "after": last_word}]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def default_chimes(line):
|
|
100
|
+
"""Without a list in film.json: a bell on the title card, when the film has one, and a fuller chord after the last word."""
|
|
101
|
+
title = [{"at": 0.15, "gain": 0.075}] if line.intro > 0 else []
|
|
102
|
+
return title + [{"scene": -1, "beat": -1, "edge": "end", "offset": 0.2, "gain": 0.1, "full": True}]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def pad_level(voice, seconds, line, levels, swells):
|
|
106
|
+
"""How loud the pad is over time, as a share of the level of the voice, and where the voice speaks."""
|
|
107
|
+
level, spoke = soundbed.level(voice, RATE, levels["under"], levels["pause"])
|
|
108
|
+
placed = []
|
|
109
|
+
for index, spec in enumerate(swells):
|
|
110
|
+
where = f"film.json: mix.swells[{index}]"
|
|
111
|
+
center = anchor(spec, line, where)
|
|
112
|
+
rise = swell(seconds, center, spec.get("width", 2.0), spec.get("db", levels["card"]))
|
|
113
|
+
if "after" in spec:
|
|
114
|
+
# The swell must not start under the last words, so it is cut before this time.
|
|
115
|
+
rise = rise * (seconds > anchor(spec["after"], line, f'{where} "after"'))
|
|
116
|
+
level = np.maximum(level, rise)
|
|
117
|
+
placed.append(center)
|
|
118
|
+
# The pad fades in over the first 1.5 seconds and out over the last 2.
|
|
119
|
+
level *= np.clip(seconds / 1.5, 0, 1) * np.clip((line.total - seconds) / 2.0, 0, 1)
|
|
120
|
+
return level, spoke, placed
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def bells(samples, line, chimes):
|
|
124
|
+
"""All the chimes as one track, and the time of each. A chime with "full" gets a second, lower bell."""
|
|
125
|
+
track, placed = np.zeros(samples, dtype=np.float32), []
|
|
126
|
+
for index, spec in enumerate(chimes):
|
|
127
|
+
where = f"film.json: mix.chimes[{index}]"
|
|
128
|
+
at = anchor(spec, line, where)
|
|
129
|
+
until = anchor(spec["until"], line, f'{where} "until"') if "until" in spec else None
|
|
130
|
+
if not 0 <= at < line.total:
|
|
131
|
+
film.fail(f"{where}: the chime is at {at:.2f} s, and the film is {line.total:.2f} s long. Give a time inside the film")
|
|
132
|
+
if until is not None and until - at < 1.0:
|
|
133
|
+
film.fail(f"{where}: the chime starts at {at:.2f} s and must be silent from {until:.2f} s. A bell needs one second to ring")
|
|
134
|
+
bell = chime(samples, at, tuple(spec.get("notes", BELL)), spec.get("gain", 0.11), until)
|
|
135
|
+
if until is not None and until < line.total:
|
|
136
|
+
assert np.abs(bell[int(until * RATE):]).max() == 0, "the bell is silent from the time that the film gives"
|
|
137
|
+
track = track + bell
|
|
138
|
+
if spec.get("full"):
|
|
139
|
+
# The last card gets a fuller chord: the same bell, and one an octave lower.
|
|
140
|
+
track = track + chime(samples, at + 0.03, LOW_BELL, 0.07)
|
|
141
|
+
placed.append((at, until))
|
|
142
|
+
return track, placed
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def read_effect_wav(data, where):
|
|
146
|
+
"""Decode the mono source at its authored level and resample it to the mix rate."""
|
|
147
|
+
try:
|
|
148
|
+
with sf.SoundFile(io.BytesIO(data)) as source:
|
|
149
|
+
if source.format not in ("WAV", "WAVEX", "RF64") or source.channels != 1 or not source.frames:
|
|
150
|
+
film.fail(f"{where}: give a nonempty mono WAV")
|
|
151
|
+
rate = source.samplerate
|
|
152
|
+
samples = source.read(dtype="float32")
|
|
153
|
+
except (ValueError, RuntimeError) as error:
|
|
154
|
+
film.fail(f"{where}: cannot read the WAV ({error})")
|
|
155
|
+
if not np.isfinite(samples).all():
|
|
156
|
+
film.fail(f"{where}: the WAV contains nonfinite samples")
|
|
157
|
+
return resample_poly(samples, RATE, rate) if rate != RATE else samples
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def effects(samples, line, entries, wavs):
|
|
161
|
+
"""Place each clip with its own gain and cutoff, in the order the author gives."""
|
|
162
|
+
track, placed = np.zeros(samples, dtype=np.float32), []
|
|
163
|
+
for index, spec in enumerate(entries):
|
|
164
|
+
where = f"film.json: mix.effects[{index}]"
|
|
165
|
+
at = anchor(spec, line, where)
|
|
166
|
+
until = anchor(spec["until"], line, f'{where} "until"') if "until" in spec else None
|
|
167
|
+
if not 0 <= at < line.total:
|
|
168
|
+
film.fail(f"{where}: the effect is at {at:.2f} s, and the film is {line.total:.2f} s long. Give a time inside the film")
|
|
169
|
+
if until is not None and until <= at:
|
|
170
|
+
film.fail(f"{where}: the effect starts at {at:.2f} s and must be silent from {until:.2f} s. Give an until time after the start")
|
|
171
|
+
start = int(at * RATE)
|
|
172
|
+
end = int(min(line.total, until) * RATE) if until is not None else samples
|
|
173
|
+
if "wav" in spec:
|
|
174
|
+
name = spec["wav"]
|
|
175
|
+
clip = read_effect_wav(wavs[name], f"{where}.wav ({name})")
|
|
176
|
+
gain = spec.get("gain", 1.0)
|
|
177
|
+
else:
|
|
178
|
+
clip = soundbed.sound_effect(spec["sound"], RATE)
|
|
179
|
+
gain = spec.get("gain", checks.SOUND_EFFECTS[spec["sound"]]["gain"])
|
|
180
|
+
clip = clip[:max(0, end - start)] * gain
|
|
181
|
+
# A cut fades only this clip. It must not change effects that already occupy the track.
|
|
182
|
+
if len(clip) and end - start <= len(clip):
|
|
183
|
+
fade = min(len(clip), int(0.02 * RATE))
|
|
184
|
+
clip[-fade:] *= np.linspace(1, 0, fade, dtype=np.float32) if fade > 1 else 0
|
|
185
|
+
track[start:start + len(clip)] += clip
|
|
186
|
+
placed.append((at, until))
|
|
187
|
+
return track, placed
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def write_sound(root, mix, voice, bed):
|
|
191
|
+
"""The mix at -16 LUFS for the render, and the three tracks that a check can read."""
|
|
192
|
+
stills = root / "stills"
|
|
193
|
+
stills.mkdir(exist_ok=True)
|
|
194
|
+
sf.write(stills / "mix-raw.wav", mix, RATE, subtype="PCM_24")
|
|
195
|
+
subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", str(stills / "mix-raw.wav"), "-af", "loudnorm=I=-16:TP=-1.5:LRA=11", "-ar", str(RATE), "-c:a", "pcm_s24le", str(root / "narration.wav")], check=True)
|
|
196
|
+
sf.write(stills / "voice-only.wav", voice, RATE, subtype="PCM_16")
|
|
197
|
+
sf.write(stills / "bed-only.wav", bed, RATE, subtype="PCM_24")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def report(line, levels, mix, voice, bed, level, spoke, track, parts, swells, chimes, named=False):
|
|
201
|
+
"""Print the levels, because nobody hears the mix before the film is rendered."""
|
|
202
|
+
rms = soundbed.decibels
|
|
203
|
+
quiet = ~spoke & (level >= 10 ** (levels["pause"] / 20) * 0.98)
|
|
204
|
+
print(f"film {line.total:.2f}s beats {len(line.beats)} peak {np.abs(mix).max():.3f}")
|
|
205
|
+
print(f"voice RMS while speaking {rms(voice[spoke]):.1f} dB; bed RMS under voice {rms(bed[spoke, 0]):.1f} dB; gap {rms(voice[spoke]) - rms(bed[spoke, 0]):.1f} dB")
|
|
206
|
+
if line.intro > 0:
|
|
207
|
+
intro = int(line.intro * RATE)
|
|
208
|
+
print(f"intro: bed RMS {rms(bed[:intro, 0]):.1f} dB, {'effect' if named else 'chime'} peak {np.abs(track[:intro]).max():.3f}")
|
|
209
|
+
long_pauses = f"{rms(bed[quiet, 0]):.1f} dB" if quiet.any() else "none"
|
|
210
|
+
print(f"bed RMS in long pauses {long_pauses}; {'effects' if named else 'chimes'} at {', '.join(f'{at:.2f}s' for at, _ in chimes)}; swells at {', '.join(f'{at:.2f}s' for at in swells)}")
|
|
211
|
+
for at, until in chimes:
|
|
212
|
+
if until is not None and named:
|
|
213
|
+
print(f"the effect of {at:.2f}s is silent from {until:.2f}s")
|
|
214
|
+
elif until is not None:
|
|
215
|
+
# What sounds under the words that follow the bell: the voice, the pad at its usual level, and no bell.
|
|
216
|
+
a, b = int((until + 0.5) * RATE), int((until + 1.0) * RATE)
|
|
217
|
+
print(f"the bell of {at:.2f}s is silent from {until:.2f}s; from {until + 0.5:.2f}s to {until + 1.0:.2f}s, voice RMS {rms(voice[a:b]):.1f} dB, bed RMS {rms(bed[a:b, 0]):.1f} dB, bell peak {np.abs(track[a:b]).max():.4f}")
|
|
218
|
+
lengths = [end - start for start, end, _, _ in parts]
|
|
219
|
+
print(f"pad: {len(parts)} chords, {min(lengths):.1f} to {max(lengths):.1f} s each; " + " ".join(chord for _, _, chord, _ in parts))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def main(found, _words, _options):
|
|
223
|
+
root = found.root
|
|
224
|
+
line = film.place(found.timing)
|
|
225
|
+
settings = found.mix
|
|
226
|
+
levels = settings["levels"]
|
|
227
|
+
samples = int(line.total * RATE)
|
|
228
|
+
voice = lay_voice(root, line, samples)
|
|
229
|
+
|
|
230
|
+
# The pad has no loop: the chord changes where a beat starts, a seeded random walk picks each chord,
|
|
231
|
+
# and each scene has its own texture.
|
|
232
|
+
parts = soundbed.plan(line.total, [beat.start for beat in line.beats], list(line.scenes.values()), seed=settings["seed"])
|
|
233
|
+
pad = soundbed.pad(line.total, RATE, parts)
|
|
234
|
+
seconds = np.arange(samples, dtype=np.float32) / RATE
|
|
235
|
+
swells = settings["swells"] if settings["swells"] is not None else default_swells(line, levels["card"])
|
|
236
|
+
level, spoke, swell_times = pad_level(voice, seconds, line, levels, swells)
|
|
237
|
+
bed = pad * (level * 10 ** (soundbed.decibels(voice[spoke]) / 20))[:, None]
|
|
238
|
+
|
|
239
|
+
named = settings["effects"] is not None
|
|
240
|
+
# Decode and fingerprint the same bytes even if the author edits a WAV during the mix.
|
|
241
|
+
wavs = film.effect_wavs(found, Path.read_bytes)
|
|
242
|
+
if named:
|
|
243
|
+
track, chime_times = effects(samples, line, settings["effects"], wavs)
|
|
244
|
+
else:
|
|
245
|
+
chimes = settings["chimes"] if settings["chimes"] is not None else default_chimes(line)
|
|
246
|
+
track, chime_times = bells(samples, line, chimes)
|
|
247
|
+
# The voice is the same in both channels. The pad differs between them, and the bells are a little quieter at the right.
|
|
248
|
+
mix = np.stack([voice + bed[:, 0] + track, voice + bed[:, 1] + track * 0.92], axis=1)
|
|
249
|
+
peak = np.abs(mix).max()
|
|
250
|
+
if peak > 0.97:
|
|
251
|
+
mix = mix / peak * 0.97
|
|
252
|
+
write_sound(root, mix, voice, bed)
|
|
253
|
+
# A build reads this record, and leaves a mix that is made from the timing and the film.json of now.
|
|
254
|
+
film.write_mix_record(found, {name: hashlib.sha256(data).hexdigest() for name, data in wavs.items()})
|
|
255
|
+
report(line, levels, mix, voice, bed, level, spoke, track, parts, swell_times, chime_times, named)
|
|
256
|
+
under = soundbed.decibels(voice[spoke]) - soundbed.decibels(bed[spoke, 0])
|
|
257
|
+
summary = f"narration.wav is written: {line.total:.2f} s, {len(line.beats)} takes, the pad {under:.1f} dB under the voice."
|
|
258
|
+
if under < CLEAR_VOICE:
|
|
259
|
+
film.verdict(f"{summary} That is too near: under {CLEAR_VOICE} dB the pad covers the voice. Lower mix.levels.under, or move a swell away from the voice, and mix again.")
|
|
260
|
+
else:
|
|
261
|
+
film.verdict(f"{summary} Next: {found.next('check', '--motion')}")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
if __name__ == "__main__":
|
|
265
|
+
film.run(main, "mix", "mix.py <folder>")
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/usr/bin/env -S uv run --script
|
|
2
|
+
# /// script
|
|
3
|
+
# requires-python = ">=3.12"
|
|
4
|
+
# dependencies = []
|
|
5
|
+
# ///
|
|
6
|
+
"""Probe a movie: its streams, its duration, its chapters, its captions and its hash. Prints JSON.
|
|
7
|
+
|
|
8
|
+
With a timing in the folder, it also says if the picture of the movie was rendered with that timing.
|
|
9
|
+
|
|
10
|
+
Usage: probe.py <folder> [movie.mp4 | --draft]. The default is the delivered film in dist/."""
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import subprocess
|
|
14
|
+
|
|
15
|
+
import checks
|
|
16
|
+
import film
|
|
17
|
+
|
|
18
|
+
# QuickTime can refuse a film of 1080p at 60 frames per second above this H.264 level.
|
|
19
|
+
HIGHEST_LEVEL = 42
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def ffprobe(movie, *args):
|
|
23
|
+
return json.loads(subprocess.run(["ffprobe", "-v", "error", "-of", "json", *args, str(movie)], capture_output=True, text=True, check=True).stdout)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def caption_cues(movie):
|
|
27
|
+
"""The number of caption cues in the first subtitle stream."""
|
|
28
|
+
srt = subprocess.run(["ffmpeg", "-v", "error", "-i", str(movie), "-map", "0:s:0", "-f", "srt", "-"], capture_output=True, text=True).stdout
|
|
29
|
+
return srt.count(" --> ")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def in_step(root, movie, info):
|
|
33
|
+
"""The picture of the movie against timing.json: if the movie has a valid record of the timing of its
|
|
34
|
+
picture, and where that timing differs from timing.json. None without a timing, and for a movie with no picture."""
|
|
35
|
+
timing = checks.usable_timing(root)
|
|
36
|
+
if timing is None or not any(stream["codec_type"] == "video" for stream in info["streams"]):
|
|
37
|
+
return None
|
|
38
|
+
rendered, why = film.picture_timing(movie)
|
|
39
|
+
return {"record": why or "valid", "differs": film.timeline_changes(rendered, timing) if rendered else []}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def probe(movie, root):
|
|
43
|
+
info = ffprobe(movie, "-show_format", "-show_streams", "-show_chapters")
|
|
44
|
+
streams = {stream["codec_type"]: stream for stream in info["streams"]}
|
|
45
|
+
video, audio, subs = streams.get("video"), streams.get("audio"), streams.get("subtitle")
|
|
46
|
+
return {
|
|
47
|
+
"path": str(movie),
|
|
48
|
+
"sha256": hashlib.sha256(movie.read_bytes()).hexdigest(),
|
|
49
|
+
"size_mb": round(movie.stat().st_size / 1e6, 1),
|
|
50
|
+
"duration_seconds": round(float(info["format"]["duration"]), 2),
|
|
51
|
+
"title": info["format"].get("tags", {}).get("title"),
|
|
52
|
+
"video": video and {key: video.get(key) for key in ("codec_name", "profile", "level", "pix_fmt", "width", "height", "r_frame_rate", "nb_frames")},
|
|
53
|
+
"audio": audio and {key: audio.get(key) for key in ("codec_name", "sample_rate", "channels", "duration")},
|
|
54
|
+
"captions": subs and {"codec_name": subs.get("codec_name"), "cues": caption_cues(movie)},
|
|
55
|
+
"chapters": [{"start": round(float(chapter["start_time"]), 2), "title": chapter.get("tags", {}).get("title")} for chapter in info.get("chapters", [])],
|
|
56
|
+
"timing": in_step(root, movie, info),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def missing(found):
|
|
61
|
+
"""What a narrated film has and this movie does not."""
|
|
62
|
+
video, lacks = found["video"], []
|
|
63
|
+
if not video:
|
|
64
|
+
lacks.append("it has no picture")
|
|
65
|
+
elif video.get("codec_name") == "h264" and (video.get("level") or 0) > HIGHEST_LEVEL:
|
|
66
|
+
lacks.append(f"its H.264 level is {video['level']}, and QuickTime can refuse a film over {HIGHEST_LEVEL}")
|
|
67
|
+
if not found["audio"]:
|
|
68
|
+
lacks.append("it has no sound")
|
|
69
|
+
if not found["captions"] or not found["captions"]["cues"]:
|
|
70
|
+
lacks.append("it has no captions")
|
|
71
|
+
if not found["chapters"]:
|
|
72
|
+
lacks.append("it has no chapters")
|
|
73
|
+
if found["timing"] and found["timing"]["record"] != "valid":
|
|
74
|
+
lacks.append(f"it {found['timing']['record']}")
|
|
75
|
+
elif found["timing"] and found["timing"]["differs"]:
|
|
76
|
+
lacks.append(f"its picture was rendered with another timing than timing.json has now ({', '.join(found['timing']['differs'])})")
|
|
77
|
+
return lacks
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def main(found, words, options):
|
|
81
|
+
movie = found.movie(words, options)
|
|
82
|
+
result = probe(movie, found.root)
|
|
83
|
+
print(json.dumps(result, indent=1))
|
|
84
|
+
video = result["video"] or {}
|
|
85
|
+
summary = f"{movie.name}: {video.get('width')}x{video.get('height')}, {result['duration_seconds']} s, {len(result['chapters'])} chapters, {(result['captions'] or {}).get('cues', 0)} caption cues"
|
|
86
|
+
lacks = missing(result)
|
|
87
|
+
timing = result["timing"]
|
|
88
|
+
draft = " --draft" if movie.name.endswith("-draft.mp4") else ""
|
|
89
|
+
if timing and timing["record"] != "valid" and len(lacks) == 1:
|
|
90
|
+
# The one thing that is wrong is the record: the line is the line of the resound and sync steps.
|
|
91
|
+
film.verdict(f"{summary}. {film.no_record(found, movie, timing['record'])}", passed=False)
|
|
92
|
+
elif lacks:
|
|
93
|
+
film.verdict(f"{summary}. Not as a narrated film is built: {'; '.join(lacks)}. Build it again: {found.next('build')}{draft}", passed=False)
|
|
94
|
+
else:
|
|
95
|
+
film.verdict(f"{summary}. It has a picture, sound, captions and chapters. Next: {found.next('sync')}{draft}")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == "__main__":
|
|
99
|
+
film.run(main, "probe", "probe.py <folder> [movie.mp4 | --draft]", flags=("--draft",), words=True)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#!/usr/bin/env -S uv run --script
|
|
2
|
+
# /// script
|
|
3
|
+
# requires-python = ">=3.12"
|
|
4
|
+
# dependencies = []
|
|
5
|
+
# ///
|
|
6
|
+
"""Keep with a movie the timing that its picture was rendered with, as <name>.timing.json beside it.
|
|
7
|
+
|
|
8
|
+
The build runs this step after each render. The resound step and the sync step read the record: a sound
|
|
9
|
+
of another timing must not go into this movie, because the picture and the captions follow the voice.
|
|
10
|
+
|
|
11
|
+
The build deletes the old record before it renders, and writes the new one after the render. So a build
|
|
12
|
+
that stops at any point leaves a movie with no record, and never a new picture beside an old record.
|
|
13
|
+
|
|
14
|
+
Usage: record.py <folder> <movie.mp4> --timing <sha256 of timing.json before the render>"""
|
|
15
|
+
import film
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main(found, words, options):
|
|
19
|
+
if len(words) != 1:
|
|
20
|
+
film.fail("give the movie that was rendered: record.py <folder> <movie.mp4>")
|
|
21
|
+
movie = found.movie(words, options, writes=True)
|
|
22
|
+
before = options.get("--timing")
|
|
23
|
+
if not before:
|
|
24
|
+
film.fail("give the hash of timing.json as it was before the render: --timing <sha256>. Without it, nothing says that this timing is the timing of the picture")
|
|
25
|
+
# The render read timing.json when it started. A file that changed since then is not the timing of this picture.
|
|
26
|
+
if film.file_hash(found.root / "timing.json") != before:
|
|
27
|
+
film.fail(f"timing.json changed while the film was rendered, so nothing says which timing the picture of {movie.name} has. Build again: {found.next('build')}")
|
|
28
|
+
film.write_picture_record(movie, found.timing)
|
|
29
|
+
film.verdict(f"{film.picture_record(movie).name} is written beside {movie.name}: the timing that its picture was rendered with.")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
if __name__ == "__main__":
|
|
33
|
+
film.run(main, "record", "record.py <folder> <movie.mp4> --timing <sha256>", values=("--timing",), words=True)
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!/usr/bin/env -S uv run --script
|
|
2
|
+
# /// script
|
|
3
|
+
# requires-python = ">=3.12"
|
|
4
|
+
# dependencies = []
|
|
5
|
+
# ///
|
|
6
|
+
"""Put a new narration.wav into a rendered movie, without a new render of the picture.
|
|
7
|
+
|
|
8
|
+
Use it after a change that touches only the sound: the levels, the seed, a chime. The timing of the
|
|
9
|
+
voice must not change, because the picture and the captions follow the voice. So the step compares the
|
|
10
|
+
timing that the picture was rendered with and the timing of the new sound, scene by scene, and stops
|
|
11
|
+
when they differ. The build keeps the timing of a picture beside its movie, as <name>.timing.json. A
|
|
12
|
+
movie with no record that is valid for its picture is refused too: build it again.
|
|
13
|
+
|
|
14
|
+
The picture and the captions are copied as they are. The title and the chapters are read from the movie
|
|
15
|
+
and written again. The new movie is written under a name of its own, it is checked (its streams, its
|
|
16
|
+
length, its chapters and its picture), and only then one rename puts it in the place of the old one.
|
|
17
|
+
|
|
18
|
+
Usage: resound.py <folder> [movie.mp4 | --draft]. The default is the delivered film in dist/."""
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import subprocess
|
|
22
|
+
import tempfile
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import film
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def ffprobe(movie, *args):
|
|
29
|
+
return json.loads(subprocess.run(["ffprobe", "-v", "error", "-of", "json", *args, str(movie)], capture_output=True, text=True, check=True).stdout)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def escape(value):
|
|
33
|
+
"""A value of an FFmpeg metadata file: these four characters and a line break need a backslash."""
|
|
34
|
+
return "".join("\\" + char if char in "=;#\\\n" else char for char in value)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def metadata(info):
|
|
38
|
+
"""The title and the chapters of the movie, as an FFmpeg metadata file."""
|
|
39
|
+
lines = [";FFMETADATA1"]
|
|
40
|
+
title = info["format"].get("tags", {}).get("title")
|
|
41
|
+
if title:
|
|
42
|
+
lines.append(f"title={escape(title)}")
|
|
43
|
+
for chapter in info.get("chapters", []):
|
|
44
|
+
start, end = round(float(chapter["start_time"]) * 1000), round(float(chapter["end_time"]) * 1000)
|
|
45
|
+
lines += ["[CHAPTER]", "TIMEBASE=1/1000", f"START={start}", f"END={end}", f"title={escape(chapter.get('tags', {}).get('title', ''))}"]
|
|
46
|
+
return "\n".join(lines) + "\n"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def chapter_track_entries(movie):
|
|
50
|
+
"""The entries of the chapter track of an MP4, which is its first data stream. A broken track has fewer than the film has chapters."""
|
|
51
|
+
streams = ffprobe(movie, "-select_streams", "d:0", "-show_entries", "stream=nb_frames").get("streams", [])
|
|
52
|
+
return int(streams[0].get("nb_frames", 0)) if streams else 0
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def same_length(movie_seconds, sound_seconds, shown):
|
|
56
|
+
"""Stop when the new sound is not as long as the movie: the voice moved, and the picture follows the voice."""
|
|
57
|
+
if abs(movie_seconds - sound_seconds) > film.SAME_LENGTH:
|
|
58
|
+
film.fail(f"the movie is {movie_seconds:.2f} s long and narration.wav is {sound_seconds:.2f} s long. The timing of the voice changed, "
|
|
59
|
+
f"so the picture needs a new render: vizmatic narrated build {shown}. The movie was not replaced")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def temporary(movie, suffix):
|
|
63
|
+
"""A file of this run alone, beside the movie. Two runs at once, such as one for the delivered movie and
|
|
64
|
+
one for the draft, must never write to one file."""
|
|
65
|
+
handle, name = tempfile.mkstemp(dir=movie.parent, prefix=f".{movie.stem}.", suffix=suffix)
|
|
66
|
+
os.close(handle)
|
|
67
|
+
return Path(name)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def kinds(info):
|
|
71
|
+
"""How many streams of each kind a movie has."""
|
|
72
|
+
found = {}
|
|
73
|
+
for stream in info.get("streams", []):
|
|
74
|
+
found[stream["codec_type"]] = found.get(stream["codec_type"], 0) + 1
|
|
75
|
+
return found
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def defects(before, after, picture_before, picture_after, chapter_track):
|
|
79
|
+
"""What is wrong with the new movie, against the movie that it replaces: `before` and `after` are what
|
|
80
|
+
ffprobe says of the two. An empty list means that every stream, the length, the chapters and the
|
|
81
|
+
picture are as they were, and that the movie has sound."""
|
|
82
|
+
found = []
|
|
83
|
+
had, has = kinds(before), kinds(after)
|
|
84
|
+
for kind in ("video", "subtitle"):
|
|
85
|
+
if has.get(kind, 0) != had.get(kind, 0):
|
|
86
|
+
found.append(f"it has {has.get(kind, 0)} {kind} streams, and the movie has {had.get(kind, 0)}")
|
|
87
|
+
if has.get("audio", 0) != 1:
|
|
88
|
+
found.append(f"it has {has.get('audio', 0)} sound streams, and a movie has 1")
|
|
89
|
+
length = float(after.get("format", {}).get("duration", 0))
|
|
90
|
+
if abs(length - float(before["format"]["duration"])) > film.SAME_LENGTH:
|
|
91
|
+
found.append(f"it is {length:.2f} s long, and the movie is {float(before['format']['duration']):.2f} s long")
|
|
92
|
+
chapters = lambda info: [(round(float(chapter["start_time"]), 3), chapter.get("tags", {}).get("title")) for chapter in info.get("chapters", [])]
|
|
93
|
+
wanted, written = chapters(before), chapters(after)
|
|
94
|
+
if written != wanted or chapter_track != len(wanted):
|
|
95
|
+
found.append(f"it has {min(len(written), chapter_track)} chapters, and the movie has {len(wanted)}")
|
|
96
|
+
if picture_after != picture_before:
|
|
97
|
+
found.append("its picture is not the picture of the movie")
|
|
98
|
+
return found
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def millisecond_scale():
|
|
102
|
+
"""The option that sets the time scale of the movie header to one millisecond.
|
|
103
|
+
|
|
104
|
+
The chapter track of an MP4 counts in that scale. FFmpeg 7 and later take for it the least common
|
|
105
|
+
multiple of the time scales of the tracks: 48,000,000 for a film with a picture, sound at 48 kHz and
|
|
106
|
+
captions. A chapter longer than 44 seconds then does not fit in 32 bits, and the chapter track breaks.
|
|
107
|
+
One millisecond is the scale that FFmpeg wrote up to version 6, and it holds a chapter of any length.
|
|
108
|
+
An FFmpeg that has no such option, which is version 4, writes one millisecond always."""
|
|
109
|
+
listed = subprocess.run(["ffmpeg", "-hide_banner", "-h", "muxer=mp4"], capture_output=True, text=True).stdout
|
|
110
|
+
return ["-movie_timescale", "1000"] if "movie_timescale" in listed else []
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def aac_options():
|
|
114
|
+
"""Disable noise substitution when supported, to avoid decoder-dependent spikes."""
|
|
115
|
+
listed = subprocess.run(["ffmpeg", "-hide_banner", "-h", "encoder=aac"], capture_output=True, text=True)
|
|
116
|
+
return ["-aac_pns", "0"] if listed.returncode == 0 and "-aac_pns" in listed.stdout else []
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def remux(movie, sound, info, made):
|
|
120
|
+
"""Write the movie again with the new sound, into `made`. The picture is copied as it is."""
|
|
121
|
+
notes = temporary(movie, ".resound.txt")
|
|
122
|
+
try:
|
|
123
|
+
notes.write_text(metadata(info))
|
|
124
|
+
subprocess.run([
|
|
125
|
+
"ffmpeg", "-y", "-loglevel", "warning", "-i", str(movie), "-i", str(sound), "-f", "ffmetadata", "-i", str(notes),
|
|
126
|
+
"-map", "0:v:0", "-map", "1:a:0", "-map", "0:s?", "-map_metadata", "2", "-map_chapters", "2",
|
|
127
|
+
"-af", f"asetpts=PTS-STARTPTS,apad,atrim=duration={info['format']['duration']}",
|
|
128
|
+
"-c:v", "copy", "-c:s", "copy",
|
|
129
|
+
"-c:a", "aac", "-b:a", "384k", "-ar", "48000", "-ac", "2", *aac_options(), *millisecond_scale(), "-movflags", "+faststart", "-f", "mp4", str(made),
|
|
130
|
+
], check=True)
|
|
131
|
+
finally:
|
|
132
|
+
notes.unlink(missing_ok=True)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def main(found, words, options):
|
|
136
|
+
movie = found.movie(words, options, writes=True)
|
|
137
|
+
draft = " --draft" if movie.name.endswith("-draft.mp4") else ""
|
|
138
|
+
sound = found.root / "narration.wav"
|
|
139
|
+
if not sound.exists():
|
|
140
|
+
film.fail(f"narration.wav is missing. Run the mix step first: {found.next('mix')}")
|
|
141
|
+
current, why = film.mix_state(found)
|
|
142
|
+
if not current:
|
|
143
|
+
film.fail(f"{why}, so the sound is not known to be the sound of timing.json. Run the mix step first: {found.next('mix')}. The movie was not replaced")
|
|
144
|
+
info = ffprobe(movie, "-show_format", "-show_streams", "-show_chapters")
|
|
145
|
+
same_length(float(info["format"]["duration"]), float(ffprobe(sound, "-show_format")["format"]["duration"]), found.shown)
|
|
146
|
+
rendered, why = film.picture_timing(movie)
|
|
147
|
+
if rendered is None:
|
|
148
|
+
film.fail(f"{film.no_record(found, movie, why)}. The movie was not replaced")
|
|
149
|
+
differ = film.timeline_changes(rendered, found.timing)
|
|
150
|
+
if differ:
|
|
151
|
+
film.fail(f"the picture of {movie.name} was rendered with another timing than timing.json has now: {', '.join(differ)}. The voice moved, "
|
|
152
|
+
f"and the picture and the captions follow the voice, so the picture needs a new render: {found.next('build')}{draft}. The movie was not replaced")
|
|
153
|
+
picture = film.picture_of(movie)
|
|
154
|
+
made = temporary(movie, ".resound.mp4")
|
|
155
|
+
try:
|
|
156
|
+
remux(movie, sound, info, made)
|
|
157
|
+
wrong = defects(info, ffprobe(made, "-show_format", "-show_streams", "-show_chapters"), picture, film.picture_of(made), chapter_track_entries(made))
|
|
158
|
+
if wrong:
|
|
159
|
+
film.fail(f"the movie with the new sound is not right: {'; '.join(wrong)}. {movie.name} was not replaced")
|
|
160
|
+
# One rename: a reader of the movie sees the old file or the new one, and never a part of one.
|
|
161
|
+
os.replace(made, movie)
|
|
162
|
+
finally:
|
|
163
|
+
made.unlink(missing_ok=True)
|
|
164
|
+
film.verdict(f"{movie.name} has the new sound. The new file was checked before it took the place of the old one: its picture, its captions, "
|
|
165
|
+
f"its {len(info.get('chapters', []))} chapters and its length are as they were. Its picture was rendered with the timing of timing.json, "
|
|
166
|
+
f"by the record that the build keeps beside it. Next: {found.next('sync')}{draft}")
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
if __name__ == "__main__":
|
|
170
|
+
film.run(main, "resound", "resound.py <folder> [movie.mp4 | --draft]", flags=("--draft",), words=True)
|