polyphon 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +374 -2
- package/README.zh-TW.md +375 -0
- package/dist/engine/automation.d.ts +24 -0
- package/dist/engine/automation.d.ts.map +1 -0
- package/dist/engine/automation.js +55 -0
- package/dist/engine/automation.js.map +1 -0
- package/dist/engine/compile.d.ts +36 -0
- package/dist/engine/compile.d.ts.map +1 -0
- package/dist/engine/compile.js +103 -0
- package/dist/engine/compile.js.map +1 -0
- package/dist/engine/jukebox.d.ts +113 -0
- package/dist/engine/jukebox.d.ts.map +1 -0
- package/dist/engine/jukebox.js +287 -0
- package/dist/engine/jukebox.js.map +1 -0
- package/dist/engine/mixer.d.ts +29 -0
- package/dist/engine/mixer.d.ts.map +1 -0
- package/dist/engine/mixer.js +141 -0
- package/dist/engine/mixer.js.map +1 -0
- package/dist/engine/output.d.ts +14 -0
- package/dist/engine/output.d.ts.map +1 -0
- package/dist/engine/output.js +51 -0
- package/dist/engine/output.js.map +1 -0
- package/dist/engine/playback.d.ts +97 -0
- package/dist/engine/playback.d.ts.map +1 -0
- package/dist/engine/playback.js +311 -0
- package/dist/engine/playback.js.map +1 -0
- package/dist/engine/prepare.d.ts +6 -0
- package/dist/engine/prepare.d.ts.map +1 -0
- package/dist/engine/prepare.js +12 -0
- package/dist/engine/prepare.js.map +1 -0
- package/dist/engine/render.d.ts +14 -0
- package/dist/engine/render.d.ts.map +1 -0
- package/dist/engine/render.js +60 -0
- package/dist/engine/render.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/instruments/bowed.d.ts +357 -0
- package/dist/instruments/bowed.d.ts.map +1 -0
- package/dist/instruments/bowed.js +502 -0
- package/dist/instruments/bowed.js.map +1 -0
- package/dist/instruments/builtin.d.ts +261 -0
- package/dist/instruments/builtin.d.ts.map +1 -0
- package/dist/instruments/builtin.js +160 -0
- package/dist/instruments/builtin.js.map +1 -0
- package/dist/instruments/definition.d.ts +46 -0
- package/dist/instruments/definition.d.ts.map +1 -0
- package/dist/instruments/definition.js +5 -0
- package/dist/instruments/definition.js.map +1 -0
- package/dist/instruments/dsp/ambience.d.ts +17 -0
- package/dist/instruments/dsp/ambience.d.ts.map +1 -0
- package/dist/instruments/dsp/ambience.js +146 -0
- package/dist/instruments/dsp/ambience.js.map +1 -0
- package/dist/instruments/dsp/core.d.ts +22 -0
- package/dist/instruments/dsp/core.d.ts.map +1 -0
- package/dist/instruments/dsp/core.js +100 -0
- package/dist/instruments/dsp/core.js.map +1 -0
- package/dist/instruments/dsp/drums.d.ts +112 -0
- package/dist/instruments/dsp/drums.d.ts.map +1 -0
- package/dist/instruments/dsp/drums.js +566 -0
- package/dist/instruments/dsp/drums.js.map +1 -0
- package/dist/instruments/dsp/modal.d.ts +30 -0
- package/dist/instruments/dsp/modal.d.ts.map +1 -0
- package/dist/instruments/dsp/modal.js +54 -0
- package/dist/instruments/dsp/modal.js.map +1 -0
- package/dist/instruments/dsp/piano.d.ts +9 -0
- package/dist/instruments/dsp/piano.d.ts.map +1 -0
- package/dist/instruments/dsp/piano.js +230 -0
- package/dist/instruments/dsp/piano.js.map +1 -0
- package/dist/instruments/dsp/pluck.d.ts +53 -0
- package/dist/instruments/dsp/pluck.d.ts.map +1 -0
- package/dist/instruments/dsp/pluck.js +195 -0
- package/dist/instruments/dsp/pluck.js.map +1 -0
- package/dist/instruments/dsp/reverb.d.ts +16 -0
- package/dist/instruments/dsp/reverb.d.ts.map +1 -0
- package/dist/instruments/dsp/reverb.js +47 -0
- package/dist/instruments/dsp/reverb.js.map +1 -0
- package/dist/instruments/dsp/staccato.d.ts +18 -0
- package/dist/instruments/dsp/staccato.d.ts.map +1 -0
- package/dist/instruments/dsp/staccato.js +147 -0
- package/dist/instruments/dsp/staccato.js.map +1 -0
- package/dist/instruments/play.d.ts +21 -0
- package/dist/instruments/play.d.ts.map +1 -0
- package/dist/instruments/play.js +140 -0
- package/dist/instruments/play.js.map +1 -0
- package/dist/instruments/registry.d.ts +63 -0
- package/dist/instruments/registry.d.ts.map +1 -0
- package/dist/instruments/registry.js +50 -0
- package/dist/instruments/registry.js.map +1 -0
- package/dist/instruments/samples.d.ts +30 -0
- package/dist/instruments/samples.d.ts.map +1 -0
- package/dist/instruments/samples.js +109 -0
- package/dist/instruments/samples.js.map +1 -0
- package/dist/instruments/voices.d.ts +76 -0
- package/dist/instruments/voices.d.ts.map +1 -0
- package/dist/instruments/voices.js +1061 -0
- package/dist/instruments/voices.js.map +1 -0
- package/dist/midi.d.ts +15 -0
- package/dist/midi.d.ts.map +1 -0
- package/dist/midi.js +280 -0
- package/dist/midi.js.map +1 -0
- package/dist/notation/format.d.ts +12 -0
- package/dist/notation/format.d.ts.map +1 -0
- package/dist/notation/format.js +102 -0
- package/dist/notation/format.js.map +1 -0
- package/dist/notation/harmony.d.ts +83 -0
- package/dist/notation/harmony.d.ts.map +1 -0
- package/dist/notation/harmony.js +191 -0
- package/dist/notation/harmony.js.map +1 -0
- package/dist/notation/melody.d.ts +25 -0
- package/dist/notation/melody.d.ts.map +1 -0
- package/dist/notation/melody.js +138 -0
- package/dist/notation/melody.js.map +1 -0
- package/dist/notation/part.d.ts +34 -0
- package/dist/notation/part.d.ts.map +1 -0
- package/dist/notation/part.js +15 -0
- package/dist/notation/part.js.map +1 -0
- package/dist/notation/transform.d.ts +28 -0
- package/dist/notation/transform.d.ts.map +1 -0
- package/dist/notation/transform.js +67 -0
- package/dist/notation/transform.js.map +1 -0
- package/dist/song.d.ts +140 -0
- package/dist/song.d.ts.map +1 -0
- package/dist/song.js +45 -0
- package/dist/song.js.map +1 -0
- package/dist/tempo.d.ts +36 -0
- package/dist/tempo.d.ts.map +1 -0
- package/dist/tempo.js +81 -0
- package/dist/tempo.js.map +1 -0
- package/dist/theory.d.ts +70 -0
- package/dist/theory.d.ts.map +1 -0
- package/dist/theory.js +183 -0
- package/dist/theory.js.map +1 -0
- package/dist/transcribe.d.ts +81 -0
- package/dist/transcribe.d.ts.map +1 -0
- package/dist/transcribe.js +433 -0
- package/dist/transcribe.js.map +1 -0
- package/package.json +57 -4
- package/src/engine/automation.ts +65 -0
- package/src/engine/compile.ts +124 -0
- package/src/engine/jukebox.ts +322 -0
- package/src/engine/mixer.ts +166 -0
- package/src/engine/output.ts +59 -0
- package/src/engine/playback.ts +349 -0
- package/src/engine/prepare.ts +15 -0
- package/src/engine/render.ts +71 -0
- package/src/index.ts +28 -0
- package/src/instruments/bowed.ts +660 -0
- package/src/instruments/builtin.ts +180 -0
- package/src/instruments/definition.ts +52 -0
- package/src/instruments/dsp/ambience.ts +142 -0
- package/src/instruments/dsp/core.ts +97 -0
- package/src/instruments/dsp/drums.ts +565 -0
- package/src/instruments/dsp/modal.ts +76 -0
- package/src/instruments/dsp/piano.ts +226 -0
- package/src/instruments/dsp/pluck.ts +240 -0
- package/src/instruments/dsp/reverb.ts +56 -0
- package/src/instruments/dsp/staccato.ts +141 -0
- package/src/instruments/play.ts +151 -0
- package/src/instruments/registry.ts +100 -0
- package/src/instruments/samples.ts +119 -0
- package/src/instruments/voices.ts +1168 -0
- package/src/midi.ts +285 -0
- package/src/notation/format.ts +96 -0
- package/src/notation/harmony.ts +233 -0
- package/src/notation/melody.ts +133 -0
- package/src/notation/part.ts +42 -0
- package/src/notation/transform.ts +78 -0
- package/src/song.ts +170 -0
- package/src/tempo.ts +108 -0
- package/src/theory.ts +208 -0
- package/src/transcribe.ts +487 -0
|
@@ -0,0 +1,487 @@
|
|
|
1
|
+
// Melodies by ear: a pitch tracker for one voice (singing, humming, whistling) and a transcriber that writes what it heard as notes
|
|
2
|
+
// on the beat grid. Clean singing needs no trained model: a voice's pitch shows in the sound's own periodicity (the YIN method), and
|
|
3
|
+
// a metronome fixes where the beats are, so the rhythm comes down to rounding to the grid.
|
|
4
|
+
|
|
5
|
+
import { formatMelody } from './notation/format.ts';
|
|
6
|
+
import { stepsPerBar, VELOCITY, type Meter, type Note, type Part } from './notation/part.ts';
|
|
7
|
+
|
|
8
|
+
/** The pitch heard over a few hundredths of a second */
|
|
9
|
+
export interface PitchFrame {
|
|
10
|
+
/** Seconds from the first sample pushed, at the middle of the frame */
|
|
11
|
+
time: number;
|
|
12
|
+
/** MIDI pitch, fractional (69.2 is A4 a little sharp); null when no steady pitch was heard */
|
|
13
|
+
pitch: number | null;
|
|
14
|
+
/** How periodic the sound is, 0-1 (a held vowel is above 0.9, breath and clicks well below) */
|
|
15
|
+
clarity: number;
|
|
16
|
+
/** Loudness in dB (0 is full scale) */
|
|
17
|
+
level: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Voices are tracked at about this rate: fundamentals up to C6 with room to spare, at a third of the work of 48 kHz */
|
|
21
|
+
const TRACKING_RATE = 16000;
|
|
22
|
+
/** The pitch range tracked, in Hz: a low bass hum to a high whistle */
|
|
23
|
+
const LOWEST = 55;
|
|
24
|
+
const HIGHEST = 1400;
|
|
25
|
+
/** YIN's threshold on the normalized difference: the first dip under it is the period */
|
|
26
|
+
const THRESHOLD = 0.15;
|
|
27
|
+
/** Frames less periodic than this have no pitch */
|
|
28
|
+
const MIN_CLARITY = 0.6;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Follows the pitch of one voice through audio pushed to it in pieces of any size, as a microphone delivers them, every 10 ms.
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* const tracker = new PitchTracker(context.sampleRate);
|
|
35
|
+
* worklet.port.onmessage = ({ data }) => draw(tracker.push(data));
|
|
36
|
+
*/
|
|
37
|
+
export class PitchTracker {
|
|
38
|
+
/** Every frame so far */
|
|
39
|
+
readonly frames: PitchFrame[] = [];
|
|
40
|
+
/** Input samples averaged into each tracked one */
|
|
41
|
+
private readonly factor: number;
|
|
42
|
+
private readonly rate: number;
|
|
43
|
+
/** Samples in YIN's integration window, the longest period, and between frames */
|
|
44
|
+
private readonly window: number;
|
|
45
|
+
private readonly maxLag: number;
|
|
46
|
+
private readonly minLag: number;
|
|
47
|
+
private readonly hop: number;
|
|
48
|
+
/** Tracked samples not yet passed by the frames, the first of them sample number `base` */
|
|
49
|
+
private buffer = new Float32Array(4096);
|
|
50
|
+
private length = 0;
|
|
51
|
+
private base = 0;
|
|
52
|
+
/** The first sample of the next frame */
|
|
53
|
+
private next = 0;
|
|
54
|
+
/** The input samples being averaged into the next tracked one */
|
|
55
|
+
private sum = 0;
|
|
56
|
+
private count = 0;
|
|
57
|
+
private readonly difference: Float64Array;
|
|
58
|
+
|
|
59
|
+
constructor(sampleRate: number) {
|
|
60
|
+
this.factor = Math.max(1, Math.floor(sampleRate / TRACKING_RATE));
|
|
61
|
+
this.rate = sampleRate / this.factor;
|
|
62
|
+
this.window = Math.round(0.032 * this.rate);
|
|
63
|
+
this.maxLag = Math.ceil(this.rate / LOWEST);
|
|
64
|
+
this.minLag = Math.floor(this.rate / HIGHEST);
|
|
65
|
+
this.hop = Math.round(0.01 * this.rate);
|
|
66
|
+
this.difference = new Float64Array(this.maxLag + 1);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Add samples; returns the frames they completed */
|
|
70
|
+
push(samples: Float32Array): PitchFrame[] {
|
|
71
|
+
for (const sample of samples) {
|
|
72
|
+
this.sum += sample;
|
|
73
|
+
if (++this.count < this.factor) continue;
|
|
74
|
+
if (this.length === this.buffer.length) {
|
|
75
|
+
const grown = new Float32Array(this.buffer.length * 2);
|
|
76
|
+
grown.set(this.buffer);
|
|
77
|
+
this.buffer = grown;
|
|
78
|
+
}
|
|
79
|
+
this.buffer[this.length++] = this.sum / this.factor;
|
|
80
|
+
this.sum = this.count = 0;
|
|
81
|
+
}
|
|
82
|
+
const added: PitchFrame[] = [];
|
|
83
|
+
const span = this.window + this.maxLag;
|
|
84
|
+
while (this.next + span <= this.base + this.length) {
|
|
85
|
+
const start = this.next - this.base;
|
|
86
|
+
added.push(this.analyse(this.buffer.subarray(start, start + span), (this.next + this.window / 2) / this.rate));
|
|
87
|
+
this.next += this.hop;
|
|
88
|
+
}
|
|
89
|
+
const passed = this.next - this.base;
|
|
90
|
+
this.buffer.copyWithin(0, passed, this.length);
|
|
91
|
+
this.length -= passed;
|
|
92
|
+
this.base = this.next;
|
|
93
|
+
this.frames.push(...added);
|
|
94
|
+
return added;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private analyse(x: Float32Array, time: number): PitchFrame {
|
|
98
|
+
const { window, minLag, maxLag, difference: d } = this;
|
|
99
|
+
let energy = 0;
|
|
100
|
+
for (let j = 0; j < window; j++) energy += x[j] * x[j];
|
|
101
|
+
const level = 10 * Math.log10(energy / window + 1e-12);
|
|
102
|
+
// The difference between the signal and itself delayed by each lag, normalized by its running mean
|
|
103
|
+
d[0] = 1;
|
|
104
|
+
let running = 0;
|
|
105
|
+
for (let lag = 1; lag <= maxLag; lag++) {
|
|
106
|
+
let sum = 0;
|
|
107
|
+
for (let j = 0; j < window; j++) {
|
|
108
|
+
const delta = x[j] - x[j + lag];
|
|
109
|
+
sum += delta * delta;
|
|
110
|
+
}
|
|
111
|
+
running += sum;
|
|
112
|
+
d[lag] = running > 0 ? (sum * lag) / running : 1;
|
|
113
|
+
}
|
|
114
|
+
// The first dip under the threshold (at its lowest point), else the deepest dip there is
|
|
115
|
+
let lag = -1;
|
|
116
|
+
for (let l = minLag; l < maxLag; l++) {
|
|
117
|
+
if (d[l] >= THRESHOLD) continue;
|
|
118
|
+
while (l + 1 < maxLag && d[l + 1] < d[l]) l++;
|
|
119
|
+
lag = l;
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
if (lag < 0) {
|
|
123
|
+
lag = minLag;
|
|
124
|
+
for (let l = minLag + 1; l < maxLag; l++) if (d[l] < d[lag]) lag = l;
|
|
125
|
+
}
|
|
126
|
+
const clarity = Math.min(1, Math.max(0, 1 - d[lag]));
|
|
127
|
+
if (clarity < MIN_CLARITY) return { time, pitch: null, clarity, level };
|
|
128
|
+
// A parabola through the dip and its neighbours finds the period between samples
|
|
129
|
+
const [a, b, c] = [d[lag - 1], d[lag], d[lag + 1]];
|
|
130
|
+
const bend = a - 2 * b + c;
|
|
131
|
+
const period = lag + (bend > 0 ? (a - c) / (2 * bend) : 0);
|
|
132
|
+
return { time, pitch: 69 + 12 * Math.log2(this.rate / period / 440), clarity, level };
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export interface TranscribeOptions {
|
|
137
|
+
/** The tempo the melody was sung at: the metronome's. Left out, it is found from the notes themselves (singing without a click). */
|
|
138
|
+
bpm?: number;
|
|
139
|
+
/** Time signature (default '4/4') */
|
|
140
|
+
meter?: Meter;
|
|
141
|
+
/** When a bar starts, in seconds on the frames' clock: the metronome's first downbeat (found with the tempo when that is left out) */
|
|
142
|
+
start?: number;
|
|
143
|
+
/** The shortest note written, in steps: 1 for sixteenths (default), 2 for eighths */
|
|
144
|
+
grid?: number;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export interface Transcription {
|
|
148
|
+
/** The notes on the beat grid, from the start of the first bar sung in */
|
|
149
|
+
part: Part;
|
|
150
|
+
/** The part written as a melody string, spelled in its key */
|
|
151
|
+
melody: string;
|
|
152
|
+
/** The tempo: as given, or as found */
|
|
153
|
+
bpm: number;
|
|
154
|
+
/** Where the part's first step falls, in seconds on the frames' clock (to line the notes up with what was heard) */
|
|
155
|
+
start: number;
|
|
156
|
+
/** The key the notes suggest, such as 'G major' or 'E minor' (empty when nothing was heard) */
|
|
157
|
+
key: string;
|
|
158
|
+
/** How far the singing sat from concert pitch, in cents (+20: a little sharp). The notes are corrected for it. */
|
|
159
|
+
tuning: number;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** A stretch of steady pitch, before it is put on the grid */
|
|
163
|
+
interface Heard {
|
|
164
|
+
/** Seconds on the frames' clock */
|
|
165
|
+
start: number;
|
|
166
|
+
end: number;
|
|
167
|
+
/** MIDI pitch: fractional as heard, then settled on a semitone */
|
|
168
|
+
pitch: number;
|
|
169
|
+
/** It runs on from the note before, the voice moving straight from one to the other without a break or a dip */
|
|
170
|
+
tied: boolean;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Write a sung melody as notes. The pitches are corrected for a voice that sits sharp or flat (or drifts) and rounded to the key where
|
|
175
|
+
* a note falls between two. The timing is put on the grid of a metronome at `bpm` whose bars start at `start`, or of the tempo the
|
|
176
|
+
* notes themselves keep; a note between two grid lines leans to the stronger beat, and runs on to the next note unless a clear rest
|
|
177
|
+
* is left between them.
|
|
178
|
+
*
|
|
179
|
+
* @example
|
|
180
|
+
* transcribe(tracker.frames, { bpm: 90, start: 2.67 }).melody; // 'r:12 G4:2 G4 | A4:4 G4 C5 B4 | ...'
|
|
181
|
+
* transcribe(tracker.frames).bpm; // 96.8, found from the notes
|
|
182
|
+
*/
|
|
183
|
+
export function transcribe(frames: readonly PitchFrame[], options: TranscribeOptions = {}): Transcription {
|
|
184
|
+
const { meter, grid = 1 } = options;
|
|
185
|
+
const bar = stepsPerBar(meter);
|
|
186
|
+
// A slide into a note shorter than this belongs to the note, not one of its own
|
|
187
|
+
const slide = options.bpm ? Math.min(0.09, (0.6 * 15) / options.bpm) : 0.08;
|
|
188
|
+
const heard = listen(frames, slide);
|
|
189
|
+
if (!heard.length) return { part: { notes: [], duration: 0 }, melody: '', bpm: options.bpm ?? 0, start: options.start ?? 0, key: '', tuning: 0 };
|
|
190
|
+
const { pitches, key, tuning } = tune(heard);
|
|
191
|
+
// Pieces of one note heard apart (the voice wavered past the band between them) are one note again
|
|
192
|
+
const sung: Heard[] = [];
|
|
193
|
+
heard.forEach((h, i) => {
|
|
194
|
+
const previous = sung[sung.length - 1];
|
|
195
|
+
if (previous && h.tied && previous.pitch === pitches[i]) previous.end = h.end;
|
|
196
|
+
else sung.push({ ...h, pitch: pitches[i] });
|
|
197
|
+
});
|
|
198
|
+
const { bpm, start } = options.bpm ? { bpm: options.bpm, start: options.start ?? 0 } : findTempo(sung, bar);
|
|
199
|
+
const { timed, shift } = align(sung, bpm, start, grid, bar);
|
|
200
|
+
// From the bar the first note falls in, which may be a bar before the first downbeat (a pickup in the count-in)
|
|
201
|
+
const origin = Math.floor(timed[0].start / bar) * bar;
|
|
202
|
+
const notes: Note[] = timed.map(({ start, end, index }) => ({ time: start - origin, duration: end - start, pitches: [sung[index].pitch], velocity: VELOCITY.normal }));
|
|
203
|
+
const last = notes[notes.length - 1];
|
|
204
|
+
const part = { notes, duration: Math.ceil((last.time + last.duration) / bar) * bar };
|
|
205
|
+
return { part, melody: formatMelody(part, meter, key), bpm, start: start + ((origin + shift) * 15) / bpm, key, tuning };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Steady stretches of pitch: voiced runs, split where the pitch moves to a new note or the same note starts again */
|
|
209
|
+
function listen(frames: readonly PitchFrame[], slide: number): Heard[] {
|
|
210
|
+
if (frames.length < 2) return [];
|
|
211
|
+
const hop = frames[1].time - frames[0].time;
|
|
212
|
+
const sorted = frames.flatMap((f) => (f.pitch === null ? [] : [f.level])).sort((a, b) => a - b);
|
|
213
|
+
if (!sorted.length) return [];
|
|
214
|
+
const loudest = sorted[Math.floor(sorted.length * 0.98)];
|
|
215
|
+
// Quieter than this is the room, or the metronome leaking in, not the voice
|
|
216
|
+
const floor = Math.max(loudest - 35, -70);
|
|
217
|
+
const voiced = (f: PitchFrame) => f.pitch !== null && f.level >= floor;
|
|
218
|
+
const heard: Heard[] = [];
|
|
219
|
+
for (let i = 0; i < frames.length; ) {
|
|
220
|
+
if (!voiced(frames[i])) {
|
|
221
|
+
i++;
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
let j = i;
|
|
225
|
+
while (j < frames.length && voiced(frames[j])) j++;
|
|
226
|
+
heard.push(...notesIn(frames.slice(i, j), hop, Math.round(slide / hop)));
|
|
227
|
+
i = j;
|
|
228
|
+
}
|
|
229
|
+
return heard;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** The shortest stretch of voice taken for a note, in frames */
|
|
233
|
+
const MIN_FRAMES = 4;
|
|
234
|
+
|
|
235
|
+
/** The notes in one unbroken run of voice; `slide` is the most frames a slide into or out of a note lasts */
|
|
236
|
+
function notesIn(run: readonly PitchFrame[], hop: number, slide: number): Heard[] {
|
|
237
|
+
// A median of five steadies the line and removes single frames an octave out
|
|
238
|
+
const pitch = run.map((_, i) => median(run.slice(Math.max(0, i - 2), i + 3).map((f) => f.pitch!)));
|
|
239
|
+
const level = run.map((_, i) => mean(run.slice(Math.max(0, i - 1), i + 2).map((f) => f.level)));
|
|
240
|
+
// Steady frames: the pitch is not sliding from one note to another (vibrato stays well under this)
|
|
241
|
+
const steady = pitch.map((_, i) => Math.abs(pitch[Math.min(i + 1, pitch.length - 1)] - pitch[Math.max(i - 1, 0)]) / 2 < 0.3);
|
|
242
|
+
const length = (s: Segment) => s.to - s.from;
|
|
243
|
+
// A note's pitch is where the voice settles: its first frames may still be scooping up to it
|
|
244
|
+
const at = (s: Segment) => median(pitch.slice(s.from + Math.floor(0.3 * (s.to - s.from)), s.to));
|
|
245
|
+
const notes: Heard[] = [];
|
|
246
|
+
// Each new start of the voice (a dip and a rise, or a sudden attack) begins a note; within one, the pitch may still move on
|
|
247
|
+
for (const [first, last] of articulations(run, level, Math.max(MIN_FRAMES, slide))) {
|
|
248
|
+
// Segments of steady frames within a semitone's band of their own median
|
|
249
|
+
let segments: Segment[] = [];
|
|
250
|
+
let from = -1;
|
|
251
|
+
let members: number[] = [];
|
|
252
|
+
const close = (to: number) => {
|
|
253
|
+
if (from >= 0 && to - from >= MIN_FRAMES) segments.push({ from, to });
|
|
254
|
+
from = -1;
|
|
255
|
+
members = [];
|
|
256
|
+
};
|
|
257
|
+
for (let i = first; i < last; i++) {
|
|
258
|
+
if (!steady[i]) {
|
|
259
|
+
close(i);
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
if (from >= 0 && Math.abs(pitch[i] - median(members)) > 0.6) close(i);
|
|
263
|
+
if (from < 0) from = i;
|
|
264
|
+
members.push(pitch[i]);
|
|
265
|
+
}
|
|
266
|
+
close(last);
|
|
267
|
+
// A segment no longer than a slide is the voice moving between notes, unless there is nothing longer; at the start a scoop up
|
|
268
|
+
// to the note, and at the end a fall from it, can last twice as long
|
|
269
|
+
const longest = Math.max(0, ...segments.map(length));
|
|
270
|
+
segments = segments.filter((s) => length(s) > slide || length(s) === longest);
|
|
271
|
+
for (const [end, next] of [[0, 1], [segments.length - 1, segments.length - 2]]) {
|
|
272
|
+
const [a, b] = [segments[end], segments[next]];
|
|
273
|
+
if (a && b && length(a) <= 2 * slide && length(a) < length(b) && Math.abs(at(a) - at(b)) < 2.5) segments.splice(end, 1);
|
|
274
|
+
}
|
|
275
|
+
if (!segments.length && last - first >= MIN_FRAMES) segments = [{ from: first, to: last }];
|
|
276
|
+
// Neighbours at the same pitch are one note: a vibrato wider than the band can split one
|
|
277
|
+
for (let k = segments.length - 1; k > 0; k--) {
|
|
278
|
+
if (Math.abs(at(segments[k - 1]) - at(segments[k])) < 0.5) {
|
|
279
|
+
segments[k - 1].to = segments[k].to;
|
|
280
|
+
segments.splice(k, 1);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
// Notes change halfway through the slide between them; the first starts with the voice and the last ends with it
|
|
284
|
+
segments.forEach((segment, k) => {
|
|
285
|
+
const start = k === 0 ? first : Math.round((segments[k - 1].to + segment.from) / 2);
|
|
286
|
+
const end = k === segments.length - 1 ? last : Math.round((segment.to + segments[k + 1].from) / 2);
|
|
287
|
+
notes.push({ start: run[start].time - hop / 2, end: run[end - 1].time + hop / 2, pitch: at(segment), tied: k > 0 });
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
return notes;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Frames `from` up to `to` */
|
|
294
|
+
interface Segment {
|
|
295
|
+
from: number;
|
|
296
|
+
to: number;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Where the same pitch starts again without a break: the voice dips by 6 dB and comes back ("da da"), or the sound jumps up by
|
|
301
|
+
* 3 dB within 40 ms and stays up (a new attack, the way an instrument strikes a repeated note; a voice swells far more slowly).
|
|
302
|
+
* `level` is smoothed; the attack is read from the frames' own levels, as smoothing blunts it.
|
|
303
|
+
*/
|
|
304
|
+
function articulations(run: readonly PitchFrame[], level: readonly number[], shortest: number): [number, number][] {
|
|
305
|
+
const parts: [number, number][] = [];
|
|
306
|
+
let start = 0;
|
|
307
|
+
for (let i = shortest; i < run.length - shortest; i++) {
|
|
308
|
+
const before = Math.max(...level.slice(Math.max(start, i - 12), i));
|
|
309
|
+
const after = Math.max(...level.slice(i + 1, i + 13));
|
|
310
|
+
const dip = level[i] <= level[i - 1] && level[i] <= level[i + 1] && level[i] < before - 6 && level[i] < after - 6;
|
|
311
|
+
const raw = run[i].level;
|
|
312
|
+
const ahead = run.slice(i + 1, i + 9).map((f) => f.level);
|
|
313
|
+
const attack = raw <= run[i - 1].level && Math.max(...ahead.slice(0, 4)) - raw >= 3 && mean(ahead) - raw >= 2;
|
|
314
|
+
if (!dip && !attack) continue;
|
|
315
|
+
parts.push([start, i]);
|
|
316
|
+
start = dip ? i + 1 : i;
|
|
317
|
+
i += shortest - 1;
|
|
318
|
+
}
|
|
319
|
+
parts.push([start, run.length]);
|
|
320
|
+
return parts;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const MAJOR = [6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88];
|
|
324
|
+
const MINOR = [6.33, 2.68, 3.52, 5.38, 2.6, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17];
|
|
325
|
+
const MAJOR_TONICS = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'F#', 'G', 'Ab', 'A', 'Bb', 'B'];
|
|
326
|
+
const MINOR_TONICS = ['C', 'C#', 'D', 'Eb', 'E', 'F', 'F#', 'G', 'G#', 'A', 'Bb', 'B'];
|
|
327
|
+
/** The notes of each mode, relative to the tonic (minor with both its sixths and sevenths) */
|
|
328
|
+
const SCALES = { major: [0, 2, 4, 5, 7, 9, 11], minor: [0, 2, 3, 5, 7, 8, 9, 10, 11] };
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Settle each note on a semitone. The voice's offset from concert pitch is measured around each note (over its neighbours, so a
|
|
332
|
+
* voice that drifts flat is followed), and a note that still falls between two semitones takes the one in the key.
|
|
333
|
+
*/
|
|
334
|
+
function tune(heard: readonly Heard[]): { pitches: number[]; key: string; tuning: number } {
|
|
335
|
+
const offsets = heard.map((_, i) => {
|
|
336
|
+
let [x, y] = [0, 0];
|
|
337
|
+
for (let j = Math.max(0, i - 4); j <= Math.min(heard.length - 1, i + 4); j++) {
|
|
338
|
+
const weight = heard[j].end - heard[j].start;
|
|
339
|
+
const angle = 2 * Math.PI * (heard[j].pitch - Math.round(heard[j].pitch));
|
|
340
|
+
x += weight * Math.cos(angle);
|
|
341
|
+
y += weight * Math.sin(angle);
|
|
342
|
+
}
|
|
343
|
+
return Math.atan2(y, x) / (2 * Math.PI);
|
|
344
|
+
});
|
|
345
|
+
const corrected = heard.map((h, i) => h.pitch - offsets[i]);
|
|
346
|
+
// The key whose profile best matches how long each pitch class sounds (Krumhansl and Kessler's)
|
|
347
|
+
const weights = new Array<number>(12).fill(0);
|
|
348
|
+
corrected.forEach((p, i) => (weights[((Math.round(p) % 12) + 12) % 12] += heard[i].end - heard[i].start));
|
|
349
|
+
let best = { score: -Infinity, tonic: 0, mode: 'major' as keyof typeof SCALES };
|
|
350
|
+
for (let tonic = 0; tonic < 12; tonic++) {
|
|
351
|
+
for (const [mode, profile] of [['major', MAJOR], ['minor', MINOR]] as const) {
|
|
352
|
+
const score = correlation(weights, profile.map((_, k) => profile[(k - tonic + 12) % 12]));
|
|
353
|
+
if (score > best.score) best = { score, tonic, mode };
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
const inKey = (p: number) => SCALES[best.mode].includes((((p - best.tonic) % 12) + 12) % 12);
|
|
357
|
+
const pitches = corrected.map((p) => {
|
|
358
|
+
const [low, high] = [Math.floor(p), Math.ceil(p)];
|
|
359
|
+
const between = Math.abs(p - Math.round(p)) > 0.3;
|
|
360
|
+
if (between && inKey(low) !== inKey(high)) return inKey(low) ? low : high;
|
|
361
|
+
return Math.round(p);
|
|
362
|
+
});
|
|
363
|
+
const key = `${(best.mode === 'major' ? MAJOR_TONICS : MINOR_TONICS)[best.tonic]} ${best.mode}`;
|
|
364
|
+
return { pitches, key, tuning: Math.round(100 * median(offsets)) };
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* The tempo and a downbeat of notes sung without a click. The tempo is the sixteenth-note grid (between 70 and 140 beats a minute)
|
|
369
|
+
* the starts fit best, favouring grids on which the notes last whole eighths; the downbeat is the grid line that puts the longest
|
|
370
|
+
* notes on the strongest beats.
|
|
371
|
+
*/
|
|
372
|
+
function findTempo(heard: readonly Heard[], bar: number): { bpm: number; start: number } {
|
|
373
|
+
const onsets = heard.map((h) => h.start);
|
|
374
|
+
/** How closely the starts sit on a grid `step` seconds apart (0-1), and where its lines fall */
|
|
375
|
+
const fit = (step: number) => {
|
|
376
|
+
let [x, y] = [0, 0];
|
|
377
|
+
for (const t of onsets) {
|
|
378
|
+
x += Math.cos((2 * Math.PI * t) / step);
|
|
379
|
+
y += Math.sin((2 * Math.PI * t) / step);
|
|
380
|
+
}
|
|
381
|
+
return { closeness: Math.hypot(x, y) / onsets.length, phase: ((Math.atan2(y, x) / (2 * Math.PI)) * step + step) % step };
|
|
382
|
+
};
|
|
383
|
+
const evens = (step: number) => onsets.slice(1).filter((t, i) => Math.round((t - onsets[i]) / step) % 2 === 0).length / Math.max(1, onsets.length - 1);
|
|
384
|
+
const score = (step: number) => fit(step).closeness + 0.25 * evens(step);
|
|
385
|
+
// Coarsely across the range, then finely around the best: over a minute of singing, a tenth of a percent is a sixteenth adrift
|
|
386
|
+
let best = 15 / 140;
|
|
387
|
+
for (let step = best; step <= 15 / 70; step *= 1.001) if (score(step) > score(best)) best = step;
|
|
388
|
+
for (let step = best / 1.001, around = best; step <= around * 1.001; step *= 1.00002) if (fit(step).closeness > fit(best).closeness) best = step;
|
|
389
|
+
const { phase } = fit(best);
|
|
390
|
+
// Where each note starts on the grid, and how long it lasts
|
|
391
|
+
const notes = heard.map((h) => ({ at: Math.round((h.start - phase) / best), length: h.end - h.start }));
|
|
392
|
+
const strength = (position: number) => (position === 0 ? 1 : position * 2 === bar ? 0.6 : position % 4 === 0 ? 0.4 : position % 2 === 0 ? 0.15 : 0);
|
|
393
|
+
let downbeat = 0;
|
|
394
|
+
let strongest = -Infinity;
|
|
395
|
+
for (let line = 0; line < bar; line++) {
|
|
396
|
+
const total = notes.reduce((sum, n) => sum + n.length * strength((((n.at - line) % bar) + bar) % bar), 0);
|
|
397
|
+
if (total > strongest) [downbeat, strongest] = [line, total];
|
|
398
|
+
}
|
|
399
|
+
return { bpm: 15 / best, start: phase + downbeat * best };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** A heard note on the grid, in steps from the first bar's downbeat */
|
|
403
|
+
interface Timed {
|
|
404
|
+
start: number;
|
|
405
|
+
end: number;
|
|
406
|
+
/** Which heard note */
|
|
407
|
+
index: number;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** How weak a place in the beat is: a beat, an eighth off it, a sixteenth off it */
|
|
411
|
+
const weakness = (step: number) => [0, 0.12, 0.06, 0.12][((step % 4) + 4) % 4];
|
|
412
|
+
|
|
413
|
+
/** The grid line a start goes to: the nearer of the two around it, weighed against how weak a place each is */
|
|
414
|
+
function snap(position: number, grid: number): number {
|
|
415
|
+
const below = Math.floor(position / grid) * grid;
|
|
416
|
+
const cost = (at: number) => Math.abs(position - at) + weakness(at);
|
|
417
|
+
return cost(below) <= cost(below + grid) ? below : below + grid;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Put the notes on the grid. A steady lead or lag behind the beat (the singer's, or the audio hardware's) is taken out first: the
|
|
422
|
+
* shift, up to a step either way, that brings the starts nearest the grid and the strongest beats. A note lasts until the next one
|
|
423
|
+
* unless the silence before that is a rest worth writing.
|
|
424
|
+
*/
|
|
425
|
+
function align(heard: readonly Heard[], bpm: number, start: number, grid: number, bar: number): { timed: Timed[]; shift: number } {
|
|
426
|
+
const step = 15 / bpm;
|
|
427
|
+
const raw = heard.map((h) => ({ start: (h.start - start) / step, end: (h.end - start) / step }));
|
|
428
|
+
const cost = (shift: number) =>
|
|
429
|
+
raw.reduce((sum, n) => {
|
|
430
|
+
const at = snap(n.start - shift, grid);
|
|
431
|
+
return sum + Math.abs(n.start - shift - at) + weakness(at);
|
|
432
|
+
}, 0.02 * raw.length * Math.abs(shift));
|
|
433
|
+
let shift = 0;
|
|
434
|
+
for (let s = -1; s <= 1; s += 0.02) if (cost(s) < cost(shift)) shift = s;
|
|
435
|
+
const timed: Timed[] = [];
|
|
436
|
+
raw.forEach(({ start: s, end: e }, index) => {
|
|
437
|
+
const at = snap(s - shift, grid);
|
|
438
|
+
const previous = timed[timed.length - 1];
|
|
439
|
+
if (previous && at <= previous.start) {
|
|
440
|
+
// Two notes on one grid line: the shorter was a slip of the voice
|
|
441
|
+
if (e - s <= previous.end - previous.start) return;
|
|
442
|
+
timed.pop();
|
|
443
|
+
}
|
|
444
|
+
timed.push({ start: at, end: e - shift, index });
|
|
445
|
+
});
|
|
446
|
+
const eighths = Math.max(grid, 2);
|
|
447
|
+
timed.forEach((note, i) => {
|
|
448
|
+
// Where a note let go ends: on the beat it was let go up to half a beat before, or else in the eighth it was let go in (voices
|
|
449
|
+
// let go a little early, and fade a little late)
|
|
450
|
+
const beat = Math.ceil(note.end / 4) * 4;
|
|
451
|
+
const released = Math.max(note.start + grid, beat - note.end <= 2 ? beat : Math.ceil((note.end - 0.6) / eighths) * eighths);
|
|
452
|
+
const next = timed[i + 1]?.start;
|
|
453
|
+
if (next === undefined) {
|
|
454
|
+
// The last note rings on to the bar line if it gets within three sixteenths of it
|
|
455
|
+
const barline = Math.ceil(note.end / bar) * bar;
|
|
456
|
+
note.end = barline - note.end <= 3 ? barline : released;
|
|
457
|
+
return;
|
|
458
|
+
}
|
|
459
|
+
// A rest is written for a silence of half a beat and a third of a second, as long as a beat or two fifths of the time from
|
|
460
|
+
// one note to the next
|
|
461
|
+
const silence = next - note.end;
|
|
462
|
+
const rest = silence >= 2 && silence * step >= 1 / 3 && (silence >= 4 || silence >= 0.4 * (next - note.start));
|
|
463
|
+
note.end = rest ? Math.min(released, next) : next;
|
|
464
|
+
});
|
|
465
|
+
return { timed, shift };
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function median(values: readonly number[]): number {
|
|
469
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
470
|
+
const mid = sorted.length >> 1;
|
|
471
|
+
return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function mean(values: readonly number[]): number {
|
|
475
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function correlation(a: readonly number[], b: readonly number[]): number {
|
|
479
|
+
const [ma, mb] = [mean(a), mean(b)];
|
|
480
|
+
let [ab, aa, bb] = [0, 0, 0];
|
|
481
|
+
for (let i = 0; i < a.length; i++) {
|
|
482
|
+
ab += (a[i] - ma) * (b[i] - mb);
|
|
483
|
+
aa += (a[i] - ma) ** 2;
|
|
484
|
+
bb += (b[i] - mb) ** 2;
|
|
485
|
+
}
|
|
486
|
+
return aa && bb ? ab / Math.sqrt(aa * bb) : 0;
|
|
487
|
+
}
|