pitch-transcription 4.1.2 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/dist/index.d.ts +0 -37
- package/dist/index.js +0 -75
- package/dist/notes-BZsEo1k5.js +0 -312
- package/dist/notes.d.ts +0 -79
- package/dist/notes.js +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pitch-transcription",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"description": "The pitched notes in a recording — Spotify's Basic Pitch model on current TensorFlow.js, its readings turned into notes with pitch bends and written as MIDI.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"audio",
|
|
@@ -53,12 +53,12 @@
|
|
|
53
53
|
"typecheck": "tsc"
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
|
-
"@esposter/shared": "4.
|
|
56
|
+
"@esposter/shared": "4.2.0"
|
|
57
57
|
},
|
|
58
58
|
"devDependencies": {
|
|
59
59
|
"@arethetypeswrong/core": "^0.18.5",
|
|
60
|
-
"@esposter/configuration": "4.
|
|
61
|
-
"@esposter/shared-node": "4.
|
|
60
|
+
"@esposter/configuration": "4.2.0",
|
|
61
|
+
"@esposter/shared-node": "4.2.0",
|
|
62
62
|
"@tensorflow/tfjs": "^4.22.0",
|
|
63
63
|
"@types/node": "^26.6.5",
|
|
64
64
|
"publint": "^0.3.25",
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
"peerDependencies": {
|
|
69
69
|
"@tensorflow/tfjs": "^4.22.0"
|
|
70
70
|
},
|
|
71
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "fef4971cf6a621a2c1850f32fcec19ee4ec0de99"
|
|
72
72
|
}
|
package/dist/index.d.ts
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import { ANNOTATIONS_FPS, ANNOTATIONS_FRAMES_PER_WINDOW, ANNOTATIONS_SEMITONES, AUDIO_SAMPLE_RATE, AUDIO_WINDOW_LENGTH_SECONDS, AUDIO_WINDOW_SAMPLES, CONTOURS_TENSOR, CONTOUR_BINS, CONTOUR_BINS_PER_SEMITONE, ENERGY_TOLERANCE, FFT_HOP, FRAMES_TENSOR, FRAME_THRESHOLD, MIDI_OFFSET, MIDI_PITCH_BEND_SEMITONES, MIDI_PITCH_BEND_STEPS, MIN_NOTE_LENGTH, MODEL_URL, MODEL_WEIGHTS_URL, ModelReadings, NoteCreationOptions, NoteEvent, NoteEventTime, ONSETS_TENSOR, ONSET_THRESHOLD, OVERLAPPING_FRAMES, OVERLAP_SAMPLES, PITCH_BEND_STANDARD_DEVIATION, PITCH_BEND_TOLERANCE, WINDOW_HOP_SAMPLES, WINDOW_OFFSET_SECONDS, addPitchBends, convertNotesToSeconds, createNotes, writeMidi } from "./notes.js";
|
|
2
|
-
import { GraphModel, Tensor2D, Tensor3D } from "@tensorflow/tfjs";
|
|
3
|
-
//#region src/models/AudioChannels.d.ts
|
|
4
|
-
export interface AudioChannels {
|
|
5
|
-
getChannelData: (channel: number) => Float32Array;
|
|
6
|
-
numberOfChannels: number;
|
|
7
|
-
sampleRate: number;
|
|
8
|
-
}
|
|
9
|
-
//#endregion
|
|
10
|
-
//#region src/services/clearPitch.d.ts
|
|
11
|
-
export declare const clearPitch: (energy: number[][], frame: number, pitch: number) => void;
|
|
12
|
-
//#endregion
|
|
13
|
-
//#region src/services/constrainFrequency.d.ts
|
|
14
|
-
export declare const constrainFrequency: (readings: number[][], maxFrequency?: number, minFrequency?: number) => number[][];
|
|
15
|
-
//#endregion
|
|
16
|
-
//#region src/services/findLargestReading.d.ts
|
|
17
|
-
export declare const findLargestReading: (readings: number[][]) => number;
|
|
18
|
-
//#endregion
|
|
19
|
-
//#region src/services/findOnsetPeaks.d.ts
|
|
20
|
-
export declare const findOnsetPeaks: (onsets: number[][], threshold: number) => [number, number][];
|
|
21
|
-
//#endregion
|
|
22
|
-
//#region src/services/inferOnsets.d.ts
|
|
23
|
-
export declare const inferOnsets: (onsets: number[][], frames: number[][]) => number[][];
|
|
24
|
-
//#endregion
|
|
25
|
-
//#region src/services/mixDownChannels.d.ts
|
|
26
|
-
export declare const mixDownChannels: (audio: AudioChannels | Float32Array) => Float32Array;
|
|
27
|
-
//#endregion
|
|
28
|
-
//#region src/services/prepareAudioWindows.d.ts
|
|
29
|
-
export declare const prepareAudioWindows: (samples: Float32Array) => Tensor3D;
|
|
30
|
-
//#endregion
|
|
31
|
-
//#region src/services/readModel.d.ts
|
|
32
|
-
export declare const readModel: (model: Pick<GraphModel, "execute">, audio: AudioChannels | Float32Array, onWindow?: (readings: ModelReadings, progress: number) => void) => Promise<ModelReadings>;
|
|
33
|
-
//#endregion
|
|
34
|
-
//#region src/services/unwrapWindowReadings.d.ts
|
|
35
|
-
export declare const unwrapWindowReadings: (readings: Tensor3D) => Tensor2D;
|
|
36
|
-
//#endregion
|
|
37
|
-
export { ANNOTATIONS_FPS, ANNOTATIONS_FRAMES_PER_WINDOW, ANNOTATIONS_SEMITONES, AUDIO_SAMPLE_RATE, AUDIO_WINDOW_LENGTH_SECONDS, AUDIO_WINDOW_SAMPLES, CONTOURS_TENSOR, CONTOUR_BINS, CONTOUR_BINS_PER_SEMITONE, ENERGY_TOLERANCE, FFT_HOP, FRAMES_TENSOR, FRAME_THRESHOLD, MIDI_OFFSET, MIDI_PITCH_BEND_SEMITONES, MIDI_PITCH_BEND_STEPS, MIN_NOTE_LENGTH, MODEL_URL, MODEL_WEIGHTS_URL, ModelReadings, NoteCreationOptions, NoteEvent, NoteEventTime, ONSETS_TENSOR, ONSET_THRESHOLD, OVERLAPPING_FRAMES, OVERLAP_SAMPLES, PITCH_BEND_STANDARD_DEVIATION, PITCH_BEND_TOLERANCE, WINDOW_HOP_SAMPLES, WINDOW_OFFSET_SECONDS, addPitchBends, convertNotesToSeconds, createNotes, writeMidi };
|
package/dist/index.js
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import { A as ONSET_THRESHOLD, C as MIDI_OFFSET, D as MODEL_URL, E as MIN_NOTE_LENGTH, F as WINDOW_HOP_SAMPLES, I as WINDOW_OFFSET_SECONDS, M as OVERLAP_SAMPLES, N as PITCH_BEND_STANDARD_DEVIATION, O as MODEL_WEIGHTS_URL, P as PITCH_BEND_TOLERANCE, S as FRAME_THRESHOLD, T as MIDI_PITCH_BEND_STEPS, _ as CONTOUR_BINS, a as findOnsetPeaks, b as FFT_HOP, c as convertNotesToSeconds, d as ANNOTATIONS_FRAMES_PER_WINDOW, f as ANNOTATIONS_SEMITONES, g as CONTOURS_TENSOR, h as AUDIO_WINDOW_SAMPLES, i as findLargestReading, j as OVERLAPPING_FRAMES, k as ONSETS_TENSOR, l as addPitchBends, m as AUDIO_WINDOW_LENGTH_SECONDS, n as createNotes, o as constrainFrequency, p as AUDIO_SAMPLE_RATE, r as inferOnsets, s as clearPitch, t as writeMidi, u as ANNOTATIONS_FPS, v as CONTOUR_BINS_PER_SEMITONE, w as MIDI_PITCH_BEND_SEMITONES, x as FRAMES_TENSOR, y as ENERGY_TOLERANCE } from "./notes-BZsEo1k5.js";
|
|
2
|
-
import { InvalidOperationError, Operation, withFinalizerAsync } from "@esposter/shared";
|
|
3
|
-
import { concat1d, dispose, expandDims, signal, slice, tensor1d, tidy, zeros } from "@tensorflow/tfjs";
|
|
4
|
-
//#region src/services/mixDownChannels.ts
|
|
5
|
-
const mixDownChannels = (audio) => {
|
|
6
|
-
if (audio instanceof Float32Array) return audio;
|
|
7
|
-
if (audio.sampleRate !== 22050) throw new InvalidOperationError(Operation.Read, mixDownChannels.name, `audio at ${audio.sampleRate} Hz, the model reads ${AUDIO_SAMPLE_RATE} Hz`);
|
|
8
|
-
if (audio.numberOfChannels === 1) return audio.getChannelData(0);
|
|
9
|
-
const mixed = new Float32Array(audio.getChannelData(0).length);
|
|
10
|
-
for (let channel = 0; channel < audio.numberOfChannels; channel++) for (const [index, sample] of audio.getChannelData(channel).entries()) mixed[index] = (mixed[index] ?? 0) + sample / audio.numberOfChannels;
|
|
11
|
-
return mixed;
|
|
12
|
-
};
|
|
13
|
-
//#endregion
|
|
14
|
-
//#region src/services/prepareAudioWindows.ts
|
|
15
|
-
const prepareAudioWindows = (samples) => tidy(() => {
|
|
16
|
-
const padded = concat1d([zeros([OVERLAP_SAMPLES / 2]), tensor1d(samples)]);
|
|
17
|
-
return expandDims(signal.frame(padded, AUDIO_WINDOW_SAMPLES, WINDOW_HOP_SAMPLES, true, 0), -1);
|
|
18
|
-
});
|
|
19
|
-
//#endregion
|
|
20
|
-
//#region src/services/unwrapWindowReadings.ts
|
|
21
|
-
const unwrapWindowReadings = (readings) => {
|
|
22
|
-
const [windowCount, frameCount, binCount] = readings.shape;
|
|
23
|
-
return readings.slice([
|
|
24
|
-
0,
|
|
25
|
-
15,
|
|
26
|
-
0
|
|
27
|
-
], [
|
|
28
|
-
-1,
|
|
29
|
-
frameCount - 30,
|
|
30
|
-
-1
|
|
31
|
-
]).reshape([windowCount * (frameCount - 30), binCount]);
|
|
32
|
-
};
|
|
33
|
-
//#endregion
|
|
34
|
-
//#region src/services/readModel.ts
|
|
35
|
-
const readModel = (model, audio, onWindow) => {
|
|
36
|
-
const samples = mixDownChannels(audio);
|
|
37
|
-
const frameCount = Math.floor(samples.length * ANNOTATIONS_FPS / AUDIO_SAMPLE_RATE);
|
|
38
|
-
const windows = prepareAudioWindows(samples);
|
|
39
|
-
return withFinalizerAsync(async () => {
|
|
40
|
-
const readings = {
|
|
41
|
-
contours: [],
|
|
42
|
-
frames: [],
|
|
43
|
-
onsets: []
|
|
44
|
-
};
|
|
45
|
-
for (let index = 0; index < windows.shape[0] && readings.frames.length < frameCount; index++) {
|
|
46
|
-
const remainingFrames = frameCount - readings.frames.length;
|
|
47
|
-
const outputs = tidy(() => {
|
|
48
|
-
const window = slice(windows, index, 1);
|
|
49
|
-
return model.execute(window, [
|
|
50
|
-
FRAMES_TENSOR,
|
|
51
|
-
ONSETS_TENSOR,
|
|
52
|
-
CONTOURS_TENSOR
|
|
53
|
-
]).map((output) => {
|
|
54
|
-
const unwrapped = unwrapWindowReadings(output);
|
|
55
|
-
return unwrapped.slice(0, Math.min(remainingFrames, unwrapped.shape[0]));
|
|
56
|
-
});
|
|
57
|
-
});
|
|
58
|
-
const [frames = [], onsets = [], contours = []] = await Promise.all(outputs.map((output) => output.array()));
|
|
59
|
-
dispose(outputs);
|
|
60
|
-
readings.frames.push(...frames);
|
|
61
|
-
readings.onsets.push(...onsets);
|
|
62
|
-
readings.contours.push(...contours);
|
|
63
|
-
onWindow?.({
|
|
64
|
-
contours,
|
|
65
|
-
frames,
|
|
66
|
-
onsets
|
|
67
|
-
}, readings.frames.length / frameCount);
|
|
68
|
-
}
|
|
69
|
-
return readings;
|
|
70
|
-
}, () => {
|
|
71
|
-
windows.dispose();
|
|
72
|
-
});
|
|
73
|
-
};
|
|
74
|
-
//#endregion
|
|
75
|
-
export { ANNOTATIONS_FPS, ANNOTATIONS_FRAMES_PER_WINDOW, ANNOTATIONS_SEMITONES, AUDIO_SAMPLE_RATE, AUDIO_WINDOW_LENGTH_SECONDS, AUDIO_WINDOW_SAMPLES, CONTOURS_TENSOR, CONTOUR_BINS, CONTOUR_BINS_PER_SEMITONE, ENERGY_TOLERANCE, FFT_HOP, FRAMES_TENSOR, FRAME_THRESHOLD, MIDI_OFFSET, MIDI_PITCH_BEND_SEMITONES, MIDI_PITCH_BEND_STEPS, MIN_NOTE_LENGTH, MODEL_URL, MODEL_WEIGHTS_URL, ONSETS_TENSOR, ONSET_THRESHOLD, OVERLAPPING_FRAMES, OVERLAP_SAMPLES, PITCH_BEND_STANDARD_DEVIATION, PITCH_BEND_TOLERANCE, WINDOW_HOP_SAMPLES, WINDOW_OFFSET_SECONDS, addPitchBends, clearPitch, constrainFrequency, convertNotesToSeconds, createNotes, findLargestReading, findOnsetPeaks, inferOnsets, mixDownChannels, prepareAudioWindows, readModel, unwrapWindowReadings, writeMidi };
|
package/dist/notes-BZsEo1k5.js
DELETED
|
@@ -1,312 +0,0 @@
|
|
|
1
|
-
import { takeOne } from "@esposter/shared";
|
|
2
|
-
//#region src/constants.ts
|
|
3
|
-
const AUDIO_SAMPLE_RATE = 22050;
|
|
4
|
-
const FFT_HOP = 256;
|
|
5
|
-
const AUDIO_WINDOW_LENGTH_SECONDS = 2;
|
|
6
|
-
const AUDIO_WINDOW_SAMPLES = AUDIO_SAMPLE_RATE * 2 - 256;
|
|
7
|
-
const ANNOTATIONS_FPS = Math.floor(AUDIO_SAMPLE_RATE / 256);
|
|
8
|
-
const ANNOTATIONS_FRAMES_PER_WINDOW = 172;
|
|
9
|
-
const OVERLAPPING_FRAMES = 30;
|
|
10
|
-
const OVERLAP_SAMPLES = 7680;
|
|
11
|
-
const WINDOW_HOP_SAMPLES = 36164;
|
|
12
|
-
const WINDOW_OFFSET_SECONDS = .010326077097505668;
|
|
13
|
-
const MIDI_OFFSET = 21;
|
|
14
|
-
const ANNOTATIONS_SEMITONES = 88;
|
|
15
|
-
const CONTOUR_BINS_PER_SEMITONE = 3;
|
|
16
|
-
const CONTOUR_BINS = 264;
|
|
17
|
-
const FRAMES_TENSOR = "Identity_1";
|
|
18
|
-
const ONSETS_TENSOR = "Identity_2";
|
|
19
|
-
const CONTOURS_TENSOR = "Identity";
|
|
20
|
-
const MODEL_URL = new URL("../model/model.json", import.meta.url);
|
|
21
|
-
const MODEL_WEIGHTS_URL = new URL("group1-shard1of1.bin", MODEL_URL);
|
|
22
|
-
const ONSET_THRESHOLD = .5;
|
|
23
|
-
const FRAME_THRESHOLD = .3;
|
|
24
|
-
const MIN_NOTE_LENGTH = 11;
|
|
25
|
-
const ENERGY_TOLERANCE = 11;
|
|
26
|
-
const PITCH_BEND_TOLERANCE = 25;
|
|
27
|
-
const PITCH_BEND_STANDARD_DEVIATION = 5;
|
|
28
|
-
const MIDI_PITCH_BEND_SEMITONES = 2;
|
|
29
|
-
const MIDI_PITCH_BEND_STEPS = 8192;
|
|
30
|
-
//#endregion
|
|
31
|
-
//#region src/services/addPitchBends.ts
|
|
32
|
-
const PITCH_BEND_WEIGHTS = Array.from({ length: 51 }, (_value, offset) => Math.exp(-((offset - 25) ** 2) / 50));
|
|
33
|
-
const addPitchBends = (contours, notes) => notes.map((note) => {
|
|
34
|
-
const bin = (note.pitchMidi - 21) * 3;
|
|
35
|
-
const start = Math.max(bin - 25, 0);
|
|
36
|
-
const end = Math.min(bin + 25 + 1, 264);
|
|
37
|
-
const pitchBends = contours.slice(note.startFrame, note.startFrame + note.durationFrames).map((row) => {
|
|
38
|
-
let bestBin = start;
|
|
39
|
-
let bestReading = -Infinity;
|
|
40
|
-
for (let contourBin = start; contourBin < end; contourBin++) {
|
|
41
|
-
const reading = takeOne(row, contourBin) * takeOne(PITCH_BEND_WEIGHTS, contourBin - bin + 25);
|
|
42
|
-
if (reading <= bestReading) continue;
|
|
43
|
-
bestBin = contourBin;
|
|
44
|
-
bestReading = reading;
|
|
45
|
-
}
|
|
46
|
-
return bestBin - bin;
|
|
47
|
-
});
|
|
48
|
-
return {
|
|
49
|
-
...note,
|
|
50
|
-
pitchBends
|
|
51
|
-
};
|
|
52
|
-
});
|
|
53
|
-
//#endregion
|
|
54
|
-
//#region src/services/convertNotesToSeconds.ts
|
|
55
|
-
const toSeconds = (frame) => frame * 256 / AUDIO_SAMPLE_RATE - WINDOW_OFFSET_SECONDS * Math.floor(frame / 172);
|
|
56
|
-
const convertNotesToSeconds = (notes) => notes.map(({ amplitude, durationFrames, pitchBends, pitchMidi, startFrame }) => {
|
|
57
|
-
const startTimeSeconds = toSeconds(startFrame);
|
|
58
|
-
return {
|
|
59
|
-
amplitude,
|
|
60
|
-
durationSeconds: toSeconds(startFrame + durationFrames) - startTimeSeconds,
|
|
61
|
-
pitchBends,
|
|
62
|
-
pitchMidi,
|
|
63
|
-
startTimeSeconds
|
|
64
|
-
};
|
|
65
|
-
});
|
|
66
|
-
//#endregion
|
|
67
|
-
//#region src/services/clearPitch.ts
|
|
68
|
-
const clearPitch = (energy, frame, pitch) => {
|
|
69
|
-
const row = takeOne(energy, frame);
|
|
70
|
-
row[pitch] = 0;
|
|
71
|
-
if (pitch + 1 < row.length) row[pitch + 1] = 0;
|
|
72
|
-
if (pitch > 0) row[pitch - 1] = 0;
|
|
73
|
-
};
|
|
74
|
-
//#endregion
|
|
75
|
-
//#region src/services/constrainFrequency.ts
|
|
76
|
-
const toPitchIndex = (frequency) => Math.round(12 * Math.log2(frequency / 440) + 69 - 21);
|
|
77
|
-
const constrainFrequency = (readings, maxFrequency, minFrequency) => {
|
|
78
|
-
if (maxFrequency === void 0 && minFrequency === void 0) return readings;
|
|
79
|
-
const end = maxFrequency === void 0 ? Infinity : toPitchIndex(maxFrequency);
|
|
80
|
-
const start = minFrequency === void 0 ? 0 : toPitchIndex(minFrequency);
|
|
81
|
-
return readings.map((row) => row.map((reading, pitch) => pitch >= start && pitch < end ? reading : 0));
|
|
82
|
-
};
|
|
83
|
-
//#endregion
|
|
84
|
-
//#region src/services/findOnsetPeaks.ts
|
|
85
|
-
const findOnsetPeaks = (onsets, threshold) => {
|
|
86
|
-
const peaks = [];
|
|
87
|
-
for (let frame = 1; frame < onsets.length - 1; frame++) {
|
|
88
|
-
const previous = takeOne(onsets, frame - 1);
|
|
89
|
-
const row = takeOne(onsets, frame);
|
|
90
|
-
const next = takeOne(onsets, frame + 1);
|
|
91
|
-
for (let pitch = 0; pitch < row.length; pitch++) {
|
|
92
|
-
const reading = takeOne(row, pitch);
|
|
93
|
-
if (reading > threshold && reading > takeOne(previous, pitch) && reading > takeOne(next, pitch)) peaks.push([frame, pitch]);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
return peaks;
|
|
97
|
-
};
|
|
98
|
-
//#endregion
|
|
99
|
-
//#region src/services/findLargestReading.ts
|
|
100
|
-
const findLargestReading = (readings) => {
|
|
101
|
-
let largest = 0;
|
|
102
|
-
for (const row of readings) for (const reading of row) largest = Math.max(largest, reading);
|
|
103
|
-
return largest;
|
|
104
|
-
};
|
|
105
|
-
//#endregion
|
|
106
|
-
//#region src/services/inferOnsets.ts
|
|
107
|
-
const getJump = (row, previous, beforePrevious, pitch) => {
|
|
108
|
-
const reading = takeOne(row, pitch);
|
|
109
|
-
return Math.max(0, Math.min(reading - takeOne(previous, pitch), reading - takeOne(beforePrevious, pitch)));
|
|
110
|
-
};
|
|
111
|
-
const inferOnsets = (onsets, frames) => {
|
|
112
|
-
let largestJump = 0;
|
|
113
|
-
for (let frame = 2; frame < frames.length; frame++) {
|
|
114
|
-
const row = takeOne(frames, frame);
|
|
115
|
-
const previous = takeOne(frames, frame - 1);
|
|
116
|
-
const beforePrevious = takeOne(frames, frame - 2);
|
|
117
|
-
for (let pitch = 0; pitch < row.length; pitch++) largestJump = Math.max(largestJump, getJump(row, previous, beforePrevious, pitch));
|
|
118
|
-
}
|
|
119
|
-
if (largestJump === 0) return onsets;
|
|
120
|
-
const scale = findLargestReading(onsets) / largestJump;
|
|
121
|
-
return onsets.map((onsetRow, frame) => {
|
|
122
|
-
const row = takeOne(frames, frame);
|
|
123
|
-
const previous = frame < 2 ? row : takeOne(frames, frame - 1);
|
|
124
|
-
const beforePrevious = frame < 2 ? row : takeOne(frames, frame - 2);
|
|
125
|
-
return onsetRow.map((reading, pitch) => Math.max(reading, scale * getJump(row, previous, beforePrevious, pitch)));
|
|
126
|
-
});
|
|
127
|
-
};
|
|
128
|
-
//#endregion
|
|
129
|
-
//#region src/services/createNotes.ts
|
|
130
|
-
const getMeanReading = (frames, start, end, pitch) => {
|
|
131
|
-
let sum = 0;
|
|
132
|
-
for (let frame = start; frame < end; frame++) sum += takeOne(takeOne(frames, frame), pitch);
|
|
133
|
-
return sum / (end - start);
|
|
134
|
-
};
|
|
135
|
-
const createNotes = ({ frames, onsets }, { energyTolerance = 11, frameThreshold = FRAME_THRESHOLD, isInferringOnsets = true, isMelodiaTrick = true, maxFrequency, minFrequency, minNoteLength = 11, onsetThreshold = ONSET_THRESHOLD } = {}) => {
|
|
136
|
-
const frameCount = frames.length;
|
|
137
|
-
const constrainedFrames = constrainFrequency(frames, maxFrequency, minFrequency);
|
|
138
|
-
const constrainedOnsets = constrainFrequency(onsets, maxFrequency, minFrequency);
|
|
139
|
-
const peakOnsets = isInferringOnsets ? inferOnsets(constrainedOnsets, constrainedFrames) : constrainedOnsets;
|
|
140
|
-
const energy = constrainedFrames.map((row) => [...row]);
|
|
141
|
-
const notes = [];
|
|
142
|
-
for (const [startFrame, pitch] of findOnsetPeaks(peakOnsets, onsetThreshold).toReversed()) {
|
|
143
|
-
if (startFrame >= frameCount - 1) continue;
|
|
144
|
-
let end = startFrame + 1;
|
|
145
|
-
let quietFrames = 0;
|
|
146
|
-
for (; end < frameCount - 1 && quietFrames < energyTolerance; end++) quietFrames = takeOne(takeOne(energy, end), pitch) < frameThreshold ? quietFrames + 1 : 0;
|
|
147
|
-
end -= quietFrames;
|
|
148
|
-
if (end - startFrame <= minNoteLength) continue;
|
|
149
|
-
for (let frame = startFrame; frame < end; frame++) clearPitch(energy, frame, pitch);
|
|
150
|
-
notes.push({
|
|
151
|
-
amplitude: getMeanReading(constrainedFrames, startFrame, end, pitch),
|
|
152
|
-
durationFrames: end - startFrame,
|
|
153
|
-
pitchMidi: pitch + 21,
|
|
154
|
-
startFrame
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
|
-
if (!isMelodiaTrick) return notes;
|
|
158
|
-
const candidates = [];
|
|
159
|
-
for (const [frame, row] of energy.entries()) for (let pitch = 0; pitch < row.length; pitch++) {
|
|
160
|
-
const reading = takeOne(row, pitch);
|
|
161
|
-
if (reading > frameThreshold) candidates.push([
|
|
162
|
-
reading,
|
|
163
|
-
frame,
|
|
164
|
-
pitch
|
|
165
|
-
]);
|
|
166
|
-
}
|
|
167
|
-
const sortedCandidates = candidates.toSorted(([firstReading, firstFrame, firstPitch], [secondReading, secondFrame, secondPitch]) => secondReading - firstReading || firstFrame - secondFrame || firstPitch - secondPitch);
|
|
168
|
-
for (const [reading, middle, pitch] of sortedCandidates) {
|
|
169
|
-
const middleRow = takeOne(energy, middle);
|
|
170
|
-
if (takeOne(middleRow, pitch) !== reading) continue;
|
|
171
|
-
middleRow[pitch] = 0;
|
|
172
|
-
let frame = middle + 1;
|
|
173
|
-
let quietFrames = 0;
|
|
174
|
-
for (; frame < frameCount - 1 && quietFrames < energyTolerance; frame++) {
|
|
175
|
-
quietFrames = takeOne(takeOne(energy, frame), pitch) < frameThreshold ? quietFrames + 1 : 0;
|
|
176
|
-
clearPitch(energy, frame, pitch);
|
|
177
|
-
}
|
|
178
|
-
const end = frame - 1 - quietFrames;
|
|
179
|
-
frame = middle - 1;
|
|
180
|
-
quietFrames = 0;
|
|
181
|
-
for (; frame > 0 && quietFrames < energyTolerance; frame--) {
|
|
182
|
-
quietFrames = takeOne(takeOne(energy, frame), pitch) < frameThreshold ? quietFrames + 1 : 0;
|
|
183
|
-
clearPitch(energy, frame, pitch);
|
|
184
|
-
}
|
|
185
|
-
const start = frame + 1 + quietFrames;
|
|
186
|
-
if (end - start <= minNoteLength) continue;
|
|
187
|
-
notes.push({
|
|
188
|
-
amplitude: getMeanReading(constrainedFrames, start, end, pitch),
|
|
189
|
-
durationFrames: end - start,
|
|
190
|
-
pitchMidi: pitch + 21,
|
|
191
|
-
startFrame: start
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
return notes;
|
|
195
|
-
};
|
|
196
|
-
//#endregion
|
|
197
|
-
//#region src/services/writeMidi.ts
|
|
198
|
-
const MIDI_TICKS_PER_BEAT = 480;
|
|
199
|
-
const MIDI_TICKS_PER_SECOND = MIDI_TICKS_PER_BEAT * 2;
|
|
200
|
-
const writeVariableLength = (value) => {
|
|
201
|
-
const bytes = [value & 127];
|
|
202
|
-
for (let rest = value >> 7; rest > 0; rest >>= 7) bytes.unshift(rest & 127 | 128);
|
|
203
|
-
return bytes;
|
|
204
|
-
};
|
|
205
|
-
const toTicks = (seconds) => Math.round(seconds * MIDI_TICKS_PER_SECOND);
|
|
206
|
-
const toUint16 = (value) => [value >> 8 & 255, value & 255];
|
|
207
|
-
const toUint32 = (value) => [
|
|
208
|
-
value >>> 24 & 255,
|
|
209
|
-
value >> 16 & 255,
|
|
210
|
-
value >> 8 & 255,
|
|
211
|
-
value & 255
|
|
212
|
-
];
|
|
213
|
-
const writeChunk = (type, data) => [
|
|
214
|
-
...type,
|
|
215
|
-
...toUint32(data.length),
|
|
216
|
-
...data
|
|
217
|
-
];
|
|
218
|
-
const HEADER_CHUNK_TYPE = [
|
|
219
|
-
77,
|
|
220
|
-
84,
|
|
221
|
-
104,
|
|
222
|
-
100
|
|
223
|
-
];
|
|
224
|
-
const TRACK_CHUNK_TYPE = [
|
|
225
|
-
77,
|
|
226
|
-
84,
|
|
227
|
-
114,
|
|
228
|
-
107
|
|
229
|
-
];
|
|
230
|
-
const EMPTY_TRACK_NAME = [
|
|
231
|
-
255,
|
|
232
|
-
3,
|
|
233
|
-
0
|
|
234
|
-
];
|
|
235
|
-
const END_OF_TRACK = [
|
|
236
|
-
255,
|
|
237
|
-
47,
|
|
238
|
-
0
|
|
239
|
-
];
|
|
240
|
-
const PIANO_PROGRAM_CHANGE = [192, 0];
|
|
241
|
-
const writeTrack = (events) => {
|
|
242
|
-
const data = [];
|
|
243
|
-
let lastTick = 0;
|
|
244
|
-
for (const { bytes, tick } of events.toSorted((firstEvent, secondEvent) => firstEvent.tick - secondEvent.tick)) {
|
|
245
|
-
data.push(...writeVariableLength(tick - lastTick), ...bytes);
|
|
246
|
-
lastTick = tick;
|
|
247
|
-
}
|
|
248
|
-
data.push(0, ...END_OF_TRACK);
|
|
249
|
-
return writeChunk(TRACK_CHUNK_TYPE, data);
|
|
250
|
-
};
|
|
251
|
-
const writeNoteEvents = (notes) => notes.map(({ amplitude, durationSeconds, pitchMidi, startTimeSeconds }) => ({
|
|
252
|
-
amplitude,
|
|
253
|
-
offTick: toTicks(startTimeSeconds) + toTicks(durationSeconds),
|
|
254
|
-
onTick: toTicks(startTimeSeconds),
|
|
255
|
-
pitchMidi
|
|
256
|
-
})).toSorted((firstNote, secondNote) => firstNote.onTick - secondNote.onTick).flatMap(({ amplitude, offTick, onTick, pitchMidi }) => [{
|
|
257
|
-
bytes: [
|
|
258
|
-
144,
|
|
259
|
-
pitchMidi,
|
|
260
|
-
Math.floor(amplitude * 127)
|
|
261
|
-
],
|
|
262
|
-
tick: onTick
|
|
263
|
-
}, {
|
|
264
|
-
bytes: [
|
|
265
|
-
128,
|
|
266
|
-
pitchMidi,
|
|
267
|
-
0
|
|
268
|
-
],
|
|
269
|
-
tick: offTick
|
|
270
|
-
}]);
|
|
271
|
-
const writePitchBendEvents = (notes) => notes.flatMap(({ durationSeconds, pitchBends = [], startTimeSeconds }) => pitchBends.map((bend, index) => {
|
|
272
|
-
const share = bend / 3 / 2;
|
|
273
|
-
const value = MIDI_PITCH_BEND_STEPS + Math.max(-8192, Math.min(8191, Math.round(share * MIDI_PITCH_BEND_STEPS)));
|
|
274
|
-
return {
|
|
275
|
-
bytes: [
|
|
276
|
-
224,
|
|
277
|
-
value & 127,
|
|
278
|
-
value >> 7 & 127
|
|
279
|
-
],
|
|
280
|
-
tick: toTicks(startTimeSeconds + index * durationSeconds / pitchBends.length)
|
|
281
|
-
};
|
|
282
|
-
}));
|
|
283
|
-
const writeMidi = (notes) => {
|
|
284
|
-
const header = writeChunk(HEADER_CHUNK_TYPE, [
|
|
285
|
-
...toUint16(1),
|
|
286
|
-
...toUint16(2),
|
|
287
|
-
...toUint16(MIDI_TICKS_PER_BEAT)
|
|
288
|
-
]);
|
|
289
|
-
const tempoTrack = writeTrack([{
|
|
290
|
-
bytes: EMPTY_TRACK_NAME,
|
|
291
|
-
tick: 0
|
|
292
|
-
}]);
|
|
293
|
-
const noteTrack = writeTrack([
|
|
294
|
-
{
|
|
295
|
-
bytes: EMPTY_TRACK_NAME,
|
|
296
|
-
tick: 0
|
|
297
|
-
},
|
|
298
|
-
{
|
|
299
|
-
bytes: PIANO_PROGRAM_CHANGE,
|
|
300
|
-
tick: 0
|
|
301
|
-
},
|
|
302
|
-
...writeNoteEvents(notes),
|
|
303
|
-
...writePitchBendEvents(notes)
|
|
304
|
-
]);
|
|
305
|
-
return new Uint8Array([
|
|
306
|
-
...header,
|
|
307
|
-
...tempoTrack,
|
|
308
|
-
...noteTrack
|
|
309
|
-
]);
|
|
310
|
-
};
|
|
311
|
-
//#endregion
|
|
312
|
-
export { ONSET_THRESHOLD as A, MIDI_OFFSET as C, MODEL_URL as D, MIN_NOTE_LENGTH as E, WINDOW_HOP_SAMPLES as F, WINDOW_OFFSET_SECONDS as I, OVERLAP_SAMPLES as M, PITCH_BEND_STANDARD_DEVIATION as N, MODEL_WEIGHTS_URL as O, PITCH_BEND_TOLERANCE as P, FRAME_THRESHOLD as S, MIDI_PITCH_BEND_STEPS as T, CONTOUR_BINS as _, findOnsetPeaks as a, FFT_HOP as b, convertNotesToSeconds as c, ANNOTATIONS_FRAMES_PER_WINDOW as d, ANNOTATIONS_SEMITONES as f, CONTOURS_TENSOR as g, AUDIO_WINDOW_SAMPLES as h, findLargestReading as i, OVERLAPPING_FRAMES as j, ONSETS_TENSOR as k, addPitchBends as l, AUDIO_WINDOW_LENGTH_SECONDS as m, createNotes as n, constrainFrequency as o, AUDIO_SAMPLE_RATE as p, inferOnsets as r, clearPitch as s, writeMidi as t, ANNOTATIONS_FPS as u, CONTOUR_BINS_PER_SEMITONE as v, MIDI_PITCH_BEND_SEMITONES as w, FRAMES_TENSOR as x, ENERGY_TOLERANCE as y };
|
package/dist/notes.d.ts
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
//#region src/constants.d.ts
|
|
2
|
-
export declare const AUDIO_SAMPLE_RATE = 22050;
|
|
3
|
-
export declare const FFT_HOP = 256;
|
|
4
|
-
export declare const AUDIO_WINDOW_LENGTH_SECONDS = 2;
|
|
5
|
-
export declare const AUDIO_WINDOW_SAMPLES: number;
|
|
6
|
-
export declare const ANNOTATIONS_FPS: number;
|
|
7
|
-
export declare const ANNOTATIONS_FRAMES_PER_WINDOW: number;
|
|
8
|
-
export declare const OVERLAPPING_FRAMES = 30;
|
|
9
|
-
export declare const OVERLAP_SAMPLES: number;
|
|
10
|
-
export declare const WINDOW_HOP_SAMPLES: number;
|
|
11
|
-
export declare const WINDOW_OFFSET_SECONDS: number;
|
|
12
|
-
export declare const MIDI_OFFSET = 21;
|
|
13
|
-
export declare const ANNOTATIONS_SEMITONES = 88;
|
|
14
|
-
export declare const CONTOUR_BINS_PER_SEMITONE = 3;
|
|
15
|
-
export declare const CONTOUR_BINS: number;
|
|
16
|
-
export declare const FRAMES_TENSOR = "Identity_1";
|
|
17
|
-
export declare const ONSETS_TENSOR = "Identity_2";
|
|
18
|
-
export declare const CONTOURS_TENSOR = "Identity";
|
|
19
|
-
export declare const MODEL_URL: URL;
|
|
20
|
-
export declare const MODEL_WEIGHTS_URL: URL;
|
|
21
|
-
export declare const ONSET_THRESHOLD = .5;
|
|
22
|
-
export declare const FRAME_THRESHOLD = .3;
|
|
23
|
-
export declare const MIN_NOTE_LENGTH = 11;
|
|
24
|
-
export declare const ENERGY_TOLERANCE = 11;
|
|
25
|
-
export declare const PITCH_BEND_TOLERANCE = 25;
|
|
26
|
-
export declare const PITCH_BEND_STANDARD_DEVIATION = 5;
|
|
27
|
-
export declare const MIDI_PITCH_BEND_SEMITONES = 2;
|
|
28
|
-
export declare const MIDI_PITCH_BEND_STEPS: number;
|
|
29
|
-
//#endregion
|
|
30
|
-
//#region src/models/ModelReadings.d.ts
|
|
31
|
-
interface ModelReadings {
|
|
32
|
-
contours: number[][];
|
|
33
|
-
frames: number[][];
|
|
34
|
-
onsets: number[][];
|
|
35
|
-
}
|
|
36
|
-
//#endregion
|
|
37
|
-
//#region src/models/NoteCreationOptions.d.ts
|
|
38
|
-
interface NoteCreationOptions {
|
|
39
|
-
energyTolerance?: number;
|
|
40
|
-
frameThreshold?: number;
|
|
41
|
-
isInferringOnsets?: boolean;
|
|
42
|
-
isMelodiaTrick?: boolean;
|
|
43
|
-
maxFrequency?: number;
|
|
44
|
-
minFrequency?: number;
|
|
45
|
-
minNoteLength?: number;
|
|
46
|
-
onsetThreshold?: number;
|
|
47
|
-
}
|
|
48
|
-
//#endregion
|
|
49
|
-
//#region src/models/NoteEvent.d.ts
|
|
50
|
-
interface NoteEvent {
|
|
51
|
-
amplitude: number;
|
|
52
|
-
durationFrames: number;
|
|
53
|
-
pitchBends?: number[];
|
|
54
|
-
pitchMidi: number;
|
|
55
|
-
startFrame: number;
|
|
56
|
-
}
|
|
57
|
-
//#endregion
|
|
58
|
-
//#region src/models/NoteEventTime.d.ts
|
|
59
|
-
interface NoteEventTime {
|
|
60
|
-
amplitude: number;
|
|
61
|
-
durationSeconds: number;
|
|
62
|
-
pitchBends?: number[];
|
|
63
|
-
pitchMidi: number;
|
|
64
|
-
startTimeSeconds: number;
|
|
65
|
-
}
|
|
66
|
-
//#endregion
|
|
67
|
-
//#region src/services/addPitchBends.d.ts
|
|
68
|
-
export declare const addPitchBends: (contours: number[][], notes: NoteEvent[]) => NoteEvent[];
|
|
69
|
-
//#endregion
|
|
70
|
-
//#region src/services/convertNotesToSeconds.d.ts
|
|
71
|
-
export declare const convertNotesToSeconds: (notes: NoteEvent[]) => NoteEventTime[];
|
|
72
|
-
//#endregion
|
|
73
|
-
//#region src/services/createNotes.d.ts
|
|
74
|
-
export declare const createNotes: ({ frames, onsets }: Pick<ModelReadings, "frames" | "onsets">, { energyTolerance, frameThreshold, isInferringOnsets, isMelodiaTrick, maxFrequency, minFrequency, minNoteLength, onsetThreshold }?: NoteCreationOptions) => NoteEvent[];
|
|
75
|
-
//#endregion
|
|
76
|
-
//#region src/services/writeMidi.d.ts
|
|
77
|
-
export declare const writeMidi: (notes: NoteEventTime[]) => Uint8Array;
|
|
78
|
-
//#endregion
|
|
79
|
-
export type { ModelReadings, NoteCreationOptions, NoteEvent, NoteEventTime };
|
package/dist/notes.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export { u as ANNOTATIONS_FPS, d as ANNOTATIONS_FRAMES_PER_WINDOW, f as ANNOTATIONS_SEMITONES, p as AUDIO_SAMPLE_RATE, m as AUDIO_WINDOW_LENGTH_SECONDS, h as AUDIO_WINDOW_SAMPLES, g as CONTOURS_TENSOR, _ as CONTOUR_BINS, v as CONTOUR_BINS_PER_SEMITONE, y as ENERGY_TOLERANCE, b as FFT_HOP, x as FRAMES_TENSOR, S as FRAME_THRESHOLD, C as MIDI_OFFSET, w as MIDI_PITCH_BEND_SEMITONES, T as MIDI_PITCH_BEND_STEPS, E as MIN_NOTE_LENGTH, D as MODEL_URL, O as MODEL_WEIGHTS_URL, k as ONSETS_TENSOR, A as ONSET_THRESHOLD, j as OVERLAPPING_FRAMES, M as OVERLAP_SAMPLES, N as PITCH_BEND_STANDARD_DEVIATION, P as PITCH_BEND_TOLERANCE, F as WINDOW_HOP_SAMPLES, I as WINDOW_OFFSET_SECONDS, l as addPitchBends, c as convertNotesToSeconds, n as createNotes, t as writeMidi } from "./notes-BZsEo1k5.js";
|