spessasynth_core 1.1.3 → 1.1.4

package/src/synthetizer/audio_engine/main_processor.js

@@ -0,0 +1,804 @@
+import { SpessaSynthInfo } from "../../utils/loggin.js";
+import { consoleColors } from "../../utils/other.js";
+import { voiceKilling } from "./engine_methods/stopping_notes/voice_killing.js";
+import {
+    ALL_CHANNELS_OR_DIFFERENT_ACTION,
+    DEFAULT_PERCUSSION,
+    DEFAULT_SYNTH_MODE,
+    VOICE_CAP
+} from "../synth_constants.js";
+import { stbvorbis } from "../../externals/stbvorbis_sync/stbvorbis_sync.min.js";
+import { VOLUME_ENVELOPE_SMOOTHING_FACTOR } from "./engine_components/volume_envelope.js";
+import { systemExclusive } from "./engine_methods/system_exclusive.js";
+import { masterParameterType, setMasterParameter } from "./engine_methods/controller_control/master_parameters.js";
+import { resetAllControllers } from "./engine_methods/controller_control/reset_controllers.js";
+import { WorkletSoundfontManager } from "./engine_components/soundfont_manager.js";
+import { interpolationTypes } from "./engine_components/wavetable_oscillator.js";
+import { KeyModifierManager } from "./engine_components/key_modifier_manager.js";
+import { getVoices } from "./engine_components/voice.js";
+import { PAN_SMOOTHING_FACTOR } from "./engine_components/stereo_panner.js";
+import { stopAllChannels } from "./engine_methods/stopping_notes/stop_all_channels.js";
+import { setEmbeddedSoundFont } from "./engine_methods/soundfont_management/set_embedded_sound_font.js";
+import { reloadSoundFont } from "./engine_methods/soundfont_management/reload_sound_font.js";
+import { clearSoundFont } from "./engine_methods/soundfont_management/clear_sound_font.js";
+import { sendPresetList } from "./engine_methods/soundfont_management/send_preset_list.js";
+import { getPreset } from "./engine_methods/soundfont_management/get_preset.js";
+import { transposeAllChannels } from "./engine_methods/tuning_control/transpose_all_channels.js";
+import { setMasterTuning } from "./engine_methods/tuning_control/set_master_tuning.js";
+import { applySynthesizerSnapshot } from "./snapshot/apply_synthesizer_snapshot.js";
+import { createMidiChannel } from "./engine_methods/create_midi_channel.js";
+import { FILTER_SMOOTHING_FACTOR } from "./engine_components/lowpass_filter.js";
+import { getEvent, messageTypes } from "../../midi/midi_message.js";
+import { IndexedByteArray } from "../../utils/indexed_array.js";
+
+
+/**
+ * @typedef {"gm"|"gm2"|"gs"|"xg"} SynthSystem
+ */
+
+/**
+ * main_processor.js
+ * purpose: the core synthesis engine
+ */
+
+
+/**
+ * @typedef {Object} NoteOnCallback
+ * @property {number} midiNote - The MIDI note number.
+ * @property {number} channel - The MIDI channel number.
+ * @property {number} velocity - The velocity of the note.
+ */
+
+/**
+ * @typedef {Object} NoteOffCallback
+ * @property {number} midiNote - The MIDI note number.
+ * @property {number} channel - The MIDI channel number.
+ */
+
+/**
+ * @typedef {Object} DrumChangeCallback
+ * @property {number} channel - The MIDI channel number.
+ * @property {boolean} isDrumChannel - Indicates if the channel is a drum channel.
+ */
+
+/**
+ * @typedef {Object} ProgramChangeCallback
+ * @property {number} channel - The MIDI channel number.
+ * @property {number} program - The program number.
+ * @property {number} bank - The bank number.
+ */
+
+/**
+ * @typedef {Object} ControllerChangeCallback
+ * @property {number} channel - The MIDI channel number.
+ * @property {number} controllerNumber - The controller number.
+ * @property {number} controllerValue - The value of the controller.
+ */
+
+/**
+ * @typedef {Object} MuteChannelCallback
+ * @property {number} channel - The MIDI channel number.
+ * @property {boolean} isMuted - Indicates if the channel is muted.
+ */
+
+/**
+ * @typedef {Object} PresetListChangeCallbackSingle
+ * @property {string} presetName - The name of the preset.
+ * @property {number} bank - The bank number.
+ * @property {number} program - The program number.
+ */
+
+/**
+ * @typedef {PresetListChangeCallbackSingle[]} PresetListChangeCallback - A list of preset objects.
+ */
+
+/**
+ * @typedef {Object} SynthDisplayCallback
+ * @property {Uint8Array} displayData - The data to display.
+ * @property {SynthDisplayType} displayType - The type of display.
+ */
+
+/**
+ * @typedef {Object} PitchWheelCallback
+ * @property {number} channel - The MIDI channel number.
+ * @property {number} MSB - The most significant byte of the pitch-wheel value.
+ * @property {number} LSB - The least significant byte of the pitch-wheel value.
+ */
+
+/**
+ * @typedef {Object} ChannelPressureCallback
+ * @property {number} channel - The MIDI channel number.
+ * @property {number} pressure - The pressure value.
+ */
+
+/**
+ * @typedef {Error} SoundfontErrorCallback - The error message for soundfont errors.
+ */
+
+/**
+ * @typedef {
+ *     NoteOnCallback |
+ *     NoteOffCallback |
+ *     DrumChangeCallback |
+ *     ProgramChangeCallback |
+ *     ControllerChangeCallback |
+ *     MuteChannelCallback |
+ *     PresetListChangeCallback |
+ *     PitchWheelCallback |
+ *     SoundfontErrorCallback |
+ *     ChannelPressureCallback |
+ *     SynthDisplayCallback |
+ *     undefined
+ * } EventCallbackData
+ */
+
+/**
+ * @typedef {
+ * "noteon"|
+ * "noteoff"|
+ * "pitchwheel"|
+ * "controllerchange"|
+ * "programchange"|
+ * "channelpressure"|
+ * "polypressure" |
+ * "drumchange"|
+ * "stopall"|
+ * "newchannel"|
+ * "mutechannel"|
+ * "presetlistchange"|
+ * "allcontrollerreset"|
+ * "soundfonterror"|
+ * "synthdisplay"} EventTypes
+ */
+
+
+/**
+ * @typedef {Object} SynthMethodOptions
+ * @property {number} time - the audio context time when the event should execute, in seconds.
+ */
+
+// if the note is released faster than that, it forced to last that long
+// this is used mostly for drum channels, where a lot of midis like to send instant note off after a note on
+export const MIN_NOTE_LENGTH = 0.03;
+// this sounds way nicer for an instant hi-hat cutoff
+export const MIN_EXCLUSIVE_LENGTH = 0.07;
+
+export const SYNTHESIZER_GAIN = 1.0;
+
+
+// the core synthesis engine of spessasynth.
+class SpessaSynthProcessor
+{
+    
+    /**
+     * Cached voices for all presets for this synthesizer.
+     * Nesting goes like this:
+     * this.cachedVoices[bankNumber][programNumber][midiNote][velocity] = a list of Voices for that.
+     * @type {Voice[][][][][]}
+     */
+    cachedVoices = [];
+    
+    /**
+     * Synth's device id: -1 means all
+     * @type {number}
+     */
+    deviceID = ALL_CHANNELS_OR_DIFFERENT_ACTION;
+    
+    /**
+     * Synth's event queue from the main thread
+     * @type {{callback: function(), time: number}[]}
+     */
+    eventQueue = [];
+    
+    /**
+     * Interpolation type used
+     * @type {interpolationTypes}
+     */
+    interpolationType = interpolationTypes.fourthOrder;
+    
+    /**
+     * Global transposition in semitones
+     * @type {number}
+     */
+    transposition = 0;
+    
+    /**
+     * this.tunings[program][key] = tuning
+     * @type {MTSProgramTuning[]}
+     */
+    tunings = [];
+    
+    
+    /**
+     * Bank offset for things like embedded RMIDIS. Added for every program change
+     * @type {number}
+     */
+    soundfontBankOffset = 0;
+    
+    
+    /**
+     * The volume gain, set by user
+     * @type {number}
+     */
+    masterGain = SYNTHESIZER_GAIN;
+    
+    /**
+     * The volume gain, set by MIDI sysEx
+     * @type {number}
+     */
+    midiVolume = 1;
+    
+    /**
+     * Reverb linear gain
+     * @type {number}
+     */
+    reverbGain = 1;
+    /**
+     * Chorus linear gain
+     * @type {number}
+     */
+    chorusGain = 1;
+    
+    /**
+     * Maximum number of voices allowed at once
+     * @type {number}
+     */
+    voiceCap = VOICE_CAP;
+    
+    /**
+     * (-1 to 1)
+     * @type {number}
+     */
+    pan = 0.0;
+    /**
+     * the pan of the left channel
+     * @type {number}
+     */
+    panLeft = 0.5;
+    
+    /**
+     * the pan of the right channel
+     * @type {number}
+     */
+    panRight = 0.5;
+    
+    /**
+     * forces note killing instead of releasing
+     * @type {boolean}
+     */
+    highPerformanceMode = false;
+    
+    /**
+     * Handlese custom key overrides: velocity and preset
+     * @type {KeyModifierManager}
+     */
+    keyModifierManager = new KeyModifierManager();
+    
+    /**
+     * Overrides the main soundfont (embedded, for example)
+     * @type {BasicSoundBank}
+     */
+    overrideSoundfont = undefined;
+    
+    /**
+     * contains all the channels with their voices on the processor size
+     * @type {MidiAudioChannel[]}
+     */
+    midiAudioChannels = [];
+    
+    /**
+     * Controls the bank selection & SysEx
+     * @type {SynthSystem}
+     */
+    system = DEFAULT_SYNTH_MODE;
+    /**
+     * Current total voices amount
+     * @type {number}
+     */
+    totalVoicesAmount = 0;
+    
+    /**
+     * Synth's default (reset) preset
+     * @type {BasicPreset}
+     */
+    defaultPreset;
+    
+    defaultPresetUsesOverride = false;
+    
+    /**
+     * Synth's default (reset) drum preset
+     * @type {BasicPreset}
+     */
+    drumPreset;
+    
+    defaultDrumsUsesOverride = false;
+    
+    /**
+     * Controls if the worklet processor is fully initialized
+     * @type {Promise<boolean>}
+     */
+    processorInitialized = stbvorbis.isInitialized;
+    
+    /**
+     * Current audio time
+     * @type {number}
+     */
+    currentSynthTime = 0;
+    
+    /**
+     * in hertz
+     * @type {number}
+     */
+    sampleRate;
+    
+    /**
+     * @typedef {Object} CallbacksTypedef
+     * @property {function?} ready
+     * @property {function(EventTypes, EventCallbackData)?} eventCall
+     * @property {function(ChannelProperty, number)?} channelPropertyChange
+     * @property {function(masterParameterType, number|string)?} masterParameterChange
+     */
+    
+    /**
+     * Creates a new worklet synthesis system. contains all channels
+     * @param midiChannels {number}
+     * @param soundfont {ArrayBuffer}
+     * @param enableEventSystem {boolean}
+     * @param callbacks {CallbacksTypedef}
+     * @param sampleRate {number}
+     * @param initialTime {number}
+     * @param effectsEnabled {boolean}
+     * @param snapshot {SynthesizerSnapshot}
+     */
+    constructor(
+        soundfont,
+        sampleRate,
+        callbacks,
+        effectsEnabled = true,
+        enableEventSystem = true,
+        initialTime = 0,
+        midiChannels = 16,
+        snapshot = undefined)
+    {
+        /**
+         * Midi output count
+         * @type {number}
+         */
+        this.midiOutputsCount = midiChannels;
+        /**
+         * are the chorus and reverb effects enabled?
+         * @type {boolean}
+         */
+        this.effectsEnabled = effectsEnabled;
+        let initialChannelCount = this.midiOutputsCount;
+        
+        /**
+         * @type {CallbacksTypedef}
+         */
+        this.callbacks = callbacks;
+        
+        this.currentSynthTime = initialTime;
+        this.sampleRate = sampleRate;
+        
+        /**
+         * Sample time in seconds
+         * @type {number}
+         */
+        this.sampleTime = 1 / sampleRate;
+        
+        this.enableEventSystem = enableEventSystem;
+        
+        
+        for (let i = 0; i < 128; i++)
+        {
+            this.tunings.push([]);
+        }
+        
+        /**
+         * @type {WorkletSoundfontManager}
+         */
+        this.soundfontManager = new WorkletSoundfontManager(
+            soundfont
+        );
+        this.sendPresetList();
+        
+        this.getDefaultPresets();
+        
+        
+        for (let i = 0; i < initialChannelCount; i++)
+        {
+            this.createWorkletChannel(false);
+        }
+        
+        this.midiAudioChannels[DEFAULT_PERCUSSION].preset = this.drumPreset;
+        this.midiAudioChannels[DEFAULT_PERCUSSION].drumChannel = true;
+        
+        // these smoothing factors were tested on 44,100 Hz, adjust them to target sample rate here
+        this.volumeEnvelopeSmoothingFactor = VOLUME_ENVELOPE_SMOOTHING_FACTOR * (44100 / sampleRate);
+        this.panSmoothingFactor = PAN_SMOOTHING_FACTOR * (44100 / sampleRate);
+        this.filterSmoothingFactor = FILTER_SMOOTHING_FACTOR * (44100 / sampleRate);
+        
+        this._snapshot = snapshot;
+        if (this?._snapshot)
+        {
+            this.applySynthesizerSnapshot(snapshot);
+        }
+        
+        this.postReady();
+    }
+    
+    /**
+     * @returns {number}
+     */
+    get currentGain()
+    {
+        return this.masterGain * this.midiVolume;
+    }
+    
+    getDefaultPresets()
+    {
+        // override this to XG, to set the default preset to NOT be XG drums!
+        const sys = this.system;
+        this.system = "xg";
+        this.defaultPreset = this.getPreset(0, 0);
+        this.defaultPresetUsesOverride = this.overrideSoundfont?.presets?.indexOf(this.defaultPreset) >= 0;
+        this.system = sys;
+        this.drumPreset = this.getPreset(128, 0);
+        this.defaultDrumsUsesOverride = this.overrideSoundfont?.presets?.indexOf(this.drumPreset) >= 0;
+    }
+    
+    /**
+     * @param value {SynthSystem}
+     */
+    setSystem(value)
+    {
+        this.system = value;
+        this?.callbacks?.masterParameterChange?.(masterParameterType.midiSystem, this.system);
+    }
+    
+    /**
+     * @param bank {number}
+     * @param program {number}
+     * @param midiNote {number}
+     * @param velocity {number}
+     * @returns {Voice[]|undefined}
+     */
+    getCachedVoice(bank, program, midiNote, velocity)
+    {
+        return this.cachedVoices?.[bank]?.[program]?.[midiNote]?.[velocity];
+    }
+    
+    /**
+     * @param bank {number}
+     * @param program {number}
+     * @param midiNote {number}
+     * @param velocity {number}
+     * @param voices {Voice[]}
+     */
+    setCachedVoice(bank, program, midiNote, velocity, voices)
+    {
+        // make sure that it exists
+        if (!this.cachedVoices)
+        {
+            this.cachedVoices = [];
+        }
+        if (!this.cachedVoices[bank])
+        {
+            this.cachedVoices[bank] = [];
+        }
+        if (!this.cachedVoices[bank][program])
+        {
+            this.cachedVoices[bank][program] = [];
+        }
+        if (!this.cachedVoices[bank][program][midiNote])
+        {
+            this.cachedVoices[bank][program][midiNote] = [];
+        }
+        
+        // cache
+        this.cachedVoices[bank][program][midiNote][velocity] = voices;
+    }
+    
+    postReady()
+    {
+        // ensure stbvorbis is fully initialized
+        this.processorInitialized.then(() =>
+        {
+            SpessaSynthInfo("%cSpessaSynth is ready!", consoleColors.recognized);
+            this?.callbacks?.ready?.();
+        });
+    }
+    
+    // noinspection JSUnusedGlobalSymbols
+    /**
+     * Renders float32 audio data to stereo outputs; buffer size of 128 is recommended
+     * All float arrays must have the same length
+     * @param outputs {Float32Array[]} output stereo channels (L, R)
+     * @param reverb {Float32Array[]} reverb stereo channels (L, R)
+     * @param chorus {Float32Array[]} chorus stereo channels (L, R)
+     */
+    renderAudio(outputs, reverb, chorus)
+    {
+        this.renderAudioSplit(reverb, chorus, Array(16).fill(outputs));
+    }
+    
+    /**
+     * Renders the float32 audio data of each channel; buffer size of 128 is recommended
+     * All float arrays must have the same length
+     * @param reverbChannels {Float32Array[]} reverb stereo channels (L, R)
+     * @param chorusChannels {Float32Array[]} chorus stereo channels (L, R)
+     * @param separateChannels {Float32Array[][]} a total of 16 stereo pairs (L, R) for each MIDI channel
+     */
+    renderAudioSplit(reverbChannels, chorusChannels, separateChannels)
+    {
+        // process event queue
+        const time = this.currentSynthTime;
+        while (this.eventQueue[0]?.time <= time)
+        {
+            this.eventQueue.shift().callback();
+        }
+        const revL = reverbChannels[0];
+        const revR = reverbChannels[1];
+        const chrL = chorusChannels[0];
+        const chrR = chorusChannels[1];
+        
+        // for every channel
+        this.totalVoicesAmount = 0;
+        this.midiAudioChannels.forEach((channel, index) =>
+        {
+            if (channel.voices.length < 1 || channel.isMuted)
+            {
+                // there's nothing to do!
+                return;
+            }
+            let voiceCount = channel.voices.length;
+            const ch = index % 16;
+            
+            // render to the appropriate output
+            channel.renderAudio(
+                separateChannels[ch][0], separateChannels[ch][1],
+                revL, revR,
+                chrL, chrR
+            );
+            
+            this.totalVoicesAmount += channel.voices.length;
+            // if voice count changed, update voice amount
+            if (channel.voices.length !== voiceCount)
+            {
+                channel.sendChannelProperty();
+            }
+        });
+        
+        // advance the time appropriately
+        this.currentSynthTime += separateChannels[0][0].length * this.sampleTime;
+    }
+    
+    // noinspection JSUnusedGlobalSymbols
+    destroySynthProcessor()
+    {
+        this.midiAudioChannels.forEach(c =>
+        {
+            delete c.midiControllers;
+            delete c.voices;
+            delete c.sustainedVoices;
+            delete c.lockedControllers;
+            delete c.preset;
+            delete c.customControllers;
+        });
+        delete this.cachedVoices;
+        delete this.midiAudioChannels;
+        this.soundfontManager.destroyManager();
+        delete this.soundfontManager;
+    }
+    
+    /**
+     * @param channel {number}
+     * @param controllerNumber {number}
+     * @param controllerValue {number}
+     * @param force {boolean}
+     */
+    controllerChange(channel, controllerNumber, controllerValue, force = false)
+    {
+        this.midiAudioChannels[channel].controllerChange(controllerNumber, controllerValue, force);
+    }
+    
+    /**
+     * @param channel {number}
+     * @param midiNote {number}
+     * @param velocity {number}
+     */
+    noteOn(channel, midiNote, velocity)
+    {
+        this.midiAudioChannels[channel].noteOn(midiNote, velocity);
+    }
+    
+    /**
+     * @param channel {number}
+     * @param midiNote {number}
+     */
+    noteOff(channel, midiNote)
+    {
+        this.midiAudioChannels[channel].noteOff(midiNote);
+    }
+    
+    /**
+     * @param channel {number}
+     * @param midiNote {number}
+     * @param pressure {number}
+     */
+    polyPressure(channel, midiNote, pressure)
+    {
+        this.midiAudioChannels[channel].polyPressure(midiNote, pressure);
+    }
+    
+    /**
+     * @param channel {number}
+     * @param pressure {number}
+     */
+    channelPressure(channel, pressure)
+    {
+        this.midiAudioChannels[channel].channelPressure(pressure);
+    }
+    
+    /**
+     * @param channel {number}
+     * @param MSB {number}
+     * @param LSB {number}
+     */
+    pitchWheel(channel, MSB, LSB)
+    {
+        this.midiAudioChannels[channel].pitchWheel(MSB, LSB);
+    }
+    
+    /**
+     * @param channel {number}
+     * @param programNumber {number}
+     */
+    programChange(channel, programNumber)
+    {
+        this.midiAudioChannels[channel].programChange(programNumber);
+    }
+    
+    // noinspection JSUnusedGlobalSymbols
+    /**
+     * @param message {Uint8Array}
+     * @param channelOffset {number}
+     * @param force {boolean} cool stuff
+     * @param options {SynthMethodOptions}
+     */
+    processMessage(message, channelOffset, force, options)
+    {
+        const call = () =>
+        {
+            const statusByteData = getEvent(message[0]);
+            
+            const channel = statusByteData.channel + channelOffset;
+            // process the event
+            switch (statusByteData.status)
+            {
+                case messageTypes.noteOn:
+                    const velocity = message[2];
+                    if (velocity > 0)
+                    {
+                        this.noteOn(channel, message[1], velocity);
+                    }
+                    else
+                    {
+                        this.noteOff(channel, message[1]);
+                    }
+                    break;
+                
+                case messageTypes.noteOff:
+                    if (force)
+                    {
+                        this.midiAudioChannels[channel].killNote(message[1]);
+                    }
+                    else
+                    {
+                        this.noteOff(channel, message[1]);
+                    }
+                    break;
+                
+                case messageTypes.pitchBend:
+                    this.pitchWheel(channel, message[2], message[1]);
+                    break;
+                
+                case messageTypes.controllerChange:
+                    this.controllerChange(channel, message[1], message[2], force);
+                    break;
+                
+                case messageTypes.programChange:
+                    this.programChange(channel, message[1]);
+                    break;
+                
+                case messageTypes.polyPressure:
+                    this.polyPressure(channel, message[0], message[1]);
+                    break;
+                
+                case messageTypes.channelPressure:
+                    this.channelPressure(channel, message[1]);
+                    break;
+                
+                case messageTypes.systemExclusive:
+                    this.systemExclusive(new IndexedByteArray(message.slice(1)), channelOffset);
+                    break;
+                
+                case messageTypes.reset:
+                    this.stopAllChannels(true);
+                    this.resetAllControllers();
+                    break;
+                
+                default:
+                    break;
+            }
+        };
+        
+        const time = options.time;
+        if (time > this.currentSynthTime)
+        {
+            this.eventQueue.push({
+                callback: call.bind(this),
+                time: time
+            });
+            this.eventQueue.sort((e1, e2) => e1.time - e2.time);
+        }
+        else
+        {
+            call();
+        }
+    }
+    
+    /**
+     * @param volume {number} 0 to 1
+     */
+    setMIDIVolume(volume)
+    {
+        // GM2 specification, section 4.1: volume is squared.
+        // though, according to my own testing, Math.E seems like a better choice
+        this.midiVolume = Math.pow(volume, Math.E);
+        this.setMasterParameter(masterParameterType.masterPan, this.pan);
+    }
+    
+    /**
+     * Calls synth event from the worklet side
+     * @param eventName {EventTypes} the event name
+     * @param eventData {EventCallbackData}
+     * @this {SpessaSynthProcessor}
+     */
+    callEvent(eventName, eventData)
+    {
+        this?.callbacks?.eventCall?.(eventName, eventData);
+    }
+}
+
+// include other methods
+// voice related
+SpessaSynthProcessor.prototype.voiceKilling = voiceKilling;
+SpessaSynthProcessor.prototype.getVoices = getVoices;
+
+// system-exclusive related
+SpessaSynthProcessor.prototype.systemExclusive = systemExclusive;
+
+// channel related
+SpessaSynthProcessor.prototype.stopAllChannels = stopAllChannels;
+SpessaSynthProcessor.prototype.createWorkletChannel = createMidiChannel;
+SpessaSynthProcessor.prototype.resetAllControllers = resetAllControllers;
+
+// master parameter related
+SpessaSynthProcessor.prototype.setMasterParameter = setMasterParameter;
+
+// tuning related
+SpessaSynthProcessor.prototype.transposeAllChannels = transposeAllChannels;
+SpessaSynthProcessor.prototype.setMasterTuning = setMasterTuning;
+
+// program related
+SpessaSynthProcessor.prototype.getPreset = getPreset;
+SpessaSynthProcessor.prototype.reloadSoundFont = reloadSoundFont;
+SpessaSynthProcessor.prototype.clearSoundFont = clearSoundFont;
+SpessaSynthProcessor.prototype.setEmbeddedSoundFont = setEmbeddedSoundFont;
+SpessaSynthProcessor.prototype.sendPresetList = sendPresetList;
+
+// snapshot related
+SpessaSynthProcessor.prototype.applySynthesizerSnapshot = applySynthesizerSnapshot;
+
+export { SpessaSynthProcessor };