speaker-calibration 2.0.0 → 2.1.0

package/src/tasks/audioRecorder.js

@@ -0,0 +1,148 @@
+import MyEventEmitter from '../myEventEmitter';
+
+/**
+ * @class provides a simple interface for recording audio from a microphone
+ * using the Media Recorder API.
+ */
+class AudioRecorder extends MyEventEmitter {
+  /** @private */
+  #mediaRecorder;
+
+  /** @private */
+  #recordedChunks = [];
+
+  /** @private */
+  #audioBlob;
+
+  /** @private */
+  #audioContext;
+
+  /** @private */
+  #recordedSignals = [];
+
+  /** @private */
+  sinkSamplingRate;
+
+  /**
+   * Decode the audio data from the recorded audio blob.
+   *
+   * @private
+   * @example
+   */
+  #saveRecording = async () => {
+    const arrayBuffer = await this.#audioBlob.arrayBuffer();
+    const audioBuffer = await this.#audioContext.decodeAudioData(arrayBuffer);
+    const data = audioBuffer.getChannelData(0);
+
+    console.log(`Decoded audio buffer with ${data.length} samples`);
+    this.#recordedSignals.push(Array.from(data));
+  };
+
+  /**
+   * Event listener triggered when data is available in the media recorder.
+   *
+   * @private
+   * @param e - The event object.
+   * @example
+   */
+  #onRecorderDataAvailable = e => {
+    if (e.data && e.data.size > 0) this.#recordedChunks.push(e.data);
+  };
+
+  /**
+   * Method to create a media recorder object and set up event listeners.
+   *
+   * @private
+   * @param stream - The stream of audio from the Listener.
+   * @example
+   */
+  #setMediaRecorder = stream => {
+    // Create a new MediaRecorder object
+    this.#mediaRecorder = new MediaRecorder(stream);
+
+    // Add event listeners
+    this.#mediaRecorder.ondataavailable = e => this.#onRecorderDataAvailable(e);
+  };
+
+  #setAudioContext = () => {
+    this.#audioContext = new (window.AudioContext ||
+      window.webkitAudioContext ||
+      window.audioContext)({
+      sampleRate: this.sinkSamplingRate,
+    });
+  };
+
+  /**
+   * Public method to start the recording process.
+   *
+   * @param stream - The stream of audio from the Listener.
+   * @example
+   */
+  startRecording = async stream => {
+    // Create a fresh audio context
+    this.#setAudioContext();
+    // Set up media recorder if needed
+    if (!this.#mediaRecorder) this.#setMediaRecorder(stream);
+    // clear recorded chunks
+    this.#recordedChunks = [];
+    // start recording
+    this.#mediaRecorder.start();
+  };
+
+  /**
+   * Method to stop the recording process.
+   *
+   * @public
+   * @example
+   */
+  stopRecording = async () => {
+    // Stop the media recorder, and wait for the data to be available
+    await new Promise(resolve => {
+      this.#mediaRecorder.onstop = () => {
+        // when the stop event is triggered, resolve the promise
+        this.#audioBlob = new Blob(this.#recordedChunks, {
+          type: 'audio/wav; codecs=opus',
+        });
+        resolve(this.#audioBlob);
+      };
+      // call stop
+      this.#mediaRecorder.stop();
+    });
+    // Now that we have data, save it
+    await this.#saveRecording();
+  };
+
+  /** .
+   * .
+   * .
+   * Public method to get the last recorded audio signal
+   *
+   * @returns
+   * @example
+   */
+  getLastRecordedSignal = () => this.#recordedSignals[this.#recordedSignals.length - 1];
+
+  /** .
+   * .
+   * .
+   * Public method to get all the recorded audio signals
+   *
+   * @returns
+   * @example
+   */
+  getAllRecordedSignals = () => this.#recordedSignals;
+
+  /** .
+   * .
+   * .
+   * Public method to set the sampling rate used by the capture device
+   *
+   * @param {Number} sinkSamplingRate - The sampling rate of the capture device
+   * @example
+   */
+  setSinkSamplingRate = sinkSamplingRate => {
+    this.sinkSamplingRate = sinkSamplingRate;
+  };
+}
+
+export default AudioRecorder;

package/src/tasks/impulse-response/impulseResponse.js

@@ -0,0 +1,436 @@
+import AudioCalibrator from '../audioCalibrator';
+import MlsGenInterface from './mlsGen/mlsGenInterface';
+
+import {sleep, csvToArray} from '../../utils';
+
+/**
+ *
+ */
+class ImpulseResponse extends AudioCalibrator {
+  /**
+   * Default constructor. Creates an instance with any number of paramters passed or the default parameters defined here.
+   *
+   * @param {Object<boolean, number, number, number>} calibratorParams  - paramter object
+   * @param {boolean} [calibratorParams.download = false]  - boolean flag to download captures
+   * @param {number} [calibratorParams.mlsOrder = 18] - order of the MLS to be generated
+   * @param {number} [calibratorParams.numCaptures = 5] - number of captures to perform
+   * @param {number} [calibratorParams.numMLSPerCapture = 4] - number of bursts of MLS per capture
+   */
+  constructor({download = false, mlsOrder = 18, numCaptures = 3, numMLSPerCapture = 3}) {
+    super(numCaptures, numMLSPerCapture);
+    this.#mlsOrder = parseInt(mlsOrder, 10);
+    this.#P = 2 ** mlsOrder - 1;
+    this.#download = download;
+  }
+
+  /** @private */
+  #download;
+
+  /** @private */
+  #mlsGenInterface;
+
+  /** @private */
+  #mlsBufferView;
+
+  /** @private */
+  invertedImpulseResponse = null;
+
+  /** @private */
+  impulseResponses = [];
+
+  /** @private */
+  #mlsOrder;
+
+  /** @private */
+  #P;
+
+  /** @private */
+  TAPER_SECS = 5;
+
+  /** @private */
+  offsetGainNode;
+
+  /** .
+   * .
+   * .
+   * Sends all the computed impulse responses to the backend server for processing
+   *
+   * @returns sets the resulting inverted impulse response to the class property
+   * @example
+   */
+  sendImpulseResponsesToServerForProcessing = async () => {
+    const computedIRs = await Promise.all(this.impulseResponses);
+    this.emit('update', {message: `computing the IIR...`});
+    return this.pyServerAPI
+      .getInverseImpulseResponse({
+        payload: computedIRs.slice(0, this.numCaptures),
+      })
+      .then(res => {
+        this.emit('update', {message: `done computing the IIR...`});
+        this.invertedImpulseResponse = res;
+      })
+      .catch(err => {
+        // this.emit('InvertedImpulseResponse', {res: false});
+        console.error(err);
+      });
+  };
+
+  /** .
+   * .
+   * .
+   * Sends the recorded signal, or a given csv string of a signal, to the back end server for processing
+   *
+   * @param {<array>String} signalCsv - Optional csv string of a previously recorded signal, if given, this signal will be processed
+   * @example
+   */
+  sendRecordingToServerForProcessing = signalCsv => {
+    const allSignals = this.getAllRecordedSignals();
+    const numSignals = allSignals.length;
+    const payload =
+      signalCsv && signalCsv.length > 0 ? csvToArray(signalCsv) : allSignals[numSignals - 1];
+
+    this.emit('update', {message: `computing the IR of the last recording...`});
+    this.impulseResponses.push(
+      this.pyServerAPI
+        .getImpulseResponse({
+          sampleRate: this.sourceSamplingRate || 96000,
+          payload,
+          P: this.#P,
+        })
+        .then(res => {
+          if (this.numSuccessfulCaptured < this.numCaptures) {
+            this.numSuccessfulCaptured += 1;
+            this.emit('update', {
+              message: `${this.numSuccessfulCaptured}/${this.numCaptures} IRs computed...`,
+            });
+          }
+          return res;
+        })
+        .catch(err => {
+          console.error(err);
+        })
+    );
+  };
+
+  /**
+   * Passed to the calibration steps function, awaits the desired amount of seconds to capture the desired number
+   * of MLS periods defined in the constructor.
+   *
+   * @example
+   */
+  #awaitDesiredMLSLength = async () => {
+    // seconds per MLS = P / SR
+    // await N * P / SR
+    this.emit('update', {
+      message: `sampling the calibration signal...`,
+    });
+    await sleep((this.#P / this.sourceSamplingRate) * this.numMLSPerCapture);
+  };
+
+  /** .
+   * .
+   * .
+   * Passed to the calibration steps function, awaits the onset of the signal to ensure a steady state
+   *
+   * @example
+   */
+  #awaitSignalOnset = async () => {
+    this.emit('update', {
+      message: `waiting for the signal to stabalize...`,
+    });
+    await sleep(this.TAPER_SECS);
+  };
+
+  /**
+   * Called immediately after a recording is captured. Used to process the resulting signal
+   * whether by sending the result to a server or by computing a result locally.
+   *
+   * @example
+   */
+  #afterMLSRecord = () => {
+    if (this.#download) {
+      this.downloadData();
+    }
+    this.sendRecordingToServerForProcessing();
+  };
+
+  #afterMLSwIIRRecord = () => {
+    if (this.#download) {
+      this.downloadData();
+    }
+    this.#stopCalibrationAudio();
+  };
+
+  /** .
+   * .
+   * .
+   * Created an S Curver Buffer to taper the signal onset
+   *
+   * @param {*} length
+   * @param {*} phase
+   * @returns
+   * @example
+   */
+  static createSCurveBuffer = (length, phase) => {
+    const curve = new Float32Array(length);
+    let i;
+    for (i = 0; i < length; i += 1) {
+      // scale the curve to be between 0-1
+      curve[i] = Math.sin((Math.PI * i) / length - phase) / 2 + 0.5;
+    }
+    return curve;
+  };
+
+  static createInverseSCurveBuffer = (length, phase) => {
+    const curve = new Float32Array(length);
+    let i;
+    let j = length - 1;
+    for (i = 0; i < length; i += 1) {
+      // scale the curve to be between 0-1
+      curve[i] = Math.sin((Math.PI * j) / length - phase) / 2 + 0.5;
+      j -= 1;
+    }
+    return curve;
+  };
+
+  /**
+   * Construct a Calibration Node with the calibration parameters.
+   *
+   * @param CALIBRATION_TONE_FREQUENCY
+   * @private
+   * @example
+   */
+  #createPureTonenNode = CALIBRATION_TONE_FREQUENCY => {
+    const audioContext = this.makeNewSourceAudioContext();
+    const oscilator = audioContext.createOscillator();
+    const gainNode = audioContext.createGain();
+
+    oscilator.frequency.value = CALIBRATION_TONE_FREQUENCY;
+    oscilator.type = 'sine';
+    gainNode.gain.value = 0.04;
+
+    oscilator.connect(gainNode);
+    gainNode.connect(audioContext.destination);
+
+    this.addCalibrationNode(oscilator);
+  };
+
+  /**
+   * Construct a Calibration Node with the calibration parameters.
+   *
+   * @param dataBuffer
+   * @private
+   * @example
+   */
+  #createCalibrationNodeFromBuffer = dataBuffer => {
+    const audioContext = this.makeNewSourceAudioContext();
+    const buffer = audioContext.createBuffer(
+      1, // number of channels
+      dataBuffer.length,
+      audioContext.sampleRate // sample rate
+    );
+
+    const data = buffer.getChannelData(0); // get data
+    // fill the buffer with our data
+    try {
+      for (let i = 0; i < dataBuffer.length; i += 1) {
+        data[i] = dataBuffer[i];
+      }
+    } catch (error) {
+      console.error(error);
+    }
+
+    const onsetGainNode = audioContext.createGain();
+    this.offsetGainNode = audioContext.createGain();
+    const source = audioContext.createBufferSource();
+
+    source.buffer = buffer;
+    source.loop = true;
+    source.connect(onsetGainNode);
+    onsetGainNode.connect(this.offsetGainNode);
+    this.offsetGainNode.connect(audioContext.destination);
+
+    const onsetCurve = ImpulseResponse.createSCurveBuffer(this.sourceSamplingRate, Math.PI / 2);
+    onsetGainNode.gain.setValueCurveAtTime(onsetCurve, 0, this.TAPER_SECS);
+
+    this.addCalibrationNode(source);
+  };
+
+  /**
+   * Given a data buffer, creates the required calibration node
+   *
+   * @param {*} dataBufferArray
+   * @example
+   */
+  #setCalibrationNodesFromBuffer = (dataBufferArray = [this.#mlsBufferView]) => {
+    if (dataBufferArray.length === 1) {
+      this.#createCalibrationNodeFromBuffer(dataBufferArray[0]);
+    } else {
+      throw new Error('The length of the data buffer array must be 1');
+    }
+  };
+
+  #createImpulseResponseFilterGraph = (calibrationSignal, iir) => {
+    const audioCtx = this.makeNewSourceAudioContext();
+
+    // -------------------------------------------------------- IIR
+    const iirBuffer = audioCtx.createBuffer(
+      1,
+      // TODO: quality check this
+      iir.length - 1,
+      audioCtx.sampleRate
+    );
+
+    // Fill the buffer with the inverted impulse response
+    const iirChannelZeroBuffer = iirBuffer.getChannelData(0);
+    for (let i = 0; i < iirBuffer.length; i += 1) {
+      // audio needs to be in [-1.0; 1.0]
+      iirChannelZeroBuffer[i] = iir[i];
+    }
+
+    const convolverNode = audioCtx.createConvolver();
+
+    convolverNode.normalize = false;
+    convolverNode.channelCount = 1;
+    convolverNode.buffer = iirBuffer;
+
+    // ------------------------------------------------------ MLS
+    const calibrationSignalBuffer = audioCtx.createBuffer(
+      1, // number of channels
+      calibrationSignal.length,
+      audioCtx.sampleRate // sample rate
+    );
+
+    const mlsChannelZeroBuffer = calibrationSignalBuffer.getChannelData(0); // get data
+    // fill the buffer with our data
+    try {
+      for (let i = 0; i < calibrationSignal.length; i += 1) {
+        mlsChannelZeroBuffer[i] = calibrationSignal[i];
+      }
+    } catch (error) {
+      console.error(error);
+    }
+
+    const sourceNode = audioCtx.createBufferSource();
+
+    sourceNode.buffer = calibrationSignalBuffer;
+    sourceNode.loop = true;
+    sourceNode.connect(convolverNode);
+
+    convolverNode.connect(audioCtx.destination);
+
+    console.log({convolverNode, sourceNode});
+
+    this.addCalibrationNode(sourceNode);
+  };
+
+  #createIIRwMLSGraph = () => {
+    this.#createImpulseResponseFilterGraph(this.impulseResponses, [this.#mlsBufferView][0]);
+  };
+
+  /**
+   * Creates an audio context and plays it for a few seconds.
+   *
+   * @private
+   * @returns - Resolves when the audio is done playing.
+   * @example
+   */
+  #playCalibrationAudio = () => {
+    this.calibrationNodes[0].start(0);
+    this.emit('update', {message: 'playing the calibration tone...'});
+  };
+
+  /** .
+   * .
+   * .
+   * Stops the audio with tapered offset
+   *
+   * @example
+   */
+  #stopCalibrationAudio = () => {
+    this.offsetGainNode.gain.setValueAtTime(
+      this.offsetGainNode.gain.value,
+      this.sourceAudioContext.currentTime
+    );
+
+    this.offsetGainNode.gain.setTargetAtTime(0, this.sourceAudioContext.currentTime, 0.5);
+    this.emit('update', {message: 'stopping the calibration tone...'});
+  };
+
+  playMLSwithIIR = async (stream, iir) => {
+    this.invertedImpulseResponse = iir;
+    // initialize the MLSGenInterface object with it's factory method
+    await MlsGenInterface.factory(
+      this.#mlsOrder,
+      this.sinkSamplingRate,
+      this.sourceSamplingRate
+    ).then(mlsGenInterface => {
+      this.#mlsGenInterface = mlsGenInterface;
+      this.#mlsBufferView = this.#mlsGenInterface.getMLS();
+    });
+
+    // after intializating, start the calibration steps with garbage collection
+    await this.#mlsGenInterface.withGarbageCollection([
+      [
+        this.calibrationSteps,
+        [
+          stream,
+          this.#playCalibrationAudio, // play audio func (required)
+          this.#createImpulseResponseFilterGraph, // before play func
+          null, // before record
+          this.#awaitDesiredMLSLength, // during record
+          this.#afterMLSwIIRRecord, // after record
+        ],
+      ],
+    ]);
+  };
+
+  /**
+   * Public method to start the calibration process. Objects intialized from webassembly allocate new memory
+   * and must be manually freed. This function is responsible for intializing the MlsGenInterface,
+   * and wrapping the calibration steps with a garbage collection safe gaurd.
+   *
+   * @public
+   * @param stream - The stream of audio from the Listener.
+   * @example
+   */
+  startCalibration = async stream => {
+    // initialize the MLSGenInterface object with it's factory method
+    await MlsGenInterface.factory(
+      this.#mlsOrder,
+      this.sinkSamplingRate,
+      this.sourceSamplingRate
+    ).then(mlsGenInterface => {
+      this.#mlsGenInterface = mlsGenInterface;
+      this.#mlsBufferView = this.#mlsGenInterface.getMLS();
+    });
+
+    // after intializating, start the calibration steps with garbage collection
+    await this.#mlsGenInterface.withGarbageCollection([
+      () =>
+        this.calibrationSteps(
+          stream,
+          this.#playCalibrationAudio, // play audio func (required)
+          this.#setCalibrationNodesFromBuffer, // before play func
+          this.#awaitSignalOnset, // before record
+          () => this.numSuccessfulCaptured < this.numCaptures, // loop while true
+          this.#awaitDesiredMLSLength, // during record
+          this.#afterMLSRecord // after record
+        ),
+    ]);
+
+    this.#stopCalibrationAudio();
+
+    // at this stage we've captured all the required signals,
+    // and have received IRs for each one
+    // so let's send all the IRs to the server to be converted to a single IIR
+    await this.sendImpulseResponsesToServerForProcessing();
+
+    // debugging function, use to test the result of the IIR
+    // await this.playMLSwithIIR(stream, this.invertedImpulseResponse);
+
+    return this.invertedImpulseResponse;
+  };
+}
+
+export default ImpulseResponse;

package/src/tasks/impulse-response/mlsGen/mlsGen.cpp

@@ -0,0 +1,99 @@
+#include "mlsGen.hpp"
+#include <sanitizer/lsan_interface.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+MLSGen::MLSGen(long N, long srcSR, long sinkSR) {
+  MLSGen::N = N;
+  MLSGen::srcSR = srcSR;
+  MLSGen::sinkSR = sinkSR;
+  P = (1 << N) - 1;
+  mls = new bool[P];
+  tagL = new long[P];
+  tagS = new long[P];
+  generatedSignal = new float[P];
+  recordedSignal = new float[P];
+  perm = new float[P + 1];
+  resp = new float[P + 1];
+}
+
+#ifndef __EMSCRIPTEN__
+MLSGen::~MLSGen() {
+  delete[] mls;
+  delete[] tagL;
+  delete[] tagS;
+  delete[] generatedSignal;
+  delete[] recordedSignal;
+  delete[] perm;
+  delete[] resp;
+}
+#endif
+
+#ifdef __EMSCRIPTEN__
+
+using namespace emscripten;
+
+void MLSGen::Destruct() {
+  delete[] mls;
+  delete[] tagL;
+  delete[] tagS;
+  delete[] generatedSignal;
+  delete[] recordedSignal;
+  delete[] perm;
+  delete[] resp;
+}
+
+emscripten::val MLSGen::getMLS() {
+  // if mls is not generated, generate it
+  if (*(&mls + 1) - mls != P) {
+    generateMls();
+  }
+  for (int i = 0; i < P; i++) {
+    generatedSignal[i] = -2 * mls[i] + 1;
+  }
+  return emscripten::val(typed_memory_view(P, generatedSignal));
+}
+
+emscripten::val MLSGen::setRecordedSignalsMemoryView(long sizeRecordedSignals) {
+  C = sizeRecordedSignals;
+  recordedSignals = new float[C];
+  return emscripten::val(typed_memory_view(C, recordedSignals));
+}
+
+emscripten::val MLSGen::getRecordedSignalsMemoryView() {
+  return emscripten::val(typed_memory_view(C, recordedSignals));
+}
+
+emscripten::val MLSGen::getImpulseResponse() {
+  generateTagL();     // Generate tagL for the L matrix
+  generateTagS();     // Generate tagS for the S matrix
+  GenerateSignal();   // Generate the signal TEST PURPOSES
+  permuteSignal();    // Permute the signal according to tagS
+  fastHadamard();     // Do a Hadamard transform in place
+  permuteResponse();  // Permute the impulseresponse according to tagL
+  return emscripten::val(typed_memory_view(P + 1, resp));
+}
+
+// Binding code
+EMSCRIPTEN_BINDINGS(mls_gen_module) {
+  class_<MLSGen>("MLSGen")
+      .constructor<long, long, long>()
+      .function("Destruct", &MLSGen::Destruct)
+      .function("getMLS", &MLSGen::getMLS)
+      .function("getRecordedSignalsMemoryView",
+                &MLSGen::getRecordedSignalsMemoryView)
+      .function("setRecordedSignalsMemoryView",
+                &MLSGen::setRecordedSignalsMemoryView)
+      .function("getImpulseResponse", &MLSGen::getImpulseResponse);
+  function("doLeakCheck", &__lsan_do_recoverable_leak_check);
+};
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+// https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html
+// https://web.dev/webassembly-memory-debugging/