speaker-calibration 2.1.7 → 2.1.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "speaker-calibration",
-  "version": "2.1.7",
+  "version": "2.1.9",
   "description": "Speaker calibration library for auditory testing",
   "main": "dist/main.js",
   "directories": {

package/src/server/PythonServerAPI.js

@@ -6,7 +6,6 @@ class PythonServerAPI {
   static PYTHON_SERVER_URL = 'https://easyeyes-python-flask-server.herokuapp.com';
 
   static TEST_SERVER_URL = 'http://127.0.0.1:5000';
-
   /**
    * @param data- -
    * g = inverted impulse response, when convolved with the impulse
@@ -20,7 +19,7 @@ class PythonServerAPI {
    * @returns
    * @example
    */
-  getImpulseResponse = async ({payload, sampleRate, P}) => {
+  getImpulseResponse = async ({mls, payload, sampleRate, P}) => {
     const task = 'impulse-response';
     let res = null;
 
@@ -30,6 +29,7 @@ class PythonServerAPI {
       task,
       payload,
       'sample-rate': sampleRate,
+      mls,
       P,
     });
 
@@ -48,11 +48,10 @@ class PythonServerAPI {
       .catch(error => {
         throw error;
       });
-
     return res.data[task];
   };
 
-  getInverseImpulseResponse = async ({payload}) => {
+  getInverseImpulseResponse = async ({payload,mls,lowHz,highHz}) => {
     const task = 'inverse-impulse-response';
     let res = null;
 
@@ -61,6 +60,9 @@ class PythonServerAPI {
     const data = JSON.stringify({
       task,
       payload,
+      mls,
+      lowHz,
+      highHz
     });
 
     await axios({

package/src/tasks/audioCalibrator.js

@@ -31,6 +31,9 @@ class AudioCalibrator extends AudioRecorder {
   /** @private */
   sourceAudioContext;
 
+  /** @private */
+  sourceAudioContextConvolved;
+
   /** @protected */
   numCalibratingRounds = 1;
 
@@ -43,6 +46,9 @@ class AudioCalibrator extends AudioRecorder {
   /** @protected */
   calibrationNodes = [];
 
+  /** @protected */
+  calibrationNodesConvolved = [];
+
   /** @protected */
   localAudio;
 
@@ -181,6 +187,7 @@ class AudioCalibrator extends AudioRecorder {
   setSamplingRates = samplingRate => {
     this.sinkSamplingRate = samplingRate;
     this.sourceSamplingRate = samplingRate;
+
     // this.emit('update', {message: `sampling at ${samplingRate}Hz...`});
   };
 
@@ -190,6 +197,11 @@ class AudioCalibrator extends AudioRecorder {
     this.calibrationNodes.push(node);
   };
 
+  addCalibrationNodeConvolved = node => {
+    
+    this.calibrationNodesConvolved.push(node);
+  }
+
   makeNewSourceAudioContext = () => {
     const options = {
       sampleRate: this.sourceSamplingRate,
@@ -202,6 +214,20 @@ class AudioCalibrator extends AudioRecorder {
     return this.sourceAudioContext;
   };
 
+  makeNewSourceAudioContextConvolved = () => {
+    const options = {
+      sampleRate: this.sourceSamplingRate,
+    };
+
+    this.sourceAudioContextConvolved = new (window.AudioContext ||
+      window.webkitAudioContext ||
+      window.audioContext)(options);
+
+    return this.sourceAudioContextConvolved;
+  };
+
+
+
   /** .
    * .
    * .
@@ -212,7 +238,12 @@ class AudioCalibrator extends AudioRecorder {
   downloadData = () => {
     const recordings = this.getAllRecordedSignals();
     const i = recordings.length - 1;
-    saveToCSV(recordings[i], `recordedMLSignal_${i}.csv`);
+    saveToCSV(recordings[i], `recordedMLSignal_${i}_unconvolved.csv`);
+  };
+  downloadConvolvedData = () => {
+    const recordings = this.getAllRecordedSignals();
+    const i = recordings.length - 1;
+    saveToCSV(recordings[i], `recordedMLSignal_${i}_convolved.csv`);
   };
 }
 

package/src/tasks/audioRecorder.js

@@ -69,6 +69,7 @@ class AudioRecorder extends MyEventEmitter {
       window.webkitAudioContext ||
       window.audioContext)({
       sampleRate: this.sinkSamplingRate,
+      //sampleRate: 96000
     });
   };
 

package/src/tasks/impulse-response/impulseResponse.js

@@ -1,7 +1,7 @@
 import AudioCalibrator from '../audioCalibrator';
 import MlsGenInterface from './mlsGen/mlsGenInterface';
 
-import {sleep, csvToArray} from '../../utils';
+import {sleep, csvToArray, saveToCSV} from '../../utils';
 
 /**
  *
@@ -16,11 +16,14 @@ class ImpulseResponse extends AudioCalibrator {
    * @param {number} [calibratorParams.numCaptures = 5] - number of captures to perform
    * @param {number} [calibratorParams.numMLSPerCapture = 4] - number of bursts of MLS per capture
    */
-  constructor({download = false, mlsOrder = 18, numCaptures = 3, numMLSPerCapture = 3}) {
+  constructor({download = false, mlsOrder = 18, numCaptures = 3, numMLSPerCapture = 4, lowHz = 20, highHz = 10000}) {
     super(numCaptures, numMLSPerCapture);
     this.#mlsOrder = parseInt(mlsOrder, 10);
     this.#P = 2 ** mlsOrder - 1;
     this.#download = download;
+    this.#mls = [];
+    this.#lowHz = lowHz;
+    this.#highHz = highHz;
   }
 
   /** @private */
@@ -41,15 +44,30 @@ class ImpulseResponse extends AudioCalibrator {
   /** @private */
   #mlsOrder;
 
+  /** @private */
+  #lowHz;
+
+  /** @private */
+  #highHz;
+
+  /** @private */
+  #mls;
+
   /** @private */
   #P;
 
+  /** @private */
+  #audioContext;
+
   /** @private */
   TAPER_SECS = 5;
 
   /** @private */
   offsetGainNode;
 
+  /** @private */
+  convolution;
+
   /** .
    * .
    * .
@@ -60,14 +78,22 @@ class ImpulseResponse extends AudioCalibrator {
    */
   sendImpulseResponsesToServerForProcessing = async () => {
     const computedIRs = await Promise.all(this.impulseResponses);
+    const mls = this.#mls;
+    const lowHz = this.#lowHz;
+    const highHz = this.#highHz;
     this.emit('update', {message: `computing the IIR...`});
     return this.pyServerAPI
       .getInverseImpulseResponse({
         payload: computedIRs.slice(0, this.numCaptures),
+        mls,
+        lowHz,
+        highHz
       })
       .then(res => {
+        console.log(res);
         this.emit('update', {message: `done computing the IIR...`});
-        this.invertedImpulseResponse = res;
+        this.invertedImpulseResponse = res["iir"];
+        this.convolution = res["convolution"];
       })
       .catch(err => {
         // this.emit('InvertedImpulseResponse', {res: false});
@@ -86,6 +112,7 @@ class ImpulseResponse extends AudioCalibrator {
   sendRecordingToServerForProcessing = signalCsv => {
     const allSignals = this.getAllRecordedSignals();
     const numSignals = allSignals.length;
+    const mls = this.#mls;
     const payload =
       signalCsv && signalCsv.length > 0 ? csvToArray(signalCsv) : allSignals[numSignals - 1];
 
@@ -95,11 +122,13 @@ class ImpulseResponse extends AudioCalibrator {
         .getImpulseResponse({
           sampleRate: this.sourceSamplingRate || 96000,
           payload,
+          mls,
           P: this.#P,
         })
         .then(res => {
           if (this.numSuccessfulCaptured < this.numCaptures) {
             this.numSuccessfulCaptured += 1;
+            console.log("num succ capt: " + this.numSuccessfulCaptured);
             this.emit('update', {
               message: `${this.numSuccessfulCaptured}/${this.numCaptures} IRs computed...`,
             });
@@ -156,9 +185,14 @@ class ImpulseResponse extends AudioCalibrator {
 
   #afterMLSwIIRRecord = () => {
     if (this.#download) {
-      this.downloadData();
+      this.downloadConvolvedData();
+    }
+    if (this.numSuccessfulCaptured < this.numCaptures) {
+      this.numSuccessfulCaptured += 1;
+      this.emit('update', {
+        message: `${this.numSuccessfulCaptured}/${this.numCaptures} IRs computed...`,
+      });
     }
-    this.#stopCalibrationAudio();
   };
 
   /** .
@@ -234,12 +268,13 @@ class ImpulseResponse extends AudioCalibrator {
     // fill the buffer with our data
     try {
       for (let i = 0; i < dataBuffer.length; i += 1) {
-        data[i] = dataBuffer[i];
+        data[i] = dataBuffer[i]*.1;
       }
     } catch (error) {
       console.error(error);
     }
-
+    console.log("mls second, same?");
+    console.log(data);
     const onsetGainNode = audioContext.createGain();
     this.offsetGainNode = audioContext.createGain();
     const source = audioContext.createBufferSource();
@@ -252,7 +287,6 @@ class ImpulseResponse extends AudioCalibrator {
 
     const onsetCurve = ImpulseResponse.createSCurveBuffer(this.sourceSamplingRate, Math.PI / 2);
     onsetGainNode.gain.setValueCurveAtTime(onsetCurve, 0, this.TAPER_SECS);
-
     this.addCalibrationNode(source);
   };
 
@@ -264,70 +298,47 @@ class ImpulseResponse extends AudioCalibrator {
    */
   #setCalibrationNodesFromBuffer = (dataBufferArray = [this.#mlsBufferView]) => {
     if (dataBufferArray.length === 1) {
+      console.log('data buffer aray');
+      console.log(dataBufferArray);
       this.#createCalibrationNodeFromBuffer(dataBufferArray[0]);
     } else {
       throw new Error('The length of the data buffer array must be 1');
     }
-  };
-
-  #createImpulseResponseFilterGraph = (calibrationSignal, iir) => {
-    const audioCtx = this.makeNewSourceAudioContext();
-
-    // -------------------------------------------------------- IIR
-    const iirBuffer = audioCtx.createBuffer(
-      1,
-      // TODO: quality check this
-      iir.length - 1,
-      audioCtx.sampleRate
-    );
 
-    // Fill the buffer with the inverted impulse response
-    const iirChannelZeroBuffer = iirBuffer.getChannelData(0);
-    for (let i = 0; i < iirBuffer.length; i += 1) {
-      // audio needs to be in [-1.0; 1.0]
-      iirChannelZeroBuffer[i] = iir[i];
-    }
-
-    const convolverNode = audioCtx.createConvolver();
-
-    convolverNode.normalize = false;
-    convolverNode.channelCount = 1;
-    convolverNode.buffer = iirBuffer;
+    
+  };
 
-    // ------------------------------------------------------ MLS
-    const calibrationSignalBuffer = audioCtx.createBuffer(
+  /**
+   * function to put MLS filtered IIR data obtained from
+   * python server into our audio buffer to be played aloud
+   */
+  #putInPythonConv = () => {
+    const audioCtx = this.makeNewSourceAudioContextConvolved();
+    const buffer = audioCtx.createBuffer(
       1, // number of channels
-      calibrationSignal.length,
+      this.convolution.length,
       audioCtx.sampleRate // sample rate
     );
 
-    const mlsChannelZeroBuffer = calibrationSignalBuffer.getChannelData(0); // get data
+    const data = buffer.getChannelData(0); // get data
     // fill the buffer with our data
     try {
-      for (let i = 0; i < calibrationSignal.length; i += 1) {
-        mlsChannelZeroBuffer[i] = calibrationSignal[i];
+      for (let i = 0; i < this.convolution.length; i += 1) {
+        data[i] = this.convolution[i];
       }
     } catch (error) {
       console.error(error);
     }
 
-    const sourceNode = audioCtx.createBufferSource();
-
-    sourceNode.buffer = calibrationSignalBuffer;
-    sourceNode.loop = true;
-    sourceNode.connect(convolverNode);
-
-    convolverNode.connect(audioCtx.destination);
-
-    console.log({convolverNode, sourceNode});
-
-    this.addCalibrationNode(sourceNode);
-  };
+    const source = audioCtx.createBufferSource();
 
-  #createIIRwMLSGraph = () => {
-    this.#createImpulseResponseFilterGraph(this.impulseResponses, [this.#mlsBufferView][0]);
-  };
+    source.buffer = buffer;
+    source.loop = true;
+    source.connect(audioCtx.destination);
 
+    this.addCalibrationNodeConvolved(source);
+  }
+  
   /**
    * Creates an audio context and plays it for a few seconds.
    *
@@ -337,8 +348,16 @@ class ImpulseResponse extends AudioCalibrator {
    */
   #playCalibrationAudio = () => {
     this.calibrationNodes[0].start(0);
+    this.#mls = this.calibrationNodes[0].buffer.getChannelData(0);
+    console.log(this.#mls);
     this.emit('update', {message: 'playing the calibration tone...'});
-  };
+  }; 
+
+
+  #playCalibrationAudioConvolved = () => {
+    this.calibrationNodesConvolved[0].start(0);
+    this.emit('update',{message: 'playing the convolved calibration tone...'})
+  }
 
   /** .
    * .
@@ -354,34 +373,53 @@ class ImpulseResponse extends AudioCalibrator {
     );
 
     this.offsetGainNode.gain.setTargetAtTime(0, this.sourceAudioContext.currentTime, 0.5);
+    this.calibrationNodes[0].stop(0);
+    this.sourceAudioContext.close();
     this.emit('update', {message: 'stopping the calibration tone...'});
   };
 
+  #stopCalibrationAudioConvolved = () => {
+    this.offsetGainNode.gain.setValueAtTime(
+      this.offsetGainNode.gain.value,
+      this.sourceAudioContextConvolved.currentTime
+    );
+
+    this.offsetGainNode.gain.setTargetAtTime(0, this.sourceAudioContextConvolved.currentTime, 0.5);
+    //this.calibrationNodesConvolved[0].stop(0);
+    console.log("right before closing volved audio context");
+    this.sourceAudioContextConvolved.close();
+    this.emit('update', {message: 'stopping the convolved calibration tone...'});
+
+  }
+
   playMLSwithIIR = async (stream, iir) => {
+    console.log('play mls with iir');
     this.invertedImpulseResponse = iir;
     // initialize the MLSGenInterface object with it's factory method
+    
     await MlsGenInterface.factory(
-      this.#mlsOrder,
-      this.sinkSamplingRate,
-      this.sourceSamplingRate
+     this.#mlsOrder,
+     this.sinkSamplingRate,
+     this.sourceSamplingRate
     ).then(mlsGenInterface => {
-      this.#mlsGenInterface = mlsGenInterface;
-      this.#mlsBufferView = this.#mlsGenInterface.getMLS();
+     this.#mlsGenInterface = mlsGenInterface;
+     this.#mlsBufferView = this.#mlsGenInterface.getMLS();
     });
 
+    console.log('after mls factory'); //works up to here.
+    console.log(this.#mls);
     // after intializating, start the calibration steps with garbage collection
     await this.#mlsGenInterface.withGarbageCollection([
-      [
-        this.calibrationSteps,
-        [
+      () =>
+        this.calibrationSteps(
           stream,
-          this.#playCalibrationAudio, // play audio func (required)
-          this.#createImpulseResponseFilterGraph, // before play func
-          null, // before record
+          this.#playCalibrationAudioConvolved, // play audio func (required)
+          this.#putInPythonConv, // before play func
+          this.#awaitSignalOnset, // before record
+          () => this.numSuccessfulCaptured < this.numCaptures,
           this.#awaitDesiredMLSLength, // during record
           this.#afterMLSwIIRRecord, // after record
-        ],
-      ],
+        ),
     ]);
   };
 
@@ -426,8 +464,22 @@ class ImpulseResponse extends AudioCalibrator {
     // so let's send all the IRs to the server to be converted to a single IIR
     await this.sendImpulseResponsesToServerForProcessing();
 
+    if (this.#download) {
+      saveToCSV(this.invertedImpulseResponse,'IIR.csv');
+      const computedIRagain = await Promise.all(this.impulseResponses)
+        .then(res => {
+          for (let i = 0; i < res.length; i++){
+            saveToCSV(res[i], `IR_${i}`);
+          }
+        })
+      saveToCSV(this.#mls,"MLS.csv");
+      saveToCSV(this.convolution,'python_convolution_mls_iir.csv');
+    }
+
+    this.numSuccessfulCaptured = 0;
     // debugging function, use to test the result of the IIR
-    // await this.playMLSwithIIR(stream, this.invertedImpulseResponse);
+    await this.playMLSwithIIR(stream, this.invertedImpulseResponse);
+    this.#stopCalibrationAudioConvolved();
 
     return this.invertedImpulseResponse;
   };