speaker-calibration 2.0.0 → 2.1.0

package/src/peer-connection/speaker.js

@@ -0,0 +1,346 @@
+import QRCode from 'qrcode';
+import AudioPeer from './audioPeer';
+import {sleep} from '../utils';
+import {
+  UnsupportedDeviceError,
+  MissingSpeakerIdError,
+  CalibrationTimedOutError,
+} from './peerErrors';
+
+/**
+ * @class Handles the speaker's side of the connection. Responsible for initiating the connection,
+ * rendering the QRCode, and answering the call.
+ * @augments AudioPeer
+ */
+class Speaker extends AudioPeer {
+  /**
+   * Takes the url of the current site and a target element where html elements will be appended.
+   *
+   * @param params - See type definition for initParameters.
+   * @param Calibrator - An instance of the AudioCalibrator class, should not use AudioCalibrator directly, instead use an extended class available in /tasks/.
+   * @param CalibratorInstance
+   * @example
+   */
+  constructor(params, CalibratorInstance) {
+    super(params);
+
+    this.siteUrl += '/listener?';
+    this.ac = CalibratorInstance;
+    this.result = null;
+
+    /* Set up callbacks that handle any events related to our peer object. */
+    this.peer.on('open', this.#onPeerOpen);
+    this.peer.on('connection', this.#onPeerConnection);
+    this.peer.on('close', this.#onPeerClose);
+    this.peer.on('disconnected', this.#onPeerDisconnected);
+    this.peer.on('error', this.#onPeerError);
+  }
+
+  /**
+   * Async factory method that creates the Speaker object, and returns a promise that resolves to the result of the calibration.
+   *
+   * @param params - The parameters to be passed to the peer object.
+   * @param Calibrator - The class that defines the calibration process.
+   * @param CalibratorInstance
+   * @param timeOut - The amount of time to wait before timing out the connection (in milliseconds).
+   * @public
+   * @example
+   */
+  static startCalibration = async (params, CalibratorInstance, timeOut = 180000) => {
+    window.speaker = new Speaker(params, CalibratorInstance);
+    const {speaker} = window;
+
+    // wrap the calibration process in a promise so we can await it
+    return new Promise((resolve, reject) => {
+      // when a call is received
+      speaker.peer.on('call', async call => {
+        // Answer the call (one way)
+        call.answer();
+        speaker.#removeUIElems();
+        speaker.#showSpinner();
+        speaker.ac.createLocalAudio(document.getElementById(speaker.targetElement));
+        // when we start receiving audio
+        call.on('stream', async stream => {
+          window.localStream = stream;
+          window.localAudio.srcObject = stream;
+          window.localAudio.autoplay = false;
+
+          // if the sinkSamplingRate is not set sleep
+          while (!speaker.ac.sampleRatesSet()) {
+            console.log('SinkSamplingRate is undefined, sleeping');
+            await sleep(1);
+          }
+          // resolve when we have a result
+          speaker.result = await speaker.ac.startCalibration(stream, params.gainValues);
+          speaker.#removeUIElems();
+          resolve(speaker.result);
+        });
+        // if we do not receive a result within the timeout, reject
+        setTimeout(() => {
+          reject(
+            new CalibrationTimedOutError(
+              `Calibration failed to produce a result after ${
+                timeOut / 1000
+              } seconds. Please try again.`
+            )
+          );
+        }, timeOut);
+      });
+    });
+  };
+
+  static testIIR = async (params, CalibratorInstance, IIR, timeOut = 180000) => {
+    window.speaker = new Speaker(params, CalibratorInstance);
+    const {speaker} = window;
+
+    // wrap the calibration process in a promise so we can await it
+    return new Promise((resolve, reject) => {
+      // when a call is received
+      speaker.peer.on('call', async call => {
+        // Answer the call (one way)
+        call.answer();
+        speaker.#removeUIElems();
+        speaker.#showSpinner();
+        speaker.ac.createLocalAudio(document.getElementById(speaker.targetElement));
+        // when we start receiving audio
+        call.on('stream', async stream => {
+          window.localStream = stream;
+          window.localAudio.srcObject = stream;
+          window.localAudio.autoplay = false;
+
+          // if the sinkSamplingRate is not set sleep
+          while (!speaker.ac.sampleRatesSet()) {
+            console.log('SinkSamplingRate is undefined, sleeping');
+            await sleep(1);
+          }
+          // resolve when we have a result
+          speaker.result = await speaker.ac.playMLSwithIIR(stream, IIR);
+          speaker.#removeUIElems();
+          resolve(speaker.result);
+        });
+        // if we do not receive a result within the timeout, reject
+        setTimeout(() => {
+          reject(
+            new CalibrationTimedOutError(
+              `Calibration failed to produce a result after ${
+                timeOut / 1000
+              } seconds. Please try again.`
+            )
+          );
+        }, timeOut);
+      });
+    });
+  };
+
+  /**
+   * Called after the peer conncection has been opened.
+   * Generates a QR code for the connection and displays it.
+   *
+   * @private
+   * @example
+   */
+  #showQRCode = () => {
+    // Get query string, the URL parameters to specify a Listener
+    const queryStringParameters = {
+      speakerPeerId: this.peer.id,
+    };
+    const queryString = this.queryStringFromObject(queryStringParameters);
+    const uri = this.siteUrl + queryString;
+
+    // Display QR code for the participant to scan
+    const qrCanvas = document.createElement('canvas');
+    qrCanvas.setAttribute('id', 'qrCanvas');
+    console.log(uri);
+    QRCode.toCanvas(qrCanvas, uri, error => {
+      if (error) console.error(error);
+    });
+
+    // If specified HTML Id is available, show QR code there
+    if (document.getElementById(this.targetElement)) {
+      if (document.getElementById(this.targetElement)) {
+        if (process.env.NODE_ENV === 'development') {
+          const linkTag = document.createElement('a');
+          linkTag.setAttribute('href', uri);
+          linkTag.innerHTML = "Click here to connect to the speaker's microphone";
+          linkTag.target = '_blank';
+          document.getElementById(this.targetElement).appendChild(linkTag);
+        }
+      }
+      document.getElementById(this.targetElement).appendChild(qrCanvas);
+    } else {
+      // or just print it to console
+      console.log('TEST: Peer reachable at: ', uri);
+    }
+  };
+
+  #showSpinner = () => {
+    const spinner = document.createElement('div');
+    spinner.className = 'spinner-border ml-auto';
+    spinner.role = 'status';
+    spinner.ariaHidden = 'true';
+    document.getElementById(this.targetElement).appendChild(spinner);
+  };
+
+  #removeUIElems = () => {
+    const parent = document.getElementById(this.targetElement);
+    while (parent.firstChild) {
+      parent.firstChild.remove();
+    }
+  };
+
+  /**
+   * Called when the peer connection is opened.
+   * Saves the peer id and calls the QR code generator.
+   *
+   * @param peerId - The peer id of the peer connection.
+   * @param id
+   * @private
+   * @example
+   */
+  #onPeerOpen = id => {
+    // Workaround for peer.reconnect deleting previous id
+    if (id === null) {
+      console.error('Received null id from peer open');
+      this.peer.id = this.lastPeerId;
+    } else {
+      this.lastPeerId = this.peer.id;
+    }
+
+    if (id !== this.peer.id) {
+      console.warn('DEBUG Check you assumption that id === this.peer.id');
+    }
+
+    this.#showQRCode();
+  };
+
+  /**
+   * Called when the peer connection is established.
+   * Enforces a single connection.
+   *
+   * @param connection - The connection object.
+   * @private
+   * @example
+   */
+  #onPeerConnection = connection => {
+    // Allow only a single connection
+    if (this.conn && this.conn.open) {
+      connection.on('open', () => {
+        connection.send('Already connected to another client');
+        setTimeout(() => {
+          connection.close();
+        }, 500);
+      });
+      return;
+    }
+
+    this.conn = connection;
+    console.log('Connected to: ', this.conn.peer);
+    this.#ready();
+  };
+
+  /**
+   * Called when the peer connection is closed.
+   *
+   * @private
+   * @example
+   */
+  #onPeerClose = () => {
+    this.conn = null;
+    console.log('Connection destroyed');
+  };
+
+  /**
+   * Called when the peer connection is disconnected.
+   * Attempts to reconnect.
+   *
+   * @private
+   * @example
+   */
+  #onPeerDisconnected = () => {
+    console.log('Connection lost. Please reconnect');
+
+    // Workaround for peer.reconnect deleting previous id
+    this.peer.id = this.lastPeerId;
+    // eslint-disable-next-line no-underscore-dangle
+    this.peer._lastServerId = this.lastPeerId;
+    this.peer.reconnect();
+  };
+
+  /**
+   * Called when the peer connection encounters an error.
+   *
+   * @param error
+   * @private
+   * @example
+   */
+  #onPeerError = error => {
+    // TODO: check if this function is needed or not
+    console.error(error);
+  };
+
+  /**
+   * Called when data is received from the peer connection.
+   *
+   * @param data
+   * @private
+   * @example
+   */
+  #onIncomingData = data => {
+    // enforce object type
+    if (
+      !Object.prototype.hasOwnProperty.call(data, 'name') ||
+      !Object.prototype.hasOwnProperty.call(data, 'payload')
+    ) {
+      console.error('Received malformed data: ', data);
+      return;
+    }
+
+    switch (data.name) {
+      case 'samplingRate':
+        this.ac.setSamplingRates(data.payload);
+        break;
+      case UnsupportedDeviceError.name:
+      case MissingSpeakerIdError.name:
+        throw data.payload;
+        break;
+      default:
+        break;
+    }
+  };
+
+  /**
+   * Called when the peer connection is #ready.
+   *
+   * @private
+   * @example
+   */
+  #ready = () => {
+    // Perform callback with data
+    this.conn.on('data', this.#onIncomingData);
+    this.conn.on('close', () => {
+      console.log('Connection reset<br>Awaiting connection...');
+      this.conn = null;
+    });
+  };
+
+  /** .
+   * .
+   * .
+   * Debug method for downloading the recorded audio
+   *
+   * @public
+   * @example
+   */
+  downloadData = () => {
+    this.ac.downloadData();
+  };
+}
+
+/* 
+Referenced links:
+https://stackoverflow.com/questions/28016664/when-you-pass-this-as-an-argument/28016676#28016676
+https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes
+https://stackoverflow.com/questions/879152/how-do-i-make-javascript-beep [3]
+*/
+
+export default Speaker;

package/src/server/PythonServerAPI.js

@@ -0,0 +1,117 @@
+import axios from 'axios';
+/**
+ *
+ */
+class PythonServerAPI {
+  static PYTHON_SERVER_URL = 'https://easyeyes-python-flask-server.herokuapp.com';
+
+  static TEST_SERVER_URL = 'http://127.0.0.1:5000';
+
+  /**
+   * @param data- -
+   * g = inverted impulse response, when convolved with the impulse
+   * reponse, they cancel out.
+   * @param data.payload
+   * @param data.sampleRate
+   * @param data.P
+   * @param data-.payload
+   * @param data-.sampleRate
+   * @param data-.P
+   * @returns
+   * @example
+   */
+  getImpulseResponse = async ({payload, sampleRate, P}) => {
+    const task = 'impulse-response';
+    let res = null;
+
+    console.log({payload});
+
+    const data = JSON.stringify({
+      task,
+      payload,
+      'sample-rate': sampleRate,
+      P,
+    });
+
+    await axios({
+      method: 'post',
+      baseURL: PythonServerAPI.PYTHON_SERVER_URL,
+      url: `/task/${task}`,
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      data,
+    })
+      .then(response => {
+        res = response;
+      })
+      .catch(error => {
+        throw error;
+      });
+
+    return res.data[task];
+  };
+
+  getInverseImpulseResponse = async ({payload}) => {
+    const task = 'inverse-impulse-response';
+    let res = null;
+
+    console.log({payload});
+
+    const data = JSON.stringify({
+      task,
+      payload,
+    });
+
+    await axios({
+      method: 'post',
+      baseURL: PythonServerAPI.PYTHON_SERVER_URL,
+      url: `/task/${task}`,
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      data,
+    })
+      .then(response => {
+        res = response;
+      })
+      .catch(error => {
+        throw error;
+      });
+
+    return res.data[task];
+  };
+
+  getVolumeCalibration = async ({payload, sampleRate}) => {
+    const task = 'volume';
+    let res = null;
+
+    console.log({payload});
+
+    const data = JSON.stringify({
+      task,
+      payload,
+      'sample-rate': sampleRate,
+    });
+
+    await axios({
+      method: 'post',
+      baseURL: PythonServerAPI.PYTHON_SERVER_URL,
+      url: `/task/${task}`,
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      data,
+    })
+      .then(response => {
+        res = response;
+      })
+      .catch(error => {
+        throw error;
+      });
+
+    return res.data[task];
+  };
+}
+
+export default PythonServerAPI;

package/src/tasks/audioCalibrator.js

@@ -0,0 +1,218 @@
+/* eslint-disable no-await-in-loop */
+import AudioRecorder from './audioRecorder';
+import PythonServerAPI from '../server/PythonServerAPI';
+import {sleep, saveToCSV} from '../utils';
+
+/**
+ * .
+ * .
+ * .
+ * Provides methods for calibrating the user's speakers
+ *
+ * @extends AudioRecorder
+ */
+class AudioCalibrator extends AudioRecorder {
+  /**
+   *
+   * @param numCaptures
+   * @param numMLSPerCapture
+   * @example
+   */
+  constructor(numCaptures = 1, numMLSPerCapture = 1) {
+    super();
+    this.numCaptures = numCaptures;
+    this.numMLSPerCapture = numMLSPerCapture;
+    this.pyServerAPI = new PythonServerAPI();
+  }
+
+  /** @private */
+  #isCalibrating = false;
+
+  /** @private */
+  sourceAudioContext;
+
+  /** @protected */
+  numCalibratingRounds = 1;
+
+  /** @protected */
+  numSuccessfulCaptured = 0;
+
+  /** @private */
+  sourceSamplingRate;
+
+  /** @protected */
+  calibrationNodes = [];
+
+  /** @protected */
+  localAudio;
+
+  /**
+   * Called when a call is received.
+   * Creates a local audio DOM element and attaches it to the page.
+   *
+   * @param targetElement
+   * @example
+   */
+  createLocalAudio = targetElement => {
+    this.localAudio = document.createElement('audio');
+    this.localAudio.setAttribute('id', 'localAudio');
+    targetElement.appendChild(this.localAudio);
+  };
+
+  /**
+   *
+   * @param {MediaStream} stream
+   * @param {Function} playCalibrationAudio - (async) function that plays the calibration audio
+   * @param {*} beforePlay - (async) function that is called before playing the audio
+   * @param {*} beforeRecord - (async) function that is called before recording
+   * @param {*} duringRecord - (async) function that is called while recording
+   * @param {*} afterRecord  - (async) function that is called after recording
+   * @example
+   */
+  calibrationSteps = async (
+    stream,
+    playCalibrationAudio,
+    beforePlay = async () => {},
+    beforeRecord = async () => {},
+    loopCondition = () => false,
+    duringRecord = async () => {},
+    afterRecord = async () => {}
+  ) => {
+    this.numSuccessfulCaptured = 0;
+
+    // do something before playing such as using the MLS to fill the buffers
+    console.warn('beforePlay');
+    await beforePlay();
+
+    // play calibration audio
+    console.warn('playCalibrationAudio');
+    playCalibrationAudio();
+
+    // do something before recording such as awaiting a certain amount of time
+    console.warn('beforeRecord');
+    await beforeRecord();
+
+    // calibration loop
+    while (loopCondition()) {
+      // start recording
+      console.warn('startRecording');
+      await this.startRecording(stream);
+
+      // do something during the recording such as sleep n amount of time
+      console.warn('duringRecord');
+      await duringRecord();
+
+      // when done, stop recording
+      console.warn('stopRecording');
+      await this.stopRecording();
+
+      // do something after recording such as start processing values
+      console.warn('afterRecord');
+      await afterRecord();
+
+      // eslint-disable-next-line no-await-in-loop
+      await sleep(1);
+    }
+  };
+
+  /**
+   *
+   * @param {MediaStream} stream
+   * @param {Function} playCalibrationAudio - (async) function that plays the calibration audio
+   * @param {*} beforeRecord - (async) function that is called before recording
+   * @param {*} afterRecord  - (async) function that is called after recording
+   * @param {Number} gainValue - the gain value to set the gain node to
+   */
+  volumeCalibrationSteps = async (
+    stream,
+    playCalibrationAudio,
+    beforeRecord = () => {},
+    afterRecord = () => {},
+    gainValue
+  ) => {
+    this.numCalibratingRoundsCompleted = 0;
+
+    // calibration loop
+    while (!this.#isCalibrating && this.numCalibratingRoundsCompleted < this.numCalibratingRounds) {
+      // before recording
+      await beforeRecord(gainValue);
+
+      // start recording
+      await this.startRecording(stream);
+
+      // play calibration audio
+      console.log(`Calibration Round ${this.numCalibratingRoundsCompleted}`);
+      await playCalibrationAudio();
+
+      // when done, stop recording
+      console.log('Calibration Round Complete');
+      await this.stopRecording();
+
+      // after recording
+      await afterRecord();
+
+      this.calibrationNodes = [];
+
+      // eslint-disable-next-line no-await-in-loop
+      await sleep(2);
+      this.numCalibratingRoundsCompleted += 1;
+    }
+  };
+
+  /**
+   * Getter for the isCalibrating property.
+   *
+   * @public
+   * @returns - True if the audio is being calibrated, false otherwise.
+   * @example
+   */
+  getCalibrationStatus = () => this.#isCalibrating;
+
+  /** .
+   * .
+   * .
+   * Set the sampling rate to the value received from the listener
+   *
+   * @param {*} sinkSamplingRate
+   * @param samplingRate
+   * @example
+   */
+  setSamplingRates = samplingRate => {
+    this.sinkSamplingRate = samplingRate;
+    this.sourceSamplingRate = samplingRate;
+    this.emit('update', {message: `sampling at ${samplingRate}Hz...`});
+  };
+
+  sampleRatesSet = () => this.sourceSamplingRate && this.sinkSamplingRate;
+
+  addCalibrationNode = node => {
+    this.calibrationNodes.push(node);
+  };
+
+  makeNewSourceAudioContext = () => {
+    const options = {
+      sampleRate: this.sourceSamplingRate,
+    };
+
+    this.sourceAudioContext = new (window.AudioContext ||
+      window.webkitAudioContext ||
+      window.audioContext)(options);
+
+    return this.sourceAudioContext;
+  };
+
+  /** .
+   * .
+   * .
+   * Download the result of the calibration roudns
+   *
+   * @example
+   */
+  downloadData = () => {
+    const recordings = this.getAllRecordedSignals();
+    const i = recordings.length - 1;
+    saveToCSV(recordings[i], `recordedMLSignal_${i}.csv`);
+  };
+}
+
+export default AudioCalibrator;