remote-calibrator 0.3.0-rc.3 → 0.5.0-beta.10

package/src/WebGazer4RC/package.json

@@ -20,8 +20,8 @@
   ],
   "dependencies": {
     "@tensorflow-models/facemesh": "^0.0.5",
-    "@tensorflow/tfjs": "^3.8.0",
-    "localforage": "^1.9.0",
+    "@tensorflow/tfjs": "^3.11.0",
+    "localforage": "^1.10.0",
     "numeric": "1.2.6",
     "regression": "2.0.1"
   }

package/src/WebGazer4RC/src/index.mjs

@@ -53,14 +53,21 @@ webgazer.params.paused = false;
 
 webgazer.params.greedyLearner = false;
 webgazer.params.framerate = 60;
-webgazer.params.showGazeDot = false
+webgazer.params.showGazeDot = false;
+
+webgazer.params.getLatestVideoFrameTimestamp = () => {};
+webgazer.params.activeCamera = {
+  label: '',
+  id: '',
+};
 /* -------------------------------------------------------------------------- */
+
+let videoInputs = []
+
 // registered callback for loop
 var nopCallback = function(data) {};
 var callback = nopCallback;
 
-let learning = false // Regression
-
 //Types that regression systems should handle
 //Describes the source of data so that regression systems may ignore or handle differently the various generating events
 var eventTypes = ['click', 'move'];
@@ -280,6 +287,31 @@ var k = 0;
 let _now = null
 let _last = -1
 
+// From getting the video frame to get data
+let _oneLoopFinished = true
+
+async function gazePrep(forcedPrep = false) {
+  paintCurrentFrame(videoElementCanvas, videoElementCanvas.width, videoElementCanvas.height);
+
+  // [20200617 xk] TODO: this call should be made async somehow. will take some work.
+  if (!webgazer.params.paused || forcedPrep) {
+    latestEyeFeatures = await getPupilFeatures(videoElementCanvas, videoElementCanvas.width, videoElementCanvas.height);
+    // console.log(videoElementCanvas, videoElementCanvas.width, videoElementCanvas.height);
+  }
+
+  // Draw face overlay
+  if (webgazer.params.showFaceOverlay) {
+    // Get tracker object
+    var tracker = webgazer.getTracker();
+    faceOverlay.getContext('2d').clearRect(0, 0, videoElement.videoWidth, videoElement.videoHeight);
+    tracker.drawFaceOverlay(faceOverlay.getContext('2d'), tracker.getPositions());
+  }
+
+  // Feedback box
+  // Check that the eyes are inside of the validation box
+  if (webgazer.params.showFaceFeedbackBox) checkEyesInValidationBox();
+}
+
 async function loop() {
   _now = window.performance.now()
 
@@ -291,22 +323,11 @@ async function loop() {
     // Paint the latest video frame into the canvas which will be analyzed by WebGazer
     // [20180729 JT] Why do we need to do this? clmTracker does this itself _already_, which is just duplicating the work.
     // Is it because other trackers need a canvas instead of an img/video element?
-    paintCurrentFrame(videoElementCanvas, videoElementCanvas.width, videoElementCanvas.height);
-
-    // [20200617 xk] TODO: this call should be made async somehow. will take some work.
-    latestEyeFeatures = await getPupilFeatures(videoElementCanvas, videoElementCanvas.width, videoElementCanvas.height);
-
-    // Draw face overlay
-    if (webgazer.params.showFaceOverlay) {
-      // Get tracker object
-      var tracker = webgazer.getTracker();
-      faceOverlay.getContext('2d').clearRect(0, 0, videoElement.videoWidth, videoElement.videoHeight);
-      tracker.drawFaceOverlay(faceOverlay.getContext('2d'), tracker.getPositions());
+    if (_oneLoopFinished) {
+      _oneLoopFinished = false
+      webgazer.params.getLatestVideoFrameTimestamp(new Date())
     }
-
-    // Feedback box
-    // Check that the eyes are inside of the validation box
-    if (webgazer.params.showFaceFeedbackBox) checkEyesInValidationBox();
+    await gazePrep()
   }
 
   if (!webgazer.params.paused) {
@@ -323,8 +344,9 @@ async function loop() {
 
       // [20200623 xk] callback to function passed into setGazeListener(fn)
       callback(latestGazeData);
+      _oneLoopFinished = true
 
-      if( latestGazeData ) {
+      if (latestGazeData) {
         // [20200608 XK] Smoothing across the most recent 4 predictions, do we need this with Kalman filter?
         smoothingVals.push(latestGazeData);
         var x = 0;
@@ -350,7 +372,9 @@ async function loop() {
 
     }
   } else {
-    gazeDot.style.transform = `translate(-15px, -15px)` // Move out of the display
+    try {
+      gazeDot.style.transform = `translate(-15px, -15px)` // Move out of the display
+    } catch (error) {}
   }
 
   requestAnimationFrame(loop);
@@ -503,7 +527,8 @@ async function init(initMode = 'all', stream) {
     videoContainerElement = document.createElement('div');
     videoContainerElement.id = webgazer.params.videoContainerId;
     videoContainerElement.style.display = 'block';
-    videoContainerElement.style.visibility = webgazer.params.showVideo ? 'visible' : 'hidden';
+    // videoContainerElement.style.visibility = webgazer.params.showVideo ? 'visible' : 'hidden';
+    videoContainerElement.style.opacity = webgazer.params.showVideo ? 0.8 : 0;
     // videoContainerElement.style.position = 'fixed';
     videoContainerElement.style.left = '10px';
     videoContainerElement.style.bottom = '10px';
@@ -516,7 +541,7 @@ async function init(initMode = 'all', stream) {
     videoElement.srcObject = stream;
     videoElement.autoplay = true;
     videoElement.style.display = 'block';
-    videoElement.style.visibility = webgazer.params.showVideo ? 'visible' : 'hidden';
+    // videoElement.style.visibility = webgazer.params.showVideo ? 'visible' : 'hidden';
     videoElement.style.position = 'absolute';
     // We set these to stop the video appearing too large when it is added for the very first time
     videoElement.style.width = webgazer.params.videoViewerWidth + 'px';
@@ -526,7 +551,8 @@ async function init(initMode = 'all', stream) {
     videoElementCanvas = document.createElement('canvas');
     videoElementCanvas.id = webgazer.params.videoElementCanvasId;
     videoElementCanvas.style.display = 'block';
-    videoElementCanvas.style.visibility = 'hidden';
+    videoElementCanvas.style.opacity = 0;
+    // videoElementCanvas.style.visibility = 'hidden';
 
     // Face overlay
     // Shows the CLM tracking result
@@ -658,24 +684,72 @@ webgazer.begin = function(onFail) {
   //   loadGlobalData();
   // }
 
-  onFail = onFail || function() {console.log('No stream')};
+  // onFail = onFail || function() {console.log('No stream')};
 
   // if (debugVideoLoc) {
   //   init(debugVideoLoc);
   //   return webgazer;
   // }
 
-  return webgazer._begin(false)
+  return webgazer._begin(false, onFail)
 };
 
 /**
  * Start the video element.
  */
-webgazer.beginVideo = function () {
-  webgazer._begin(true)
+webgazer.beginVideo = function (onFail) {
+  webgazer._begin(true, onFail)
+}
+
+/* ------------------------------ Video switch ------------------------------ */
+
+const _foldString = (str) => {
+  if (str.length < 8) return str
+  else return str.slice(0, 8) + '...'
+}
+
+const _setUpActiveCameraSwitch = (inputs) => {
+  const parent = videoContainerElement;
+  
+  const selectElement = document.createElement('select')
+  selectElement.className = selectElement.id = 'webgazer-videoinput-select'
+  selectElement.name = 'videoinput'
+  inputs.forEach((input, ind) => {
+    selectElement.innerHTML += `<option value="${input.deviceId + '%' + input.label}"${ind === 0 ? ' selected' : ''} name="1">${_foldString(input.label)}</option>`;
+  })
+
+  selectElement.onchange = e => {
+    const [id, label] = selectElement.value.split('%')
+    webgazer.params.activeCamera.label = label
+    webgazer.params.activeCamera.id = id
+
+    webgazer.setCameraConstraints(_setUpConstraints(webgazer.params.camConstraints));
+  }
+  
+  parent.appendChild(selectElement);
+}
+
+const _gotSources = (sources) => {
+  videoInputs = []
+  sources.forEach(device => {
+    if (device.kind === 'videoinput') videoInputs.push(device)
+  });
+
+  webgazer.params.activeCamera.label = videoInputs[0].label
+  webgazer.params.activeCamera.id = videoInputs[0].deviceId
 }
 
-webgazer._begin = function (videoOnly) {
+const _setUpConstraints = (originalConstraints) => {
+  if (!webgazer.params.activeCamera.id) return originalConstraints
+  return {
+    video: {
+      ...originalConstraints.video,
+      deviceId: webgazer.params.activeCamera.id,
+    }
+  }
+}
+
+webgazer._begin = function (videoOnly, onVideoFail) {
   // SETUP VIDEO ELEMENTS
   // Sets .mediaDevices.getUserMedia depending on browser
   if (!webgazer.params.videoIsOn) {
@@ -684,15 +758,31 @@ webgazer._begin = function (videoOnly) {
     return new Promise(async (resolve, reject) => {
       let stream;
       try {
-        stream = await navigator.mediaDevices.getUserMedia( webgazer.params.camConstraints );
-        init(videoOnly ? 'video' : 'all', stream);
-        //
-        webgazer.params.videoIsOn = true
-        //
-        if (!videoOnly) resolve(webgazer);
+        if (typeof navigator.mediaDevices !== 'undefined')
+          navigator.mediaDevices.enumerateDevices().then(async (sources) => {
+            _gotSources(sources);
+            if (videoInputs.length === 0) {
+              onVideoFail(videoInputs);
+              throw 'No camera';
+            }
+
+            try {
+              stream = await navigator.mediaDevices.getUserMedia( _setUpConstraints(webgazer.params.camConstraints) );
+            } catch (error) {
+              onVideoFail(videoInputs);
+              throw error;
+            }
+
+            init(videoOnly ? 'video' : 'all', stream).then(() => {
+              if (videoInputs.length > 1) _setUpActiveCameraSwitch(videoInputs)
+            });
+            ////
+            webgazer.params.videoIsOn = true
+            ////
+            if (!videoOnly) resolve(webgazer);
+          });
       } catch(err) {
         console.log(err);
-        onFail();
         videoElement = null;
         stream = null;
         reject(err);
@@ -749,6 +839,7 @@ webgazer.resume = async function() {
     return webgazer;
   }
   webgazer.params.paused = false;
+  _oneLoopFinished = true
   await loop();
   return webgazer;
 };
@@ -834,13 +925,15 @@ webgazer.showVideoPreview = function(val) {
  * @param {*} bool
  * @return {webgazer} this
  */
-webgazer.showVideo = function(val) {
+webgazer.showVideo = function(val, opacity = 0.8) {
   webgazer.params.showVideo = val;
-  if(videoElement) {
-    videoElement.style.visibility = val ? 'visible' : 'hidden';
-  }
-  if(videoContainerElement) {
-    videoContainerElement.style.visibility = val ? 'visible' : 'hidden';
+  // if (videoElement) {
+  //   videoElement.style.visibility = val ? 'visible' : 'hidden';
+  //   videoElement.style.opacity = val ? 1 : 0;
+  // }
+  if (videoContainerElement) {
+    // videoContainerElement.style.visibility = val ? 'visible' : 'hidden';
+    videoContainerElement.style.opacity = val ? opacity : 0;
   }
   return webgazer;
 };
@@ -886,7 +979,7 @@ webgazer.showPredictionPoints = function(val) {
 };
 
 /**
- * Set whether localprevious calibration data (from localforage) should be loaded.
+ * Set whether previous calibration data (from localforage) should be loaded.
  * Default true.
  * 
  * NOTE: Should be called before webgazer.begin() -- see www/js/main.js for example
@@ -922,26 +1015,36 @@ webgazer.applyKalmanFilter = function(val) {
  * Warning: Setting a large video resolution will decrease performance, and may require
  */
 webgazer.setCameraConstraints = async function(constraints) {
-  var videoTrack,videoSettings;
+  // var videoTrack, videoSettings;
   webgazer.params.camConstraints = constraints;
 
   // If the camera stream is already up...
   if(videoStream)
   {
     webgazer.pause();
-    videoTrack = videoStream.getVideoTracks()[0];
+    // videoTrack = videoStream.getVideoTracks()[0];
     try {
-      await videoTrack.applyConstraints( webgazer.params.camConstraints );
-      videoSettings = videoTrack.getSettings();
-      setInternalVideoBufferSizes( videoSettings.width, videoSettings.height );
+      // await videoTrack.applyConstraints( webgazer.params.camConstraints );
+      videoStream.getVideoTracks().forEach(track => {
+        track.stop();
+      })
+      const stream = await navigator.mediaDevices.getUserMedia( webgazer.params.camConstraints );
+      setTimeout(() => {
+        const videoTrack = stream.getVideoTracks()[0];
+        const videoSettings = videoTrack.getSettings();
+        videoStream = stream;
+        videoElement.srcObject = stream;
+        setInternalVideoBufferSizes( videoSettings.width, videoSettings.height );
+        console.log('New constraints applied');
+      }, 1500);
     } catch(err) {
-      console.log( err );
+      console.log(err);
       return;
     }
     // Reset and recompute sizes of the video viewer.
     // This is only to adjust the feedback box, say, if the aspect ratio of the video has changed.
-    webgazer.setVideoViewerSize( webgazer.params.videoViewerWidth, webgazer.params.videoViewerHeight )
-    webgazer.getTracker().reset();
+    // webgazer.setVideoViewerSize( webgazer.params.videoViewerWidth, webgazer.params.videoViewerHeight )
+    // webgazer.getTracker().reset();
     await webgazer.resume();
   }
 }
@@ -990,6 +1093,10 @@ webgazer.setVideoViewerSize = function(w, h) {
   videoElement.style.width = w + 'px';
   videoElement.style.height = h + 'px';
 
+  // Change video container
+  videoContainerElement.style.width = w + 'px';
+  videoContainerElement.style.height = h + 'px';
+
   // Change the face overlay
   faceOverlay.style.width = w + 'px';
   faceOverlay.style.height = h + 'px';
@@ -1176,7 +1283,9 @@ webgazer.getRegression = function() {
  * Requests an immediate prediction
  * @return {object} prediction data object
  */
-webgazer.getCurrentPrediction = function(regIndex) {
+webgazer.getCurrentPrediction = async function(regIndex) {
+  webgazer.params.getLatestVideoFrameTimestamp(new Date())
+  await gazePrep(true)
   return getPrediction(regIndex);
 };
 
@@ -1204,7 +1313,7 @@ webgazer.getVideoPreviewToCameraResolutionRatio = function() {
 }
 
 /*
- * Gets the fifty most recent tracker preditions
+ * Gets the fifty most recent tracker predictions
  */
 webgazer.getStoredPoints = function() {
   return [xPast50, yPast50];

package/src/WebGazer4RC/test/webgazer_test.js

@@ -51,7 +51,7 @@ describe('webgazer function', async() => {
 			assert.equal(videoAvailable,true);
 			assert.equal(isReady,true);
 		});
-		//modifying visibility params
+		// modifying visibility params
 		it('webgazerVideoFeed should display', async() => {
 			let video_display = await page.evaluate(async() => {
 				return document.getElementById('webgazerVideoFeed').style.display

package/src/check/checkScreenSize.js

@@ -0,0 +1,84 @@
+import Swal from 'sweetalert2'
+
+import RemoteCalibrator from '../core'
+import { swalInfoOptions } from '../components/swalOptions'
+import { constructInstructions, safeExecuteFunc } from '../components/utils'
+import { takeInput } from '../components/input'
+
+RemoteCalibrator.prototype._checkScreenSize = async function (
+  screenSizeCallback,
+  screenSizeData
+) {
+  const RC = this
+  this._addBackground()
+
+  if (this.equipment) {
+    // Asked
+    checkScreenSize(this, screenSizeCallback, screenSizeData)
+    ////
+  } else {
+    const { CM, IN_D, IN_F } = RC._CONST.UNITS
+    const haveEquipmentOptions = {}
+    haveEquipmentOptions[CM] = 'cm'
+    haveEquipmentOptions[IN_D] = 'in (Decimal, e.g. 11.5 in)'
+    haveEquipmentOptions[IN_F] = 'in (Fractional, e.g. 12 3/16 in)'
+
+    const { value: result } = await Swal.fire({
+      ...swalInfoOptions(RC, {
+        showIcon: false,
+      }),
+      title: 'Check the calibration',
+      input: 'select',
+      inputOptions: {
+        'I have an appropriate measuring device in units': haveEquipmentOptions,
+        "I don't have an appropriate measuring device": {
+          none: 'No device',
+        },
+      },
+      inputPlaceholder: 'Select an option',
+      // showCancelButton: true,
+      inputValidator: value => {
+        return new Promise(resolve => {
+          const hasEquipment = value !== 'none'
+
+          RC.newEquipmentData = {
+            value: {
+              has: hasEquipment,
+              unit: hasEquipment ? value : null,
+              equipment: hasEquipment ? '' : null,
+            },
+            timestamp: new Date(),
+          }
+
+          resolve()
+        })
+      },
+    })
+
+    if (result) checkScreenSize(this, screenSizeCallback, screenSizeData)
+  }
+}
+
+const checkScreenSize = async (RC, screenSizeCallback, screenSizeData) => {
+  const quit = () => {
+    RC._removeBackground()
+    safeExecuteFunc(screenSizeCallback, screenSizeData)
+  }
+
+  if (RC.equipment && RC.equipment.value.has) {
+    // ! Has equipment
+    RC._replaceBackground(
+      constructInstructions(
+        '📏 ' + 'Check Screen Size Measure',
+        'Measure the length of the arrow, and put your answer in the text box. You only need to put the numbers. When finished, press OK.'
+      )
+    )
+
+    const input = await takeInput(RC)
+
+    if (input) {
+      window.console.log(input)
+    }
+  }
+  quit()
+}

package/src/components/buttons.js

@@ -1,12 +1,19 @@
 import { phrases } from '../i18n'
 import '../css/buttons.scss'
 
-export const addButtons = (RCL, parent, { go, cancel }, showCancelButton) => {
+export const addButtons = (
+  RCL,
+  parent,
+  { go, cancel, custom },
+  showCancelButton,
+  absolutePositioning = true
+) => {
   const buttons = document.createElement('div')
-  buttons.className = 'rc-buttons'
+  buttons.className =
+    'rc-buttons' + (absolutePositioning ? ' rc-absolute-buttons' : '')
   buttons.id = 'rc-buttons'
 
-  let goButton, cancelButton
+  let goButton, cancelButton, customButton
 
   if (go) {
     goButton = document.createElement('button')
@@ -24,9 +31,19 @@ export const addButtons = (RCL, parent, { go, cancel }, showCancelButton) => {
     buttons.appendChild(cancelButton)
   }
 
+  if (custom) {
+    const { callback, content } = custom
+
+    customButton = document.createElement('button')
+    customButton.className = 'rc-button rc-custom-button'
+    customButton.onclick = callback
+    customButton.innerHTML = content
+    buttons.appendChild(customButton)
+  }
+
   parent.appendChild(buttons)
 
-  return [buttons, goButton, cancelButton]
+  return [buttons, goButton, cancelButton, customButton]
 }
 
 export const removeButtons = parent => {

package/src/components/input.js

@@ -0,0 +1,82 @@
+import { addButtons } from './buttons'
+
+import Arrow from '../media/two-sided-horizontal.svg'
+
+const _exampleMeasure = {
+  cm: '22.5',
+  inDecimal: '11.5',
+  inFractional: ['12', '3/16'],
+}
+
+export const takeInput = async RC => {
+  const unit = RC.equipment.value.unit
+  const unitDisplay = unit === RC._CONST.UNITS.CM ? 'cm' : 'in'
+
+  const formItem = `<div class="rc-form">
+  <div class="rc-form-inputs">
+  ${
+    unit === RC._CONST.UNITS.IN_F
+      ? ``
+      : `<input type="text" class="rc-form-input" placeholder="Your measure, e.g. ${_exampleMeasure[unit]}" /><span>${unitDisplay}</span>`
+  }
+  </div>
+  </div>`
+  // <button class="rc-form-submit rc-button rc-custom-button" disabled>OK</button>
+
+  const instruction = RC.background.querySelector('.calibration-instruction')
+  instruction.innerHTML += formItem
+  const formElement = instruction.querySelector('.rc-form')
+  const formInputElement = instruction.querySelector('.rc-form-input')
+  // const formSubmitElement = instruction.querySelector('.rc-form-submit')
+
+  const addedButtons = addButtons(
+    RC.L,
+    formElement,
+    {
+      go: () => {},
+      cancel: () => {},
+    },
+    true,
+    false
+  )
+  const goButton = addedButtons[1]
+  const cancelButton = addedButtons[2]
+
+  formInputElement.oninput = () => {
+    if (validInput(formInputElement.value)) {
+      formInputElement.classList.remove('rc-input-error')
+      goButton.disabled = false
+    } else {
+      formInputElement.classList.add('rc-input-error')
+      goButton.disabled = true
+    }
+  }
+
+  // Arrow
+  const arrow = document.createElement('div')
+  RC.background.appendChild(arrow)
+  arrow.outerHTML = Arrow
+
+  return new Promise(resolve => {
+    goButton.onclick = () => {
+      onFormSubmit(formInputElement.value, resolve)
+    }
+    cancelButton.onclick = () => {
+      resolve(null)
+    }
+  })
+}
+
+const onFormSubmit = (value, resolve) => {
+  if (validInput(value)) {
+    resolve(value)
+  }
+}
+
+// const onFormCancel = e => {}
+
+/* -------------------------------------------------------------------------- */
+
+const validInput = text => {
+  return !isNaN(text)
+}

package/src/components/keyBinder.js

@@ -2,19 +2,18 @@
  * Bind keys, e.g., SPACE (' '), ESC ('Escape')
  * keys is an object of keys of the keys, pointing to its binding functions
  */
-export function bindKeys(keys) {
+export function bindKeys(keys, eventType = 'keydown') {
   const bindingFunctions = e => {
     if (e.key in keys) {
       e.preventDefault()
-      keys[e.key]()
+      keys[e.key](e)
     }
   }
 
-  document.body.addEventListener('keydown', bindingFunctions)
-
+  document.body.addEventListener(eventType, bindingFunctions)
   return bindingFunctions
 }
 
-export function unbindKeys(event) {
-  document.body.removeEventListener('keydown', event)
+export function unbindKeys(event, eventType = 'keydown') {
+  document.body.removeEventListener(eventType, event)
 }

package/src/components/language.js

@@ -29,3 +29,8 @@ function constructLangData(lang) {
     timestamp: new Date(),
   }
 }
+
+export function spaceForLanguage(L) {
+  console.log(phrases.EE_languageUseSpace[L])
+  return phrases.EE_languageUseSpace[L] === '1'
+}

package/src/components/mediaPermission.js

@@ -0,0 +1,21 @@
+export const checkPermissions = tasks => {
+  // if (navigator.permissions)
+  // navigator.permissions
+  //   .query({ name: 'camera' })
+  //   .then(permissionObj => {
+  //     console.log(permissionObj.state)
+  //   })
+  //   .catch(error => {
+  //     console.error(error)
+  //   })
+  let now = Date.now()
+  navigator.mediaDevices
+    .getUserMedia({ video: true })
+    .then(function (stream) {
+      // Permitted
+      console.log('[GOT]', Date.now() - now)
+    })
+    .catch(function (err) {
+      console.error('[ERROR] ', Date.now() - now)
+    })
+}

package/src/components/onCanvas.js

@@ -28,9 +28,9 @@ export function _circle(RC, ctx, x, y, frameCount, sparkle = true) {
   ctx.arc(x, y, circleR >> 1, 0, Math.PI * 2)
   ctx.closePath()
 
-  if (!sparkle) ctx.fillStyle = RC._CONST.COLOR.DARK_RED
+  if (!sparkle) ctx.fillStyle = RC._CONST.COLOR.RED
   else {
-    if (frameCount % 4 < 2) ctx.fillStyle = RC._CONST.COLOR.DARK_RED
+    if (frameCount % 4 < 2) ctx.fillStyle = RC._CONST.COLOR.RED
     else ctx.fillStyle = '#fff'
   }