remote-calibrator 0.3.0-rc.3 → 0.5.0-beta.10

package/src/distance/distanceTrack.js

@@ -6,9 +6,11 @@ import {
   constructInstructions,
   blurAll,
   sleep,
+  safeExecuteFunc,
 } from '../components/utils'
 import { iRepeat } from '../components/iRepeat'
 import { phrases } from '../i18n'
+import { spaceForLanguage } from '../components/language'
 
 const originalStyles = {
   video: false,
@@ -41,17 +43,26 @@ RemoteCalibrator.prototype.trackDistance = function (
     {
       fullscreen: false,
       repeatTesting: 2,
-      sparkle: true,
+      sparkle: false,
       pipWidthPx:
         this._CONST.N.VIDEO_W[this.isMobile.value ? 'MOBILE' : 'DESKTOP'],
       showVideo: true,
       showFaceOverlay: false,
       decimalPlace: 1,
-      framerate: 3, // track rate
-      nearPoint: true, // New 0.0.6
-      showNearPoint: false, // New 0.0.6
-      headline: '🙂 ' + phrases.RC_headTrackingTitle[this.L],
-      description: phrases.RC_headTrackingIntro[this.L],
+      framerate: 3, // tracking rate
+      desiredDistanceCm: undefined,
+      desiredDistanceTolerance: 1.2,
+      desiredDistanceMonitor: false,
+      desiredDistanceMonitorCancelable: false,
+      nearPoint: true,
+      showNearPoint: false,
+      headline: '🙂 ' + phrases.RC_distanceTrackingTitle[this.L],
+      description:
+        phrases.RC_distanceTrackingIntroStart[this.L] +
+        spaceForLanguage(this.L) +
+        phrases.RC_viewingDistanceIntro[this.L] +
+        spaceForLanguage(this.L) +
+        phrases.RC_distanceTrackingIntroEnd[this.L],
     },
     options
   )
@@ -85,9 +96,7 @@ RemoteCalibrator.prototype.trackDistance = function (
     if (this.gazeTracker.checkInitialized('gaze', false))
       this.showGazer(originalGazer)
 
-    if (callbackStatic && typeof callbackStatic === 'function')
-      callbackStatic(distData)
-
+    safeExecuteFunc(callbackStatic, distData)
     stdDist.current = distData
   }
 
@@ -112,6 +121,12 @@ RemoteCalibrator.prototype.trackDistance = function (
   trackingOptions.nearPoint = options.nearPoint
   trackingOptions.showNearPoint = options.showNearPoint
 
+  trackingOptions.desiredDistanceCm = options.desiredDistanceCm
+  trackingOptions.desiredDistanceTolerance = options.desiredDistanceTolerance
+  trackingOptions.desiredDistanceMonitor = options.desiredDistanceMonitor
+  trackingOptions.desiredDistanceMonitorCancelable =
+    options.desiredDistanceMonitorCancelable
+
   originalStyles.video = options.showVideo
 
   this.gazeTracker._init(
@@ -123,41 +138,52 @@ RemoteCalibrator.prototype.trackDistance = function (
     'distance'
   )
 
+  this._trackingSetupFinishedStatus.distance = false
+
+  const trackingConfig = {
+    options: options,
+    callbackStatic: callbackStatic,
+    callbackTrack: callbackTrack,
+  }
+
   if (options.nearPoint) {
     startTrackingPupils(
       this,
       () => {
-        this.measurePD({}, _)
+        this._measurePD({}, _)
       },
-      callbackTrack
+      callbackTrack,
+      trackingConfig
     )
   } else {
-    startTrackingPupils(this, _, callbackTrack)
+    startTrackingPupils(this, _, callbackTrack, trackingConfig)
   }
 }
 
 /* -------------------------------------------------------------------------- */
 
-const startTrackingPupils = async (RC, beforeCallbackTrack, callbackTrack) => {
+const startTrackingPupils = async (
+  RC,
+  beforeCallbackTrack,
+  callbackTrack,
+  trackingConfig
+) => {
   RC.gazeTracker.beginVideo({ pipWidthPx: trackingOptions.pipWidthPx }, () => {
     RC._removeFloatInstructionElement()
-    beforeCallbackTrack()
-    _tracking(RC, trackingOptions, callbackTrack)
+    safeExecuteFunc(beforeCallbackTrack)
+    _tracking(RC, trackingOptions, callbackTrack, trackingConfig)
   })
 }
 
 const eyeDist = (a, b) => {
-  return Math.sqrt(
-    Math.pow(a[0] - b[0], 2) +
-      Math.pow(a[1] - b[1], 2) +
-      Math.pow(a[2] - b[2], 2)
-  )
+  return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2])
 }
 
 const cyclopean = (video, a, b) => {
-  let [aX, aY] = [video.videoWidth - a[0], a[1]]
-  let [bX, bY] = [video.videoWidth - b[0], b[1]]
-  return [(aX + bX) / 2, (aY + bY) / 2]
+  return [
+    (-a[0] - b[0] + video.videoWidth) / 2,
+    (-a[1] - b[1] + video.videoHeight) / 2,
+  ]
 }
 
 /* -------------------------------------------------------------------------- */
@@ -167,6 +193,10 @@ const trackingOptions = {
   framerate: 3,
   nearPoint: true,
   showNearPoint: false,
+  desiredDistanceCm: undefined,
+  desiredDistanceTolerance: 1.2,
+  desiredDistanceMonitor: false,
+  desiredDistanceMonitorCancelable: false,
 }
 
 const stdDist = { current: null }
@@ -177,22 +207,31 @@ let iRepeatOptions = { framerate: 20, break: true }
 let nearPointDot = null
 /* -------------------------------------------------------------------------- */
 
-const _tracking = async (RC, trackingOptions, callbackTrack) => {
+let readyToGetFirstData = false
+let averageDist = 0
+let distCount = 1
+
+const _tracking = async (
+  RC,
+  trackingOptions,
+  callbackTrack,
+  trackingConfig
+) => {
   const video = document.querySelector('#webgazerVideoFeed')
 
   const _ = async () => {
     // const canvas = RC.gazeTracker.webgazer.videoCanvas
     let model, faces
 
-    // Get the average of 2 estimates for one measure
-    let averageDist = 0
-    let distCount = 1
+    // Get the average of 5 estimates for one measure
+    averageDist = 0
+    distCount = 1
     const targetCount = 5
 
     model = await RC.gazeTracker.webgazer.getTracker().model
 
     // Near point
-    let ppi = RC.screenPpi ? RC.screenPpi.value : 108
+    let ppi = RC.screenPpi ? RC.screenPpi.value : RC._CONST.N.PPI_DONT_USE
     if (!RC.screenPpi && trackingOptions.nearPoint)
       console.error(
         'Screen size measurement is required to get accurate near point tracking.'
@@ -215,29 +254,54 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
       })
     }
 
+    readyToGetFirstData = false
+    const {
+      desiredDistanceCm,
+      desiredDistanceTolerance,
+      desiredDistanceMonitor,
+      desiredDistanceMonitorCancelable,
+    } = trackingOptions
+
+    // Always enable correct on a fresh start
+    RC._distanceTrackNudging.distanceCorrectEnabled = true
+    RC._distanceTrackNudging.distanceDesired = desiredDistanceCm
+    RC._distanceTrackNudging.distanceAllowedRatio = desiredDistanceTolerance
+
     viewingDistanceTrackingFunction = async () => {
+      //
+      const videoTimestamp = new Date().getTime()
+      //
       faces = await model.estimateFaces(video)
       if (faces.length) {
         // There's at least one face in video
-        const mesh = faces[0].scaledMesh
+        RC._tackingVideoFrameTimestamps.distance += videoTimestamp
         // https://github.com/tensorflow/tfjs-models/blob/master/facemesh/mesh_map.jpg
+        const mesh = faces[0].scaledMesh
+
         if (targetCount === distCount) {
           averageDist += eyeDist(mesh[133], mesh[362])
-          averageDist /= 5
+          averageDist /= targetCount
+          RC._tackingVideoFrameTimestamps.distance /= targetCount
 
+          // TODO Add more samples for the first estimate
           if (stdDist.current !== null) {
             if (!stdFactor) {
               // ! First time estimate
               // Face_Known_Px  *  Distance_Known_Cm  =  Face_Now_Px  *  Distance_x_Cm
               // Get the factor to be used for future predictions
               stdFactor = averageDist * stdDist.current.value
-              // FINISH
+              // ! FINISH
               RC._removeBackground()
+              RC._trackingSetupFinishedStatus.distance = true
+              readyToGetFirstData = true
             }
 
             /* -------------------------------------------------------------------------- */
 
             const timestamp = new Date()
+            const latency = Math.round(
+              timestamp.getTime() - RC._tackingVideoFrameTimestamps.distance
+            )
 
             const data = (RC.newViewingDistanceData = {
               value: toFixedNumber(
@@ -245,9 +309,20 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
                 trackingOptions.decimalPlace
               ),
               timestamp: timestamp,
-              method: 'Facemesh Predict',
+              method: RC._CONST.VIEW_METHOD.F,
+              latencyMs: latency,
             })
 
+            if (readyToGetFirstData || desiredDistanceMonitor) {
+              // ! Check distance
+              if (desiredDistanceCm)
+                RC.checkDistance(
+                  desiredDistanceMonitorCancelable,
+                  trackingConfig
+                )
+              readyToGetFirstData = false
+            }
+
             /* -------------------------------------------------------------------------- */
 
             // Near point
@@ -260,7 +335,8 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
                 mesh,
                 averageDist,
                 timestamp,
-                ppi
+                ppi,
+                latency
               )
             }
 
@@ -272,15 +348,18 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
                 value: {
                   viewingDistanceCm: data.value,
                   nearPointCm: nPData ? nPData.value : [null, null],
+                  latencyMs: latency,
                 },
                 timestamp: timestamp,
-                method: 'Facemesh Predict',
+                method: RC._CONST.VIEW_METHOD.F,
               })
             }
           }
 
           averageDist = 0
           distCount = 1
+
+          RC._tackingVideoFrameTimestamps.distance = 0
         } else {
           averageDist += eyeDist(mesh[133], mesh[362])
           ++distCount
@@ -289,11 +368,11 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
     }
 
     iRepeatOptions.break = false
-    iRepeatOptions.framerate = targetCount * trackingOptions.framerate // Default 3 * 5
+    iRepeatOptions.framerate = targetCount * trackingOptions.framerate // Default 5 * 3
     iRepeat(viewingDistanceTrackingFunction, iRepeatOptions)
   }
 
-  sleep(500).then(_)
+  sleep(1000).then(_)
 }
 
 const _getNearPoint = (
@@ -303,45 +382,42 @@ const _getNearPoint = (
   mesh,
   averageDist,
   timestamp,
-  ppi
+  ppi,
+  latency
 ) => {
-  let m = cyclopean(video, mesh[133], mesh[362])
-  let offsetToVideoMid = [
-    m[0] - video.videoWidth / 2,
-    video.videoHeight / 2 - m[1],
-  ]
-
-  const videoFactor = video.videoHeight / video.clientHeight
-  offsetToVideoMid.forEach((e, i) => {
-    // Average interpupillary distance - 6.4cm
-    offsetToVideoMid[i] =
-      ((RC.PDCm ? RC.PDCm.value : 6.4) * e) /
-      (averageDist * (videoFactor / 2)) /* Should this be videoFactor? */
+  let offsetToVideoCenter = cyclopean(video, mesh[133], mesh[362])
+  offsetToVideoCenter.forEach((offset, i) => {
+    // Average inter-pupillary distance - 6.4cm
+    offsetToVideoCenter[i] =
+      ((RC.PDCm ? RC.PDCm.value : RC._CONST.N.PD_DONT_USE) * offset) /
+      averageDist
   })
 
   let nPData = (RC.newNearPointData = {
     value: {
-      x: toFixedNumber(offsetToVideoMid[0], trackingOptions.decimalPlace),
+      x: toFixedNumber(offsetToVideoCenter[0], trackingOptions.decimalPlace),
       y: toFixedNumber(
-        offsetToVideoMid[1] + 0.5, // Commonly the webcam is 0.5cm above the screen
+        offsetToVideoCenter[1] + ((screen.height / 2) * 2.54) / ppi, // Commonly the webcam is 0.5cm above the screen
         trackingOptions.decimalPlace
       ),
+      latencyMs: latency,
     },
     timestamp: timestamp,
   })
 
   // SHOW
+  const dotR = 5
   if (trackingOptions.showNearPoint) {
     let offsetX = (nPData.value.x * ppi) / 2.54
     let offsetY = (nPData.value.y * ppi) / 2.54
     Object.assign(nearPointDot.style, {
-      left: `${screen.width / 2 - window.screenLeft - 5 + offsetX}px`,
+      left: `${screen.width / 2 - window.screenLeft + offsetX - dotR}px`,
       top: `${
         screen.height / 2 -
         window.screenTop -
-        5 -
-        (RC.isFullscreen.value ? 0 : 50) -
-        offsetY
+        (window.outerHeight - window.innerHeight) -
+        offsetY -
+        dotR
       }px`,
     })
   }
@@ -353,6 +429,7 @@ RemoteCalibrator.prototype.pauseDistance = function () {
   if (this.gazeTracker.checkInitialized('distance', true)) {
     iRepeatOptions.break = true
     if (nearPointDot) nearPointDot.style.display = 'none'
+    this._tackingVideoFrameTimestamps.distance = 0
     return this
   }
   return null
@@ -362,6 +439,11 @@ RemoteCalibrator.prototype.resumeDistance = function () {
   if (this.gazeTracker.checkInitialized('distance', true)) {
     iRepeatOptions.break = false
     if (nearPointDot) nearPointDot.style.display = 'block'
+
+    averageDist = 0
+    distCount = 1
+    this._tackingVideoFrameTimestamps.distance = 0
+
     iRepeat(viewingDistanceTrackingFunction, iRepeatOptions)
     return this
   }
@@ -379,21 +461,30 @@ RemoteCalibrator.prototype.endDistance = function (endAll = false, _r = true) {
     trackingOptions.nearPoint = true
     trackingOptions.showNearPoint = false
 
+    trackingOptions.desiredDistanceCm = undefined
+    trackingOptions.desiredDistanceTolerance = 1.2
+    trackingOptions.desiredDistanceMonitor = false
+    trackingOptions.desiredDistanceMonitorCancelable = false
+
     stdDist.current = null
     stdFactor = null
     viewingDistanceTrackingFunction = null
 
+    readyToGetFirstData = false
+    this._tackingVideoFrameTimestamps.distance = 0
+
     // Near point
     if (nearPointDot) {
       document.body.removeChild(nearPointDot)
       nearPointDot = null
     }
 
-    if (_r) this.gazeTracker.end('distance', endAll)
+    // Nudger
+    this.endNudger()
 
+    if (_r) this.gazeTracker.end('distance', endAll)
     return this
   }
-
   return null
 }
 
@@ -409,6 +500,7 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
 
   let v = document.querySelector('#webgazerVideoFeed')
   let m = await this.gazeTracker.webgazer.getTracker().model
+  const videoTimestamp = new Date().getTime()
   let f = await m.estimateFaces(v)
 
   if (f.length) {
@@ -416,11 +508,15 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
     const dist = eyeDist(mesh[133], mesh[362])
 
     let timestamp = new Date()
+    //
+    const latency = timestamp.getTime() - videoTimestamp
+    //
 
     const data = (this.newViewingDistanceData = {
       value: toFixedNumber(stdFactor / dist, trackingOptions.decimalPlace),
       timestamp: timestamp,
-      method: 'Facemesh Predict',
+      method: this._CONST.VIEW_METHOD.F,
+      latencyMs: latency,
     })
 
     let nPData
@@ -432,19 +528,20 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
         mesh,
         dist,
         timestamp,
-        this.screenPpi ? this.screenPpi.value : 108
+        this.screenPpi ? this.screenPpi.value : this._CONST.N.PPI_DONT_USE,
+        latency
       )
     }
 
-    if (c)
-      c({
-        value: {
-          viewingDistanceCm: data.value,
-          nearPointCm: nPData ? nPData.value : null,
-        },
-        timestamp: timestamp,
-        method: 'Facemesh Predict',
-      })
+    safeExecuteFunc(c, {
+      value: {
+        viewingDistanceCm: data.value,
+        nearPointCm: nPData ? nPData.value : null,
+        latencyMs: latency,
+      },
+      timestamp: timestamp,
+      method: this._CONST.VIEW_METHOD.F,
+    })
     return data
   }
 

package/src/{interpupillaryDistance.js → distance/interPupillaryDistance.js}

@@ -1,19 +1,20 @@
 import Swal from 'sweetalert2'
 
-import RemoteCalibrator from './core'
+import RemoteCalibrator from '../core'
 
 import {
   blurAll,
   constructInstructions,
+  safeExecuteFunc,
   toFixedNumber,
-} from './components/utils'
-import { swalInfoOptions } from './components/swalOptions'
-import Arrow from './media/arrow.svg'
-import PD from './media/pd.png?width=480&height=240'
-import { bindKeys, unbindKeys } from './components/keyBinder'
-import { addButtons } from './components/buttons'
-import { setDefaultVideoPosition } from './components/video'
-import { phrases } from './i18n'
+} from '../components/utils'
+import { swalInfoOptions } from '../components/swalOptions'
+import Arrow from '../media/arrow.svg'
+import PD from '../media/pd.png?width=480&height=240'
+import { bindKeys, unbindKeys } from '../components/keyBinder'
+import { addButtons } from '../components/buttons'
+import { setDefaultVideoPosition } from '../components/video'
+import { phrases } from '../i18n'
 
 // let selfVideo = false // No WebGazer video available and an extra video element needs to be created
 
@@ -29,7 +30,7 @@ const originalStyles = {
 const videoWidthFactor = 0.9
 const videoHeightFactor = 0.3
 
-RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
+RemoteCalibrator.prototype._measurePD = function (options = {}, callback) {
   ////
   if (!this.checkInitialized()) return
   blurAll()
@@ -39,7 +40,7 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
     {
       fullscreen: false,
       headline: '👁️ ' + phrases.RC_nearPointTitle[this.L],
-      description: phrases.RC_nearPointIntroCaption[this.L],
+      description: phrases.RC_nearPointIntro[this.L],
       shortDescription: phrases.RC_nearPointIntro[this.L],
     },
     options
@@ -50,9 +51,11 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
   this._replaceBackground()
 
   this._replaceBackground(
-    constructInstructions(options.headline, options.shortDescription)
+    constructInstructions(options.headline, options.shortDescription, true)
   )
-  const screenPpi = this.screenPpi ? this.screenPpi.value : 108
+  const screenPpi = this.screenPpi
+    ? this.screenPpi.value
+    : this._CONST.N.PPI_DONT_USE
 
   let [videoWidth, videoHeight] = setupVideo(this)
   let [ruler, rulerListener] = setupRuler(
@@ -64,7 +67,7 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
 
   const RC = this
 
-  const breakFunction = () => {
+  const breakFunction = (toBreak = true) => {
     ruler.removeEventListener('mousedown', rulerListener)
     this._removeBackground()
 
@@ -77,7 +80,7 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
       height: originalStyles.videoHeight,
       width: originalStyles.videoWidth,
       opacity: originalStyles.opacity,
-      borderRadius: '0px',
+      borderRadius: '5px',
     })
     setDefaultVideoPosition(
       RC,
@@ -99,6 +102,11 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
     originalStyles.gaze = false
     originalStyles.faceOverlay = false
 
+    if (!RC._trackingSetupFinishedStatus.distance && toBreak) {
+      RC._trackingSetupFinishedStatus.distance = true
+      RC.endDistance()
+    }
+
     unbindKeys(bindKeysFunction)
   }
 
@@ -110,14 +118,14 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
       }
       this.newPDData = newPDData
 
-      breakFunction()
-      callback()
+      breakFunction(false)
+      safeExecuteFunc(callback, newPDData)
     }
   }
 
-  // if (callback && typeof callback === 'function') callback()
   const bindKeysFunction = bindKeys({
     Escape: breakFunction,
+    Enter: finishFunction,
     ' ': finishFunction,
   })
   addButtons(
@@ -133,7 +141,7 @@ RemoteCalibrator.prototype.measurePD = function (options = {}, callback) {
   // TODO To be removed
   setTimeout(() => {
     Swal.fire({
-      ...swalInfoOptions(this),
+      ...swalInfoOptions(this, { showIcon: false }),
       icon: undefined,
       imageUrl: PD,
       imageWidth: 480,

package/src/gaze/gaze.js

@@ -1,6 +1,6 @@
 import RemoteCalibrator from '../core'
 
-import { blurAll } from '../components/utils'
+import { blurAll, safeExecuteFunc } from '../components/utils'
 import { gazeCalibrationPrepare } from './gazeCalibration'
 import { phrases } from '../i18n'
 
@@ -97,17 +97,14 @@ RemoteCalibrator.prototype.trackGaze = function (
     description: options.description,
   }
   this.gazeTracker.begin(gazeTrackerBeginOptions, () => {
+    this._trackingSetupFinishedStatus.gaze = false
     this.calibrateGaze(calibrateGazeOptions, onCalibrationEnded)
   })
 
   // Calibration
 
-  const onCalibrationEnded = () => {
-    if (
-      callbackOnCalibrationEnd &&
-      typeof callbackOnCalibrationEnd === 'function'
-    )
-      callbackOnCalibrationEnd()
+  const onCalibrationEnded = data => {
+    safeExecuteFunc(callbackOnCalibrationEnd, data)
 
     // ! greedyLearner
     if (!this.gazeTracker.webgazer.params.greedyLearner) {

package/src/gaze/gazeAccuracy.js

@@ -3,7 +3,12 @@ import Swal from 'sweetalert2/dist/sweetalert2.js'
 import RemoteCalibrator from '../core'
 
 import { _cross } from '../components/onCanvas'
-import { blurAll, sleep, toFixedNumber } from '../components/utils'
+import {
+  blurAll,
+  safeExecuteFunc,
+  sleep,
+  toFixedNumber,
+} from '../components/utils'
 import { swalInfoOptions } from '../components/swalOptions'
 
 let inGetAccuracy = false
@@ -57,7 +62,7 @@ RemoteCalibrator.prototype.getGazeAccuracy = function (
   _resizeCanvas()
 
   Swal.fire({
-    ...swalInfoOptions(this),
+    ...swalInfoOptions(this, { showIcon: true }),
     // title: text.getGazeAccuracy.headline,
     html: `We will measure your gaze accuracy. Please do not move the mouse and look at the fixation at the middle of the screen for the next 5 seconds.`,
   }).then(() => {
@@ -88,8 +93,8 @@ RemoteCalibrator.prototype.getGazeAccuracy = function (
 
       if (averageDegree < options.thresholdDeg)
         // Success
-        callbackSuccess()
-      else callbackFail()
+        safeExecuteFunc(callbackSuccess)
+      else safeExecuteFunc(callbackFail)
 
       ctx.clearRect(0, 0, canvas.width, canvas.height)
       resizeObserver.unobserve(this.background)