remote-calibrator 0.3.0 → 0.5.0-beta.3

package/src/distance/distanceTrack.js

@@ -48,11 +48,14 @@ RemoteCalibrator.prototype.trackDistance = function (
       showVideo: true,
       showFaceOverlay: false,
       decimalPlace: 1,
-      framerate: 3, // track rate
-      nearPoint: true, // New 0.0.6
-      showNearPoint: false, // New 0.0.6
-      headline: '🙂 ' + phrases.RC_headTrackingTitle[this.L],
-      description: phrases.RC_headTrackingIntro[this.L],
+      framerate: 3, // tracking rate
+      desiredDistanceCm: undefined,
+      desiredDistanceTolerance: 0.1,
+      desiredDistanceMonitor: false,
+      nearPoint: true,
+      showNearPoint: false,
+      headline: '🙂 ' + phrases.RC_distanceTrackingTitle[this.L],
+      description: phrases.RC_distanceTrackingIntro[this.L],
     },
     options
   )
@@ -111,6 +114,10 @@ RemoteCalibrator.prototype.trackDistance = function (
   trackingOptions.nearPoint = options.nearPoint
   trackingOptions.showNearPoint = options.showNearPoint
 
+  trackingOptions.desiredDistanceCm = options.desiredDistanceCm
+  trackingOptions.desiredDistanceTolerance = options.desiredDistanceTolerance
+  trackingOptions.desiredDistanceMonitor = options.desiredDistanceMonitor
+
   originalStyles.video = options.showVideo
 
   this.gazeTracker._init(
@@ -148,17 +155,14 @@ const startTrackingPupils = async (RC, beforeCallbackTrack, callbackTrack) => {
 }
 
 const eyeDist = (a, b) => {
-  return Math.sqrt(
-    Math.pow(a[0] - b[0], 2) +
-      Math.pow(a[1] - b[1], 2) +
-      Math.pow(a[2] - b[2], 2)
-  )
+  return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2])
 }
 
 const cyclopean = (video, a, b) => {
-  let [aX, aY] = [video.videoWidth - a[0], a[1]]
-  let [bX, bY] = [video.videoWidth - b[0], b[1]]
-  return [(aX + bX) / 2, (aY + bY) / 2]
+  return [
+    (-a[0] - b[0] + video.videoWidth) / 2,
+    (-a[1] - b[1] + video.videoHeight) / 2,
+  ]
 }
 
 /* -------------------------------------------------------------------------- */
@@ -168,6 +172,9 @@ const trackingOptions = {
   framerate: 3,
   nearPoint: true,
   showNearPoint: false,
+  desiredDistanceCm: undefined,
+  desiredDistanceTolerance: 0.1,
+  desiredDistanceMonitor: false,
 }
 
 const stdDist = { current: null }
@@ -178,6 +185,10 @@ let iRepeatOptions = { framerate: 20, break: true }
 let nearPointDot = null
 /* -------------------------------------------------------------------------- */
 
+let readyToGetFirstData = false
+let averageDist = 0
+let distCount = 1
+
 const _tracking = async (RC, trackingOptions, callbackTrack) => {
   const video = document.querySelector('#webgazerVideoFeed')
 
@@ -185,15 +196,15 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
     // const canvas = RC.gazeTracker.webgazer.videoCanvas
     let model, faces
 
-    // Get the average of 2 estimates for one measure
-    let averageDist = 0
-    let distCount = 1
+    // Get the average of 5 estimates for one measure
+    averageDist = 0
+    distCount = 1
     const targetCount = 5
 
     model = await RC.gazeTracker.webgazer.getTracker().model
 
     // Near point
-    let ppi = RC.screenPpi ? RC.screenPpi.value : 108
+    let ppi = RC.screenPpi ? RC.screenPpi.value : RC._CONST.N.PPI_DONT_USE
     if (!RC.screenPpi && trackingOptions.nearPoint)
       console.error(
         'Screen size measurement is required to get accurate near point tracking.'
@@ -216,16 +227,30 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
       })
     }
 
+    readyToGetFirstData = false
+    const {
+      desiredDistanceCm,
+      desiredDistanceTolerance,
+      desiredDistanceMonitor,
+    } = trackingOptions
+
     viewingDistanceTrackingFunction = async () => {
+      //
+      const videoTimestamp = new Date().getTime()
+      //
       faces = await model.estimateFaces(video)
       if (faces.length) {
         // There's at least one face in video
-        const mesh = faces[0].scaledMesh
+        RC._tackingVideoFrameTimestamps.distance += videoTimestamp
         // https://github.com/tensorflow/tfjs-models/blob/master/facemesh/mesh_map.jpg
+        const mesh = faces[0].scaledMesh
+
         if (targetCount === distCount) {
           averageDist += eyeDist(mesh[133], mesh[362])
-          averageDist /= 5
+          averageDist /= targetCount
+          RC._tackingVideoFrameTimestamps.distance /= targetCount
 
+          // TODO Add more samples for the first estimate
           if (stdDist.current !== null) {
             if (!stdFactor) {
               // ! First time estimate
@@ -235,11 +260,15 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
               // ! FINISH
               RC._removeBackground()
               RC._trackingSetupFinishedStatus.distance = true
+              readyToGetFirstData = true
             }
 
             /* -------------------------------------------------------------------------- */
 
             const timestamp = new Date()
+            const latency = Math.round(
+              timestamp.getTime() - RC._tackingVideoFrameTimestamps.distance
+            )
 
             const data = (RC.newViewingDistanceData = {
               value: toFixedNumber(
@@ -248,8 +277,16 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
               ),
               timestamp: timestamp,
               method: RC._CONST.VIEW_METHOD.F,
+              latencyMs: latency,
             })
 
+            if (readyToGetFirstData || desiredDistanceMonitor) {
+              // Check distance
+              if (desiredDistanceCm)
+                RC.checkDistance(desiredDistanceCm, desiredDistanceTolerance)
+              readyToGetFirstData = false
+            }
+
             /* -------------------------------------------------------------------------- */
 
             // Near point
@@ -262,7 +299,8 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
                 mesh,
                 averageDist,
                 timestamp,
-                ppi
+                ppi,
+                latency
               )
             }
 
@@ -274,6 +312,7 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
                 value: {
                   viewingDistanceCm: data.value,
                   nearPointCm: nPData ? nPData.value : [null, null],
+                  latencyMs: latency,
                 },
                 timestamp: timestamp,
                 method: RC._CONST.VIEW_METHOD.F,
@@ -283,6 +322,8 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
 
           averageDist = 0
           distCount = 1
+
+          RC._tackingVideoFrameTimestamps.distance = 0
         } else {
           averageDist += eyeDist(mesh[133], mesh[362])
           ++distCount
@@ -291,7 +332,7 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
     }
 
     iRepeatOptions.break = false
-    iRepeatOptions.framerate = targetCount * trackingOptions.framerate // Default 3 * 5
+    iRepeatOptions.framerate = targetCount * trackingOptions.framerate // Default 5 * 3
     iRepeat(viewingDistanceTrackingFunction, iRepeatOptions)
   }
 
@@ -305,45 +346,42 @@ const _getNearPoint = (
   mesh,
   averageDist,
   timestamp,
-  ppi
+  ppi,
+  latency
 ) => {
-  let m = cyclopean(video, mesh[133], mesh[362])
-  let offsetToVideoMid = [
-    m[0] - video.videoWidth / 2,
-    video.videoHeight / 2 - m[1],
-  ]
-
-  const videoFactor = video.videoHeight / video.clientHeight
-  offsetToVideoMid.forEach((e, i) => {
-    // Average interpupillary distance - 6.4cm
-    offsetToVideoMid[i] =
-      ((RC.PDCm ? RC.PDCm.value : 6.4) * e) /
-      (averageDist * (videoFactor / 2)) /* Should this be videoFactor? */
+  let offsetToVideoCenter = cyclopean(video, mesh[133], mesh[362])
+  offsetToVideoCenter.forEach((offset, i) => {
+    // Average inter-pupillary distance - 6.4cm
+    offsetToVideoCenter[i] =
+      ((RC.PDCm ? RC.PDCm.value : RC._CONST.N.PD_DONT_USE) * offset) /
+      averageDist
   })
 
   let nPData = (RC.newNearPointData = {
     value: {
-      x: toFixedNumber(offsetToVideoMid[0], trackingOptions.decimalPlace),
+      x: toFixedNumber(offsetToVideoCenter[0], trackingOptions.decimalPlace),
       y: toFixedNumber(
-        offsetToVideoMid[1] + 0.5, // Commonly the webcam is 0.5cm above the screen
+        offsetToVideoCenter[1] + ((screen.height / 2) * 2.54) / ppi, // Commonly the webcam is 0.5cm above the screen
         trackingOptions.decimalPlace
       ),
+      latencyMs: latency,
     },
     timestamp: timestamp,
   })
 
   // SHOW
+  const dotR = 5
   if (trackingOptions.showNearPoint) {
     let offsetX = (nPData.value.x * ppi) / 2.54
     let offsetY = (nPData.value.y * ppi) / 2.54
     Object.assign(nearPointDot.style, {
-      left: `${screen.width / 2 - window.screenLeft - 5 + offsetX}px`,
+      left: `${screen.width / 2 - window.screenLeft + offsetX - dotR}px`,
       top: `${
         screen.height / 2 -
         window.screenTop -
-        5 -
-        (RC.isFullscreen.value ? 0 : 50) -
-        offsetY
+        (window.outerHeight - window.innerHeight) -
+        offsetY -
+        dotR
       }px`,
     })
   }
@@ -355,6 +393,7 @@ RemoteCalibrator.prototype.pauseDistance = function () {
   if (this.gazeTracker.checkInitialized('distance', true)) {
     iRepeatOptions.break = true
     if (nearPointDot) nearPointDot.style.display = 'none'
+    this._tackingVideoFrameTimestamps.distance = 0
     return this
   }
   return null
@@ -364,6 +403,11 @@ RemoteCalibrator.prototype.resumeDistance = function () {
   if (this.gazeTracker.checkInitialized('distance', true)) {
     iRepeatOptions.break = false
     if (nearPointDot) nearPointDot.style.display = 'block'
+
+    averageDist = 0
+    distCount = 1
+    this._tackingVideoFrameTimestamps.distance = 0
+
     iRepeat(viewingDistanceTrackingFunction, iRepeatOptions)
     return this
   }
@@ -381,10 +425,17 @@ RemoteCalibrator.prototype.endDistance = function (endAll = false, _r = true) {
     trackingOptions.nearPoint = true
     trackingOptions.showNearPoint = false
 
+    trackingOptions.desiredDistanceCm = undefined
+    trackingOptions.desiredDistanceTolerance = 0.1
+    trackingOptions.desiredDistanceMonitor = false
+
     stdDist.current = null
     stdFactor = null
     viewingDistanceTrackingFunction = null
 
+    readyToGetFirstData = false
+    this._tackingVideoFrameTimestamps.distance = 0
+
     // Near point
     if (nearPointDot) {
       document.body.removeChild(nearPointDot)
@@ -411,6 +462,7 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
 
   let v = document.querySelector('#webgazerVideoFeed')
   let m = await this.gazeTracker.webgazer.getTracker().model
+  const videoTimestamp = new Date().getTime()
   let f = await m.estimateFaces(v)
 
   if (f.length) {
@@ -418,11 +470,15 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
     const dist = eyeDist(mesh[133], mesh[362])
 
     let timestamp = new Date()
+    //
+    const latency = timestamp.getTime() - videoTimestamp
+    //
 
     const data = (this.newViewingDistanceData = {
       value: toFixedNumber(stdFactor / dist, trackingOptions.decimalPlace),
       timestamp: timestamp,
       method: this._CONST.VIEW_METHOD.F,
+      latencyMs: latency,
     })
 
     let nPData
@@ -434,7 +490,8 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
         mesh,
         dist,
         timestamp,
-        this.screenPpi ? this.screenPpi.value : 108
+        this.screenPpi ? this.screenPpi.value : this._CONST.N.PPI_DONT_USE,
+        latency
       )
     }
 
@@ -442,6 +499,7 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
       value: {
         viewingDistanceCm: data.value,
         nearPointCm: nPData ? nPData.value : null,
+        latencyMs: latency,
       },
       timestamp: timestamp,
       method: this._CONST.VIEW_METHOD.F,

package/src/{interpupillaryDistance.js → distance/interPupillaryDistance.js}

@@ -1,20 +1,20 @@
 import Swal from 'sweetalert2'
 
-import RemoteCalibrator from './core'
+import RemoteCalibrator from '../core'
 
 import {
   blurAll,
   constructInstructions,
   safeExecuteFunc,
   toFixedNumber,
-} from './components/utils'
-import { swalInfoOptions } from './components/swalOptions'
-import Arrow from './media/arrow.svg'
-import PD from './media/pd.png?width=480&height=240'
-import { bindKeys, unbindKeys } from './components/keyBinder'
-import { addButtons } from './components/buttons'
-import { setDefaultVideoPosition } from './components/video'
-import { phrases } from './i18n'
+} from '../components/utils'
+import { swalInfoOptions } from '../components/swalOptions'
+import Arrow from '../media/arrow.svg'
+import PD from '../media/pd.png?width=480&height=240'
+import { bindKeys, unbindKeys } from '../components/keyBinder'
+import { addButtons } from '../components/buttons'
+import { setDefaultVideoPosition } from '../components/video'
+import { phrases } from '../i18n'
 
 // let selfVideo = false // No WebGazer video available and an extra video element needs to be created
 
@@ -40,7 +40,7 @@ RemoteCalibrator.prototype._measurePD = function (options = {}, callback) {
     {
       fullscreen: false,
       headline: '👁️ ' + phrases.RC_nearPointTitle[this.L],
-      description: phrases.RC_nearPointIntroCaption[this.L],
+      description: phrases.RC_nearPointIntro[this.L],
       shortDescription: phrases.RC_nearPointIntro[this.L],
     },
     options
@@ -51,9 +51,11 @@ RemoteCalibrator.prototype._measurePD = function (options = {}, callback) {
   this._replaceBackground()
 
   this._replaceBackground(
-    constructInstructions(options.headline, options.shortDescription)
+    constructInstructions(options.headline, options.shortDescription, true)
   )
-  const screenPpi = this.screenPpi ? this.screenPpi.value : 108
+  const screenPpi = this.screenPpi
+    ? this.screenPpi.value
+    : this._CONST.N.PPI_DONT_USE
 
   let [videoWidth, videoHeight] = setupVideo(this)
   let [ruler, rulerListener] = setupRuler(

package/src/gaze/gazeTracker.js

@@ -100,6 +100,8 @@ GazeTracker.prototype._init = function (
       // this.webgazer.saveDataAcrossSessions(false)
       this.webgazer.params.greedyLearner = greedyLearner
       this.webgazer.params.framerate = framerate
+      this.webgazer.params.getLatestVideoFrameTimestamp =
+        this._getLatestVideoTimestamp.bind(this)
       this.showGazer(showGazer)
     }
 
@@ -127,12 +129,15 @@ GazeTracker.prototype.checkInitialized = function (task, warning = false) {
 }
 
 GazeTracker.prototype.getData = function (d) {
+  let t = new Date()
   return {
     value: {
       x: toFixedNumber(d.x, this._toFixedN),
       y: toFixedNumber(d.y, this._toFixedN),
+      latencyMs:
+        t.getTime() - this.calibrator._tackingVideoFrameTimestamps.gaze, // latency
     },
-    timestamp: new Date(),
+    timestamp: t,
   }
 }
 
@@ -156,6 +161,8 @@ GazeTracker.prototype.end = function (type, endAll = false) {
     this._endGaze()
     if (endEverything && this.checkInitialized('distance'))
       this.calibrator.endDistance(false, false)
+
+    this.calibrator._tackingVideoFrameTimestamps.gaze = 0
   } else {
     // Distance
     this.defaultDistanceTrackCallback = null
@@ -188,8 +195,16 @@ GazeTracker.prototype._endGaze = function () {
 
   this.webgazer.params.greedyLearner = false
   this.webgazer.params.framerate = 60
+
+  this.webgazer.params.getLatestVideoFrameTimestamp = () => {}
+}
+
+GazeTracker.prototype._getLatestVideoTimestamp = function (t) {
+  this.calibrator._tackingVideoFrameTimestamps.gaze = t.getTime()
 }
 
+/* -------------------------------------------------------------------------- */
+
 GazeTracker.prototype.startStoringPoints = function () {
   this.webgazer.params.storingPoints = true
 }