remote-calibrator 0.3.0 → 0.5.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/README.md +29 -19
  3. package/homepage/example.js +9 -3
  4. package/i18n/fetch-languages-sheets.js +5 -4
  5. package/lib/RemoteCalibrator.min.js +1 -1
  6. package/lib/RemoteCalibrator.min.js.LICENSE.txt +1 -1
  7. package/lib/RemoteCalibrator.min.js.map +1 -1
  8. package/package.json +15 -15
  9. package/src/WebGazer4RC/.gitattributes +10 -0
  10. package/src/WebGazer4RC/LICENSE.md +15 -0
  11. package/src/WebGazer4RC/README.md +142 -0
  12. package/src/WebGazer4RC/gnu-lgpl-v3.0.md +163 -0
  13. package/src/WebGazer4RC/gplv3.md +636 -0
  14. package/src/WebGazer4RC/package-lock.json +1133 -0
  15. package/src/WebGazer4RC/package.json +28 -0
  16. package/src/WebGazer4RC/src/dom_util.mjs +27 -0
  17. package/src/WebGazer4RC/src/facemesh.mjs +150 -0
  18. package/src/WebGazer4RC/src/index.mjs +1235 -0
  19. package/src/WebGazer4RC/src/mat.mjs +301 -0
  20. package/src/WebGazer4RC/src/params.mjs +29 -0
  21. package/src/WebGazer4RC/src/pupil.mjs +109 -0
  22. package/src/WebGazer4RC/src/ridgeReg.mjs +104 -0
  23. package/src/WebGazer4RC/src/ridgeRegThreaded.mjs +161 -0
  24. package/src/WebGazer4RC/src/ridgeWeightedReg.mjs +125 -0
  25. package/src/WebGazer4RC/src/ridgeWorker.mjs +135 -0
  26. package/src/WebGazer4RC/src/util.mjs +348 -0
  27. package/src/WebGazer4RC/src/util_regression.mjs +240 -0
  28. package/src/WebGazer4RC/src/worker_scripts/mat.js +306 -0
  29. package/src/WebGazer4RC/src/worker_scripts/util.js +398 -0
  30. package/src/WebGazer4RC/test/regression_test.js +182 -0
  31. package/src/WebGazer4RC/test/run_tests_and_server.sh +24 -0
  32. package/src/WebGazer4RC/test/util_test.js +60 -0
  33. package/src/WebGazer4RC/test/webgazerExtract_test.js +40 -0
  34. package/src/WebGazer4RC/test/webgazer_test.js +160 -0
  35. package/src/WebGazer4RC/test/www_page_test.js +41 -0
  36. package/src/const.js +3 -0
  37. package/src/core.js +8 -0
  38. package/src/css/distance.scss +40 -0
  39. package/src/css/panel.scss +32 -1
  40. package/src/distance/distance.js +4 -4
  41. package/src/distance/distanceCheck.js +115 -0
  42. package/src/distance/distanceTrack.js +99 -41
  43. package/src/{interpupillaryDistance.js → distance/interPupillaryDistance.js} +14 -12
  44. package/src/gaze/gazeTracker.js +16 -1
  45. package/src/i18n.js +1 -1
  46. package/src/index.js +2 -1
  47. package/src/panel.js +32 -3
  48. package/webpack.config.js +4 -4
@@ -48,11 +48,14 @@ RemoteCalibrator.prototype.trackDistance = function (
48
48
  showVideo: true,
49
49
  showFaceOverlay: false,
50
50
  decimalPlace: 1,
51
- framerate: 3, // track rate
52
- nearPoint: true, // New 0.0.6
53
- showNearPoint: false, // New 0.0.6
54
- headline: '🙂 ' + phrases.RC_headTrackingTitle[this.L],
55
- description: phrases.RC_headTrackingIntro[this.L],
51
+ framerate: 3, // tracking rate
52
+ desiredDistanceCm: undefined,
53
+ desiredDistanceTolerance: 0.1,
54
+ desiredDistanceMonitor: false,
55
+ nearPoint: true,
56
+ showNearPoint: false,
57
+ headline: '🙂 ' + phrases.RC_distanceTrackingTitle[this.L],
58
+ description: phrases.RC_distanceTrackingIntro[this.L],
56
59
  },
57
60
  options
58
61
  )
@@ -111,6 +114,10 @@ RemoteCalibrator.prototype.trackDistance = function (
111
114
  trackingOptions.nearPoint = options.nearPoint
112
115
  trackingOptions.showNearPoint = options.showNearPoint
113
116
 
117
+ trackingOptions.desiredDistanceCm = options.desiredDistanceCm
118
+ trackingOptions.desiredDistanceTolerance = options.desiredDistanceTolerance
119
+ trackingOptions.desiredDistanceMonitor = options.desiredDistanceMonitor
120
+
114
121
  originalStyles.video = options.showVideo
115
122
 
116
123
  this.gazeTracker._init(
@@ -148,17 +155,14 @@ const startTrackingPupils = async (RC, beforeCallbackTrack, callbackTrack) => {
148
155
  }
149
156
 
150
157
  const eyeDist = (a, b) => {
151
- return Math.sqrt(
152
- Math.pow(a[0] - b[0], 2) +
153
- Math.pow(a[1] - b[1], 2) +
154
- Math.pow(a[2] - b[2], 2)
155
- )
158
+ return Math.hypot(a[0] - b[0], a[1] - b[1], a[2] - b[2])
156
159
  }
157
160
 
158
161
  const cyclopean = (video, a, b) => {
159
- let [aX, aY] = [video.videoWidth - a[0], a[1]]
160
- let [bX, bY] = [video.videoWidth - b[0], b[1]]
161
- return [(aX + bX) / 2, (aY + bY) / 2]
162
+ return [
163
+ (-a[0] - b[0] + video.videoWidth) / 2,
164
+ (-a[1] - b[1] + video.videoHeight) / 2,
165
+ ]
162
166
  }
163
167
 
164
168
  /* -------------------------------------------------------------------------- */
@@ -168,6 +172,9 @@ const trackingOptions = {
168
172
  framerate: 3,
169
173
  nearPoint: true,
170
174
  showNearPoint: false,
175
+ desiredDistanceCm: undefined,
176
+ desiredDistanceTolerance: 0.1,
177
+ desiredDistanceMonitor: false,
171
178
  }
172
179
 
173
180
  const stdDist = { current: null }
@@ -178,6 +185,10 @@ let iRepeatOptions = { framerate: 20, break: true }
178
185
  let nearPointDot = null
179
186
  /* -------------------------------------------------------------------------- */
180
187
 
188
+ let readyToGetFirstData = false
189
+ let averageDist = 0
190
+ let distCount = 1
191
+
181
192
  const _tracking = async (RC, trackingOptions, callbackTrack) => {
182
193
  const video = document.querySelector('#webgazerVideoFeed')
183
194
 
@@ -185,15 +196,15 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
185
196
  // const canvas = RC.gazeTracker.webgazer.videoCanvas
186
197
  let model, faces
187
198
 
188
- // Get the average of 2 estimates for one measure
189
- let averageDist = 0
190
- let distCount = 1
199
+ // Get the average of 5 estimates for one measure
200
+ averageDist = 0
201
+ distCount = 1
191
202
  const targetCount = 5
192
203
 
193
204
  model = await RC.gazeTracker.webgazer.getTracker().model
194
205
 
195
206
  // Near point
196
- let ppi = RC.screenPpi ? RC.screenPpi.value : 108
207
+ let ppi = RC.screenPpi ? RC.screenPpi.value : RC._CONST.N.PPI_DONT_USE
197
208
  if (!RC.screenPpi && trackingOptions.nearPoint)
198
209
  console.error(
199
210
  'Screen size measurement is required to get accurate near point tracking.'
@@ -216,16 +227,30 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
216
227
  })
217
228
  }
218
229
 
230
+ readyToGetFirstData = false
231
+ const {
232
+ desiredDistanceCm,
233
+ desiredDistanceTolerance,
234
+ desiredDistanceMonitor,
235
+ } = trackingOptions
236
+
219
237
  viewingDistanceTrackingFunction = async () => {
238
+ //
239
+ const videoTimestamp = new Date().getTime()
240
+ //
220
241
  faces = await model.estimateFaces(video)
221
242
  if (faces.length) {
222
243
  // There's at least one face in video
223
- const mesh = faces[0].scaledMesh
244
+ RC._tackingVideoFrameTimestamps.distance += videoTimestamp
224
245
  // https://github.com/tensorflow/tfjs-models/blob/master/facemesh/mesh_map.jpg
246
+ const mesh = faces[0].scaledMesh
247
+
225
248
  if (targetCount === distCount) {
226
249
  averageDist += eyeDist(mesh[133], mesh[362])
227
- averageDist /= 5
250
+ averageDist /= targetCount
251
+ RC._tackingVideoFrameTimestamps.distance /= targetCount
228
252
 
253
+ // TODO Add more samples for the first estimate
229
254
  if (stdDist.current !== null) {
230
255
  if (!stdFactor) {
231
256
  // ! First time estimate
@@ -235,11 +260,15 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
235
260
  // ! FINISH
236
261
  RC._removeBackground()
237
262
  RC._trackingSetupFinishedStatus.distance = true
263
+ readyToGetFirstData = true
238
264
  }
239
265
 
240
266
  /* -------------------------------------------------------------------------- */
241
267
 
242
268
  const timestamp = new Date()
269
+ const latency = Math.round(
270
+ timestamp.getTime() - RC._tackingVideoFrameTimestamps.distance
271
+ )
243
272
 
244
273
  const data = (RC.newViewingDistanceData = {
245
274
  value: toFixedNumber(
@@ -248,8 +277,16 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
248
277
  ),
249
278
  timestamp: timestamp,
250
279
  method: RC._CONST.VIEW_METHOD.F,
280
+ latencyMs: latency,
251
281
  })
252
282
 
283
+ if (readyToGetFirstData || desiredDistanceMonitor) {
284
+ // Check distance
285
+ if (desiredDistanceCm)
286
+ RC.checkDistance(desiredDistanceCm, desiredDistanceTolerance)
287
+ readyToGetFirstData = false
288
+ }
289
+
253
290
  /* -------------------------------------------------------------------------- */
254
291
 
255
292
  // Near point
@@ -262,7 +299,8 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
262
299
  mesh,
263
300
  averageDist,
264
301
  timestamp,
265
- ppi
302
+ ppi,
303
+ latency
266
304
  )
267
305
  }
268
306
 
@@ -274,6 +312,7 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
274
312
  value: {
275
313
  viewingDistanceCm: data.value,
276
314
  nearPointCm: nPData ? nPData.value : [null, null],
315
+ latencyMs: latency,
277
316
  },
278
317
  timestamp: timestamp,
279
318
  method: RC._CONST.VIEW_METHOD.F,
@@ -283,6 +322,8 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
283
322
 
284
323
  averageDist = 0
285
324
  distCount = 1
325
+
326
+ RC._tackingVideoFrameTimestamps.distance = 0
286
327
  } else {
287
328
  averageDist += eyeDist(mesh[133], mesh[362])
288
329
  ++distCount
@@ -291,7 +332,7 @@ const _tracking = async (RC, trackingOptions, callbackTrack) => {
291
332
  }
292
333
 
293
334
  iRepeatOptions.break = false
294
- iRepeatOptions.framerate = targetCount * trackingOptions.framerate // Default 3 * 5
335
+ iRepeatOptions.framerate = targetCount * trackingOptions.framerate // Default 5 * 3
295
336
  iRepeat(viewingDistanceTrackingFunction, iRepeatOptions)
296
337
  }
297
338
 
@@ -305,45 +346,42 @@ const _getNearPoint = (
305
346
  mesh,
306
347
  averageDist,
307
348
  timestamp,
308
- ppi
349
+ ppi,
350
+ latency
309
351
  ) => {
310
- let m = cyclopean(video, mesh[133], mesh[362])
311
- let offsetToVideoMid = [
312
- m[0] - video.videoWidth / 2,
313
- video.videoHeight / 2 - m[1],
314
- ]
315
-
316
- const videoFactor = video.videoHeight / video.clientHeight
317
- offsetToVideoMid.forEach((e, i) => {
318
- // Average interpupillary distance - 6.4cm
319
- offsetToVideoMid[i] =
320
- ((RC.PDCm ? RC.PDCm.value : 6.4) * e) /
321
- (averageDist * (videoFactor / 2)) /* Should this be videoFactor? */
352
+ let offsetToVideoCenter = cyclopean(video, mesh[133], mesh[362])
353
+ offsetToVideoCenter.forEach((offset, i) => {
354
+ // Average inter-pupillary distance - 6.4cm
355
+ offsetToVideoCenter[i] =
356
+ ((RC.PDCm ? RC.PDCm.value : RC._CONST.N.PD_DONT_USE) * offset) /
357
+ averageDist
322
358
  })
323
359
 
324
360
  let nPData = (RC.newNearPointData = {
325
361
  value: {
326
- x: toFixedNumber(offsetToVideoMid[0], trackingOptions.decimalPlace),
362
+ x: toFixedNumber(offsetToVideoCenter[0], trackingOptions.decimalPlace),
327
363
  y: toFixedNumber(
328
- offsetToVideoMid[1] + 0.5, // Commonly the webcam is 0.5cm above the screen
364
+ offsetToVideoCenter[1] + ((screen.height / 2) * 2.54) / ppi, // Commonly the webcam is 0.5cm above the screen
329
365
  trackingOptions.decimalPlace
330
366
  ),
367
+ latencyMs: latency,
331
368
  },
332
369
  timestamp: timestamp,
333
370
  })
334
371
 
335
372
  // SHOW
373
+ const dotR = 5
336
374
  if (trackingOptions.showNearPoint) {
337
375
  let offsetX = (nPData.value.x * ppi) / 2.54
338
376
  let offsetY = (nPData.value.y * ppi) / 2.54
339
377
  Object.assign(nearPointDot.style, {
340
- left: `${screen.width / 2 - window.screenLeft - 5 + offsetX}px`,
378
+ left: `${screen.width / 2 - window.screenLeft + offsetX - dotR}px`,
341
379
  top: `${
342
380
  screen.height / 2 -
343
381
  window.screenTop -
344
- 5 -
345
- (RC.isFullscreen.value ? 0 : 50) -
346
- offsetY
382
+ (window.outerHeight - window.innerHeight) -
383
+ offsetY -
384
+ dotR
347
385
  }px`,
348
386
  })
349
387
  }
@@ -355,6 +393,7 @@ RemoteCalibrator.prototype.pauseDistance = function () {
355
393
  if (this.gazeTracker.checkInitialized('distance', true)) {
356
394
  iRepeatOptions.break = true
357
395
  if (nearPointDot) nearPointDot.style.display = 'none'
396
+ this._tackingVideoFrameTimestamps.distance = 0
358
397
  return this
359
398
  }
360
399
  return null
@@ -364,6 +403,11 @@ RemoteCalibrator.prototype.resumeDistance = function () {
364
403
  if (this.gazeTracker.checkInitialized('distance', true)) {
365
404
  iRepeatOptions.break = false
366
405
  if (nearPointDot) nearPointDot.style.display = 'block'
406
+
407
+ averageDist = 0
408
+ distCount = 1
409
+ this._tackingVideoFrameTimestamps.distance = 0
410
+
367
411
  iRepeat(viewingDistanceTrackingFunction, iRepeatOptions)
368
412
  return this
369
413
  }
@@ -381,10 +425,17 @@ RemoteCalibrator.prototype.endDistance = function (endAll = false, _r = true) {
381
425
  trackingOptions.nearPoint = true
382
426
  trackingOptions.showNearPoint = false
383
427
 
428
+ trackingOptions.desiredDistanceCm = undefined
429
+ trackingOptions.desiredDistanceTolerance = 0.1
430
+ trackingOptions.desiredDistanceMonitor = false
431
+
384
432
  stdDist.current = null
385
433
  stdFactor = null
386
434
  viewingDistanceTrackingFunction = null
387
435
 
436
+ readyToGetFirstData = false
437
+ this._tackingVideoFrameTimestamps.distance = 0
438
+
388
439
  // Near point
389
440
  if (nearPointDot) {
390
441
  document.body.removeChild(nearPointDot)
@@ -411,6 +462,7 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
411
462
 
412
463
  let v = document.querySelector('#webgazerVideoFeed')
413
464
  let m = await this.gazeTracker.webgazer.getTracker().model
465
+ const videoTimestamp = new Date().getTime()
414
466
  let f = await m.estimateFaces(v)
415
467
 
416
468
  if (f.length) {
@@ -418,11 +470,15 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
418
470
  const dist = eyeDist(mesh[133], mesh[362])
419
471
 
420
472
  let timestamp = new Date()
473
+ //
474
+ const latency = timestamp.getTime() - videoTimestamp
475
+ //
421
476
 
422
477
  const data = (this.newViewingDistanceData = {
423
478
  value: toFixedNumber(stdFactor / dist, trackingOptions.decimalPlace),
424
479
  timestamp: timestamp,
425
480
  method: this._CONST.VIEW_METHOD.F,
481
+ latencyMs: latency,
426
482
  })
427
483
 
428
484
  let nPData
@@ -434,7 +490,8 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
434
490
  mesh,
435
491
  dist,
436
492
  timestamp,
437
- this.screenPpi ? this.screenPpi.value : 108
493
+ this.screenPpi ? this.screenPpi.value : this._CONST.N.PPI_DONT_USE,
494
+ latency
438
495
  )
439
496
  }
440
497
 
@@ -442,6 +499,7 @@ RemoteCalibrator.prototype.getDistanceNow = async function (callback = null) {
442
499
  value: {
443
500
  viewingDistanceCm: data.value,
444
501
  nearPointCm: nPData ? nPData.value : null,
502
+ latencyMs: latency,
445
503
  },
446
504
  timestamp: timestamp,
447
505
  method: this._CONST.VIEW_METHOD.F,
@@ -1,20 +1,20 @@
1
1
  import Swal from 'sweetalert2'
2
2
 
3
- import RemoteCalibrator from './core'
3
+ import RemoteCalibrator from '../core'
4
4
 
5
5
  import {
6
6
  blurAll,
7
7
  constructInstructions,
8
8
  safeExecuteFunc,
9
9
  toFixedNumber,
10
- } from './components/utils'
11
- import { swalInfoOptions } from './components/swalOptions'
12
- import Arrow from './media/arrow.svg'
13
- import PD from './media/pd.png?width=480&height=240'
14
- import { bindKeys, unbindKeys } from './components/keyBinder'
15
- import { addButtons } from './components/buttons'
16
- import { setDefaultVideoPosition } from './components/video'
17
- import { phrases } from './i18n'
10
+ } from '../components/utils'
11
+ import { swalInfoOptions } from '../components/swalOptions'
12
+ import Arrow from '../media/arrow.svg'
13
+ import PD from '../media/pd.png?width=480&height=240'
14
+ import { bindKeys, unbindKeys } from '../components/keyBinder'
15
+ import { addButtons } from '../components/buttons'
16
+ import { setDefaultVideoPosition } from '../components/video'
17
+ import { phrases } from '../i18n'
18
18
 
19
19
  // let selfVideo = false // No WebGazer video available and an extra video element needs to be created
20
20
 
@@ -40,7 +40,7 @@ RemoteCalibrator.prototype._measurePD = function (options = {}, callback) {
40
40
  {
41
41
  fullscreen: false,
42
42
  headline: '👁️ ' + phrases.RC_nearPointTitle[this.L],
43
- description: phrases.RC_nearPointIntroCaption[this.L],
43
+ description: phrases.RC_nearPointIntro[this.L],
44
44
  shortDescription: phrases.RC_nearPointIntro[this.L],
45
45
  },
46
46
  options
@@ -51,9 +51,11 @@ RemoteCalibrator.prototype._measurePD = function (options = {}, callback) {
51
51
  this._replaceBackground()
52
52
 
53
53
  this._replaceBackground(
54
- constructInstructions(options.headline, options.shortDescription)
54
+ constructInstructions(options.headline, options.shortDescription, true)
55
55
  )
56
- const screenPpi = this.screenPpi ? this.screenPpi.value : 108
56
+ const screenPpi = this.screenPpi
57
+ ? this.screenPpi.value
58
+ : this._CONST.N.PPI_DONT_USE
57
59
 
58
60
  let [videoWidth, videoHeight] = setupVideo(this)
59
61
  let [ruler, rulerListener] = setupRuler(
@@ -100,6 +100,8 @@ GazeTracker.prototype._init = function (
100
100
  // this.webgazer.saveDataAcrossSessions(false)
101
101
  this.webgazer.params.greedyLearner = greedyLearner
102
102
  this.webgazer.params.framerate = framerate
103
+ this.webgazer.params.getLatestVideoFrameTimestamp =
104
+ this._getLatestVideoTimestamp.bind(this)
103
105
  this.showGazer(showGazer)
104
106
  }
105
107
 
@@ -127,12 +129,15 @@ GazeTracker.prototype.checkInitialized = function (task, warning = false) {
127
129
  }
128
130
 
129
131
  GazeTracker.prototype.getData = function (d) {
132
+ let t = new Date()
130
133
  return {
131
134
  value: {
132
135
  x: toFixedNumber(d.x, this._toFixedN),
133
136
  y: toFixedNumber(d.y, this._toFixedN),
137
+ latencyMs:
138
+ t.getTime() - this.calibrator._tackingVideoFrameTimestamps.gaze, // latency
134
139
  },
135
- timestamp: new Date(),
140
+ timestamp: t,
136
141
  }
137
142
  }
138
143
 
@@ -156,6 +161,8 @@ GazeTracker.prototype.end = function (type, endAll = false) {
156
161
  this._endGaze()
157
162
  if (endEverything && this.checkInitialized('distance'))
158
163
  this.calibrator.endDistance(false, false)
164
+
165
+ this.calibrator._tackingVideoFrameTimestamps.gaze = 0
159
166
  } else {
160
167
  // Distance
161
168
  this.defaultDistanceTrackCallback = null
@@ -188,8 +195,16 @@ GazeTracker.prototype._endGaze = function () {
188
195
 
189
196
  this.webgazer.params.greedyLearner = false
190
197
  this.webgazer.params.framerate = 60
198
+
199
+ this.webgazer.params.getLatestVideoFrameTimestamp = () => {}
200
+ }
201
+
202
+ GazeTracker.prototype._getLatestVideoTimestamp = function (t) {
203
+ this.calibrator._tackingVideoFrameTimestamps.gaze = t.getTime()
191
204
  }
192
205
 
206
+ /* -------------------------------------------------------------------------- */
207
+
193
208
  GazeTracker.prototype.startStoringPoints = function () {
194
209
  this.webgazer.params.storingPoints = true
195
210
  }