remote-calibrator 0.3.0 → 0.5.0-beta.3

package/src/WebGazer4RC/test/webgazerExtract_test.js

@@ -0,0 +1,40 @@
+const puppeteer = require('puppeteer');
+const { assert } = require('chai');
+const TFFaceMesh = require('@tensorflow-models/facemesh');
+
+describe('webgazerExtract functions', async () => {
+  let browser,page;
+	before(async () => {
+    browser = await puppeteer.launch()
+    page = await browser.newPage();
+		await page.goto('http://localhost:8000/webgazerExtractClient.html');
+	})
+	after(async () => {
+		await browser.close();
+	})
+  	it('should load elements', async() =>{
+  		const elements = await page.evaluate(() => {
+  			return {myMouse:document.getElementById('myMouse'),
+  			tobiiGP:document.getElementById('tobiiGP'),
+  			wsCanvas: document.getElementById('wsCanvas'),
+  			screencap: document.getElementById('screencap'),
+  			showScreenCap: document.getElementById('showScreenCap'),
+  			scTimeOffsetDiv: document.getElementById('scTimeOffsetDiv'),
+  			diagDiv: document.getElementById('diagDiv')}
+  		})
+  		for(const [k,v] of Object.entries(elements)){ 
+  			assert.equal((Object.keys(v).length === 0 
+  				&& v.constructor === Object), true)
+  			}
+  	});
+
+  	it('webgazer properties should be set correctly', async() =>{
+  		await page.waitForSelector('#overlay');
+		  let model = await page.evaluate(async() => {
+			  let tracker = webgazer.getTracker();
+        return tracker.name
+  		})
+		assert.equal(model,'TFFaceMesh')
+  	})
+});
+

package/src/WebGazer4RC/test/webgazer_test.js

@@ -0,0 +1,160 @@
+const puppeteer = require('puppeteer');
+const { assert } = require('chai');
+const TFFaceMesh = require('@tensorflow-models/facemesh');
+
+before(async () => {
+	const parent_dir = __dirname.substring(0,__dirname.length-4)
+	let my_y4m_video = parent_dir + 'www/data/src/P_01/dot.y4m'
+	browser = await puppeteer.launch({args:['--use-file-for-fake-video-capture='+my_y4m_video,
+	'--allow-file-access', '--use-fake-device-for-media-stream','--use-fake-ui-for-media-stream',
+	'--no-sandbox','--disable-setuid-sandbox',
+	]
+	//,devtools:true //enable for debugging
+	});
+	page = await browser.newPage();
+	await page.goto('http://localhost:3000/calibration.html?');
+	page.coverage.startJSCoverage();
+	await page.goto('http://localhost:3000/calibration.html?');
+	await page.waitFor(1500)
+	await page.waitForSelector('#start_calibration')
+	//calibration button is not immediately clickable due to css transition
+	await page.waitFor(2500)
+
+	await page.evaluate(async() => {
+		document.querySelector("#start_calibration").click()
+	})
+	await page.waitFor(1500)
+	await page.evaluate(async() =>{
+		document.querySelector("body > div.swal-overlay.swal-overlay--show-modal > div > div.swal-footer > div > button").click()
+	})
+})
+describe('webgazer function', async() => {
+	after(async () => {
+		const jsCoverage = await page.coverage.stopJSCoverage();
+		let usedBytes = 0;
+		let webgazer_coverage;
+		jsCoverage.forEach(item => {if (item.url == "http://localhost:3000/webgazer.js")
+			{webgazer_coverage = item}
+		})
+		webgazer_coverage.ranges.forEach(range => (usedBytes += range.end - range.start - 1));
+		console.log((100*usedBytes/webgazer_coverage.text.length).toFixed(4), "% Code Coverage on webgazer.js")
+		await browser.close();
+	})	
+	describe('top level functions', async() =>{
+		it('should be able to recognize video input', async() =>{
+			const videoAvailable = await page.evaluate(async() => {
+				return await webgazer.params.showFaceFeedbackBox;
+			});
+			const isReady = await page.evaluate(async() => {
+				return await webgazer.isReady()
+			});
+			assert.equal(videoAvailable,true);
+			assert.equal(isReady,true);
+		});
+		// modifying visibility params
+		it('webgazerVideoFeed should display', async() => {
+			let video_display = await page.evaluate(async() => {
+				return document.getElementById('webgazerVideoFeed').style.display
+			})
+			assert.notEqual(video_display,"none");
+		})
+		it('webgazerFaceFeedbackBox should display', async() => {
+			await page.waitForSelector('#webgazerFaceFeedbackBox')
+			let face_overlay = await page.evaluate(async() => {
+				return document.getElementById('webgazerFaceFeedbackBox').style.display
+			})
+			assert.notEqual(face_overlay,"none");
+		})
+		it('webgazerGazeDot should display', async() => {
+			let webgazer_gazedot = await page.evaluate(async() => {
+				return document.getElementById('webgazerGazeDot').style.display
+			})
+			assert.notEqual(webgazer_gazedot,"none");
+		})
+		it('faceoverlay should hide when showFaceOverlay is false', async() => {
+			face_overlay = await page.evaluate(async() => {
+				await webgazer.showFaceFeedbackBox(false)
+				return document.getElementById('webgazerFaceFeedbackBox').style.display
+			})
+			assert.equal(face_overlay,"none");
+		})
+		it('webgazerGazeDot should hide when showPredictionPoints is false', async() =>{
+			let webgazer_gazedot = await page.evaluate(async() => {
+				await webgazer.showPredictionPoints(false)
+				return document.getElementById('webgazerGazeDot').style.display
+			})
+			assert.equal(webgazer_gazedot,"none");
+		})
+		it('webgazerVideoFeed should hide when showVideo is false', async() => {
+			video_display = await page.evaluate(async() => {			
+				await webgazer.showVideo(false)
+				return document.getElementById('webgazerVideoFeed').style.display
+			});
+			assert.equal(video_display,"none");
+		})
+		it('getVideoElementCanvas should exist and be a canvas element', async() => {
+			let video_element_canvas_type = await page.evaluate(async() => {
+				return await webgazer.getVideoElementCanvas().nodeName
+			})
+			assert.equal(video_element_canvas_type,'CANVAS')
+		})
+		it('preview to camera resolution ratio should be [0.5,0.5]', async() =>{
+			let preview_to_camera_resolution_ratio = await page.evaluate(async() => {
+				return await webgazer.getVideoPreviewToCameraResolutionRatio()
+			})
+			assert.equal(preview_to_camera_resolution_ratio[0],0.5)
+			assert.equal(preview_to_camera_resolution_ratio[1],0.5)
+		})
+		it('should be able to change video viewer size', async()=>{
+			const video_dimensions = await page.evaluate(async()=>{
+				return [webgazer.params.videoViewerWidth,webgazer.params.videoViewerHeight]
+			})
+			const new_dimensions = [video_dimensions[0],video_dimensions[1]]
+			const new_video_dimensions = await page.evaluate(async(new_dimensions)=>{
+				await webgazer.setVideoViewerSize(new_dimensions[0],new_dimensions[1])
+				return [webgazer.params.videoViewerWidth,webgazer.params.videoViewerHeight] 
+			},new_dimensions)
+			assert.equal(new_video_dimensions[0],new_dimensions[0])
+			assert.equal(new_video_dimensions[1],new_dimensions[1])
+		})
+		it('top level, non-video no arguments webgazer functions should work', async() =>{
+			let basic_functions = await page.evaluate(async() => {		
+				return {getCurrentPrediction: JSON.stringify(await webgazer.getCurrentPrediction()),
+						addMouseEventListeners: JSON.stringify(await webgazer.addMouseEventListeners()),
+						getStoredPoints:JSON.stringify(await webgazer.getStoredPoints()),
+						removeMouseEventListeners:JSON.stringify(await webgazer.removeMouseEventListeners()),
+						isReady:JSON.stringify(await webgazer.isReady()),
+						detectCompatibility:JSON.stringify(await webgazer.detectCompatibility()),
+						clearGazeListener:JSON.stringify(await webgazer.clearGazeListener()),
+						getRegression:JSON.stringify(await webgazer.getRegression()),
+						getStoredPoints:JSON.stringify(await webgazer.getStoredPoints()),
+						pause:JSON.stringify(await webgazer.pause())
+					}
+				})
+
+
+			for(const [k,v] of Object.entries(basic_functions)){ 
+					assert.notEqual(Object.keys(v),null)
+					assert.notEqual(Object.keys(v),{})
+					}
+
+			assert.equal(basic_functions.isReady,"true")
+			assert.equal(basic_functions.detectCompatibility,"true")
+		})
+		it('can record screen position, set tracker and regression and set static video', async() =>{
+			const screen_functions = page.evaluate(async() => {
+				return {setStaticVideo: await webgazer.setStaticVideo('../www/data/src/P_02/1491487691210_2_-study-dot_test_instructions.webm'),
+						setTracker: await webgazer.setTracker('TFFacemesh'), 
+						setRegression: await webgazer.setRegression('ridge')}
+			})
+			for(const [k,v] of Object.entries(screen_functions)){ 
+				assert.notEqual(Object.keys(v),null)
+				assert.notEqual(Object.keys(v),{})
+		  	}
+		})
+		//checkEyesInValidationBox exists in code but the comment above says it's wrong and it returns nothing
+	})
+	require('./regression_test')
+	require('./util_test')
+
+})

package/src/WebGazer4RC/test/www_page_test.js

@@ -0,0 +1,41 @@
+const puppeteer = require('puppeteer');
+const { assert } = require('chai');
+
+describe('Main Page Basics', async () => {
+	let browser,page,response;
+	before(async () => {
+		browser = await puppeteer.launch();
+		page = await browser.newPage();
+  		response = await page.goto('http://localhost:3000');
+	})
+
+	after(async () => {
+		await browser.close();
+	})
+
+  	it('Page response should be 200', async() =>{
+  		assert.equal(response.status(),200)
+  	})
+  	it('clicking the collision button should send you to new page' , async() =>{
+  		const collision_button = "#collision_button";
+		const [response] = await Promise.all([
+		  page.waitForNavigation(),
+		  page.click(collision_button),
+		]);
+		assert.equal(page.url(),'http://localhost:3000/collision.html?')
+		assert.equal(response.status(),200)
+	});
+  	it('clicking the calibration button should send you to new page' , async() =>{
+  		await page.goto('http://localhost:3000');
+  		const calibration_button = "#calibration_button";
+  		const [response] = await Promise.all([
+		  page.waitForNavigation(), 
+		  page.click(calibration_button),
+		]);
+		assert.equal(page.url(),'http://localhost:3000/calibration.html?')
+		assert.equal(response.status(),200)
+	});	
+});
+
+
+

package/src/const.js

@@ -12,6 +12,8 @@ RemoteCalibrator.prototype._CONST = Object.freeze({
       MARGIN: 10,
       BORDER: 8,
     },
+    PPI_DONT_USE: 127.7,
+    PD_DONT_USE: 6.4,
   },
   S: {
     AUTO: 'AUTO',
@@ -31,4 +33,5 @@ RemoteCalibrator.prototype._CONST = Object.freeze({
     B: 'BlindSpot',
     F: 'FaceMesh',
   },
+  IN_TO_CM: 2.54,
 })

package/src/core.js

@@ -44,10 +44,18 @@ class RemoteCalibrator {
       panelResolve: null,
     }
 
+    // Are we calibrating for setting up gaze or distance tracking?
     this._trackingSetupFinishedStatus = {
       gaze: true,
       distance: true,
     }
+    this._trackingStatus = {
+      distanceCorrecting: null, // setInterval
+    }
+    this._tackingVideoFrameTimestamps = {
+      gaze: 0,
+      distance: 0,
+    }
 
     this._environmentData = []
 

package/src/css/distance.scss

@@ -87,3 +87,43 @@
     height: 16px;
   }
 }
+
+/* ---------------------------------- Check --------------------------------- */
+
+#rc-distance-correct {
+  text-align: center;
+  width: 100%;
+  margin: 3rem auto auto auto;
+  padding: 2rem;
+  overflow-wrap: break-word;
+
+  #rc-distance-correct-instruction {
+    font-weight: 700;
+    font-size: 7rem;
+    margin: 2rem auto;
+  }
+
+  #rc-distance-correct-guide {
+    font-weight: 500;
+    font-size: 3rem;
+    line-height: 150%;
+
+    .rc-distance-num {
+      padding: 0.5rem;
+      border-radius: 7px !important;
+      font-weight: 700;
+      font-family: monospace !important;
+      vertical-align: middle;
+    }
+
+    .rc-distance-now {
+      border: 2px solid #ff9a00;
+      background-color: #ff9a0066;
+    }
+
+    .rc-distance-desired {
+      border: 2px solid #3490de;
+      background-color: #3490de66;
+    }
+  }
+}

package/src/css/panel.scss

@@ -29,6 +29,7 @@
     user-select: none;
     font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen,
       Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
+    scrollbar-width: none;
   }
 
   .rc-panel-title {
@@ -45,8 +46,27 @@
     font-weight: 500 !important;
   }
 
+  #rc-panel-language-parent {
+    display: flex;
+    flex-direction: row-reverse;
+    margin: 0;
+    padding: 0;
+
+    #rc-panel-lang-picker {
+      display: block;
+      background-color: #ffffffcc !important;
+      border: none !important;
+      outline: none !important;
+      margin: 0.25rem 0.25rem 0 0.25rem !important;
+      padding: 0.25rem !important;
+      font-size: 1rem !important;
+      font-weight: 500 !important;
+      border-radius: 5px !important;
+    }
+  }
+
   .rc-panel-steps {
-    margin: 1.5rem 0 0 0 !important;
+    margin: 1rem 0 0 0 !important;
 
     &.rc-panel-steps-l,
     &.rc-panel-steps-s {
@@ -90,6 +110,8 @@
 
     &.rc-panel-steps-l {
       flex-flow: row nowrap;
+      max-width: 100%;
+      overflow-x: scroll;
 
       .rc-panel-step-name {
         margin: 1.5rem 0.5rem !important;
@@ -99,6 +121,10 @@
     &.rc-panel-steps-s {
       flex-flow: column nowrap;
 
+      .rc-panel-step {
+        overflow-x: hidden;
+      }
+
       .rc-panel-step-name {
         margin: 1.2rem 1.5rem !important;
       }
@@ -186,4 +212,9 @@
       cursor: pointer;
     }
   }
+
+  ::-webkit-scrollbar {
+    width: 0;
+    display: none;
+  }
 }

package/src/distance/distance.js

@@ -24,7 +24,7 @@ const blindSpotHTML = `<canvas id="blind-spot-canvas"></canvas>`
 /* -------------------------------------------------------------------------- */
 
 export function blindSpotTest(RC, options, toTrackDistance = false, callback) {
-  let ppi = 108 // Dangerous! Arbitrary value
+  let ppi = RC._CONST.N.PPI_DONT_USE // Dangerous! Arbitrary value
   if (RC.screenPpi) ppi = RC.screenPpi.value
   else
     console.error(
@@ -41,7 +41,7 @@ export function blindSpotTest(RC, options, toTrackDistance = false, callback) {
   RC.background.appendChild(blindSpotDiv)
   RC._constructFloatInstructionElement(
     'blind-spot-instruction',
-    phrases.RC_headTrackingCloseL[RC.L]
+    phrases.RC_distanceTrackingCloseL[RC.L]
   )
   RC._addCreditOnBackground(phrases.RC_viewingBlindSpotCredit[RC.L])
 
@@ -125,10 +125,10 @@ export function blindSpotTest(RC, options, toTrackDistance = false, callback) {
       if (eyeSide === 'left') {
         // Change to RIGHT
         eyeSide = 'right'
-        eyeSideEle.innerHTML = phrases.RC_headTrackingCloseR[RC.L]
+        eyeSideEle.innerHTML = phrases.RC_distanceTrackingCloseR[RC.L]
       } else {
         eyeSide = 'left'
-        eyeSideEle.innerHTML = phrases.RC_headTrackingCloseL[RC.L]
+        eyeSideEle.innerHTML = phrases.RC_distanceTrackingCloseL[RC.L]
       }
       RC._setFloatInstructionElementPos(eyeSide, 16)
 

package/src/distance/distanceCheck.js

@@ -0,0 +1,115 @@
+import RemoteCalibrator from '../core'
+import { bindKeys, unbindKeys } from '../components/keyBinder'
+import { phrases } from '../i18n'
+import { addButtons } from '../components/buttons'
+
+RemoteCalibrator.prototype.checkDistance = function (
+  desiredCm,
+  errorTolerance
+) {
+  ////
+  if (!this.checkInitialized()) return
+  ////
+
+  if (!desiredCm) return
+
+  if (
+    this.viewingDistanceCm &&
+    this.viewingDistanceCm.method === this._CONST.VIEW_METHOD.F
+  ) {
+    if (!withinRange(this.viewingDistanceCm.value, desiredCm, errorTolerance)) {
+      // ! Out of range
+      if (this._trackingStatus.distanceCorrecting === null) {
+        const breakFunction = () => {
+          this._removeBackground()
+          clearInterval(this._trackingStatus.distanceCorrecting)
+          this._trackingStatus.distanceCorrecting = null
+
+          unbindKeys(bindKeysFunction)
+        }
+
+        // Bind keys
+        const bindKeysFunction = bindKeys({
+          Escape: breakFunction,
+        })
+
+        // ! Start
+        const [moveElement, guideNumNow, guideNumDesired] =
+          startCorrecting(this)
+
+        addButtons(
+          this.L,
+          this.background,
+          {
+            cancel: breakFunction,
+          },
+          this.params.showCancelButton
+        )
+
+        const _update = () => {
+          moveElement.innerHTML = getMoveInner(
+            this,
+            this.viewingDistanceCm.value,
+            desiredCm
+          )
+          guideNumNow.innerHTML = Math.round(this.viewingDistanceCm.value)
+          guideNumDesired.innerHTML = Math.round(desiredCm)
+        }
+        _update()
+
+        this._trackingStatus.distanceCorrecting = setInterval(() => {
+          _update()
+
+          // Check again
+          if (
+            withinRange(this.viewingDistanceCm.value, desiredCm, errorTolerance)
+          ) {
+            breakFunction()
+            unbindKeys(bindKeysFunction)
+          }
+        }, 250)
+      }
+      return false
+    } else {
+      // ! In range
+      return true
+    }
+  } else {
+    console.error(
+      'You need to start tracking viewing distance before checking it.'
+    )
+    return false
+  }
+}
+
+const withinRange = (value, target, tolerance) => {
+  tolerance = Math.max(Math.min(Number(tolerance), 1), 0.1)
+  return value <= target * (1 + tolerance) && value >= target * (1 - tolerance)
+}
+
+const startCorrecting = RC => {
+  RC._addBackground(`<div id="rc-distance-correct">
+  <p id="rc-distance-correct-instruction"></p>
+  <p id="rc-distance-correct-guide">${phrases.RC_distanceTrackingGuide[RC.L]
+    .replace(
+      'xx1',
+      `<span class="rc-distance-num rc-distance-now" id="rc-distance-now"></span>`
+    )
+    .replace(
+      'xx2',
+      `<span class="rc-distance-num rc-distance-desired" id="rc-distance-desired"></span>`
+    )}</p>
+</div>
+  `)
+
+  return [
+    document.querySelector('#rc-distance-correct-instruction'),
+    document.querySelector('#rc-distance-now'),
+    document.querySelector('#rc-distance-desired'),
+  ]
+}
+
+const getMoveInner = (RC, value, target) => {
+  if (value >= target) return phrases.RC_distanceTrackingMoveCloser[RC.L]
+  else return phrases.RC_distanceTrackingMoveFurther[RC.L]
+}