remote-calibrator 0.2.1 → 0.2.2-beta.0

package/src/WebGazer4RC/src/ridgeWeightedReg.mjs

@@ -0,0 +1,125 @@
+import util from './util.mjs';
+import util_regression from './util_regression.mjs';
+import params from './params.mjs'
+
+const reg = {};
+
+/**
+ * Constructor of RidgeWeightedReg object
+ * @constructor
+ */
+reg.RidgeWeightedReg = function() {
+    this.init();
+};
+
+/**
+ * Initialize new arrays and initialize Kalman filter.
+ */
+reg.RidgeWeightedReg.prototype.init = util_regression.InitRegression
+
+/**
+ * Add given data from eyes
+ * @param {Object} eyes - eyes where extract data to add
+ * @param {Object} screenPos - The current screen point
+ * @param {Object} type - The type of performed action
+ */
+reg.RidgeWeightedReg.prototype.addData = util_regression.addData
+
+/**
+ * Try to predict coordinates from pupil data
+ * after apply linear regression on data set
+ * @param {Object} eyesObj - The current user eyes object
+ * @returns {Object}
+ */
+reg.RidgeWeightedReg.prototype.predict = function(eyesObj) {
+    if (!eyesObj || this.eyeFeaturesClicks.length === 0) {
+        return null;
+    }
+    var acceptTime = performance.now() - this.trailTime;
+    var trailX = [];
+    var trailY = [];
+    var trailFeat = [];
+    for (var i = 0; i < this.trailDataWindow; i++) {
+        if (this.trailTimes.get(i) > acceptTime) {
+            trailX.push(this.screenXTrailArray.get(i));
+            trailY.push(this.screenYTrailArray.get(i));
+            trailFeat.push(this.eyeFeaturesTrail.get(i));
+        }
+    }
+
+    var len = this.eyeFeaturesClicks.data.length;
+    var weightedEyeFeats = Array(len);
+    var weightedXArray = Array(len);
+    var weightedYArray = Array(len);
+    for (var i = 0; i < len; i++) {
+        var weight = Math.sqrt( 1 / (len - i) ); // access from oldest to newest so should start with low weight and increase steadily
+        //abstraction is leaking...
+        var trueIndex = this.eyeFeaturesClicks.getTrueIndex(i);
+        for (var j = 0; j < this.eyeFeaturesClicks.data[trueIndex].length; j++) {
+            var val = this.eyeFeaturesClicks.data[trueIndex][j] * weight;
+            if (weightedEyeFeats[trueIndex] !== undefined){
+                weightedEyeFeats[trueIndex].push(val);
+            } else {
+                weightedEyeFeats[trueIndex] = [val];
+            }
+        }
+        weightedXArray[trueIndex] = this.screenXClicksArray.get(i).slice(0, this.screenXClicksArray.get(i).length);
+        weightedYArray[trueIndex] = this.screenYClicksArray.get(i).slice(0, this.screenYClicksArray.get(i).length);
+        weightedXArray[i][0] = weightedXArray[i][0] * weight;
+        weightedYArray[i][0] = weightedYArray[i][0] * weight;
+    }
+
+    var screenXArray = weightedXArray.concat(trailX);
+    var screenYArray = weightedYArray.concat(trailY);
+    var eyeFeatures = weightedEyeFeats.concat(trailFeat);
+
+    var coefficientsX = util_regression.ridge(screenXArray, eyeFeatures, this.ridgeParameter);
+    var coefficientsY = util_regression.ridge(screenYArray, eyeFeatures, this.ridgeParameter);
+
+    var eyeFeats = util.getEyeFeats(eyesObj);
+    var predictedX = 0;
+    for(var i=0; i< eyeFeats.length; i++){
+        predictedX += eyeFeats[i] * coefficientsX[i];
+    }
+    var predictedY = 0;
+    for(var i=0; i< eyeFeats.length; i++){
+        predictedY += eyeFeats[i] * coefficientsY[i];
+    }
+
+    predictedX = Math.floor(predictedX);
+    predictedY = Math.floor(predictedY);
+
+    if (params.applyKalmanFilter) {
+        // Update Kalman model, and get prediction
+        var newGaze = [predictedX, predictedY]; // [20200607 xk] Should we use a 1x4 vector?
+        newGaze = this.kalman.update(newGaze);
+
+        return {
+            x: newGaze[0],
+            y: newGaze[1]
+        };
+    } else {
+        return {
+            x: predictedX,
+            y: predictedY
+        };
+    }
+};
+
+reg.RidgeWeightedReg.prototype.setData = util_regression.setData;
+
+/**
+ * Return the data
+ * @returns {Array.<Object>|*}
+ */
+reg.RidgeWeightedReg.prototype.getData = function() {
+    return this.dataClicks.data;
+};
+
+/**
+ * The RidgeWeightedReg object name
+ * @type {string}
+ */
+reg.RidgeWeightedReg.prototype.name = 'ridgeWeighted';
+
+export default reg;

package/src/WebGazer4RC/src/ridgeWorker.mjs

@@ -0,0 +1,135 @@
+'use strict';
+
+console.log('thread starting');
+
+// Add src/util.mjs and src/mat.mjs to the same directory as your html file
+importScripts('./worker_scripts/util.js', './worker_scripts/mat.js'); // [20200708] Figure out how to make all of this wrap up neatly
+var ridgeParameter = Math.pow(10,-5);
+var resizeWidth = 10;
+var resizeHeight = 6;
+var dataWindow = 700;
+var trailDataWindow = 10;
+var trainInterval = 500;
+
+var screenXClicksArray = new self.webgazer.util.DataWindow(dataWindow);
+var screenYClicksArray = new self.webgazer.util.DataWindow(dataWindow);
+var eyeFeaturesClicks = new self.webgazer.util.DataWindow(dataWindow);
+var dataClicks = new self.webgazer.util.DataWindow(dataWindow);
+
+var screenXTrailArray = new self.webgazer.util.DataWindow(trailDataWindow);
+var screenYTrailArray = new self.webgazer.util.DataWindow(trailDataWindow);
+var eyeFeaturesTrail = new self.webgazer.util.DataWindow(trailDataWindow);
+var dataTrail = new self.webgazer.util.DataWindow(trailDataWindow);
+
+/**
+ * Performs ridge regression, according to the Weka code.
+ * @param {Array} y - corresponds to screen coordinates (either x or y) for each of n click events
+ * @param {Array.<Array.<Number>>} X - corresponds to gray pixel features (120 pixels for both eyes) for each of n clicks
+ * @param {Array} k - ridge parameter
+ * @return{Array} regression coefficients
+ */
+function ridge(y, X, k){
+    var nc = X[0].length;
+    var m_Coefficients = new Array(nc);
+    var xt = self.webgazer.mat.transpose(X);
+    var solution = new Array();
+    var success = true;
+    do{
+        var ss = self.webgazer.mat.mult(xt,X);
+        // Set ridge regression adjustment
+        for (var i = 0; i < nc; i++) {
+            ss[i][i] = ss[i][i] + k;
+        }
+
+        // Carry out the regression
+        var bb = self.webgazer.mat.mult(xt,y);
+        for(var i = 0; i < nc; i++) {
+            m_Coefficients[i] = bb[i][0];
+        }
+        try{
+            var n = (m_Coefficients.length !== 0 ? m_Coefficients.length/m_Coefficients.length: 0);
+            if (m_Coefficients.length*n !== m_Coefficients.length){
+                console.log('Array length must be a multiple of m')
+            }
+            solution = (ss.length === ss[0].length ? (self.webgazer.mat.LUDecomposition(ss,bb)) : (self.webgazer.mat.QRDecomposition(ss,bb)));
+
+            for (var i = 0; i < nc; i++){
+                m_Coefficients[i] = solution[i][0];
+            }
+            success = true;
+        }
+        catch (ex){
+            k *= 10;
+            console.log(ex);
+            success = false;
+        }
+    } while (!success);
+    return m_Coefficients;
+}
+
+//TODO: still usefull ???
+/**
+ *
+ * @returns {Number}
+ */
+function getCurrentFixationIndex() {
+    var index = 0;
+    var recentX = this.screenXTrailArray.get(0);
+    var recentY = this.screenYTrailArray.get(0);
+    for (var i = this.screenXTrailArray.length - 1; i >= 0; i--) {
+        var currX = this.screenXTrailArray.get(i);
+        var currY = this.screenYTrailArray.get(i);
+        var euclideanDistance = Math.sqrt(Math.pow((currX-recentX),2)+Math.pow((currY-recentY),2));
+        if (euclideanDistance > 72){
+            return i+1;
+        }
+    }
+    return i;
+}
+
+/**
+ * Event handler, it store screen position to allow training
+ * @param {Event} event - the receive event
+ */
+self.onmessage = function(event) {
+    var data = event.data;
+    var screenPos = data['screenPos'];
+    var eyes = data['eyes'];
+    var type = data['type'];
+    if (type === 'click') {
+        self.screenXClicksArray.push([screenPos[0]]);
+        self.screenYClicksArray.push([screenPos[1]]);
+
+        self.eyeFeaturesClicks.push(eyes);
+    } else if (type === 'move') {
+        self.screenXTrailArray.push([screenPos[0]]);
+        self.screenYTrailArray.push([screenPos[1]]);
+
+        self.eyeFeaturesTrail.push(eyes);
+        self.dataTrail.push({'eyes':eyes, 'screenPos':screenPos, 'type':type});
+    }
+    self.needsTraining = true;
+};
+
+/**
+ * Compute coefficient from training data
+ */
+function retrain() {
+    if (self.screenXClicksArray.length === 0) {
+        return;
+    }
+    if (!self.needsTraining) {
+        return;
+    }
+    var screenXArray = self.screenXClicksArray.data.concat(self.screenXTrailArray.data);
+    var screenYArray = self.screenYClicksArray.data.concat(self.screenYTrailArray.data);
+    var eyeFeatures = self.eyeFeaturesClicks.data.concat(self.eyeFeaturesTrail.data);
+
+    var coefficientsX = ridge(screenXArray, eyeFeatures, ridgeParameter);
+    var coefficientsY = ridge(screenYArray, eyeFeatures, ridgeParameter);
+    self.postMessage({'X':coefficientsX, 'Y': coefficientsY});
+    self.needsTraining = false;
+}
+
+setInterval(retrain, trainInterval);
+

package/src/WebGazer4RC/src/util.mjs

@@ -0,0 +1,348 @@
+const util = {};
+
+var resizeWidth = 10;
+var resizeHeight = 6;
+
+//not used !?
+/**
+ * Eye class, represents an eye patch detected in the video stream
+ * @param {ImageData} patch - the image data corresponding to an eye
+ * @param {Number} imagex - x-axis offset from the top-left corner of the video canvas
+ * @param {Number} imagey - y-axis offset from the top-left corner of the video canvas
+ * @param {Number} width  - width of the eye patch
+ * @param {Number} height - height of the eye patch
+ */
+util.Eye = function(patch, imagex, imagey, width, height) {
+    this.patch = patch;
+    this.imagex = imagex;
+    this.imagey = imagey;
+    this.width = width;
+    this.height = height;
+};
+
+/**
+ * Compute eyes size as gray histogram
+ * @param {Object} eyes - The eyes where looking for gray histogram
+ * @returns {Array.<T>} The eyes gray level histogram
+ */
+util.getEyeFeats = function(eyes) {
+    var resizedLeft = this.resizeEye(eyes.left, resizeWidth, resizeHeight);
+    var resizedRight = this.resizeEye(eyes.right, resizeWidth, resizeHeight);
+    
+    var leftGray = this.grayscale(resizedLeft.data, resizedLeft.width, resizedLeft.height);
+    var rightGray = this.grayscale(resizedRight.data, resizedRight.width, resizedRight.height);
+
+    var histLeft = [];
+    this.equalizeHistogram(leftGray, 5, histLeft);
+    var histRight = [];
+    this.equalizeHistogram(rightGray, 5, histRight);
+
+    // var leftGrayArray = Array.prototype.slice.call(histLeft);
+    // var rightGrayArray = Array.prototype.slice.call(histRight);
+
+    return histLeft.concat(histRight);
+}
+//Data Window class
+//operates like an array but 'wraps' data around to keep the array at a fixed windowSize
+/**
+ * DataWindow class - Operates like an array, but 'wraps' data around to keep the array at a fixed windowSize
+ * @param {Number} windowSize - defines the maximum size of the window
+ * @param {Array} data - optional data to seed the DataWindow with
+ **/
+util.DataWindow = function(windowSize, data) {
+    this.data = [];
+    this.windowSize = windowSize;
+    this.index = 0;
+    this.length = 0;
+    if(data){
+        this.data = data.slice(data.length-windowSize,data.length);
+        this.length = this.data.length;
+    }
+};
+
+/**
+ * [push description]
+ * @param  {*} entry - item to be inserted. It either grows the DataWindow or replaces the oldest item
+ * @return {DataWindow} this
+ */
+util.DataWindow.prototype.push = function(entry) {
+    if (this.data.length < this.windowSize) {
+        this.data.push(entry);
+        this.length = this.data.length;
+        return this;
+    }
+
+    //replace oldest entry by wrapping around the DataWindow
+    this.data[this.index] = entry;
+    this.index = (this.index + 1) % this.windowSize;
+    return this;
+};
+
+/**
+ * Get the element at the ind position by wrapping around the DataWindow
+ * @param  {Number} ind index of desired entry
+ * @return {*}
+ */
+util.DataWindow.prototype.get = function(ind) {
+    return this.data[this.getTrueIndex(ind)];
+};
+
+/**
+ * Gets the true this.data array index given an index for a desired element
+ * @param {Number} ind - index of desired entry
+ * @return {Number} index of desired entry in this.data
+ */
+util.DataWindow.prototype.getTrueIndex = function(ind) {
+    if (this.data.length < this.windowSize) {
+        return ind;
+    } else {
+        //wrap around ind so that we can traverse from oldest to newest
+        return (ind + this.index) % this.windowSize;
+    }
+};
+
+/**
+ * Append all the contents of data
+ * @param {Array} data - to be inserted
+ */
+util.DataWindow.prototype.addAll = function(data) {
+    for (var i = 0; i < data.length; i++) {
+        this.push(data[i]);
+    }
+};
+
+
+//Helper functions
+/**
+ * Grayscales an image patch. Can be used for the whole canvas, detected face, detected eye, etc.
+ *
+ * Code from tracking.js by Eduardo Lundgren, et al.
+ * https://github.com/eduardolundgren/tracking.js/blob/master/src/tracking.js
+ *
+ * Software License Agreement (BSD License) Copyright (c) 2014, Eduardo A. Lundgren Melo. All rights reserved.
+ * Redistribution and use of this software in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+ * The name of Eduardo A. Lundgren Melo may not be used to endorse or promote products derived from this software without specific prior written permission of Eduardo A. Lundgren Melo.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @param  {Array} pixels - image data to be grayscaled
+ * @param  {Number} width  - width of image data to be grayscaled
+ * @param  {Number} height - height of image data to be grayscaled
+ * @return {Array} grayscaledImage
+ */
+util.grayscale = function(pixels, width, height){
+    var gray = new Uint8ClampedArray(pixels.length >> 2);
+    var p = 0;
+    var w = 0;
+    for (var i = 0; i < height; i++) {
+        for (var j = 0; j < width; j++) {
+            var value = pixels[w] * 0.299 + pixels[w + 1] * 0.587 + pixels[w + 2] * 0.114;
+            gray[p++] = value;
+
+            w += 4;
+        }
+    }
+    return gray;
+};
+
+/**
+ * Increase contrast of an image.
+ *
+ * Code from Martin Tschirsich, Copyright (c) 2012.
+ * https://github.com/mtschirs/js-objectdetect/blob/gh-pages/js/objectdetect.js
+ *
+ * @param {Array} src - grayscale integer array
+ * @param {Number} step - sampling rate, control performance
+ * @param {Array} dst - array to hold the resulting image
+ */
+util.equalizeHistogram = function(src, step, dst) {
+    var srcLength = src.length;
+    if (!dst) dst = src;
+    if (!step) step = 5;
+
+    // Compute histogram and histogram sum:
+    var hist = Array(256).fill(0);
+
+    for (var i = 0; i < srcLength; i += step) {
+        ++hist[src[i]];
+    }
+
+    // Compute integral histogram:
+    var norm = 255 * step / srcLength,
+        prev = 0;
+    for (var j = 0; j < 256; ++j) {
+        var h = hist[j];
+        prev = h += prev;
+        hist[j] = h * norm; // For non-integer src: ~~(h * norm + 0.5);
+    }
+
+    // Equalize image:
+    for (var k = 0; k < srcLength; ++k) {
+        dst[k] = hist[src[k]];
+    }
+    return dst;
+};
+
+//not used !?
+util.threshold = function(data, threshold) {
+    for (let i = 0; i < data.length; i++) {
+        data[i] = (data[i] > threshold) ? 255 : 0;
+    }
+    return data;
+};
+
+//not used !?
+util.correlation = function(data1, data2) {
+    const length = Math.min(data1.length, data2.length);
+    let count = 0;
+    for (let i = 0; i < length; i++) {
+        if (data1[i] === data2[i]) {
+            count++;
+        }
+    }
+    return count / Math.max(data1.length, data2.length);
+};
+
+/**
+ * Gets an Eye object and resizes it to the desired resolution
+ * @param  {webgazer.util.Eye} eye - patch to be resized
+ * @param  {Number} resizeWidth - desired width
+ * @param  {Number} resizeHeight - desired height
+ * @return {webgazer.util.Eye} resized eye patch
+ */
+util.resizeEye = function(eye, resizeWidth, resizeHeight) {
+
+    var canvas = document.createElement('canvas');
+    canvas.width = eye.width;
+    canvas.height = eye.height;
+
+    canvas.getContext('2d').putImageData(eye.patch,0,0);
+
+    var tempCanvas = document.createElement('canvas');
+
+    tempCanvas.width = resizeWidth;
+    tempCanvas.height = resizeHeight;
+
+    // save the canvas into temp canvas
+    tempCanvas.getContext('2d').drawImage(canvas, 0, 0, canvas.width, canvas.height, 0, 0, resizeWidth, resizeHeight);
+
+    return tempCanvas.getContext('2d').getImageData(0, 0, resizeWidth, resizeHeight);
+};
+
+/**
+ * Checks if the prediction is within the boundaries of the viewport and constrains it
+ * @param  {Array} prediction [x,y] - predicted gaze coordinates
+ * @return {Array} constrained coordinates
+ */
+util.bound = function(prediction){
+    prediction.x = _constrain(prediction.x, 0, window.innerWidth);
+    prediction.y = _constrain(prediction.y, 0, window.innerHeight);
+    return prediction
+};
+
+function _constrain(a, b0, b1) {
+    return (a < b0) ? b0 : (a > b1) ? b1 : a
+}
+
+//not used !?
+/**
+ * Write statistics in debug paragraph panel
+ * @param {HTMLElement} para - The <p> tag where write data
+ * @param {Object} stats - The stats data to output
+ */
+function debugBoxWrite(para, stats) {
+    var str = '';
+    for (var key in stats) {
+        str += key + ': ' + stats[key] + '\n';
+    }
+    para.innerText = str;
+}
+
+//not used !?
+/**
+ * Constructor of DebugBox object,
+ * it insert an paragraph inside a div to the body, in view to display debug data
+ * @param {Number} interval - The log interval
+ * @constructor
+ */
+util.DebugBox = function(interval) {
+    this.para = document.createElement('p');
+    this.div = document.createElement('div');
+    this.div.appendChild(this.para);
+    document.body.appendChild(this.div);
+
+    this.buttons = {};
+    this.canvas = {};
+    this.stats = {};
+    var updateInterval = interval || 300;
+    (function(localThis) {
+        setInterval(function() {
+            debugBoxWrite(localThis.para, localThis.stats);
+        }, updateInterval);
+    }(this));
+};
+
+//not used !?
+/**
+ * Add stat data for log
+ * @param {String} key - The data key
+ * @param {*} value - The value
+ */
+util.DebugBox.prototype.set = function(key, value) {
+    this.stats[key] = value;
+};
+
+//not used !?
+/**
+ * Initialize stats in case where key does not exist, else
+ * increment value for key
+ * @param {String} key - The key to process
+ * @param {Number} incBy - Value to increment for given key (default: 1)
+ * @param {Number} init - Initial value in case where key does not exist (default: 0)
+ */
+util.DebugBox.prototype.inc = function(key, incBy, init) {
+    if (!this.stats[key]) {
+        this.stats[key] = init || 0;
+    }
+    this.stats[key] += incBy || 1;
+};
+
+//not used !?
+/**
+ * Create a button and register the given function to the button click event
+ * @param {String} name - The button name to link
+ * @param {Function} func - The onClick callback
+ */
+util.DebugBox.prototype.addButton = function(name, func) {
+    if (!this.buttons[name]) {
+        this.buttons[name] = document.createElement('button');
+        this.div.appendChild(this.buttons[name]);
+    }
+    var button = this.buttons[name];
+    this.buttons[name] = button;
+    button.addEventListener('click', func);
+    button.innerText = name;
+};
+
+//not used !?
+/**
+ * Search for a canvas elemenet with name, or create on if not exist.
+ * Then send the canvas element as callback parameter.
+ * @param {String} name - The canvas name to send/create
+ * @param {Function} func - The callback function where send canvas
+ */
+util.DebugBox.prototype.show = function(name, func) {
+    if (!this.canvas[name]) {
+        this.canvas[name] = document.createElement('canvas');
+        this.div.appendChild(this.canvas[name]);
+    }
+    var canvas = this.canvas[name];
+    canvas.getContext('2d').clearRect(0,0, canvas.width, canvas.height);
+    func(canvas);
+};
+
+export default util;