remote-calibrator 0.3.0 → 0.5.0-beta.3

package/src/WebGazer4RC/src/util_regression.mjs

@@ -0,0 +1,240 @@
+import util from './util.mjs';
+import numeric from 'numeric';
+import mat from './mat.mjs';
+import params from './params.mjs';
+
+const util_regression = {};
+
+
+/**
+ * Initialize new arrays and initialize Kalman filter for regressions.
+ */
+util_regression.InitRegression = function() {
+  var dataWindow = 700;
+  var trailDataWindow = 10;
+  this.ridgeParameter = Math.pow(10,-5);
+  this.errorXArray = new util.DataWindow(dataWindow);
+  this.errorYArray = new util.DataWindow(dataWindow);
+
+
+  this.screenXClicksArray = new util.DataWindow(dataWindow);
+  this.screenYClicksArray = new util.DataWindow(dataWindow);
+  this.eyeFeaturesClicks = new util.DataWindow(dataWindow);
+
+  //sets to one second worth of cursor trail
+  this.trailTime = 1000;
+  this.trailDataWindow = this.trailTime / params.moveTickSize;
+  this.screenXTrailArray = new util.DataWindow(trailDataWindow);
+  this.screenYTrailArray = new util.DataWindow(trailDataWindow);
+  this.eyeFeaturesTrail = new util.DataWindow(trailDataWindow);
+  this.trailTimes = new util.DataWindow(trailDataWindow);
+
+  this.dataClicks = new util.DataWindow(dataWindow);
+  this.dataTrail = new util.DataWindow(trailDataWindow);
+
+  // Initialize Kalman filter [20200608 xk] what do we do about parameters?
+  // [20200611 xk] unsure what to do w.r.t. dimensionality of these matrices. So far at least
+  //               by my own anecdotal observation a 4x1 x vector seems to work alright
+  var F = [ [1, 0, 1, 0],
+    [0, 1, 0, 1],
+    [0, 0, 1, 0],
+    [0, 0, 0, 1]];
+
+  //Parameters Q and R may require some fine tuning
+  var Q = [ [1/4, 0,    1/2, 0],
+    [0,   1/4,  0,   1/2],
+    [1/2, 0,    1,   0],
+    [0,  1/2,  0,   1]];// * delta_t
+  var delta_t = 1/10; // The amount of time between frames
+  Q = numeric.mul(Q, delta_t);
+
+  var H = [ [1, 0, 0, 0, 0, 0],
+    [0, 1, 0, 0, 0, 0],
+    [0, 0, 1, 0, 0, 0],
+    [0, 0, 0, 1, 0, 0]];
+  var H = [ [1, 0, 0, 0],
+    [0, 1, 0, 0]];
+  var pixel_error = 47; //We will need to fine tune this value [20200611 xk] I just put a random value here
+
+  //This matrix represents the expected measurement error
+  var R = numeric.mul(numeric.identity(2), pixel_error);
+
+  var P_initial = numeric.mul(numeric.identity(4), 0.0001); //Initial covariance matrix
+  var x_initial = [[500], [500], [0], [0]]; // Initial measurement matrix
+
+  this.kalman = new util_regression.KalmanFilter(F, H, Q, R, P_initial, x_initial);
+}
+
+/**
+ * Kalman Filter constructor
+ * Kalman filters work by reducing the amount of noise in a models.
+ * https://blog.cordiner.net/2011/05/03/object-tracking-using-a-kalman-filter-matlab/
+ *
+ * @param {Array.<Array.<Number>>} F - transition matrix
+ * @param {Array.<Array.<Number>>} Q - process noise matrix
+ * @param {Array.<Array.<Number>>} H - maps between measurement vector and noise matrix
+ * @param {Array.<Array.<Number>>} R - defines measurement error of the device
+ * @param {Array} P_initial - the initial state
+ * @param {Array} X_initial - the initial state of the device
+ */
+util_regression.KalmanFilter = function(F, H, Q, R, P_initial, X_initial) {
+    this.F = F; // State transition matrix
+    this.Q = Q; // Process noise matrix
+    this.H = H; // Transformation matrix
+    this.R = R; // Measurement Noise
+    this.P = P_initial; //Initial covariance matrix
+    this.X = X_initial; //Initial guess of measurement
+};
+
+/**
+ * Get Kalman next filtered value and update the internal state
+ * @param {Array} z - the new measurement
+ * @return {Array}
+ */
+util_regression.KalmanFilter.prototype.update = function(z) {
+    // Here, we define all the different matrix operations we will need
+    var add = numeric.add, sub = numeric.sub, inv = numeric.inv, identity = numeric.identity;
+    var mult = mat.mult, transpose = mat.transpose;
+    //TODO cache variables like the transpose of H
+
+    // prediction: X = F * X  |  P = F * P * F' + Q
+    var X_p = mult(this.F, this.X); //Update state vector
+    var P_p = add(mult(mult(this.F,this.P), transpose(this.F)), this.Q); //Predicted covaraince
+
+    //Calculate the update values
+    var y = sub(z, mult(this.H, X_p)); // This is the measurement error (between what we expect and the actual value)
+    var S = add(mult(mult(this.H, P_p), transpose(this.H)), this.R); //This is the residual covariance (the error in the covariance)
+
+    // kalman multiplier: K = P * H' * (H * P * H' + R)^-1
+    var K = mult(P_p, mult(transpose(this.H), inv(S))); //This is the Optimal Kalman Gain
+
+    //We need to change Y into it's column vector form
+    for(var i = 0; i < y.length; i++){
+        y[i] = [y[i]];
+    }
+
+    //Now we correct the internal values of the model
+    // correction: X = X + K * (m - H * X)  |  P = (I - K * H) * P
+    this.X = add(X_p, mult(K, y));
+    this.P = mult(sub(identity(K.length), mult(K,this.H)), P_p);
+    return transpose(mult(this.H, this.X))[0]; //Transforms the predicted state back into it's measurement form
+};
+
+/**
+ * Performs ridge regression, according to the Weka code.
+ * @param {Array} y - corresponds to screen coordinates (either x or y) for each of n click events
+ * @param {Array.<Array.<Number>>} X - corresponds to gray pixel features (120 pixels for both eyes) for each of n clicks
+ * @param {Array} k - ridge parameter
+ * @return{Array} regression coefficients
+ */
+util_regression.ridge = function(y, X, k){
+    var nc = X[0].length;
+    var m_Coefficients = new Array(nc);
+    var xt = mat.transpose(X);
+    var solution = new Array();
+    var success = true;
+    do{
+        var ss = mat.mult(xt,X);
+        // Set ridge regression adjustment
+        for (var i = 0; i < nc; i++) {
+            ss[i][i] = ss[i][i] + k;
+        }
+
+        // Carry out the regression
+        var bb = mat.mult(xt,y);
+        for(var i = 0; i < nc; i++) {
+            m_Coefficients[i] = bb[i][0];
+        }
+        try{
+            var n = (m_Coefficients.length !== 0 ? m_Coefficients.length/m_Coefficients.length: 0);
+            if (m_Coefficients.length*n !== m_Coefficients.length){
+                console.log('Array length must be a multiple of m')
+            }
+            solution = (ss.length === ss[0].length ? (numeric.LUsolve(numeric.LU(ss,true),bb)) : (webgazer.mat.QRDecomposition(ss,bb)));
+
+            for (var i = 0; i < nc; i++){
+                m_Coefficients[i] = solution[i];
+            }
+            success = true;
+        }
+        catch (ex){
+            k *= 10;
+            console.log(ex);
+            success = false;
+        }
+    } while (!success);
+    return m_Coefficients;
+}
+
+/**
+ * Add given data to current data set then,
+ * replace current data member with given data
+ * @param {Array.<Object>} data - The data to set
+ */
+util_regression.setData = function(data) {
+  for (var i = 0; i < data.length; i++) {
+    // Clone data array
+    var leftData = new Uint8ClampedArray(data[i].eyes.left.patch.data);
+    var rightData = new Uint8ClampedArray(data[i].eyes.right.patch.data);
+    // Duplicate ImageData object
+    data[i].eyes.left.patch = new ImageData(leftData, data[i].eyes.left.width, data[i].eyes.left.height);
+    data[i].eyes.right.patch = new ImageData(rightData, data[i].eyes.right.width, data[i].eyes.right.height);
+
+    // Add those data objects to model
+    this.addData(data[i].eyes, data[i].screenPos, data[i].type);
+  }
+};
+
+
+//not used ?!
+//TODO: still usefull ???
+/**
+ *
+ * @returns {Number}
+ */
+util_regression.getCurrentFixationIndex = function() {
+  var index = 0;
+  var recentX = this.screenXTrailArray.get(0);
+  var recentY = this.screenYTrailArray.get(0);
+  for (var i = this.screenXTrailArray.length - 1; i >= 0; i--) {
+    var currX = this.screenXTrailArray.get(i);
+    var currY = this.screenYTrailArray.get(i);
+    var euclideanDistance = Math.sqrt(Math.pow((currX-recentX),2)+Math.pow((currY-recentY),2));
+    if (euclideanDistance > 72){
+      return i+1;
+    }
+  }
+  return i;
+}
+
+util_regression.addData = function(eyes, screenPos, type) {
+    if (!eyes) {
+        return;
+    }
+    //not doing anything with blink at present
+    // if (eyes.left.blink || eyes.right.blink) {
+    //     return;
+    // }
+    if (type === 'click') {
+        this.screenXClicksArray.push([screenPos[0]]);
+        this.screenYClicksArray.push([screenPos[1]]);
+        this.eyeFeaturesClicks.push(util.getEyeFeats(eyes));
+        this.dataClicks.push({'eyes':eyes, 'screenPos':screenPos, 'type':type});
+    } else if (type === 'move') {
+        this.screenXTrailArray.push([screenPos[0]]);
+        this.screenYTrailArray.push([screenPos[1]]);
+
+        this.eyeFeaturesTrail.push(util.getEyeFeats(eyes));
+        this.trailTimes.push(performance.now());
+        this.dataTrail.push({'eyes':eyes, 'screenPos':screenPos, 'type':type});
+    }
+
+    // [20180730 JT] Why do we do this? It doesn't return anything...
+    // But as JS is pass by reference, it still affects it.
+    //
+    // Causes problems for when we want to call 'addData' twice in a row on the same object, but perhaps with different screenPos or types (think multiple interactions within one video frame)
+    //eyes.left.patch = Array.from(eyes.left.patch.data);
+    //eyes.right.patch = Array.from(eyes.right.patch.data);
+};
+
+export default util_regression;

package/src/WebGazer4RC/src/worker_scripts/mat.js

@@ -0,0 +1,306 @@
+(function() {
+    'use strict';
+
+    self.webgazer = self.webgazer || {};
+    self.webgazer.mat = self.webgazer.mat || {};
+
+    /**
+     * Transposes an mxn array
+     * @param {Array.<Array.<Number>>} matrix - of 'M x N' dimensionality
+     * @return {Array.<Array.<Number>>} transposed matrix
+     */
+    self.webgazer.mat.transpose = function(matrix){
+        var m = matrix.length;
+        var n = matrix[0].length;
+        var transposedMatrix = new Array(n);
+
+        for (var i = 0; i < m; i++){
+            for (var j = 0; j < n; j++){
+                if (i === 0) transposedMatrix[j] = new Array(m);
+                transposedMatrix[j][i] = matrix[i][j];
+            }
+        }
+
+        return transposedMatrix;
+    };
+
+    /**
+     * Get a sub-matrix of matrix
+     * @param {Array.<Array.<Number>>} matrix - original matrix
+     * @param {Array.<Number>} r - Array of row indices
+     * @param {Number} j0 - Initial column index
+     * @param {Number} j1 - Final column index
+     * @returns {Array} The sub-matrix matrix(r(:),j0:j1)
+     */
+    self.webgazer.mat.getMatrix = function(matrix, r, j0, j1){
+        var X = new Array(r.length),
+            m = j1-j0+1;
+
+        for (var i = 0; i < r.length; i++){
+            X[i] = new Array(m);
+            for (var j = j0; j <= j1; j++){
+                X[i][j-j0] = matrix[r[i]][j];
+            }
+        }
+        return X;
+    };
+
+    /**
+     * Get a submatrix of matrix
+     * @param {Array.<Array.<Number>>} matrix - original matrix
+     * @param {Number} i0 - Initial row index
+     * @param {Number} i1 - Final row index
+     * @param {Number} j0 - Initial column index
+     * @param {Number} j1 - Final column index
+     * @return {Array} The sub-matrix matrix(i0:i1,j0:j1)
+     */
+    self.webgazer.mat.getSubMatrix = function(matrix, i0, i1, j0, j1){
+        var size = j1 - j0 + 1,
+            X = new Array(i1-i0+1);
+
+        for (var i = i0; i <= i1; i++){
+            var subI = i-i0;
+
+            X[subI] = new Array(size);
+
+            for (var j = j0; j <= j1; j++){
+                X[subI][j-j0] = matrix[i][j];
+            }
+        }
+        return X;
+    };
+
+    /**
+     * Linear algebraic matrix multiplication, matrix1 * matrix2
+     * @param {Array.<Array.<Number>>} matrix1
+     * @param {Array.<Array.<Number>>} matrix2
+     * @return {Array.<Array.<Number>>} Matrix product, matrix1 * matrix2
+     */
+    self.webgazer.mat.mult = function(matrix1, matrix2){
+
+        if (matrix2.length != matrix1[0].length){
+            console.log('Matrix inner dimensions must agree:');
+            
+        }
+
+        var X = new Array(matrix1.length),
+            Bcolj = new Array(matrix1[0].length);
+
+        for (var j = 0; j < matrix2[0].length; j++){
+            for (var k = 0; k < matrix1[0].length; k++){
+                Bcolj[k] = matrix2[k][j];
+            }
+            for (var i = 0; i < matrix1.length; i++){
+
+                if (j === 0)
+                    X[i] = new Array(matrix2[0].length);
+
+                var Arowi = matrix1[i];
+                var s = 0;
+                for (var k = 0; k < matrix1[0].length; k++){
+                    s += Arowi[k]*Bcolj[k];
+                }
+                X[i][j] = s;
+            }
+        }
+        return X;
+    };
+
+
+    /**
+     * LUDecomposition to solve A*X = B, based on WEKA code
+     * @param {Array.<Array.<Number>>} A - left matrix of equation to be solved
+     * @param {Array.<Array.<Number>>} B - right matrix of equation to be solved
+     * @return {Array.<Array.<Number>>} X so that L*U*X = B(piv,:)
+     */
+    self.webgazer.mat.LUDecomposition = function(A,B){
+        var LU = new Array(A.length);
+
+        for (var i = 0; i < A.length; i++){
+          LU[i] = new Array(A[0].length);
+            for (var j = 0; j < A[0].length; j++){
+                LU[i][j] = A[i][j];
+            }
+        }
+
+        var m = A.length;
+        var n = A[0].length;
+        var piv = new Array(m);
+        for (var i = 0; i < m; i++){
+            piv[i] = i;
+        }
+        var pivsign = 1;
+        var LUrowi = new Array();
+        var LUcolj = new Array(m);
+        // Outer loop.
+        for (var j = 0; j < n; j++){
+            // Make a copy of the j-th column to localize references.
+            for (var i = 0; i < m; i++){
+                LUcolj[i] = LU[i][j];
+            }
+            // Apply previous transformations.
+            for (var i = 0; i < m; i++){
+                LUrowi = LU[i];
+                // Most of the time is spent in the following dot product.
+                var kmax = Math.min(i,j);
+                var s = 0;
+                for (var k = 0; k < kmax; k++){
+                    s += LUrowi[k]*LUcolj[k];
+                }
+                LUrowi[j] = LUcolj[i] -= s;
+            }
+            // Find pivot and exchange if necessary.
+            var p = j;
+            for (var i = j+1; i < m; i++){
+                if (Math.abs(LUcolj[i]) > Math.abs(LUcolj[p])){
+                    p = i;
+                }
+            }
+            if (p != j){
+                for (var k = 0; k < n; k++){
+                    var t = LU[p][k];
+                    LU[p][k] = LU[j][k];
+                    LU[j][k] = t;
+                }
+                var k = piv[p];
+                piv[p] = piv[j];
+                piv[j] = k;
+                pivsign = -pivsign;
+            }
+            // Compute multipliers.
+            if (j < m & LU[j][j] != 0){
+                for (var i = j+1; i < m; i++){
+                    LU[i][j] /= LU[j][j];
+                }
+            }
+        }
+        if (B.length != m){
+            console.log('Matrix row dimensions must agree.');
+        }
+        for (var j = 0; j < n; j++){
+            if (LU[j][j] === 0){
+                console.log('Matrix is singular.')
+            }
+        }
+        var nx = B[0].length;
+        var X = self.webgazer.mat.getMatrix(B,piv,0,nx-1);
+        // Solve L*Y = B(piv,:)
+        for (var k = 0; k < n; k++){
+            for (var i = k+1; i < n; i++){
+                for (var j = 0; j < nx; j++){
+                    X[i][j] -= X[k][j]*LU[i][k];
+                }
+            }
+        }
+        // Solve U*X = Y;
+        for (var k = n-1; k >= 0; k--){
+            for (var j = 0; j < nx; j++){
+                X[k][j] /= LU[k][k];
+            }
+            for (var i = 0; i < k; i++){
+                for (var j = 0; j < nx; j++){
+                    X[i][j] -= X[k][j]*LU[i][k];
+                }
+            }
+        }
+        return X;
+    };
+
+    /**
+     * Least squares solution of A*X = B, based on WEKA code
+     * @param {Array.<Array.<Number>>} A - left side matrix to be solved
+     * @param {Array.<Array.<Number>>} B - a matrix with as many rows as A and any number of columns.
+     * @return {Array.<Array.<Number>>} X - that minimizes the two norms of QR*X-B.
+     */
+    self.webgazer.mat.QRDecomposition = function(A, B){
+        // Initialize.
+        var QR = new Array(A.length);
+
+        for (var i = 0; i < A.length; i++){
+            QR[i] = new Array(A[0].length);
+            for (var j = 0; j < A[0].length; j++){
+                QR[i][j] = A[i][j];
+            }
+        }
+        var m = A.length;
+        var n = A[0].length;
+        var Rdiag = new Array(n);
+        var nrm;
+
+        // Main loop.
+        for (var k = 0; k < n; k++){
+            // Compute 2-norm of k-th column without under/overflow.
+            nrm = 0;
+            for (var i = k; i < m; i++){
+                nrm = Math.hypot(nrm,QR[i][k]);
+            }
+            if (nrm != 0){
+                // Form k-th Householder vector.
+                if (QR[k][k] < 0){
+                    nrm = -nrm;
+                }
+                for (var i = k; i < m; i++){
+                    QR[i][k] /= nrm;
+                }
+                QR[k][k] += 1;
+
+                // Apply transformation to remaining columns.
+                for (var j = k+1; j < n; j++){
+                    var s = 0;
+                    for (var i = k; i < m; i++){
+                        s += QR[i][k]*QR[i][j];
+                    }
+                    s = -s/QR[k][k];
+                    for (var i = k; i < m; i++){
+                        QR[i][j] += s*QR[i][k];
+                    }
+                }
+            }
+            Rdiag[k] = -nrm;
+        }
+        if (B.length != m){
+            console.log('Matrix row dimensions must agree.');
+        }
+        for (var j = 0; j < n; j++){
+            if (Rdiag[j] === 0)
+                console.log('Matrix is rank deficient');
+        }
+        // Copy right hand side
+        var nx = B[0].length;
+        var X = new Array(B.length);
+        for(var i=0; i<B.length; i++){
+            X[i] = new Array(B[0].length);
+        }
+        for (var i = 0; i < B.length; i++){
+            for (var j = 0; j < B[0].length; j++){
+                X[i][j] = B[i][j];
+            }
+        }
+        // Compute Y = transpose(Q)*B
+        for (var k = 0; k < n; k++){
+            for (var j = 0; j < nx; j++){
+                var s = 0.0;
+                for (var i = k; i < m; i++){
+                    s += QR[i][k]*X[i][j];
+                }
+                s = -s/QR[k][k];
+                for (var i = k; i < m; i++){
+                    X[i][j] += s*QR[i][k];
+                }
+            }
+        }
+        // Solve R*X = Y;
+        for (var k = n-1; k >= 0; k--){
+            for (var j = 0; j < nx; j++){
+                X[k][j] /= Rdiag[k];
+            }
+            for (var i = 0; i < k; i++){
+                for (var j = 0; j < nx; j++){
+                    X[i][j] -= X[k][j]*QR[i][k];
+                }
+            }
+        }
+        return self.webgazer.mat.getSubMatrix(X,0,n-1,0,nx-1);
+    }
+    
+}());