xy-scale 1.4.44 → 1.4.45

package/README.md

@@ -29,7 +29,8 @@ Builds supervised-learning datasets and splits them into training and testing ar
 #### Parameters
 
 - `arrObj` (Array<Object>): Source dataset.
-- `trainingSplit` (Number, optional): Fraction of rows used for training. Default: `0.8`.
+- `trainSize` (Number, required).
+- `testSize` (Number, required).
 - `yCallbackFunc` (Function, optional): Builds the output object for each row. Returning `null` or `undefined` skips the row.
 - `xCallbackFunc` (Function, optional): Builds the feature object for each row. Returning `null` or `undefined` skips the row.
 - `validateRows` (Function, optional): Extra row filter executed before the callbacks.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xy-scale",
-  "version": "1.4.44",
+  "version": "1.4.45",
   "main": "./index.js",
   "type": "module",
   "scripts": {

package/src/datasets.js

@@ -1,23 +1,35 @@
 import { arrayShuffle } from "./utilities.js";
-import { validateFirstRow, validateArray, hasInvalidNumbers } from "./validators.js";
+import { validateFirstRow, validateArray, hasInvalidNumbers, validateSizes } from "./validators.js";
+import {zscore2d} from './zscore2.js'
 
 export const parseTrainingXY = ({
     arrObj = [],
-    trainingSplit = 0.8,
+    trainSize = null,
+    testSize = null,
     yCallbackFunc = row => row,
     xCallbackFunc = row => row,
     validateRows = () => true,
     shuffle = false,
     state = {},
+    showSource = false,
+    scaling = null
 }) => {
     validateArray(arrObj, { min: 2 }, 'parseTrainingXY');
     validateFirstRow(arrObj[0]);
 
+    const arrObjSize = arrObj.length;
+    
+    validateSizes({arrObjSize, trainSize, testSize});
+
+    if(![null, 'zscore'].includes(scaling)) {
+        throw new Error(`Invalid "scaling" property. Accepting null or "zscore".`)
+    }
+
+    const totalSize = trainSize + testSize;
     let flatX = [];
     let flatY = [];
     let source = [];
 
-
     let keyNamesX = null;
     let keyNamesY = null;
 
@@ -75,7 +87,11 @@ export const parseTrainingXY = ({
 
             flatX.push(rowX);
             flatY.push(rowY);
-            source.push(arrObj[x])
+
+            if(showSource) {
+                source.push(arrObj[x])
+            }
+            
 
         } catch(err) {
             throw new Error(`[BUG] - Skipped row index=${x}: ${err.message}`);
@@ -88,21 +104,31 @@ export const parseTrainingXY = ({
         for (let i = 0; i < flatX.length; i++) {
             merged[i] = {
                 x: flatX[i],
-                y: flatY[i],
-                source: source[i]
+                y: flatY[i]
             };
+
+            if(showSource) {
+                merged[i].source = source[i]
+            }
         }
 
         const shuffled = arrayShuffle(merged);
 
         flatX = new Array(shuffled.length);
         flatY = new Array(shuffled.length);
-        source = new Array(shuffled.length)
+
+        if(showSource) {
+            source = new Array(shuffled.length)
+        }
+        
 
         for (let i = 0; i < shuffled.length; i++) {
             flatX[i] = shuffled[i].x;
             flatY[i] = shuffled[i].y;
-            source[i] = shuffled[i].source
+
+            if(showSource) {
+                source[i] = shuffled[i].source;
+            }
         }
     }
 
@@ -115,14 +141,44 @@ export const parseTrainingXY = ({
         labelCounts,
     };
 
-    const splitIndex = Math.floor(flatX.length * trainingSplit);
+    const startSize = arrObjSize - totalSize
+
+    flatX.splice(0, startSize) //keeps the last items
+    flatY.splice(0, startSize) //keeps the last items
+
+    let trainX = flatX.slice(0, trainSize);
+    let trainY = flatY.slice(0, trainSize);
+    
+    let stats = null
+
+    if(scaling === 'zscore') {
+        let trainNormalized = zscore2d(trainX)
+        stats = trainNormalized.stats
+        trainX = trainNormalized.data
+        trainNormalized = null
+    }
+
+    let testX = flatX.slice(-testSize);
+
+    if(scaling === 'zscore') {
+        testX = zscore2d(testX, stats).data
+    }
+
+    let testY = flatY.slice(-testSize);
 
-    let trainX = flatX.slice(0, splitIndex);
-    let trainY = flatY.slice(0, splitIndex);
-    let testX = flatX.slice(splitIndex);
-    let testY = flatY.slice(splitIndex);
-    let trainSource = source.slice(0, splitIndex);
-    let testSource = source.slice(splitIndex);
+
+    flatX = null
+    flatY = null
+
+    let trainSource
+    let testSource
+
+    if(showSource) {
+        source.splice(0, startSize) //keeps the last items
+        trainSource = source.slice(0, trainSize);
+        testSource = source.slice(-testSize);
+        source = null
+    }
 
     return {
         trainX,
@@ -132,8 +188,9 @@ export const parseTrainingXY = ({
         configX,
         configY,
         trainSource,
-        testSource
-    };
+        testSource,
+        stats
+    }
 };
 export const parseProductionX = ({
     arrObj = [],
@@ -142,6 +199,9 @@ export const parseProductionX = ({
     validateRows = () => true,
     shuffle = false,
     state = {},
+    showSource = false,
+    scaling = null,
+    stats
 }) => {
     let flatX = [];
     let source = [];
@@ -150,6 +210,12 @@ export const parseProductionX = ({
     validateArray(arrObj, { min: 1 }, 'parseProductionX');
     validateFirstRow(arrObj[0]);
 
+    if(![null, 'zscore'].includes(scaling)) {
+        throw new Error(`Invalid "scaling" property. Accepting null or "zscore".`)
+    }
+
+    const arrObjSize = arrObj.length
+
     if (yCallbackFunc != null) {
         throw new Error('The property "yCallbackFunc" must not be set in "parseProductionX".');
     }
@@ -179,7 +245,11 @@ export const parseProductionX = ({
             }
 
             flatX.push(rowX);
-            source.push(arrObj[x])
+
+            if(showSource) {
+                source.push(arrObj[x])
+            }
+            
 
         } catch(err) {
             throw new Error(`[BUG] - Skipped row index=${x}: ${err.message}`);
@@ -191,19 +261,29 @@ export const parseProductionX = ({
 
         for (let i = 0; i < flatX.length; i++) {
             merged[i] = {
-                x: flatX[i],
-                source: source[i]
+                x: flatX[i]
             };
+
+            if(showSource) {
+                merged[i].source = source[i]
+            }
         }
 
         const shuffled = arrayShuffle(merged);
 
         flatX = new Array(shuffled.length);
-        source = new Array(shuffled.length)
+
+        if(showSource) {
+            source = new Array(shuffled.length)
+        }
+        
 
         for (let i = 0; i < shuffled.length; i++) {
             flatX[i] = shuffled[i].x;
-            source[i] = shuffled[i].source
+
+            if(showSource) {
+                source[i] = shuffled[i].source
+            }
         }
     }
 
@@ -211,6 +291,10 @@ export const parseProductionX = ({
         keyNames: keyNamesX ?? [],
     };
 
+    if(scaling === 'zscore') {
+        flatX = zscore2d(flatX, stats).data
+    }
+
     return {
         X: flatX,
         source,

package/src/validators.js

@@ -1,6 +1,8 @@
 
 export const isNumber = v => v != null && Number.isFinite(v)
 
+export const isPositiveInteger = value =>  Number.isInteger(value) && value > 0
+
 export const isKeyPairObject = param => {
   return (
     param !== null &&
@@ -132,4 +134,24 @@ export const arraysAreNotEqualSize = (list, callerName) => {
     }
 
     return false
+}
+
+export const validateSizes = ({arrObjSize, trainSize, testSize}) => {
+
+    if(!isPositiveInteger(trainSize)) {
+        throw new Error(`Invalid property: "trainSize" (${trainSize}) must be a non-negative integer.`)
+    }
+    if(!isPositiveInteger(testSize)) {
+         throw new Error(`Invalid property: "testSize" (${testSize}) must be a non-negative integer.`)
+    }
+
+    if (!Number.isInteger(arrObjSize) || arrObjSize < 0) {
+        throw new Error(`Invalid property: "arrObjSize" (${arrObjSize}) must be a non-negative integer.`)
+    }
+
+    if(arrObjSize < (trainSize + testSize)) {
+        throw new Error(`Invalid property: The sum of "trainSize" + "testSize" (${trainSize + testSize}) must not be larger than "arrObj.length" (${arrObjSize}).`)
+    }
+
+    return true
 }

package/src/zscore2.js

@@ -0,0 +1,145 @@
+const toArray = typedOrArray => Array.from(typedOrArray);
+
+const normalizeStats = (stats, cols) => {
+  const mean = new Float64Array(cols);
+  const std = new Float64Array(cols);
+  const scale = new Float64Array(cols);
+  const count = new Uint32Array(cols);
+
+  const meanIn = Array.isArray(stats?.mean) ? stats.mean : [];
+  const stdIn = Array.isArray(stats?.std) ? stats.std : [];
+  const scaleIn = Array.isArray(stats?.scale) ? stats.scale : [];
+  const countIn = Array.isArray(stats?.count) ? stats.count : [];
+
+  for (let j = 0; j < cols; j++) {
+    const m = Number(meanIn[j]);
+    const s = Number(stdIn[j]);
+    const sc = Number(scaleIn[j]);
+    const c = Number(countIn[j]);
+
+    mean[j] = Number.isFinite(m) ? m : 0;
+    std[j] = Number.isFinite(s) && s > 0 ? s : 0;
+    count[j] = Number.isFinite(c) && c > 0 ? c : 0;
+
+    if (Number.isFinite(sc) && sc > 0) {
+      scale[j] = sc;
+      if (!(std[j] > 0)) { std[j] = 1 / sc; }
+    } else if (std[j] > 0) {
+      scale[j] = 1 / std[j];
+    } else {
+      scale[j] = 0;
+    }
+  }
+
+  return { mean, std, scale, count };
+};
+
+const fitStats = (arr, cols) => {
+  const rows = arr.length;
+  const mean = new Float64Array(cols);
+  const m2 = new Float64Array(cols);
+  const count = new Uint32Array(cols);
+
+  for (let i = 0; i < rows; i++) {
+    const row = arr[i];
+
+    if (!Array.isArray(row)) {
+      throw new TypeError(`[zscore2d] Invalid row at index=${i}. Expected an array.`);
+    }
+    if (row.length !== cols) {
+      throw new Error(`[zscore2d] Inconsistent row size at index=${i}. Expected ${cols}, got ${row.length}.`);
+    }
+
+    for (let j = 0; j < cols; j++) {
+      const x = Number(row[j]);
+      if (!Number.isFinite(x)) { continue; }
+
+      count[j]++;
+
+      const delta = x - mean[j];
+      mean[j] += delta / count[j];
+      m2[j] += delta * (x - mean[j]);
+    }
+  }
+
+  const std = new Float64Array(cols);
+  const scale = new Float64Array(cols);
+
+  for (let j = 0; j < cols; j++) {
+    const variance = count[j] > 1 ? (m2[j] / count[j]) : 0;
+    const s = variance > 0 ? Math.sqrt(variance) : 0;
+    std[j] = s;
+    scale[j] = s > 0 ? (1 / s) : 0;
+  }
+
+  return { mean, std, scale, count };
+};
+
+const scaleFromStats = (arr, normalizedStats) => {
+  const rows = arr.length;
+  const cols = normalizedStats.mean.length;
+  const out = new Array(rows);
+
+  for (let i = 0; i < rows; i++) {
+    const row = arr[i];
+
+    if (!Array.isArray(row)) {
+      throw new TypeError(`[zscore2d] Invalid row at index=${i}. Expected an array.`);
+    }
+    if (row.length !== cols) {
+      throw new Error(`[zscore2d] Inconsistent row size at index=${i}. Expected ${cols}, got ${row.length}.`);
+    }
+
+    const normalized = new Array(cols);
+    for (let j = 0; j < cols; j++) {
+      const x = Number(row[j]);
+      const sc = normalizedStats.scale[j];
+
+      normalized[j] = (Number.isFinite(x) && sc > 0)
+        ? (x - normalizedStats.mean[j]) * sc
+        : 0;
+    }
+
+    out[i] = normalized;
+  }
+
+  return out;
+};
+
+export const zscore2d = (arr, stats = null) => {
+  if (!Array.isArray(arr)) {
+    throw new TypeError('[zscore2d] "arr" must be a 2D array.');
+  }
+
+  if (arr.length === 0) {
+    const normalizedStats = normalizeStats(stats ?? {}, 0);
+    return {
+      stats: {
+        mean: toArray(normalizedStats.mean),
+        std: toArray(normalizedStats.std),
+        scale: toArray(normalizedStats.scale),
+        count: toArray(normalizedStats.count)
+      },
+      data: []
+    };
+  }
+
+  const firstRow = arr[0];
+  if (!Array.isArray(firstRow)) {
+    throw new TypeError('[zscore2d] "arr" must be a 2D array of rows.');
+  }
+
+  const cols = firstRow.length;
+  const normalizedStats = stats ? normalizeStats(stats, cols) : fitStats(arr, cols);
+  const data = scaleFromStats(arr, normalizedStats);
+
+  return {
+    stats: {
+      mean: toArray(normalizedStats.mean),
+      std: toArray(normalizedStats.std),
+      scale: toArray(normalizedStats.scale),
+      count: toArray(normalizedStats.count)
+    },
+    data
+  };
+};

package/test/test.js

@@ -31,6 +31,9 @@ const test = async () => {
 
     const arrObj = indicators.getData()
 
+    const trainSize = Math.round(arrObj.length * 0.8)
+    const testSize = arrObj.length - trainSize
+
     const {
         trainX,
         trainY,
@@ -39,8 +42,10 @@ const test = async () => {
         configX,
         configY
     } = parseTrainingXY({
+        scaling: 'zscore',
         arrObj,
-        trainingSplit: 0.50,
+        trainSize,
+        testSize,
         yCallbackFunc,
         xCallbackFunc,
         validateRows: ({objRow, index}) => {