xy-scale 1.4.43 → 1.4.45

package/README.md

@@ -29,7 +29,8 @@ Builds supervised-learning datasets and splits them into training and testing ar
 #### Parameters
 
 - `arrObj` (Array<Object>): Source dataset.
-- `trainingSplit` (Number, optional): Fraction of rows used for training. Default: `0.8`.
+- `trainSize` (Number, required).
+- `testSize` (Number, required).
 - `yCallbackFunc` (Function, optional): Builds the output object for each row. Returning `null` or `undefined` skips the row.
 - `xCallbackFunc` (Function, optional): Builds the feature object for each row. Returning `null` or `undefined` skips the row.
 - `validateRows` (Function, optional): Extra row filter executed before the callbacks.

package/index.js

@@ -1,5 +1,5 @@
 import { parseTrainingXY, parseProductionX } from "./src/datasets.js"
 import {arrayToTimesteps } from "./src/timeSteps.js"
-import { arrayShuffle, arrayShuffleXY } from "./src/utilities.js"
+import { arrayShuffle } from "./src/utilities.js"
 
-export { parseTrainingXY, parseProductionX, arrayToTimesteps, arrayShuffle, arrayShuffleXY }
+export { parseTrainingXY, parseProductionX, arrayToTimesteps, arrayShuffle }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xy-scale",
-  "version": "1.4.43",
+  "version": "1.4.45",
   "main": "./index.js",
   "type": "module",
   "scripts": {

package/src/datasets.js

@@ -1,22 +1,34 @@
 import { arrayShuffle } from "./utilities.js";
-import { oversampleXY, undersampleXY } from "./balancing.js";
-import { validateFirstRow, validateArray, hasInvalidNumbers } from "./validators.js";
+import { validateFirstRow, validateArray, hasInvalidNumbers, validateSizes } from "./validators.js";
+import {zscore2d} from './zscore2.js'
 
 export const parseTrainingXY = ({
     arrObj = [],
-    trainingSplit = 0.8,
+    trainSize = null,
+    testSize = null,
     yCallbackFunc = row => row,
     xCallbackFunc = row => row,
     validateRows = () => true,
     shuffle = false,
-    balancing = '',
     state = {},
+    showSource = false,
+    scaling = null
 }) => {
     validateArray(arrObj, { min: 2 }, 'parseTrainingXY');
     validateFirstRow(arrObj[0]);
 
+    const arrObjSize = arrObj.length;
+    
+    validateSizes({arrObjSize, trainSize, testSize});
+
+    if(![null, 'zscore'].includes(scaling)) {
+        throw new Error(`Invalid "scaling" property. Accepting null or "zscore".`)
+    }
+
+    const totalSize = trainSize + testSize;
     let flatX = [];
     let flatY = [];
+    let source = [];
 
     let keyNamesX = null;
     let keyNamesY = null;
@@ -27,6 +39,7 @@ export const parseTrainingXY = ({
         try {
             if (!validateRows({ objRow: arrObj, index: x, state })) continue;
 
+
             const parsedX = xCallbackFunc({ objRow: arrObj, index: x, state });
             const parsedY = yCallbackFunc({ objRow: arrObj, index: x, state });
 
@@ -75,6 +88,11 @@ export const parseTrainingXY = ({
             flatX.push(rowX);
             flatY.push(rowY);
 
+            if(showSource) {
+                source.push(arrObj[x])
+            }
+            
+
         } catch(err) {
             throw new Error(`[BUG] - Skipped row index=${x}: ${err.message}`);
         }
@@ -88,6 +106,10 @@ export const parseTrainingXY = ({
                 x: flatX[i],
                 y: flatY[i]
             };
+
+            if(showSource) {
+                merged[i].source = source[i]
+            }
         }
 
         const shuffled = arrayShuffle(merged);
@@ -95,9 +117,18 @@ export const parseTrainingXY = ({
         flatX = new Array(shuffled.length);
         flatY = new Array(shuffled.length);
 
+        if(showSource) {
+            source = new Array(shuffled.length)
+        }
+        
+
         for (let i = 0; i < shuffled.length; i++) {
             flatX[i] = shuffled[i].x;
             flatY[i] = shuffled[i].y;
+
+            if(showSource) {
+                source[i] = shuffled[i].source;
+            }
         }
     }
 
@@ -110,27 +141,43 @@ export const parseTrainingXY = ({
         labelCounts,
     };
 
-    const splitIndex = Math.floor(flatX.length * trainingSplit);
-
-    let trainX = flatX.slice(0, splitIndex);
-    let trainY = flatY.slice(0, splitIndex);
-    let testX = flatX.slice(splitIndex);
-    let testY = flatY.slice(splitIndex);
-
-    if (balancing) {
-        let balance;
-
-        if (balancing === 'oversample') {
-            balance = oversampleXY(trainX, trainY);
-            trainX = balance.X;
-            trainY = balance.Y;
-        } else if (balancing === 'undersample') {
-            balance = undersampleXY(trainX, trainY);
-            trainX = balance.X;
-            trainY = balance.Y;
-        } else {
-            throw Error('balancing argument only accepts "", "oversample" and "undersample". Defaults to "".');
-        }
+    const startSize = arrObjSize - totalSize
+
+    flatX.splice(0, startSize) //keeps the last items
+    flatY.splice(0, startSize) //keeps the last items
+
+    let trainX = flatX.slice(0, trainSize);
+    let trainY = flatY.slice(0, trainSize);
+    
+    let stats = null
+
+    if(scaling === 'zscore') {
+        let trainNormalized = zscore2d(trainX)
+        stats = trainNormalized.stats
+        trainX = trainNormalized.data
+        trainNormalized = null
+    }
+
+    let testX = flatX.slice(-testSize);
+
+    if(scaling === 'zscore') {
+        testX = zscore2d(testX, stats).data
+    }
+
+    let testY = flatY.slice(-testSize);
+
+
+    flatX = null
+    flatY = null
+
+    let trainSource
+    let testSource
+
+    if(showSource) {
+        source.splice(0, startSize) //keeps the last items
+        trainSource = source.slice(0, trainSize);
+        testSource = source.slice(-testSize);
+        source = null
     }
 
     return {
@@ -140,7 +187,10 @@ export const parseTrainingXY = ({
         testY,
         configX,
         configY,
-    };
+        trainSource,
+        testSource,
+        stats
+    }
 };
 export const parseProductionX = ({
     arrObj = [],
@@ -149,13 +199,23 @@ export const parseProductionX = ({
     validateRows = () => true,
     shuffle = false,
     state = {},
+    showSource = false,
+    scaling = null,
+    stats
 }) => {
     let flatX = [];
+    let source = [];
     let keyNamesX = null;
 
     validateArray(arrObj, { min: 1 }, 'parseProductionX');
     validateFirstRow(arrObj[0]);
 
+    if(![null, 'zscore'].includes(scaling)) {
+        throw new Error(`Invalid "scaling" property. Accepting null or "zscore".`)
+    }
+
+    const arrObjSize = arrObj.length
+
     if (yCallbackFunc != null) {
         throw new Error('The property "yCallbackFunc" must not be set in "parseProductionX".');
     }
@@ -185,21 +245,59 @@ export const parseProductionX = ({
             }
 
             flatX.push(rowX);
+
+            if(showSource) {
+                source.push(arrObj[x])
+            }
+            
+
         } catch(err) {
             throw new Error(`[BUG] - Skipped row index=${x}: ${err.message}`);
         }
     }
 
     if (shuffle) {
-         flatX = arrayShuffle(flatX);
+        const merged = new Array(flatX.length);
+
+        for (let i = 0; i < flatX.length; i++) {
+            merged[i] = {
+                x: flatX[i]
+            };
+
+            if(showSource) {
+                merged[i].source = source[i]
+            }
+        }
+
+        const shuffled = arrayShuffle(merged);
+
+        flatX = new Array(shuffled.length);
+
+        if(showSource) {
+            source = new Array(shuffled.length)
+        }
+        
+
+        for (let i = 0; i < shuffled.length; i++) {
+            flatX[i] = shuffled[i].x;
+
+            if(showSource) {
+                source[i] = shuffled[i].source
+            }
+        }
     }
 
     const configX = {
         keyNames: keyNamesX ?? [],
     };
 
+    if(scaling === 'zscore') {
+        flatX = zscore2d(flatX, stats).data
+    }
+
     return {
         X: flatX,
+        source,
         configX,
     };
 };

package/src/utilities.js

@@ -10,28 +10,4 @@ export const arrayShuffle = X => {
     }
 
     return shuffledX;
-}
-
-export const arrayShuffleXY = (X, Y) => {
-    if (!Array.isArray(X) || !Array.isArray(Y)) {
-        throw new TypeError('Both X and Y must be arrays');
-    }
-
-    if (X.length !== Y.length) {
-        throw new Error('X and Y must have the same length');
-    }
-
-    // Copy to avoid mutating the originals
-    const shuffledX = [...X];
-    const shuffledY = [...Y];
-
-    // Shuffle both arrays using the same swaps
-    for (let i = shuffledX.length - 1; i > 0; i--) {
-        const j = Math.floor(Math.random() * (i + 1));
-
-        [shuffledX[i], shuffledX[j]] = [shuffledX[j], shuffledX[i]];
-        [shuffledY[i], shuffledY[j]] = [shuffledY[j], shuffledY[i]];
-    }
-
-    return { X: shuffledX, Y: shuffledY };
-};
+}

package/src/validators.js

@@ -1,6 +1,8 @@
 
 export const isNumber = v => v != null && Number.isFinite(v)
 
+export const isPositiveInteger = value =>  Number.isInteger(value) && value > 0
+
 export const isKeyPairObject = param => {
   return (
     param !== null &&
@@ -132,4 +134,24 @@ export const arraysAreNotEqualSize = (list, callerName) => {
     }
 
     return false
+}
+
+export const validateSizes = ({arrObjSize, trainSize, testSize}) => {
+
+    if(!isPositiveInteger(trainSize)) {
+        throw new Error(`Invalid property: "trainSize" (${trainSize}) must be a non-negative integer.`)
+    }
+    if(!isPositiveInteger(testSize)) {
+         throw new Error(`Invalid property: "testSize" (${testSize}) must be a non-negative integer.`)
+    }
+
+    if (!Number.isInteger(arrObjSize) || arrObjSize < 0) {
+        throw new Error(`Invalid property: "arrObjSize" (${arrObjSize}) must be a non-negative integer.`)
+    }
+
+    if(arrObjSize < (trainSize + testSize)) {
+        throw new Error(`Invalid property: The sum of "trainSize" + "testSize" (${trainSize + testSize}) must not be larger than "arrObj.length" (${arrObjSize}).`)
+    }
+
+    return true
 }

package/src/zscore2.js

@@ -0,0 +1,145 @@
+const toArray = typedOrArray => Array.from(typedOrArray);
+
+const normalizeStats = (stats, cols) => {
+  const mean = new Float64Array(cols);
+  const std = new Float64Array(cols);
+  const scale = new Float64Array(cols);
+  const count = new Uint32Array(cols);
+
+  const meanIn = Array.isArray(stats?.mean) ? stats.mean : [];
+  const stdIn = Array.isArray(stats?.std) ? stats.std : [];
+  const scaleIn = Array.isArray(stats?.scale) ? stats.scale : [];
+  const countIn = Array.isArray(stats?.count) ? stats.count : [];
+
+  for (let j = 0; j < cols; j++) {
+    const m = Number(meanIn[j]);
+    const s = Number(stdIn[j]);
+    const sc = Number(scaleIn[j]);
+    const c = Number(countIn[j]);
+
+    mean[j] = Number.isFinite(m) ? m : 0;
+    std[j] = Number.isFinite(s) && s > 0 ? s : 0;
+    count[j] = Number.isFinite(c) && c > 0 ? c : 0;
+
+    if (Number.isFinite(sc) && sc > 0) {
+      scale[j] = sc;
+      if (!(std[j] > 0)) { std[j] = 1 / sc; }
+    } else if (std[j] > 0) {
+      scale[j] = 1 / std[j];
+    } else {
+      scale[j] = 0;
+    }
+  }
+
+  return { mean, std, scale, count };
+};
+
+const fitStats = (arr, cols) => {
+  const rows = arr.length;
+  const mean = new Float64Array(cols);
+  const m2 = new Float64Array(cols);
+  const count = new Uint32Array(cols);
+
+  for (let i = 0; i < rows; i++) {
+    const row = arr[i];
+
+    if (!Array.isArray(row)) {
+      throw new TypeError(`[zscore2d] Invalid row at index=${i}. Expected an array.`);
+    }
+    if (row.length !== cols) {
+      throw new Error(`[zscore2d] Inconsistent row size at index=${i}. Expected ${cols}, got ${row.length}.`);
+    }
+
+    for (let j = 0; j < cols; j++) {
+      const x = Number(row[j]);
+      if (!Number.isFinite(x)) { continue; }
+
+      count[j]++;
+
+      const delta = x - mean[j];
+      mean[j] += delta / count[j];
+      m2[j] += delta * (x - mean[j]);
+    }
+  }
+
+  const std = new Float64Array(cols);
+  const scale = new Float64Array(cols);
+
+  for (let j = 0; j < cols; j++) {
+    const variance = count[j] > 1 ? (m2[j] / count[j]) : 0;
+    const s = variance > 0 ? Math.sqrt(variance) : 0;
+    std[j] = s;
+    scale[j] = s > 0 ? (1 / s) : 0;
+  }
+
+  return { mean, std, scale, count };
+};
+
+const scaleFromStats = (arr, normalizedStats) => {
+  const rows = arr.length;
+  const cols = normalizedStats.mean.length;
+  const out = new Array(rows);
+
+  for (let i = 0; i < rows; i++) {
+    const row = arr[i];
+
+    if (!Array.isArray(row)) {
+      throw new TypeError(`[zscore2d] Invalid row at index=${i}. Expected an array.`);
+    }
+    if (row.length !== cols) {
+      throw new Error(`[zscore2d] Inconsistent row size at index=${i}. Expected ${cols}, got ${row.length}.`);
+    }
+
+    const normalized = new Array(cols);
+    for (let j = 0; j < cols; j++) {
+      const x = Number(row[j]);
+      const sc = normalizedStats.scale[j];
+
+      normalized[j] = (Number.isFinite(x) && sc > 0)
+        ? (x - normalizedStats.mean[j]) * sc
+        : 0;
+    }
+
+    out[i] = normalized;
+  }
+
+  return out;
+};
+
+export const zscore2d = (arr, stats = null) => {
+  if (!Array.isArray(arr)) {
+    throw new TypeError('[zscore2d] "arr" must be a 2D array.');
+  }
+
+  if (arr.length === 0) {
+    const normalizedStats = normalizeStats(stats ?? {}, 0);
+    return {
+      stats: {
+        mean: toArray(normalizedStats.mean),
+        std: toArray(normalizedStats.std),
+        scale: toArray(normalizedStats.scale),
+        count: toArray(normalizedStats.count)
+      },
+      data: []
+    };
+  }
+
+  const firstRow = arr[0];
+  if (!Array.isArray(firstRow)) {
+    throw new TypeError('[zscore2d] "arr" must be a 2D array of rows.');
+  }
+
+  const cols = firstRow.length;
+  const normalizedStats = stats ? normalizeStats(stats, cols) : fitStats(arr, cols);
+  const data = scaleFromStats(arr, normalizedStats);
+
+  return {
+    stats: {
+      mean: toArray(normalizedStats.mean),
+      std: toArray(normalizedStats.std),
+      scale: toArray(normalizedStats.scale),
+      count: toArray(normalizedStats.count)
+    },
+    data
+  };
+};

package/test/test.js

@@ -31,6 +31,9 @@ const test = async () => {
 
     const arrObj = indicators.getData()
 
+    const trainSize = Math.round(arrObj.length * 0.8)
+    const testSize = arrObj.length - trainSize
+
     const {
         trainX,
         trainY,
@@ -39,8 +42,10 @@ const test = async () => {
         configX,
         configY
     } = parseTrainingXY({
+        scaling: 'zscore',
         arrObj,
-        trainingSplit: 0.50,
+        trainSize,
+        testSize,
         yCallbackFunc,
         xCallbackFunc,
         validateRows: ({objRow, index}) => {

package/src/balancing.js

@@ -1,187 +0,0 @@
-const isValidLabelValue = (value) => {
-  if (typeof value === "string" || typeof value === "boolean") return true;
-  if (typeof value === "number") return Number.isFinite(value);
-
-  if (Array.isArray(value)) {
-    return value.every(isValidLabelValue);
-  }
-
-  return false;
-};
-
-const makeLabelKey = (label) => {
-  if (!isValidLabelValue(label)) {
-    throw new Error(
-      "Invalid Y label. Allowed types: finite numbers, strings, booleans, or nested arrays of those."
-    );
-  }
-
-  return JSON.stringify(label);
-};
-
-const validateXY = (X, Y) => {
-  if (!Array.isArray(X) || !Array.isArray(Y)) {
-    throw new Error("X and Y must be arrays.");
-  }
-
-  if (X.length !== Y.length) {
-    throw new Error("X and Y must have the same length.");
-  }
-
-  if (X.length === 0) {
-    throw new Error("X and Y cannot be empty.");
-  }
-};
-
-const mulberry32 = (seed) => {
-  let t = seed >>> 0;
-
-  return () => {
-    t += 0x6D2B79F5;
-    let r = Math.imul(t ^ (t >>> 15), t | 1);
-    r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
-    return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
-  };
-};
-
-const resolveRandom = ({ random, seed } = {}) => {
-  if (typeof random === "function") return random;
-  if (Number.isInteger(seed)) return mulberry32(seed);
-  return Math.random;
-};
-
-const shuffleInPlace = (arr, random = Math.random) => {
-  for (let i = arr.length - 1; i > 0; i--) {
-    const j = Math.floor(random() * (i + 1));
-    [arr[i], arr[j]] = [arr[j], arr[i]];
-  }
-
-  return arr;
-};
-
-const sampleWithoutReplacement = (samples, size, random = Math.random) => {
-  if (size > samples.length) {
-    throw new Error("Cannot sample more items than available without replacement.");
-  }
-
-  const copy = [...samples];
-  shuffleInPlace(copy, random);
-  return copy.slice(0, size);
-};
-
-const sampleWithReplacement = (samples, size, random = Math.random) => {
-  if (samples.length === 0) {
-    throw new Error("Cannot sample from an empty array.");
-  }
-
-  const out = [];
-
-  for (let i = 0; i < size; i++) {
-    const idx = Math.floor(random() * samples.length);
-    out.push(samples[idx]);
-  }
-
-  return out;
-};
-
-const maybeClone = (value, clone) => {
-  if (!clone) return value;
-  return structuredClone(value);
-};
-
-const groupXYByLabel = (X, Y, { cloneX = false } = {}) => {
-  validateXY(X, Y);
-
-  const groups = new Map();
-
-  Y.forEach((label, i) => {
-    const key = makeLabelKey(label);
-
-    if (!groups.has(key)) {
-      groups.set(key, []);
-    }
-
-    groups.get(key).push({
-      x: maybeClone(X[i], cloneX),
-      y: label,
-    });
-  });
-
-  return groups;
-};
-
-export const oversampleXY = (X, Y, options = {}) => {
-  const {
-    random,
-    seed,
-    shuffleResult = true,
-    cloneX = false,
-  } = options;
-
-  const rng = resolveRandom({ random, seed });
-  const groups = groupXYByLabel(X, Y, { cloneX });
-
-  const counts = [...groups.values()].map((samples) => samples.length);
-  const maxCount = Math.max(...counts);
-
-  const combined = [];
-
-  for (const samples of groups.values()) {
-    const originals = [...samples];
-    const extrasNeeded = maxCount - originals.length;
-
-    const extras =
-      extrasNeeded > 0
-        ? sampleWithReplacement(samples, extrasNeeded, rng).map((sample) => ({
-            x: maybeClone(sample.x, cloneX),
-            y: sample.y,
-          }))
-        : [];
-
-    combined.push(...originals, ...extras);
-  }
-
-  if (shuffleResult) {
-    shuffleInPlace(combined, rng);
-  }
-
-  return {
-    X: combined.map(({ x }) => x),
-    Y: combined.map(({ y }) => y),
-  };
-};
-
-export const undersampleXY = (X, Y, options = {}) => {
-  const {
-    random,
-    seed,
-    shuffleResult = true,
-    cloneX = false,
-  } = options;
-
-  const rng = resolveRandom({ random, seed });
-  const groups = groupXYByLabel(X, Y, { cloneX });
-
-  const counts = [...groups.values()].map((samples) => samples.length);
-  const minCount = Math.min(...counts);
-
-  const combined = [];
-
-  for (const samples of groups.values()) {
-    const selected = sampleWithoutReplacement(samples, minCount, rng).map((sample) => ({
-      x: maybeClone(sample.x, cloneX),
-      y: sample.y,
-    }));
-
-    combined.push(...selected);
-  }
-
-  if (shuffleResult) {
-    shuffleInPlace(combined, rng);
-  }
-
-  return {
-    X: combined.map(({ x }) => x),
-    Y: combined.map(({ y }) => y),
-  };
-};