xy-scale 1.4.31 → 1.4.32

package/dist/xy-scale.min.js

@@ -1 +1 @@
-var XY_Scale;(()=>{"use strict";var e={d:(t,r)=>{for(var n in r)e.o(r,n)&&!e.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:r[n]})},o:(e,t)=>Object.prototype.hasOwnProperty.call(e,t),r:e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})}},t={};e.r(t),e.d(t,{arrayToTimesteps:()=>l,parseProductionX:()=>o,parseTrainingXY:()=>a});const r=(e,{min:t=-1/0,max:r=1/0},n)=>{if(!Array.isArray(e))throw new Error(`Invalid property. "${n}" expected an array.`);if(e.length<t)throw new Error(`Invalid property value. Array "${n}" expected at least ${r} items.`);if(e.length>r)throw new Error(`Invalid property value. Array "${n}" expected at max ${r} items.`);return!0},n=e=>{for(const[t,r]of Object.entries(e)){if("number"==typeof r&&Number.isNaN(r))throw new Error(`Invalid value at index 0 property "${t}": value is "${r}". Expected a numeric value.`);if(null===r)throw new Error(`Invalid value at index 0 property "${t}": value is "${r}".`)}return!0},a=({arrObj:e=[],trainingSplit:t=.8,yCallbackFunc:a=e=>e,xCallbackFunc:o=e=>e,validateRows:l=()=>!0,shuffle:s=!1,balancing:i="",state:c={}})=>{let u=[],f=[];r(e,{min:5},"parseTrainingXY"),n(e[0]);for(let t=0;t<e.length;t++){if(!l({objRow:e,index:t,state:c}))continue;const r=o({objRow:e,index:t,state:c}),n=a({objRow:e,index:t,state:c});null!=r&&null!=n&&(u.push(r),f.push(n))}if(s){const{shuffledX:e,shuffledY:t}=((e,t)=>{if(e.length!==t.length)throw new Error("X and Y arrays must have the same length");const r=Array.from({length:e.length},((e,t)=>t));for(let e=r.length-1;e>0;e--){const t=Math.floor(Math.random()*(e+1));[r[e],r[t]]=[r[t],r[e]]}return{shuffledX:r.map((t=>e[t])),shuffledY:r.map((e=>t[e]))}})(u,f);u=e,f=t}const h=u.length,p=f.length,d=new Array(h),y=new Array(p),m={keyNames:h?Object.keys(u[0]):[]},b={keyNames:p?Object.keys(f[0]):[]};for(let e=0;e<h;e++)d[e]=Object.values(u[e]);for(let e=0;e<p;e++)y[e]=Object.values(f[e]);const g=Math.floor(d.length*t);let v=d.slice(0,g),w=y.slice(0,g),j=d.slice(g),O=y.slice(g);if(i){let e;if("oversample"===i)e=((e,t)=>{const r={},n={};t.forEach(((a,o)=>{r[a]||(r[a]=0,n[a]=[]),r[a]++,n[a].push([e[o],t[o]])}));const a=Math.max(...Object.values(r)),o=[],l=[];return Object.keys(n).forEach((e=>{const t=n[e],r=t.length;for(let e=0;e<a;e++){const n=t[e%r];o.push(n[0]),l.push(n[1])}})),{X:o,Y:l}})(v,w),v=e.X,w=e.Y;else{if("undersample"!==i)throw Error('balancing argument only accepts "false", "oversample" and "undersample". Defaults to "false".');e=((e,t)=>{const r={},n={};t.forEach(((a,o)=>{r[a]||(r[a]=0,n[a]=[]),r[a]++,n[a].push([e[o],t[o]])}));const a=Math.min(...Object.values(r)),o=[],l=[];return Object.keys(n).forEach((e=>{const t=n[e];for(let e=0;e<a;e++){const r=t[e];o.push(r[0]),l.push(r[1])}})),{X:o,Y:l}})(v,w),v=e.X,w=e.Y}}return{trainX:v,trainY:w,testX:j,testY:O,configX:m,configY:b}},o=({arrObj:e=[],xCallbackFunc:t=e=>e,validateRows:a=()=>!0,shuffle:o=!1,state:l={}})=>{let s=[];r(e,{min:5},"parseProductionX"),n(e[0]);for(let r=0;r<e.length;r++){if(!a(e[r]))continue;const n=t({objRow:e,index:r,state:l});null!=n&&!1!==n&&s.push(n)}o&&(s=(e=>{const t=[...e];for(let e=t.length-1;e>0;e--){const r=Math.floor(Math.random()*(e+1));[t[e],t[r]]=[t[r],t[e]]}return t})(s));const i=s.length,c=new Array(i),u={keyNames:i?Object.keys(s[0]):[]};for(let e=0;e<i;e++)c[e]=Object.values(s[e]);return{X:c,configX:u}},l=(e,t)=>{if(0===t)return e;if(t<0)throw new Error("timeSteps must be greater than 0");const r=[];for(let n=0;n<=e.length-t;n++)r.push(e.slice(n,n+t));return r};XY_Scale=t})();
+var XY_Scale;(()=>{"use strict";var e={d:(t,r)=>{for(var n in r)e.o(r,n)&&!e.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:r[n]})},o:(e,t)=>Object.prototype.hasOwnProperty.call(e,t),r:e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})}},t={};e.r(t),e.d(t,{arrayToTimesteps:()=>i,parseProductionX:()=>s,parseTrainingXY:()=>a});const r=e=>null==e||!Number.isFinite(e),n=e=>{const t=[...e];for(let e=t.length-1;e>0;e--){const r=Math.floor(Math.random()*(e+1));[t[e],t[r]]=[t[r],t[e]]}return t},o=(e,{min:t=-1/0,max:r=1/0},n)=>{if(!Array.isArray(e))throw new Error(`Invalid property. "${n}" expected an array.`);if(e.length<t)throw new Error(`Invalid property value. Array "${n}" expected at least ${r} items.`);if(e.length>r)throw new Error(`Invalid property value. Array "${n}" expected at max ${r} items.`);return!0},l=e=>{for(const[t,r]of Object.entries(e)){if("number"==typeof r&&Number.isNaN(r))throw new Error(`Invalid value at index 0 property "${t}": value is "${r}". Expected a numeric value.`);if(null===r)throw new Error(`Invalid value at index 0 property "${t}": value is "${r}".`)}return!0},a=({arrObj:e=[],trainingSplit:t=.8,yCallbackFunc:a=e=>e,xCallbackFunc:s=e=>e,validateRows:i=()=>!0,shuffle:c=!1,balancing:u="",state:h={}})=>{let f=[],p=[];const y=[];o(e,{min:5},"parseTrainingXY"),l(e[0]);for(let t=0;t<e.length;t++){if(!i({objRow:e,index:t,state:h}))continue;const r=s({objRow:e,index:t,state:h}),n=a({objRow:e,index:t,state:h});null!=r&&null!=n&&(f.push(r),p.push(n),y.push(t))}if(c){const e=new Array(f.length);for(let t=0;t<f.length;t++)e[t]={x:f[t],y:p[t],sourceIndex:y[t]};const t=n(e);f=new Array(t.length),p=new Array(t.length);for(let e=0;e<t.length;e++)f[e]=t[e].x,p[e]=t[e].y,y[e]=t[e].sourceIndex}const d=f.length,g=p.length,w=d?Object.keys(f[0]).filter((e=>"tempIdx"!==e)):[],b=g?Object.keys(p[0]).filter((e=>"tempIdx"!==e)):[],m=new Array(d),x=new Array(g),v={keyNames:w},j={keyNames:b};for(let e=0;e<d;e++){const t=f[e],n=y[e],o=new Array(w.length);for(let e=0;e<w.length;e++){const l=w[e],a=t[l];if(r(a))throw new Error(`Invalid property value (${a}) returned from "xCallbackFunc" on index "${n}" property "${l}".`);o[e]=a}m[e]=o}for(let e=0;e<g;e++){const t=p[e],r=new Array(b.length);for(let e=0;e<b.length;e++)r[e]=t[b[e]];x[e]=r}const A=Math.floor(m.length*t);let O=m.slice(0,A),X=x.slice(0,A),$=m.slice(A),E=x.slice(A);if(u){let e;if("oversample"===u)e=((e,t)=>{const r={},n={};t.forEach(((o,l)=>{r[o]||(r[o]=0,n[o]=[]),r[o]++,n[o].push([e[l],t[l]])}));const o=Math.max(...Object.values(r)),l=[],a=[];return Object.keys(n).forEach((e=>{const t=n[e],r=t.length;for(let e=0;e<o;e++){const n=t[e%r];l.push(n[0]),a.push(n[1])}})),{X:l,Y:a}})(O,X),O=e.X,X=e.Y;else{if("undersample"!==u)throw Error('balancing argument only accepts "", "oversample" and "undersample". Defaults to "".');e=((e,t)=>{const r={},n={};t.forEach(((o,l)=>{r[o]||(r[o]=0,n[o]=[]),r[o]++,n[o].push([e[l],t[l]])}));const o=Math.min(...Object.values(r)),l=[],a=[];return Object.keys(n).forEach((e=>{const t=n[e];for(let e=0;e<o;e++){const r=t[e];l.push(r[0]),a.push(r[1])}})),{X:l,Y:a}})(O,X),O=e.X,X=e.Y}}return{trainX:O,trainY:X,testX:$,testY:E,configX:v,configY:j}},s=({arrObj:e=[],xCallbackFunc:t=e=>e,validateRows:a=()=>!0,shuffle:s=!1,state:i={}})=>{let c=[],u=[];o(e,{min:5},"parseProductionX"),l(e[0]);for(let r=0;r<e.length;r++){if(!a({objRow:e,index:r,state:i}))continue;const n=t({objRow:e,index:r,state:i});null!=n&&!1!==n&&(c.push(n),u.push(r))}if(s){const e=new Array(c.length);for(let t=0;t<c.length;t++)e[t]={x:c[t],sourceIndex:u[t]};const t=n(e);c=new Array(t.length),u=new Array(t.length);for(let e=0;e<t.length;e++)c[e]=t[e].x,u[e]=t[e].sourceIndex}const h=c.length,f=h?Object.keys(c[0]).filter((e=>"tempIdx"!==e)):[],p=new Array(h),y={keyNames:f};for(let e=0;e<h;e++){const t=c[e],n=u[e],o=new Array(f.length);for(let e=0;e<f.length;e++){const l=f[e],a=t[l];if(r(a))throw new Error(`Invalid property value (${a}) returned from "xCallbackFunc" on index "${n}" property "${l}".`);o[e]=a}p[e]=o}return{X:p,configX:y}},i=(e,t)=>{if(0===t)return e;if(t<0)throw new Error("timeSteps must be greater than 0");const r=[];for(let n=0;n<=e.length-t;n++)r.push(e.slice(n,n+t));return r};XY_Scale=t})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xy-scale",
-  "version": "1.4.31",
+  "version": "1.4.32",
   "main": "./index.js",
   "type": "module",
   "scripts": {

package/src/datasets.js

@@ -1,99 +1,140 @@
-import { arrayShuffle, xyArrayShuffle } from "./utilities.js";
+import { arrayShuffle, isBadNumber } from "./utilities.js";
 import { oversampleXY, undersampleXY } from "./balancing.js";
 import { validateFirstRow, validateArray } from "./validators.js";
 
-//ADD A PARAM max correlation that will measure the correlation between variables if defined
-
-export const parseTrainingXY = ({ 
-    arrObj = [], //array of objects
-    trainingSplit = 0.8, //numberic float between 0.01 and 0.99
-    yCallbackFunc = row => row, //accepted callback functions
-    xCallbackFunc = row => row, //accepted callback functions
-    validateRows = () => true,//accepted callback functions
-    shuffle = false,//only booleans
-    balancing = '',//accepted null, "oversample" or "undersample"
-    state = {}, //accepted object or classes
+// ADD A PARAM max correlation that will measure the correlation between variables if defined
+
+export const parseTrainingXY = ({
+    arrObj = [], // array of objects
+    trainingSplit = 0.8, // numeric float between 0.01 and 0.99
+    yCallbackFunc = row => row, // accepted callback functions
+    xCallbackFunc = row => row, // accepted callback functions
+    validateRows = () => true, // accepted callback functions
+    shuffle = false, // only booleans
+    balancing = '', // accepted '', 'oversample' or 'undersample'
+    state = {}, // accepted object or classes
 }) => {
     let X = [];
     let Y = [];
+    const sourceIndexes = [];
 
-    validateArray(arrObj, {min: 5}, 'parseTrainingXY')
-    validateFirstRow(arrObj[0])
+    validateArray(arrObj, { min: 5 }, 'parseTrainingXY');
+    validateFirstRow(arrObj[0]);
 
-    //if parsedX and parsedY is undefined or null the current row will be excluded from training or production
+    // if parsedX or parsedY is undefined/null the current row will be excluded from training
     for (let x = 0; x < arrObj.length; x++) {
+        if (!validateRows({ objRow: arrObj, index: x, state })) continue;
+
+        const parsedX = xCallbackFunc({ objRow: arrObj, index: x, state });
+        const parsedY = yCallbackFunc({ objRow: arrObj, index: x, state });
+
+        if (
+            typeof parsedX !== 'undefined' &&
+            parsedX !== null &&
+            typeof parsedY !== 'undefined' &&
+            parsedY !== null
+        ) {
+            X.push(parsedX);
+            Y.push(parsedY);
+            sourceIndexes.push(x);
+        }
+    }
 
-        if(!validateRows({ objRow: arrObj, index: x, state})) continue
+    if (shuffle) {
+        const merged = new Array(X.length);
 
-        const parsedX = xCallbackFunc({ objRow: arrObj, index: x, state})
-        const parsedY = yCallbackFunc({ objRow: arrObj, index: x, state})
-    
-        if (typeof parsedX !== 'undefined' && parsedX !== null && typeof parsedY !== 'undefined' && parsedY !== null) {
-            X.push(parsedX)
-            Y.push(parsedY)
+        for (let i = 0; i < X.length; i++) {
+            merged[i] = {
+                x: X[i],
+                y: Y[i],
+                sourceIndex: sourceIndexes[i],
+            };
         }
-    }
 
-    if(shuffle)
-    {
-        const {shuffledX, shuffledY} = xyArrayShuffle(X, Y)
-        X = shuffledX
-        Y = shuffledY
+        const shuffled = arrayShuffle(merged);
+
+        X = new Array(shuffled.length);
+        Y = new Array(shuffled.length);
+
+        for (let i = 0; i < shuffled.length; i++) {
+            X[i] = shuffled[i].x;
+            Y[i] = shuffled[i].y;
+            sourceIndexes[i] = shuffled[i].sourceIndex;
+        }
     }
 
-    const xLen = X.length
-    const yLen = Y.length
-    const flatX = new Array(xLen)
-    const flatY = new Array(yLen)
+    const xLen = X.length;
+    const yLen = Y.length;
+
+    const xKeys = xLen ? Object.keys(X[0]).filter(key => key !== 'tempIdx') : [];
+    const yKeys = yLen ? Object.keys(Y[0]).filter(key => key !== 'tempIdx') : [];
+
+    const flatX = new Array(xLen);
+    const flatY = new Array(yLen);
+
     const configX = {
-        keyNames: xLen ? Object.keys(X[0]) : []
-    }
+        keyNames: xKeys,
+    };
+
     const configY = {
-        keyNames: yLen ? Object.keys(Y[0]) : []
-    }
+        keyNames: yKeys,
+    };
 
-    for(let idx = 0; idx < xLen; idx++)
-    {
-        flatX[idx] = Object.values(X[idx])
-    }
+    for (let idx = 0; idx < xLen; idx++) {
+        const rowObj = X[idx];
+        const sourceIndex = sourceIndexes[idx];
+        const flatRow = new Array(xKeys.length);
 
-    for(let idx = 0; idx < yLen; idx++)
-    {
-        flatY[idx] = Object.values(Y[idx])
-    }
+        for (let j = 0; j < xKeys.length; j++) {
+            const key = xKeys[j];
+            const value = rowObj[key];
 
-    const splitIndex = Math.floor(flatX.length * trainingSplit)
+            if (isBadNumber(value)) {
+                throw new Error(
+                    `Invalid property value (${value}) returned from "xCallbackFunc" on index "${sourceIndex}" property "${key}".`
+                );
+            }
 
-    let trainX = flatX.slice(0, splitIndex)
-    let trainY = flatY.slice(0, splitIndex)
-    let testX = flatX.slice(splitIndex)
-    let testY = flatY.slice(splitIndex)
+            flatRow[j] = value;
+        }
 
+        flatX[idx] = flatRow;
+    }
 
-    if(balancing)
-    {
-        let balance
+    for (let idx = 0; idx < yLen; idx++) {
+        const rowObj = Y[idx];
+        const flatRow = new Array(yKeys.length);
 
-        if(balancing === 'oversample')
-        {
-            balance = oversampleXY(trainX, trainY)
-            trainX = balance.X
-            trainY = balance.Y
-        }
-        else if(balancing === 'undersample')
-        {
-            balance = undersampleXY(trainX, trainY)
-            trainX = balance.X
-            trainY = balance.Y           
-        }
-        else
-        {
-            throw Error('balancing argument only accepts "false", "oversample" and "undersample". Defaults to "false".')
+        for (let j = 0; j < yKeys.length; j++) {
+            flatRow[j] = rowObj[yKeys[j]];
         }
+
+        flatY[idx] = flatRow;
     }
 
+    const splitIndex = Math.floor(flatX.length * trainingSplit);
+
+    let trainX = flatX.slice(0, splitIndex);
+    let trainY = flatY.slice(0, splitIndex);
+    let testX = flatX.slice(splitIndex);
+    let testY = flatY.slice(splitIndex);
+
+    if (balancing) {
+        let balance;
+
+        if (balancing === 'oversample') {
+            balance = oversampleXY(trainX, trainY);
+            trainX = balance.X;
+            trainY = balance.Y;
+        } else if (balancing === 'undersample') {
+            balance = undersampleXY(trainX, trainY);
+            trainX = balance.X;
+            trainY = balance.Y;
+        } else {
+            throw Error('balancing argument only accepts "", "oversample" and "undersample". Defaults to "".');
+        }
+    }
 
-    // Split into training and testing sets
     return {
         trainX,
         trainY,
@@ -104,49 +145,82 @@ export const parseTrainingXY = ({
     };
 };
 
-
-export const parseProductionX = ({ 
-    arrObj = [], 
+export const parseProductionX = ({
+    arrObj = [],
     xCallbackFunc = row => row,
     validateRows = () => true,
     shuffle = false,
     state = {},
 }) => {
     let X = [];
+    let sourceIndexes = [];
 
-    validateArray(arrObj, {min: 5}, 'parseProductionX')
-    validateFirstRow(arrObj[0])
+    validateArray(arrObj, { min: 5 }, 'parseProductionX');
+    validateFirstRow(arrObj[0]);
 
     for (let x = 0; x < arrObj.length; x++) {
+        if (!validateRows({ objRow: arrObj, index: x, state })) continue;
 
-        if(!validateRows(arrObj[x])) continue
-
-        const parsedX = xCallbackFunc({ objRow: arrObj, index: x, state})
+        const parsedX = xCallbackFunc({ objRow: arrObj, index: x, state });
 
         if (typeof parsedX !== 'undefined' && parsedX !== null && parsedX !== false) {
-            X.push(parsedX)
+            X.push(parsedX);
+            sourceIndexes.push(x);
         }
     }
 
-    if(shuffle)
-    {
-        X = arrayShuffle(X)
+    if (shuffle) {
+        const merged = new Array(X.length);
+
+        for (let i = 0; i < X.length; i++) {
+            merged[i] = {
+                x: X[i],
+                sourceIndex: sourceIndexes[i],
+            };
+        }
+
+        const shuffled = arrayShuffle(merged);
+
+        X = new Array(shuffled.length);
+        sourceIndexes = new Array(shuffled.length);
+
+        for (let i = 0; i < shuffled.length; i++) {
+            X[i] = shuffled[i].x;
+            sourceIndexes[i] = shuffled[i].sourceIndex;
+        }
     }
 
-    const xLen = X.length
-    const flatX = new Array(xLen)
+    const xLen = X.length;
+    const xKeys = xLen ? Object.keys(X[0]).filter(key => key !== 'tempIdx') : [];
+    const flatX = new Array(xLen);
+
     const configX = {
-        keyNames: xLen ? Object.keys(X[0]) : []
-    }
+        keyNames: xKeys,
+    };
+
+    for (let idx = 0; idx < xLen; idx++) {
+        const rowObj = X[idx];
+        const sourceIndex = sourceIndexes[idx];
+        const flatRow = new Array(xKeys.length);
+
+        for (let j = 0; j < xKeys.length; j++) {
+            const key = xKeys[j];
+            const value = rowObj[key];
+
+            if (isBadNumber(value)) {
+                throw new Error(
+                    `Invalid property value (${value}) returned from "xCallbackFunc" on index "${sourceIndex}" property "${key}".`
+                );
+            }
 
-    for(let idx = 0; idx < xLen; idx++)
-    {
-        flatX[idx] = Object.values(X[idx])
+            flatRow[j] = value;
+        }
+
+        flatX[idx] = flatRow;
     }
 
-    // Split into training and testing sets
     return {
         X: flatX,
         configX,
-    }
-};
+    };
+};

package/src/utilities.js

@@ -1,23 +1,4 @@
-export const xyArrayShuffle = (X, Y) => {
-    if (X.length !== Y.length) {
-        throw new Error("X and Y arrays must have the same length");
-    }
-
-    // Create an array of indices based on the length of X or Y
-    const indices = Array.from({ length: X.length }, (_, i) => i);
-
-    // Shuffle the indices using Fisher-Yates shuffle algorithm
-    for (let i = indices.length - 1; i > 0; i--) {
-        const j = Math.floor(Math.random() * (i + 1));
-        [indices[i], indices[j]] = [indices[j], indices[i]];
-    }
-
-    // Use the shuffled indices to rearrange X and Y
-    const shuffledX = indices.map(i => X[i]);
-    const shuffledY = indices.map(i => Y[i]);
-
-    return { shuffledX, shuffledY };
-}
+export const isBadNumber = (v) => v == null || !Number.isFinite(v)
 
 export const arrayShuffle = X => {
     // Make a copy of the array to avoid mutating the original

package/test/test.js

@@ -52,7 +52,7 @@ const test = async () => {
 
             return false
         },
-        shuffle: true,
+        shuffle: false,
         balancing: null,
     });
 
@@ -86,7 +86,7 @@ const yCallbackFunc = ({ objRow, index }) => {
     if (typeof next === 'undefined') return null
     
     return {
-        result: Number(next.close > next.open)
+        result: Number(next.close > next.open) //the output shoud must be an array to preserve the key->value order
     }
 }