xy-scale 1.3.5 → 1.3.7

package/dist/xy-scale.min.js

@@ -1 +1 @@
-var XY_Scale;(()=>{"use strict";var e={d:(t,n)=>{for(var a in n)e.o(n,a)&&!e.o(t,a)&&Object.defineProperty(t,a,{enumerable:!0,get:n[a]})},o:(e,t)=>Object.prototype.hasOwnProperty.call(e,t),r:e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})}},t={};e.r(t),e.d(t,{arrayToTimesteps:()=>u,parseProductionX:()=>s,parseTrainingXY:()=>o});const n=({arrObj:e,repeat:t={},minmaxRange:n=[0,1],groups:o={},customMinMaxRanges:s=null,excludes:u=new Set})=>{const i=[...e],l=i.length,c=i[0],p="object"==typeof s&&null!==s;if(0===l)return{scaledOutput:[],scaledConfig:{}};const m=Object.keys(c),f=m.map((e=>t.hasOwnProperty(e)?Math.max(t[e],1):1)),g=f.reduce(((e,t)=>e+t),0),h={arrObjLen:l,rangeMin:n[0],rangeMax:n[1],inputTypes:{},min:{},max:{},groupMinMax:{},repeat:t,groups:o,inputKeyNames:m,outputKeyNames:new Array(g),repeatedKeyNames:f};let d=0;for(let e=0;e<h.inputKeyNames.length;e++)for(let t=0;t<h.repeatedKeyNames[e];t++)h.outputKeyNames[d++]=h.inputKeyNames[e];a(h.groups);const x=["number","boolean"];for(const e of h.inputKeyNames){if(u.has(e)){h.inputTypes[e]="excluded";continue}const t=typeof c[e],n=r(e,h.groups);if(!x.includes(t))throw new Error(`Invalid input type "${t}" provided for key "${e}". Only accepting `);h.inputTypes[e]=t,p&&s.hasOwnProperty(e)?n?h.groupMinMax[n]=s[e]:(h.min[e]=s[e].min,h.max[e]=s[e].max):n?h.groupMinMax[n]={min:1/0,max:-1/0}:(h.min[e]=1/0,h.max[e]=-1/0)}for(const e of i)for(const t of h.inputKeyNames){if("excluded"===h.inputTypes[t])continue;let n=e[t];"boolean"===h.inputTypes[t]&&(e[t]=Number(n));const a=r(t,h.groups);(!1===p||p&&!s.hasOwnProperty(t))&&(a?(h.groupMinMax[a].min=Math.min(h.groupMinMax[a].min,n),h.groupMinMax[a].max=Math.max(h.groupMinMax[a].max,n)):(h.min[t]=Math.min(h.min[t],n),h.max[t]=Math.max(h.max[t],n)))}const M=new Array(l);for(let e=0;e<l;e++){const t=i[e],n=new Array(h.outputKeyNames.length);let a=0;for(let e=0;e<h.inputKeyNames.length;e++){const o=h.inputKeyNames[e],s=t[o];if("excluded"===h.inputTypes[o]){n[a++]=s;continue}const u=r(o,h.groups);let i,l;u?(i=h.groupMinMax[u].min,l=h.groupMinMax[u].max):(i=h.min[o],l=h.max[o]);const c=l!==i?h.rangeMin+(s-i)/(l-i)*(h.rangeMax-h.rangeMin):h.rangeMin,p=h.repeatedKeyNames[e];for(let e=0;e<p;e++)n[a++]=c}M[e]=n}return{scaledOutput:M,scaledConfig:h}},a=e=>{const t=new Set,n=[];for(const[a,r]of Object.entries(e))t.add(a),n.push(a),r.forEach((e=>{t.add(e),n.push(e)}));if(t.size!==n.length)throw new Error("Duplicate value found between properties in validateUniqueProperties function.")},r=(e,t)=>{for(const[n,a]of Object.entries(t))if(a.includes(e))return n;return null},o=({arrObj:e=[],trainingSplit:t=.8,repeat:a={},yCallbackFunc:r=e=>e,xCallbackFunc:o=e=>e,validateRows:s=()=>!0,groups:u={},shuffle:i=!1,minmaxRange:l=[],balancing:c="",state:p={},customMinMaxRanges:m={},excludes:f=[]})=>{let g=[],h=[];for(let t=0;t<e.length;t++){if(!s({objRow:e,index:t,state:p}))continue;const n=o({objRow:e,index:t,state:p}),a=r({objRow:e,index:t,state:p});null!=n&&null!=a&&(g.push(n),h.push(a))}if(i){const{shuffledX:e,shuffledY:t}=((e,t)=>{if(e.length!==t.length)throw new Error("X and Y arrays must have the same length");const n=Array.from({length:e.length},((e,t)=>t));for(let e=n.length-1;e>0;e--){const t=Math.floor(Math.random()*(e+1));[n[e],n[t]]=[n[t],n[e]]}return{shuffledX:n.map((t=>e[t])),shuffledY:n.map((e=>t[e]))}})(g,h);g=e,h=t}const d=new Set(f);let{scaledOutput:x,scaledConfig:M}=n({arrObj:g,repeat:a,groups:u,minmaxRange:l,customMinMaxRanges:m,excludes:d}),{scaledOutput:y,scaledConfig:b}=n({arrObj:h,repeat:a,groups:u,minmaxRange:l,customMinMaxRanges:m,excludes:d});const O=Math.floor(x.length*t);let w=x.slice(0,O),j=y.slice(0,O),R=x.slice(O),v=y.slice(O);if(c){let e;if("oversample"===c)e=((e,t)=>{const n={},a={};t.forEach(((r,o)=>{n[r]||(n[r]=0,a[r]=[]),n[r]++,a[r].push([e[o],t[o]])}));const r=Math.max(...Object.values(n)),o=[],s=[];return Object.keys(a).forEach((e=>{const t=a[e],n=t.length;for(let e=0;e<r;e++){const a=t[e%n];o.push(a[0]),s.push(a[1])}})),{X:o,Y:s}})(w,j),w=e.X,j=e.Y;else{if("undersample"!==c)throw Error('balancing argument only accepts "false", "oversample" and "undersample". Defaults to "false".');e=((e,t)=>{const n={},a={};t.forEach(((r,o)=>{n[r]||(n[r]=0,a[r]=[]),n[r]++,a[r].push([e[o],t[o]])}));const r=Math.min(...Object.values(n)),o=[],s=[];return Object.keys(a).forEach((e=>{const t=a[e];for(let e=0;e<r;e++){const n=t[e];o.push(n[0]),s.push(n[1])}})),{X:o,Y:s}})(w,j),w=e.X,j=e.Y}}return{trainX:w,trainY:j,testX:R,testY:v,configX:M,configY:b}},s=({arrObj:e=[],repeat:t={},xCallbackFunc:a=e=>e,validateRows:r=()=>!0,groups:o={},shuffle:s=!1,minmaxRange:u=[],state:i={},customMinMaxRanges:l,excludes:c=[]})=>{let p=[];for(let t=0;t<e.length;t++){if(!r(e[t]))continue;const n=a({objRow:e,index:t,state:i});null!=n&&!1!==n&&p.push(n)}s&&(p=(e=>{const t=[...e];for(let e=t.length-1;e>0;e--){const n=Math.floor(Math.random()*(e+1));[t[e],t[n]]=[t[n],t[e]]}return t})(p));const{scaledOutput:m,scaledConfig:f}=n({arrObj:p,repeat:t,groups:o,minmaxRange:u,customMinMaxRanges:l,excludes:new Set(c)});return{X:m,configX:f}},u=(e,t)=>{if(0===t)return e;if(t<0)throw new Error("timeSteps must be greater than 0");const n=[];for(let a=0;a<=e.length-t;a++)n.push(e.slice(a,a+t));return n};XY_Scale=t})();
+var XY_Scale;(()=>{"use strict";var e={d:(t,n)=>{for(var r in n)e.o(n,r)&&!e.o(t,r)&&Object.defineProperty(t,r,{enumerable:!0,get:n[r]})},o:(e,t)=>Object.prototype.hasOwnProperty.call(e,t),r:e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})}},t={};e.r(t),e.d(t,{arrayToTimesteps:()=>l,parseProductionX:()=>u,parseTrainingXY:()=>i});const n=({arrObj:e,repeat:t={},minmaxRange:n,groups:o={},customMinMaxRanges:s=null,excludes:i=new Set})=>{const u=[...e],l=u.length,c=u[0],p="object"==typeof s&&null!==s;if(0===l)return{scaledOutput:[],scaledConfig:{}};const m=Object.keys(c),f=m.map((e=>t.hasOwnProperty(e)?Math.max(t[e],1):1)),g=f.reduce(((e,t)=>e+t),0),d={arrObjLen:l,rangeMin:n[0],rangeMax:n[1],inputTypes:{},min:{},max:{},groupMinMax:{},repeat:t,groups:o,inputKeyNames:m,outputKeyNames:new Array(g),repeatedKeyNames:f};let h=0;for(let e=0;e<d.inputKeyNames.length;e++)for(let t=0;t<d.repeatedKeyNames[e];t++)d.outputKeyNames[h++]=d.inputKeyNames[e];r(d.groups);const x=["number","boolean"];for(const e of d.inputKeyNames){if(i.has(e)){d.inputTypes[e]="excluded";continue}const t=typeof c[e],n=a(e,d.groups);if(!x.includes(t))throw new Error(`Invalid input type "${t}" provided for key "${e}". Only accepting `);d.inputTypes[e]=t,p&&s.hasOwnProperty(e)?n?d.groupMinMax[n]=s[e]:(d.min[e]=s[e].min,d.max[e]=s[e].max):n?d.groupMinMax[n]={min:1/0,max:-1/0}:(d.min[e]=1/0,d.max[e]=-1/0)}for(const e of u)for(const t of d.inputKeyNames){if("excluded"===d.inputTypes[t])continue;let n=e[t];"boolean"===d.inputTypes[t]&&(e[t]=Number(n));const r=a(t,d.groups);(!1===p||p&&!s.hasOwnProperty(t))&&(r?(d.groupMinMax[r].min=Math.min(d.groupMinMax[r].min,n),d.groupMinMax[r].max=Math.max(d.groupMinMax[r].max,n)):(d.min[t]=Math.min(d.min[t],n),d.max[t]=Math.max(d.max[t],n)))}const y=new Array(l);for(let e=0;e<l;e++){const t=u[e],n=new Array(d.outputKeyNames.length);let r=0;for(let e=0;e<d.inputKeyNames.length;e++){const o=d.inputKeyNames[e],s=t[o];if("excluded"===d.inputTypes[o]){n[r++]=s;continue}const i=a(o,d.groups);let u,l;i?(u=d.groupMinMax[i].min,l=d.groupMinMax[i].max):(u=d.min[o],l=d.max[o]);const c=l!==u?d.rangeMin+(s-u)/(l-u)*(d.rangeMax-d.rangeMin):d.rangeMin,p=d.repeatedKeyNames[e];for(let e=0;e<p;e++)n[r++]=c}y[e]=n}return{scaledOutput:y,scaledConfig:d}},r=e=>{const t=new Set,n=[];for(const[r,a]of Object.entries(e))t.add(r),n.push(r),a.forEach((e=>{t.add(e),n.push(e)}));if(t.size!==n.length)throw new Error("Duplicate value found between properties in validateUniqueProperties function.")},a=(e,t)=>{for(const[n,r]of Object.entries(t))if(r.includes(e))return n;return null},o=(e,{min:t=-1/0,max:n=1/0},r)=>{if(!Array.isArray(e))throw new Error(`Invalid property. "${r}" expected an array.`);if(e.length<t)throw new Error(`Invalid property value. Array "${r}" expected at least ${n} items.`);if(e.length>n)throw new Error(`Invalid property value. Array "${r}" expected at max ${n} items.`);return!0},s=e=>(Object.entries(e).forEach(((e,t)=>{if(null==t||Number.isNaN(t))throw new Error(`Invalid value at index ${e}: value is ${t}. Expected a defined, non-null, numeric value.`)})),!0),i=({arrObj:e=[],trainingSplit:t=.8,repeat:r={},yCallbackFunc:a=e=>e,xCallbackFunc:i=e=>e,validateRows:u=()=>!0,groups:l={},shuffle:c=!1,minmaxRange:p=[0,1],balancing:m="",state:f={},customMinMaxRanges:g={},excludes:d=[]})=>{let h=[],x=[];o(e,{min:5},"parseTrainingXY"),s(e[0]);for(let t=0;t<e.length;t++){if(!u({objRow:e,index:t,state:f}))continue;const n=i({objRow:e,index:t,state:f}),r=a({objRow:e,index:t,state:f});null!=n&&null!=r&&(h.push(n),x.push(r))}if(c){const{shuffledX:e,shuffledY:t}=((e,t)=>{if(e.length!==t.length)throw new Error("X and Y arrays must have the same length");const n=Array.from({length:e.length},((e,t)=>t));for(let e=n.length-1;e>0;e--){const t=Math.floor(Math.random()*(e+1));[n[e],n[t]]=[n[t],n[e]]}return{shuffledX:n.map((t=>e[t])),shuffledY:n.map((e=>t[e]))}})(h,x);h=e,x=t}const y=new Set(d);let{scaledOutput:M,scaledConfig:b}=n({arrObj:h,repeat:r,groups:l,minmaxRange:p,customMinMaxRanges:g,excludes:y}),{scaledOutput:w,scaledConfig:O}=n({arrObj:x,repeat:r,groups:l,minmaxRange:p,customMinMaxRanges:g,excludes:y});const v=Math.floor(M.length*t);let j=M.slice(0,v),R=w.slice(0,v),X=M.slice(v),N=w.slice(v);if(m){let e;if("oversample"===m)e=((e,t)=>{const n={},r={};t.forEach(((a,o)=>{n[a]||(n[a]=0,r[a]=[]),n[a]++,r[a].push([e[o],t[o]])}));const a=Math.max(...Object.values(n)),o=[],s=[];return Object.keys(r).forEach((e=>{const t=r[e],n=t.length;for(let e=0;e<a;e++){const r=t[e%n];o.push(r[0]),s.push(r[1])}})),{X:o,Y:s}})(j,R),j=e.X,R=e.Y;else{if("undersample"!==m)throw Error('balancing argument only accepts "false", "oversample" and "undersample". Defaults to "false".');e=((e,t)=>{const n={},r={};t.forEach(((a,o)=>{n[a]||(n[a]=0,r[a]=[]),n[a]++,r[a].push([e[o],t[o]])}));const a=Math.min(...Object.values(n)),o=[],s=[];return Object.keys(r).forEach((e=>{const t=r[e];for(let e=0;e<a;e++){const n=t[e];o.push(n[0]),s.push(n[1])}})),{X:o,Y:s}})(j,R),j=e.X,R=e.Y}}return{trainX:j,trainY:R,testX:X,testY:N,configX:b,configY:O}},u=({arrObj:e=[],repeat:t={},xCallbackFunc:r=e=>e,validateRows:a=()=>!0,groups:i={},shuffle:u=!1,minmaxRange:l=[0,1],state:c={},customMinMaxRanges:p,excludes:m=[]})=>{let f=[];o(e,{min:5},"parseProductionX"),s(e[0]);for(let t=0;t<e.length;t++){if(!a(e[t]))continue;const n=r({objRow:e,index:t,state:c});null!=n&&!1!==n&&f.push(n)}u&&(f=(e=>{const t=[...e];for(let e=t.length-1;e>0;e--){const n=Math.floor(Math.random()*(e+1));[t[e],t[n]]=[t[n],t[e]]}return t})(f));const{scaledOutput:g,scaledConfig:d}=n({arrObj:f,repeat:t,groups:i,minmaxRange:l,customMinMaxRanges:p,excludes:new Set(m)});return{X:g,configX:d}},l=(e,t)=>{if(0===t)return e;if(t<0)throw new Error("timeSteps must be greater than 0");const n=[];for(let r=0;r<=e.length-t;r++)n.push(e.slice(r,r+t));return n};XY_Scale=t})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xy-scale",
-  "version": "1.3.5",
+  "version": "1.3.7",
   "main": "./index.js",
   "type": "module",
   "scripts": {
@@ -14,7 +14,7 @@
     "@tensorflow/tfjs-node": "^4.22.0",
     "ml-confusion-matrix": "^2.0.0",
     "ml-knn": "^3.0.0",
-    "ohlcv-indicators": "^3.4.1",
+    "ohlcv-indicators": "^3.4.7",
     "webpack": "^5.96.1",
     "webpack-cli": "^5.1.4"
   },

package/src/correlation.js

@@ -0,0 +1,66 @@
+import { sum } from "@tensorflow/tfjs-node";
+
+export const analizeCorrelation = ({arrObj, corrSplit, corrExcludes, corrThreshold}) => {
+  const n = Math.floor(arrObj.length * corrSplit);
+  if (n < 2) return [];
+
+  const keys = Object.keys(arrObj[0]).filter(k => !corrExcludes.has(k));
+  const m = keys.length;
+
+  console.log({m})
+
+  // running sums, sums of squares, and cross‐sums
+  const sums   = new Float64Array(m); // by default Float64Array are filled with 0
+  const sumsq  = new Float64Array(m); // by default Float64Array are filled with 0
+  const sumxy  = Array.from({ length: m }, () => new Float64Array(m));
+
+  // 1 pass: accumulate sums, sumsq, and sumxy[i][j] (i<j)
+  for (let r = 0; r < n; r++) {
+    const row = arrObj[r];
+    for (let i = 0; i < m; i++) {
+
+      const xi = row[keys[i]];
+      
+      //errror here, the code neve reaches this part
+
+      sums[i]  += xi;
+      sumsq[i] += xi * xi;
+      for (let j = i + 1; j < m; j++) {
+        sumxy[i][j] += xi * row[keys[j]];
+      }
+    }
+  }
+
+  // precompute denominators: n*Σx² − (Σx)²
+  const denom = new Float64Array(m);
+  for (let i = 0; i < m; i++) {
+    denom[i] = n * sumsq[i] - sums[i] * sums[i];
+  }
+
+  // build result pairs
+  // collect pairs below/above your threshold
+  let lowCorrelationPairs  = `Low Correlation Pairs (< ${corrThreshold}):`;
+  let highCorrelationPairs = `High Correlation Pairs (>= ${corrThreshold}):`;
+
+  for (let i = 0; i < m; i++) {
+    for (let j = i + 1; j < m; j++) {
+      const cov  = n * sumxy[i][j] - sums[i] * sums[j];
+      const corr = cov / Math.sqrt(denom[i] * denom[j]);
+      const pair = `\n${keys[i]} - ${keys[j]}: ${corr}`
+
+      if(corr < corrThreshold)
+      {
+        lowCorrelationPairs += pair
+      }
+      else {
+        highCorrelationPairs += pair
+      }
+    }
+  }
+
+  console.log(lowCorrelationPairs)
+  console.log('---')
+  console.log(highCorrelationPairs)
+
+  return true;
+};

package/src/datasets.js

@@ -1,25 +1,38 @@
 import { scaleArrayObj } from "./scale.js";
 import { arrayShuffle, xyArrayShuffle } from "./utilities.js";
 import { oversampleXY, undersampleXY } from "./balancing.js";
+import { validateFirstRow, validateArray } from "./validators.js";
+import { analizeCorrelation } from "./correlation.js";
+import { correlation } from "ohlcv-indicators/src/studies/correlation.js";
+
+//ADD A PARAM max correlation that will measure the correlation between variables if defined
 
 export const parseTrainingXY = ({ 
-    arrObj = [], 
-    trainingSplit = 0.8, 
-    repeat = {}, 
-    yCallbackFunc = row => row, 
-    xCallbackFunc = row => row,
-    validateRows = () => true,
-    groups = {},
-    shuffle = false,
+    arrObj = [], //array of objects
+    trainingSplit = 0.8, //numberic float between 0.01 and 0.99
+    repeat = {}, //accepted key pair object with number as values
+    yCallbackFunc = row => row, //accepted callback functions
+    xCallbackFunc = row => row, //accepted callback functions
+    validateRows = () => true,//accepted callback functions
+    groups = {},//accepted object of arrays
+    shuffle = false,//only booleans
     minmaxRange = [0, 1],
-    balancing = '',
-    state = {},
+    balancing = '',//accepted null, "oversample" or "undersample"
+    state = {}, //accepted object or classes
     customMinMaxRanges = {},
-    excludes = []
+    excludes = [],//each item must be a string
+    correlation = {}
 }) => {
     let X = [];
     let Y = [];
 
+    validateArray(arrObj, {min: 5}, 'parseTrainingXY')
+    validateFirstRow(arrObj[0])
+
+    const {corrSplit = 0.2, corrExcludes = [], corrThreshold = 0.95} = correlation
+
+    analizeCorrelation({arrObj, corrSplit, corrExcludes: new Set(corrExcludes), corrThreshold})
+
     //if parsedX and parsedY is undefined or null the current row will be excluded from training or production
     for (let x = 0; x < arrObj.length; x++) {
 
@@ -48,21 +61,25 @@ export const parseTrainingXY = ({
         scaledConfig: configX
     } = scaleArrayObj({arrObj: X, repeat, groups, minmaxRange, customMinMaxRanges, excludes: excludesSet})
 
-    let {
-        scaledOutput: scaledY,
-        scaledConfig: configY,
-    } = scaleArrayObj({arrObj: Y, repeat, groups, minmaxRange, customMinMaxRanges, excludes: excludesSet})
-
 
+    const yLen = Y.length
+    const flatY = new Array(yLen)
+    const configY = {
+        keyNames: Object.keys(Y[0])
+    }
 
+    for(let idx = 0; idx < yLen; idx++)
+    {
+        flatY[idx] = Object.values(Y[idx])
+    }
 
 
     const splitIndex = Math.floor(scaledX.length * trainingSplit)
 
     let trainX = scaledX.slice(0, splitIndex)
-    let trainY = scaledY.slice(0, splitIndex)
+    let trainY = flatY.slice(0, splitIndex)
     let testX = scaledX.slice(splitIndex)
-    let testY = scaledY.slice(splitIndex)
+    let testY = flatY.slice(splitIndex)
 
 
     if(balancing)
@@ -114,6 +131,9 @@ export const parseProductionX = ({
 }) => {
     let X = [];
 
+    validateArray(arrObj, {min: 5}, 'parseProductionX')
+    validateFirstRow(arrObj[0])
+
     for (let x = 0; x < arrObj.length; x++) {
 
         if(!validateRows(arrObj[x])) continue

package/src/validators.js

@@ -0,0 +1,30 @@
+export const validateArray = (arr, {min = -Infinity, max = Infinity}, paramName) => {
+
+    if(!Array.isArray(arr))
+    {
+        throw new Error(`Invalid property. "${paramName}" expected an array.`)
+    }
+
+    if(arr.length < min)
+    {
+        throw new Error(`Invalid property value. Array "${paramName}" expected at least ${max} items.`)
+    }
+    if(arr.length > max)
+    {
+        throw new Error(`Invalid property value. Array "${paramName}" expected at max ${max} items.`)
+    }
+
+    return true
+}
+
+export const validateFirstRow = row => {
+
+    for(const [k, v] of Object.entries(row))
+    {
+      if (typeof v !== 'number' || Number.isNaN(v)) {
+        throw new Error(`Invalid value at index 0 property "${k}": value is "${v}". Expected a numeric value.`);
+      }
+    }
+    
+    return true
+}

package/test/test.js

@@ -27,7 +27,9 @@ const test = async () => {
             {fast: 'price', slow: 'bollinger_bands_upper'}
         ])
 
-    const parsedOhlcv = indicators.getData()
+    const parsedOhlcv = indicators.getData({dateFormat: 'milliseconds'})
+
+    console.log(parsedOhlcv.slice(-1))
 
     const {scaledGroups} = indicators
 
@@ -57,7 +59,8 @@ const test = async () => {
         minmaxRange: [0, 1],
         balancing: null,
         groups: scaledGroups,
-        excludes: ['high']
+        excludes: ['high'],
+        correlation: {corrExcludes: ['price_x_sma_300', 'price_x_sma_200']}
     });
 
     //console.log(configX.outputKeyNames)