vaporous 0.0.2 → 0.0.3

package/Vaporous.js

@@ -11,6 +11,8 @@ const path = require('path')
 
 const styles = fs.readFileSync(__dirname + '/styles.css')
 
+const Papa = require('papaparse')
+
 // These globals allow us to write functions from the HTML page directly without needing to stringify
 class google { }
 const document = {}
@@ -20,15 +22,10 @@ const keyFromEvent = (event, bys) => bys.map(i => event[i.bySplit]).join('|')
 const _sort = (order, data, ...keys) => {
     return data.sort((a, b) => {
         let directive = 0;
-        keys.some(key => {
-            const type = typeof a[key];
-
-            if (type === 'number') {
-                directive = order === 'asc' ? a[key] - b[key] : b[key] - a[key];
-            } else if (type === 'string') {
-                directive = order === 'asc' ? a[key].localeCompare(b[key]) : b[key].localeCompare(a[key]);
-            }
 
+        keys.some(key => {
+            directive = typeof a[key] === 'number' ? a[key] - b[key] : a[key].localeCompare(b[key])
+            if (order === 'dsc') directive = directive * -1
             if (directive !== 0) return true;
         })
 
@@ -49,13 +46,13 @@ class Vaporous {
         this.checkpoints = {}
     }
 
-    method(operation, name, method) {
+    method(operation, name, options) {
         const operations = {
             create: () => {
-                this.savedMethods[name] = method
+                this.savedMethods[name] = options
             },
             retrieve: () => {
-                this.savedMethods[name](this)
+                this.savedMethods[name](this, options)
             },
             delete: () => {
                 delete this.savedMethods[name]
@@ -127,16 +124,51 @@ class Vaporous {
         return this;
     }
 
+    async csvLoad(parser) {
+        const tasks = this.events.map(obj => {
+            const content = []
+
+            return new Promise((resolve, reject) => {
+                const thisStream = fs.createReadStream(obj._fileInput)
+
+                Papa.parse(thisStream, {
+                    header: true,
+                    skipEmptyLines: true,
+                    step: (row) => {
+                        try {
+                            const event = parser(row)
+                            if (event !== null) content.push(event)
+                        } catch (err) {
+                            reject(err)
+                        }
+                    },
+                    complete: () => {
+                        obj._raw = content
+                        resolve(this)
+                    }
+                })
+            })
+        })
+
+        await Promise.all(tasks)
+        return this;
+    }
+
     async fileLoad(delim, parser) {
         const tasks = this.events.map(obj => {
             const content = []
 
-            return new Promise(resolve => {
+            return new Promise((resolve, reject) => {
                 fs.createReadStream(obj._fileInput)
                     .pipe(split2(delim))
                     .on('data', line => {
-                        const event = parser(line)
-                        if (event !== null) content.push(event)
+                        try {
+                            const event = parser(line)
+                            if (event !== null) content.push(event)
+                        } catch (err) {
+                            throw err;
+                        }
+
                     })
                     .on('end', () => {
                         obj._raw = content;
@@ -149,8 +181,15 @@ class Vaporous {
         return this;
     }
 
-    output() {
-        console.log(this.events)
+    output(...args) {
+        if (args.length) {
+            console.log(this.events.map(event => {
+                return args.map(item => event[item])
+            }))
+        } else {
+            console.log(this.events)
+        }
+
         return this;
     }
 
@@ -238,51 +277,63 @@ class Vaporous {
     }
 
     streamstats(...args) {
+        const backwardIterate = (event, i, by, maxBoundary = 0) => {
+            let backwardIndex = 0
+            const thisKey = keyFromEvent(event, by)
+            const byKey = thisKey
+
+            while (true) {
+                const target = i - backwardIndex
+
+                if (target < 0 || target < maxBoundary) break
+
+                const newKey = keyFromEvent(this.events[target], by)
+                if (thisKey !== newKey) break
+                backwardIndex++
+            }
+
+            return { byKey, start: i - backwardIndex + 1 }
+        }
+
+
         const window = args.filter(i => i instanceof Window)
         const by = args.filter(i => i instanceof By)
 
         // Perform some validation
         if (window.length > 1) throw new Error('Only one window allowed in streamstats')
-        if (window.length > 0 && by.length > 0) throw new Error('Window and By not supported together in streamstats')
 
         this.events.forEach((event, i) => {
-            let start, byKey;
-            if (window.length > 0) {
-                start = Math.max(i - window[0].size + 1, 0)
-                byKey = ""
-            } else if (by.length > 0) {
-                let backwardIndex = 0
-                const thisKey = keyFromEvent(event, by)
-                byKey = thisKey
-                let keyChange = false
-                while (!keyChange) {
-                    const target = i - backwardIndex
-
-                    if (target < 0) {
-                        keyChange = true
-                        break
-                    }
-
-                    const newKey = keyFromEvent(this.events[target], by)
-                    if (thisKey !== newKey) {
-                        keyChange = true
-                        break
-                    }
+            let start, byKey = "";
 
-                    backwardIndex++
-                }
-                start = Math.max(i - backwardIndex + 1, 0)
-            }
+            // Refine to window size 
+            if (window.length > 0) start = Math.max(i - window[0].size + 1, 0)
+            if (by.length !== 0) ({ start, byKey } = backwardIterate(event, i, by, start))
 
             const eventRange = this.events.slice(start, i + 1)
-            Object.assign(event, this._stats(args, eventRange).map[byKey])
+            const embed = this._stats(args, eventRange).map[byKey]
+            Object.assign(event, {
+                _streamstats: embed
+            })
+        })
+
+        // We need to assign to a separate streamstats object to avoid collusions
+        // As streamstats iteratively updates the data but rlies on previous samples
+        // Modifying data in place corrupts the results of the query
+        this.events.forEach(event => {
+            Object.assign(event, event._streamstats)
+            delete event._streamstats
         })
 
         return this;
     }
 
+    delta(field, remapField, ...bys) {
+        this.streamstats(new Aggregation(field, 'range', remapField), new Window(2), ...bys)
+        return this;
+    }
+
     sort(order, ...keys) {
-        this._events = _sort(order, this.events, keys)
+        this.events = _sort(order, this.events, ...keys)
         return this;
     }
 
@@ -294,14 +345,20 @@ class Vaporous {
         return this;
     }
 
-    build(name, type, visualisationOptions) {
+    build(name, type, { tab = 'Default', columns = 2 } = {}) {
+
+        const visualisationOptions = { tab, columns }
+
         const data = JSON.stringify(this.events)
         const lastData = this.visualisationData.at(-1)
 
         if (lastData !== data) this.visualisationData.push(data)
         this.visualisations.push([name, type, visualisationOptions, this.visualisationData.length - 1, this.graphFlags[this.graphFlags.length - 1]])
 
-        if (visualisationOptions.tab && !this.tabs.includes(visualisationOptions.tab)) this.tabs.push(visualisationOptions.tab)
+        if (visualisationOptions.tab && !this.tabs.includes(visualisationOptions.tab)) {
+            this.tabs.push(visualisationOptions.tab)
+            this.tabs = this.tabs.sort((a, b) => a.localeCompare(b))
+        }
 
         return this;
     }
@@ -309,8 +366,8 @@ class Vaporous {
     checkpoint(operation, name) {
 
         const operations = {
-            create: () => this.checkpoints[name] = this.events,
-            retrieve: () => this.events = this.checkpoints[name],
+            create: () => this.checkpoints[name] = structuredClone(this.events),
+            retrieve: () => this.events = structuredClone(this.checkpoints[name]),
             delete: () => delete this.checkpoints[name]
         }
 
@@ -322,10 +379,11 @@ class Vaporous {
         const arr = []
         this.events.forEach(event => {
             if (!event[target]) return arr.push(event)
-            event[target].forEach((item) => {
+            event[target].forEach((item, i) => {
                 arr.push({
                     ...event,
-                    [target]: item
+                    [target]: item,
+                    [`_mvExpand_${target}`]: i
                 })
             })
         })
@@ -334,27 +392,55 @@ class Vaporous {
         return this;
     }
 
-    toGraph(x, y, series, trellis, options = {}) {
+    writeFile(title) {
+        fs.writeFileSync('./' + title, JSON.stringify(this.events))
+        return this;
+    }
+
+    toGraph(x, y, series, trellis = false, options = {}) {
+
+        if (!(y instanceof Array)) y = [y]
+        if (options.y2 instanceof RegExp) options.y2 = options.y2.toString()
+
+        const yAggregations = y.map(item => new Aggregation(item, 'list', item))
 
         this.stats(
-            new Aggregation(y, 'list', y),
+            ...yAggregations,
             new Aggregation(series, 'list', series),
             new Aggregation(trellis, 'values', 'trellis'),
             new By(x), trellis ? new By(trellis) : null
         )
 
-        const trellisMap = {}
+        const trellisMap = {}, columnDefinitions = {}
 
         this.table(event => {
+            const _time = event[x]
+            if (_time === null || _time === undefined) throw new Error(`To graph operation with params ${x}, ${y.join(',')} looks corrupt. x value resolves to null - the graph will not render`)
             const obj = {
-                _time: event[x]
+                _time
             }
-            event[series].forEach((series, i) => obj[series] = event[y][i])
+
+            event[series].forEach((series, i) => {
+                y.forEach(item => {
+                    const name = y.length === 1 ? series : `${series}_${item}`
+                    obj[name] = event[item][i]
+                })
+            })
 
             if (trellis) {
                 const tval = event.trellis[0]
-                if (!trellisMap[tval]) trellisMap[tval] = []
+                if (!trellisMap[tval]) {
+                    trellisMap[tval] = []
+                    columnDefinitions[tval] = {}
+                }
                 trellisMap[tval].push(obj)
+                Object.keys(obj).forEach(key => {
+                    columnDefinitions[tval][key] = true
+                })
+            } else {
+                Object.keys(obj).forEach(key => {
+                    columnDefinitions[key] = true;
+                })
             }
 
             return obj
@@ -364,7 +450,18 @@ class Vaporous {
         if (trellis) {
             graphFlags.trellis = true;
             graphFlags.trellisName = Object.keys(trellisMap)
+            graphFlags.columnDefinitions = Object.keys(trellisMap).map(tval => {
+                const adjColumns = ['_time']
+                Object.keys(columnDefinitions[tval]).forEach(col => (col !== '_time') ? adjColumns.push(col) : null)
+                return adjColumns
+            })
             this.events = Object.keys(trellisMap).map(tval => trellisMap[tval])
+        } else {
+            const adjColumns = ['_time']
+            Object.keys(columnDefinitions).forEach(col => (col !== '_time') ? adjColumns.push(col) : null)
+
+            this.events = [this.events]
+            graphFlags.columnDefinitions = [adjColumns]
         }
 
         Object.assign(graphFlags, options)
@@ -375,12 +472,22 @@ class Vaporous {
     render() {
         const classSafe = (name) => name.replace(/[^a-zA-Z0-9]/g, "_")
 
-        const createElement = (name, type, visualisationOptions, eventData, { trellis, y2, sortX, trellisName = "", y2Type, y1Type, stacked, y1Min, y2Min }) => {
+        const createElement = (name, type, visualisationOptions, eventData, { trellis, y2, sortX, trellisName = "", y2Type, y1Type, stacked, y1Min, y2Min, columnDefinitions }) => {
+
+            if (typeof y2 === 'string') {
+                y2 = y2.split("/")
+                const flags = y2.at(-1)
+                y2.pop()
+                const content = y2.splice(1).join("/")
+                y2 = new RegExp(content, flags)
+            }
+
             if (classSafe(visualisationOptions.tab) !== selectedTab) return;
 
             eventData = visualisationData[eventData]
-            if (!trellis) eventData = [eventData]
-            else {
+
+            // TODO: migrate trellis functionality from here to tograph
+            if (trellis) {
                 let pairs = trellisName.map((name, i) => [name, eventData[i]]);
                 pairs = pairs.sort((a, b) => a[0].localeCompare(b[0]))
 
@@ -389,7 +496,7 @@ class Vaporous {
                 eventData = pairs.map(p => p[1]);
             }
 
-            eventData.forEach((trellis, i) => {
+            eventData.forEach((trellisData, i) => {
                 const data = new google.visualization.DataTable();
 
                 const series = {}, axis0 = { targetAxisIndex: 0 }, axis1 = { targetAxisIndex: 1 }
@@ -398,9 +505,12 @@ class Vaporous {
                 if (y2Type) axis1.type = y2Type
 
                 // Create columns
-                const columns = Object.keys(trellis[0])
+                const columns = columnDefinitions[i]
+
                 columns.forEach((key, i) => {
-                    data.addColumn(typeof trellis[0][key], key)
+                    // TODO: we might have to iterate the dataseries to find this information - most likely update the column definition references 
+                    const colType = typeof trellisData[0][key]
+                    data.addColumn(colType === 'undefined' ? "number" : colType, key)
 
                     if (y2 && i !== 0) {
                         let match = false;
@@ -410,10 +520,10 @@ class Vaporous {
                         if (match) series[i - 1] = axis1
                     }
 
-                    if (!series[1 - i]) series[i - 1] = axis0
+                    if (!series[i - 1]) series[i - 1] = axis0
                 })
 
-                let rows = trellis.map(event => {
+                let rows = trellisData.map(event => {
                     return columns.map(key => event[key])
                 })
 
@@ -436,7 +546,7 @@ class Vaporous {
 
                 google.visualization.events.addListener(chartElement, 'select', (e) => {
                     console.log(chartElement.getSelection()[1], chartElement.getSelection()[0])
-                    tokens[name] = trellis[chartElement.getSelection()[0].row]
+                    tokens[name] = trellisData[chartElement.getSelection()[0].row]
                     console.log(tokens[name])
                 });
 
@@ -451,7 +561,8 @@ class Vaporous {
                         viewWindow: {
                             min: y1Min
                         }
-                    }
+                    },
+                    pointSize: type === 'ScatterChart' ? 2 : undefined
                 })
             })
         }
@@ -505,7 +616,7 @@ class Vaporous {
 </html>
         `)
 
-        console.log('File ouput created ', filePath)
+        console.log('File ouput created ', path.resolve(filePath))
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vaporous",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "Transition data to different structured states for analytical processing",
   "main": "Vaporous.js",
   "scripts": {
@@ -10,6 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "dayjs": "^1.11.18",
+    "papaparse": "^5.5.3",
     "split2": "^4.2.0"
   }
 }

package/readme.md

@@ -0,0 +1,22 @@
+# Vaporous
+Vaporous provides a chained query syntax for accessing unstructured data and converting it into interpretable analytics. 
+
+The tool is still in its early phases of development and is missing quality of life features for query writers
+
+
+
+## Examples
+You can find example queries [via git in the examples folder](https://github.com/lkashl/vaporous/tree/main/examples)
+
+Interactive previews are available here:
+
+- [Virtualised temperature sensor data](https://lkashl.github.io/vaporous/pages/temp_sensors.html)
+- [CSV delimited virtruvian data](https://lkashl.github.io/vaporous/pages/gym.html)
+
+
+## TODO List
+- Support web page embedded Vaporous so clients can use browser folder storage as file input
+- Add an error for if a user tries to generate a graph without first calling toGraph
+- Intercept structual errors earlier and add validation to functions - not necessarily data as this casues overhaead
+- Migrate reponsibility for tabular conversion from create element to the primary library to reduce overhead of graph generation
+

package/types/Aggregation.js

@@ -1,53 +1,57 @@
-class Aggregation {
-    constructor(field, type, outputField = field, options) {
-        this.type = type;
-        this.field = field;
-        this.outputField = outputField
-        this.options = options;
-        this.sortable = ['max', 'min', 'percentile', 'median'].includes(type)
-    }
-
-    count(values) {
-        return values.length
-    }
-
-    distinctCount(values) {
-        return new Set(values).size
-    }
-
-    list(values) {
-        return values;
-    }
-
-    values(values) {
-        return [...new Set(values)];
-    }
-
-    calculate(statObj) {
-        return this[this.type](statObj._statsRaw[this.field])
-    }
-
-    max(values) {
-        return values[values.length - 1]
-    }
-
-    min(values) {
-        return values[0]
-    }
-
-    percentile(values) {
-        const index = Math.round(this.options / 100 * (values.length - 1));
-        return values[index]
-    }
-
-    median(values) {
-        const index = Math.floor((values.length - 1) / 2);
-        return values[index]
-    }
-
-    sum(values) {
-        return values.reduce((a, b) => a + b, 0)
-    }
-}
-
+class Aggregation {
+    constructor(field, type, outputField = field, options) {
+        this.type = type;
+        this.field = field;
+        this.outputField = outputField
+        this.options = options;
+        this.sortable = ['max', 'min', 'percentile', 'median', 'range'].includes(type)
+    }
+
+    count(values) {
+        return values.length
+    }
+
+    distinctCount(values) {
+        return new Set(values).size
+    }
+
+    list(values) {
+        return values;
+    }
+
+    values(values) {
+        return [...new Set(values)];
+    }
+
+    calculate(statObj) {
+        return this[this.type](statObj._statsRaw[this.field])
+    }
+
+    max(values) {
+        return values.at(-1)
+    }
+
+    min(values) {
+        return values[0]
+    }
+
+    range(values) {
+        return values.at(-1) - values[0]
+    }
+
+    percentile(values) {
+        const index = Math.round(this.options / 100 * (values.length - 1));
+        return values[index]
+    }
+
+    median(values) {
+        const index = Math.floor((values.length - 1) / 2);
+        return values[index]
+    }
+
+    sum(values) {
+        return values.reduce((a, b) => a + b, 0)
+    }
+}
+
 module.exports = Aggregation