vaporous 0.0.2 → 0.0.4

package/Vaporous.js

@@ -11,6 +11,8 @@ const path = require('path')
 
 const styles = fs.readFileSync(__dirname + '/styles.css')
 
+const Papa = require('papaparse')
+
 // These globals allow us to write functions from the HTML page directly without needing to stringify
 class google { }
 const document = {}
@@ -20,15 +22,10 @@ const keyFromEvent = (event, bys) => bys.map(i => event[i.bySplit]).join('|')
 const _sort = (order, data, ...keys) => {
     return data.sort((a, b) => {
         let directive = 0;
-        keys.some(key => {
-            const type = typeof a[key];
-
-            if (type === 'number') {
-                directive = order === 'asc' ? a[key] - b[key] : b[key] - a[key];
-            } else if (type === 'string') {
-                directive = order === 'asc' ? a[key].localeCompare(b[key]) : b[key].localeCompare(a[key]);
-            }
 
+        keys.some(key => {
+            directive = typeof a[key] === 'number' ? a[key] - b[key] : a[key].localeCompare(b[key])
+            if (order === 'dsc') directive = directive * -1
             if (directive !== 0) return true;
         })
 
@@ -49,13 +46,13 @@ class Vaporous {
         this.checkpoints = {}
     }
 
-    method(operation, name, method) {
+    method(operation, name, options) {
         const operations = {
             create: () => {
-                this.savedMethods[name] = method
+                this.savedMethods[name] = options
             },
             retrieve: () => {
-                this.savedMethods[name](this)
+                this.savedMethods[name](this, options)
             },
             delete: () => {
                 delete this.savedMethods[name]
@@ -127,16 +124,51 @@ class Vaporous {
         return this;
     }
 
+    async csvLoad(parser) {
+        const tasks = this.events.map(obj => {
+            const content = []
+
+            return new Promise((resolve, reject) => {
+                const thisStream = fs.createReadStream(obj._fileInput)
+
+                Papa.parse(thisStream, {
+                    header: true,
+                    skipEmptyLines: true,
+                    step: (row) => {
+                        try {
+                            const event = parser(row)
+                            if (event !== null) content.push(event)
+                        } catch (err) {
+                            reject(err)
+                        }
+                    },
+                    complete: () => {
+                        obj._raw = content
+                        resolve(this)
+                    }
+                })
+            })
+        })
+
+        await Promise.all(tasks)
+        return this;
+    }
+
     async fileLoad(delim, parser) {
         const tasks = this.events.map(obj => {
             const content = []
 
-            return new Promise(resolve => {
+            return new Promise((resolve, reject) => {
                 fs.createReadStream(obj._fileInput)
                     .pipe(split2(delim))
                     .on('data', line => {
-                        const event = parser(line)
-                        if (event !== null) content.push(event)
+                        try {
+                            const event = parser(line)
+                            if (event !== null) content.push(event)
+                        } catch (err) {
+                            throw err;
+                        }
+
                     })
                     .on('end', () => {
                         obj._raw = content;
@@ -149,8 +181,15 @@ class Vaporous {
         return this;
     }
 
-    output() {
-        console.log(this.events)
+    output(...args) {
+        if (args.length) {
+            console.log(this.events.map(event => {
+                return args.map(item => event[item])
+            }))
+        } else {
+            console.log(this.events)
+        }
+
         return this;
     }
 
@@ -206,11 +245,19 @@ class Vaporous {
         const arr = Object.keys(map).map(key => {
             const result = map[key]
 
+            let sortedCache = {}
             aggregations.forEach(aggregation => {
-                if (aggregation.sortable) map[key]._statsRaw[aggregation.field].sort((a, b) => a - b)
+                const outputField = aggregation.outputField
+                const reference = map[key]._statsRaw[aggregation.field]
+
+                if (aggregation.sortable) {
+                    sortedCache[aggregation.field] = reference.slice().sort((a, b) => a - b)
+                    result[outputField] = aggregation.calculate(sortedCache[aggregation.field])
+                } else {
+                    result[outputField] = aggregation.calculate(reference)
+                }
+
 
-                const aggregationField = aggregation.outputField
-                result[aggregationField] = aggregation.calculate(map[key])
             })
 
             delete map[key]._statsRaw
@@ -238,51 +285,63 @@ class Vaporous {
     }
 
     streamstats(...args) {
+        const backwardIterate = (event, i, by, maxBoundary = 0) => {
+            let backwardIndex = 0
+            const thisKey = keyFromEvent(event, by)
+            const byKey = thisKey
+
+            while (true) {
+                const target = i - backwardIndex
+
+                if (target < 0 || target < maxBoundary) break
+
+                const newKey = keyFromEvent(this.events[target], by)
+                if (thisKey !== newKey) break
+                backwardIndex++
+            }
+
+            return { byKey, start: i - backwardIndex + 1 }
+        }
+
+
         const window = args.filter(i => i instanceof Window)
         const by = args.filter(i => i instanceof By)
 
         // Perform some validation
         if (window.length > 1) throw new Error('Only one window allowed in streamstats')
-        if (window.length > 0 && by.length > 0) throw new Error('Window and By not supported together in streamstats')
 
         this.events.forEach((event, i) => {
-            let start, byKey;
-            if (window.length > 0) {
-                start = Math.max(i - window[0].size + 1, 0)
-                byKey = ""
-            } else if (by.length > 0) {
-                let backwardIndex = 0
-                const thisKey = keyFromEvent(event, by)
-                byKey = thisKey
-                let keyChange = false
-                while (!keyChange) {
-                    const target = i - backwardIndex
-
-                    if (target < 0) {
-                        keyChange = true
-                        break
-                    }
-
-                    const newKey = keyFromEvent(this.events[target], by)
-                    if (thisKey !== newKey) {
-                        keyChange = true
-                        break
-                    }
+            let start, byKey = "";
 
-                    backwardIndex++
-                }
-                start = Math.max(i - backwardIndex + 1, 0)
-            }
+            // Refine to window size 
+            if (window.length > 0) start = Math.max(i - window[0].size + 1, 0)
+            if (by.length !== 0) ({ start, byKey } = backwardIterate(event, i, by, start))
 
             const eventRange = this.events.slice(start, i + 1)
-            Object.assign(event, this._stats(args, eventRange).map[byKey])
+            const embed = this._stats(args, eventRange).map[byKey]
+            Object.assign(event, {
+                _streamstats: embed
+            })
+        })
+
+        // We need to assign to a separate streamstats object to avoid collusions
+        // As streamstats iteratively updates the data but rlies on previous samples
+        // Modifying data in place corrupts the results of the query
+        this.events.forEach(event => {
+            Object.assign(event, event._streamstats)
+            delete event._streamstats
         })
 
         return this;
     }
 
+    delta(field, remapField, ...bys) {
+        this.streamstats(new Aggregation(field, 'range', remapField), new Window(2), ...bys)
+        return this;
+    }
+
     sort(order, ...keys) {
-        this._events = _sort(order, this.events, keys)
+        this.events = _sort(order, this.events, ...keys)
         return this;
     }
 
@@ -294,14 +353,163 @@ class Vaporous {
         return this;
     }
 
-    build(name, type, visualisationOptions) {
-        const data = JSON.stringify(this.events)
+    build(name, type, { tab = 'Default', columns = 2, y2, y1Type, y2Type, y1Stacked, y2Stacked, sortX = 'asc', xTicks = false, trellisAxis = "shared" } = {}) {
+
+        const visualisationOptions = { tab, columns }
+
+
+        let bounds = {}
+
+        const isY2 = (data) => {
+            let y2Mapped = false;
+
+            if (y2 instanceof Array) {
+                y2Mapped = y2.includes(data)
+            }
+            else if (y2 instanceof RegExp) {
+                y2Mapped = y2.test(data)
+            }
+
+            return y2Mapped
+        }
+        const graphData = this.events.map((trellis, i) => {
+            if (type === 'Table') {
+                return trellis;
+            }
+
+            const dataOptions = {}
+
+            // For every event in this trellis restructure to chart.js
+            if (sortX) trellis = _sort(sortX, trellis, '_time')
+
+            const trellisName = this.graphFlags.at(-1).trellisName?.[i] || ""
+            const columnDefinitions = this.graphFlags.at(-1).columnDefinitions[i]
+
+            trellis.forEach(event => {
+                columnDefinitions.forEach(prop => {
+                    if (!dataOptions[prop]) dataOptions[prop] = []
+                    const val = event[prop]
+                    dataOptions[prop].push(val)
+
+                    if (!bounds[prop]) bounds[prop] = {
+                        min: val,
+                        max: val
+                    }
+
+                    if (val < bounds[prop].min) bounds[prop].min = val;
+                    if (val > bounds[prop].max) bounds[prop].max = val
+
+                })
+            })
+
+
+            const _time = dataOptions._time
+            delete dataOptions._time
+
+            let y2WasMapped = false
+            const data = {
+                labels: _time,
+                datasets: Object.keys(dataOptions).map(data => {
+                    const y2Mapped = isY2(data)
+                    if (y2Mapped) y2WasMapped = y2Mapped
+
+                    const base = {
+                        label: data,
+                        yAxisID: y2Mapped ? 'y2' : undefined,
+                        data: dataOptions[data],
+                        type: y2Mapped ? y2Type : y1Type,
+                        // borderColor: 'red',
+                        // backgroundColor: 'red',
+                    }
+
+                    if (type === 'Scatter') {
+                        base.showLine = false
+                        base.pointRadius = 8
+                        base.pointStyle = 'rect'
+                    } else if (type === 'Area') {
+                        base.fill = 'origin'
+                    } else if (type === 'Line') {
+                        base.pointRadius = 0;
+                    }
+                    return base
+                })
+            };
+
+            const scales = {
+                y: {
+                    type: 'linear',
+                    display: true,
+                    position: 'left',
+                    stacked: y1Stacked
+                },
+                x: {
+                    type: 'linear',
+                    ticks: {
+                        display: xTicks
+                    }
+                }
+            }
+
+            if (y2WasMapped) scales.y2 = {
+                type: 'linear',
+                display: true,
+                position: 'right',
+                grid: {
+                    drawOnChartArea: false
+                },
+                stacked: y2Stacked
+            }
+
+            return {
+                type: 'line',
+                data: data,
+                options: {
+                    scales,
+                    responsive: true,
+                    plugins: {
+                        legend: {
+                            position: 'bottom',
+                        },
+                        title: {
+                            display: true,
+                            text: name + trellisName
+                        }
+                    }
+                }
+            }
+        })
+
+        if (trellisAxis === 'shared') {
+            // Do a second iteration to implement bounds
+            graphData.forEach(trellisGraph => {
+                Object.keys(bounds).forEach(bound => {
+                    let axis = isY2(bound) ? 'y2' : 'y'
+                    if (bound === '_time') axis = 'x';
+
+                    const thisAxis = trellisGraph.options.scales[axis]
+                    const { min, max } = bounds[bound]
+                    if (!thisAxis.min) {
+                        thisAxis.min = min
+                        thisAxis.max = max
+                    }
+                    if (min < thisAxis.min) thisAxis.min = min
+                    if (max > thisAxis.max) thisAxis.max = max
+
+                })
+
+            })
+        }
+
+        const data = JSON.stringify(graphData)
         const lastData = this.visualisationData.at(-1)
 
         if (lastData !== data) this.visualisationData.push(data)
         this.visualisations.push([name, type, visualisationOptions, this.visualisationData.length - 1, this.graphFlags[this.graphFlags.length - 1]])
 
-        if (visualisationOptions.tab && !this.tabs.includes(visualisationOptions.tab)) this.tabs.push(visualisationOptions.tab)
+        if (visualisationOptions.tab && !this.tabs.includes(visualisationOptions.tab)) {
+            this.tabs.push(visualisationOptions.tab)
+            this.tabs = this.tabs.sort((a, b) => a.localeCompare(b))
+        }
 
         return this;
     }
@@ -309,8 +517,8 @@ class Vaporous {
     checkpoint(operation, name) {
 
         const operations = {
-            create: () => this.checkpoints[name] = this.events,
-            retrieve: () => this.events = this.checkpoints[name],
+            create: () => this.checkpoints[name] = structuredClone(this.events),
+            retrieve: () => this.events = structuredClone(this.checkpoints[name]),
             delete: () => delete this.checkpoints[name]
         }
 
@@ -322,10 +530,11 @@ class Vaporous {
         const arr = []
         this.events.forEach(event => {
             if (!event[target]) return arr.push(event)
-            event[target].forEach((item) => {
+            event[target].forEach((item, i) => {
                 arr.push({
                     ...event,
-                    [target]: item
+                    [target]: item,
+                    [`_mvExpand_${target}`]: i
                 })
             })
         })
@@ -334,53 +543,102 @@ class Vaporous {
         return this;
     }
 
-    toGraph(x, y, series, trellis, options = {}) {
+    writeFile(title) {
+        fs.writeFileSync('./' + title, JSON.stringify(this.events))
+        return this;
+    }
+
+    toGraph(x, y, series, trellis = false) {
+
+        if (!(y instanceof Array)) y = [y]
+
+        const yAggregations = y.map(item => [
+            new Aggregation(item, 'list', item),
+        ]).flat()
 
         this.stats(
-            new Aggregation(y, 'list', y),
+            ...yAggregations,
             new Aggregation(series, 'list', series),
             new Aggregation(trellis, 'values', 'trellis'),
             new By(x), trellis ? new By(trellis) : null
         )
 
-        const trellisMap = {}
+        const trellisMap = {}, columnDefinitions = {}
 
         this.table(event => {
+            const _time = event[x]
+            if (_time === null || _time === undefined) throw new Error(`To graph operation with params ${x}, ${y.join(',')} looks corrupt. x value resolves to null - the graph will not render`)
             const obj = {
-                _time: event[x]
+                _time
             }
-            event[series].forEach((series, i) => obj[series] = event[y][i])
+
+            event[series].forEach((series, i) => {
+                y.forEach(item => {
+                    let name;
+                    if (y.length === 1) {
+                        if (series === undefined) name = item
+                        else name = series
+                    } else {
+                        if (series !== undefined) name = `${series}_${item}`
+                        else name = item
+                    }
+                    obj[name] = event[item][i]
+                })
+            })
 
             if (trellis) {
                 const tval = event.trellis[0]
-                if (!trellisMap[tval]) trellisMap[tval] = []
+                if (!trellisMap[tval]) {
+                    trellisMap[tval] = []
+                    columnDefinitions[tval] = {}
+                }
                 trellisMap[tval].push(obj)
+                Object.keys(obj).forEach(key => {
+                    columnDefinitions[tval][key] = true
+                })
+            } else {
+                Object.keys(obj).forEach(key => {
+                    columnDefinitions[key] = true;
+                })
             }
 
             return obj
         })
 
         const graphFlags = {}
+
         if (trellis) {
             graphFlags.trellis = true;
             graphFlags.trellisName = Object.keys(trellisMap)
+            graphFlags.columnDefinitions = Object.keys(trellisMap).map(tval => {
+                const adjColumns = ['_time']
+                Object.keys(columnDefinitions[tval]).forEach(col => (col !== '_time') ? adjColumns.push(col) : null)
+                return adjColumns
+            })
             this.events = Object.keys(trellisMap).map(tval => trellisMap[tval])
+        } else {
+            const adjColumns = ['_time']
+            Object.keys(columnDefinitions).forEach(col => (col !== '_time') ? adjColumns.push(col) : null)
+
+            this.events = [this.events]
+            graphFlags.columnDefinitions = [adjColumns]
         }
 
-        Object.assign(graphFlags, options)
         this.graphFlags.push(graphFlags)
         return this;
     }
 
-    render() {
+    render(location = './Vaporous_generation.html') {
         const classSafe = (name) => name.replace(/[^a-zA-Z0-9]/g, "_")
 
-        const createElement = (name, type, visualisationOptions, eventData, { trellis, y2, sortX, trellisName = "", y2Type, y1Type, stacked, y1Min, y2Min }) => {
+        const createElement = (name, type, visualisationOptions, eventData, { trellis, trellisName = "" }) => {
+
             if (classSafe(visualisationOptions.tab) !== selectedTab) return;
 
             eventData = visualisationData[eventData]
-            if (!trellis) eventData = [eventData]
-            else {
+
+            // TODO: migrate trellis functionality from here to tograph
+            if (trellis) {
                 let pairs = trellisName.map((name, i) => [name, eventData[i]]);
                 pairs = pairs.sort((a, b) => a[0].localeCompare(b[0]))
 
@@ -389,84 +647,41 @@ class Vaporous {
                 eventData = pairs.map(p => p[1]);
             }
 
-            eventData.forEach((trellis, i) => {
-                const data = new google.visualization.DataTable();
-
-                const series = {}, axis0 = { targetAxisIndex: 0 }, axis1 = { targetAxisIndex: 1 }
+            const columnCount = visualisationOptions.columns || 2
 
-                if (y1Type) axis0.type = y1Type
-                if (y2Type) axis1.type = y2Type
+            eventData.forEach((trellisData, i) => {
+                const parentHolder = document.createElement('div')
 
-                // Create columns
-                const columns = Object.keys(trellis[0])
-                columns.forEach((key, i) => {
-                    data.addColumn(typeof trellis[0][key], key)
 
-                    if (y2 && i !== 0) {
-                        let match = false;
-                        if (y2 instanceof Array) { match = y2.includes(key) }
-                        else if (y2 instanceof RegExp) { match = y2.test(key) }
 
-                        if (match) series[i - 1] = axis1
-                    }
-
-                    if (!series[1 - i]) series[i - 1] = axis0
-                })
-
-                let rows = trellis.map(event => {
-                    return columns.map(key => event[key])
-                })
-
-                rows = _sort(sortX, rows, 0)
-
-                data.addRows(rows);
-
-                const columnCount = visualisationOptions.columns || 2
-                const thisEntity = document.createElement('div')
-                thisEntity.className = "parentHolder"
-                thisEntity.style = `flex: 1 0 calc(${100 / columns}% - 6px); max-width: calc(${100 / columnCount}% - 6px);`
+                document.getElementById('content').appendChild(parentHolder)
 
+                parentHolder.style = `flex: 0 0 calc(${100 / columnCount}% - 8px); max-width: calc(${100 / columnCount}% - 8px);`
+                if (type === 'Table') {
+                    new Tabulator(parentHolder, { data: trellisData, autoColumns: 'full', layout: "fitDataStretch", })
+                } else {
+                    const graphEntity = document.createElement('canvas')
+                    parentHolder.appendChild(graphEntity)
+                    new Chart(graphEntity, trellisData)
+                }
 
-                const thisGraph = document.createElement('div')
-                thisGraph.className = "graphHolder"
-                thisEntity.appendChild(thisGraph)
-                document.getElementById('content').appendChild(thisEntity)
-
-                const chartElement = new google.visualization[type](thisGraph)
-
-                google.visualization.events.addListener(chartElement, 'select', (e) => {
-                    console.log(chartElement.getSelection()[1], chartElement.getSelection()[0])
-                    tokens[name] = trellis[chartElement.getSelection()[0].row]
-                    console.log(tokens[name])
-                });
-
-                const title = trellis ? name + trellisName[i] : name
-
-                chartElement.draw(data, {
-                    series, showRowNumber: false, legend: { position: 'bottom' }, title, isStacked: stacked,
-                    width: document.body.scrollWidth / columnCount - (type === "LineChart" ? 12 : 24),
-                    animation: { duration: 500, startup: true },
-                    chartArea: { width: '85%', height: '75%' },
-                    vAxis: {
-                        viewWindow: {
-                            min: y1Min
-                        }
-                    }
-                })
             })
         }
 
-        const filePath = './Vaporous_generation.html'
+        const filePath = location
         fs.writeFileSync(filePath, `
 <html>
         <head>
+        <meta name="viewport" content="width=device-width, initial-scale=0.5">
             <style>
                 ${styles}
             </style>
-    <script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+    
+            <link href="https://unpkg.com/tabulator-tables@6.3.1/dist/css/tabulator.min.css" rel="stylesheet">
+            <script type="text/javascript" src="https://unpkg.com/tabulator-tables@6.3.1/dist/js/tabulator.min.js"></script>
+            <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    
     <script type="text/javascript">
-      google.charts.load('current', {'packages':['table', 'corechart']});
-      google.charts.setOnLoadCallback(drawVis);
 
       const classSafe = ${classSafe.toString()}
 
@@ -505,7 +720,7 @@ class Vaporous {
 </html>
         `)
 
-        console.log('File ouput created ', filePath)
+        console.log('File ouput created ', path.resolve(filePath))
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vaporous",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "Transition data to different structured states for analytical processing",
   "main": "Vaporous.js",
   "scripts": {
@@ -10,6 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "dayjs": "^1.11.18",
+    "papaparse": "^5.5.3",
     "split2": "^4.2.0"
   }
 }

package/readme.md

@@ -0,0 +1,22 @@
+# Vaporous
+Vaporous provides a chained query syntax for accessing unstructured data and converting it into interpretable analytics
+
+The tool is still in its early phases of development and is missing some quality of life features
+
+The query syntax is heavily inspired by splunk with more bias towards programmitic functionality
+
+## Examples
+
+Interactive previews for two datasources are available
+
+- [Virtualised temperature sensor data](https://lkashl.github.io/vaporous/pages/temp_sensors.html)
+- [CSV delimited virtruvian data](https://lkashl.github.io/vaporous/pages/gym.html)
+
+Examples of the source queries used can be referenced in the [examples folder](https://github.com/lkashl/vaporous/tree/main/examples)
+
+## TODO List
+- Support web page embedded Vaporous so clients can use browser folder storage as file input
+- Add an error for if a user tries to generate a graph without first calling toGraph
+- Intercept structual errors earlier and add validation to functions - not necessarily data as this casues overhaead
+- Migrate reponsibility for tabular conversion from create element to the primary library to reduce overhead of graph generation
+