vaporous 0.0.10 → 0.0.12

package/Vaporous.js

@@ -85,7 +85,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     method(operation, name, options) {
-        return core.method.call(this, operation, name, options);
+        return processing.method.call(this, operation, name, options);
     }
 
     /**
@@ -108,13 +108,17 @@ class Vaporous {
 
     /**
      * Execute all queued operations
-    * @param {string}  [stageName] - Optional stage name for logging
-    * @returns {Promise<Vaporous>} - Returns this instance for chaining
+     * @param {string} [stageName] - Optional stage name for logging
+     * @returns {Promise<Vaporous>} - Returns this instance for chaining
      */
-    async begin(stageName) {
-        return await core.begin.call(this, stageName);
+    begin(stageName) {
+        return core.begin.call(this, stageName);
     }
 
+    /**
+     * Serialize the Vaporous instance to a plain object
+     * @returns {Object} - Serialized instance data
+     */
     serialise({ } = {}) {
         return core.serialise.call(this);
     }
@@ -167,6 +171,7 @@ class Vaporous {
     /**
      * Evaluate and modify each event
      * @param {Function} modifier - Function that receives an event and returns modifications to apply
+     * @param {boolean} [discard] - Whether to discard the event if modifier returns falsy
      * @returns {Vaporous} - Returns this instance for chaining
      */
     eval(modifier, discard) {
@@ -233,8 +238,8 @@ class Vaporous {
      * @param {string} target - Field name containing array to expand
      * @returns {Vaporous} - Returns this instance for chaining
      */
-    mvexpand(target) {
-        return transformations.mvexpand.call(this, target);
+    mvexpand(...targets) {
+        return transformations.mvexpand.call(this, targets);
     }
 
     // ========================================
@@ -246,15 +251,15 @@ class Vaporous {
      * @private
      */
     _fileScan(directory) {
-        return fileOperations._fileScan?.call(this, directory);
+        return fileOperations._fileScan.call(this, directory);
     }
 
     /**
      * Internal file load helper
      * @private
      */
-    async _fileLoad(events, delim, parser) {
-        return await fileOperations._fileLoad?.call(this, events, delim, parser);
+    _fileLoad(events, delim, parser) {
+        return fileOperations._fileLoad.call(this, events, delim, parser);
     }
 
     /**
@@ -263,7 +268,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     fileScan(directory) {
-        return fileOperations.fileScan?.call(this, directory) || this;
+        return fileOperations.fileScan.call(this, directory);
     }
 
     /**
@@ -271,8 +276,8 @@ class Vaporous {
      * @param {Function} parser - Parser function for CSV rows
      * @returns {Vaporous} - Returns this instance for chaining
      */
-    async csvLoad(parser) {
-        return await fileOperations.csvLoad?.call(this, parser) || this;
+    csvLoad(parser) {
+        return fileOperations.csvLoad.call(this, parser);
     }
 
     /**
@@ -281,8 +286,8 @@ class Vaporous {
      * @param {Function} parser - Parser function for lines
      * @returns {Vaporous} - Returns this instance for chaining
      */
-    async fileLoad(delim, parser) {
-        return await fileOperations.fileLoad?.call(this, delim, parser) || this;
+    fileLoad(delim, parser) {
+        return fileOperations.fileLoad.call(this, delim, parser);
     }
 
     /**
@@ -291,7 +296,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     writeFile(title) {
-        return fileOperations.writeFile?.call(this, title) || this;
+        return fileOperations.writeFile.call(this, title);
     }
 
     /**
@@ -300,7 +305,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     output(...args) {
-        return fileOperations.output?.call(this, ...args) || this;
+        return fileOperations.output.call(this, ...args);
     }
 
     // ========================================
@@ -321,7 +326,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     stats(...args) {
-        return statistics.stats?.call(this, ...args) || this;
+        return statistics.stats.call(this, ...args);
     }
 
     /**
@@ -330,7 +335,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     eventstats(...args) {
-        return statistics.eventstats?.call(this, ...args) || this;
+        return statistics.eventstats.call(this, ...args);
     }
 
     /**
@@ -338,7 +343,7 @@ class Vaporous {
      * @private
      */
     _streamstats(...args) {
-        return statistics._streamstats?.call(this, ...args) || this;
+        return statistics._streamstats.call(this, ...args);
     }
 
     /**
@@ -347,7 +352,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     streamstats(...args) {
-        return statistics.streamstats?.call(this, ...args) || this;
+        return statistics.streamstats.call(this, ...args);
     }
 
     /**
@@ -358,7 +363,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     delta(field, remapField, ...bys) {
-        return statistics.delta?.call(this, field, remapField, ...bys) || this;
+        return statistics.delta.call(this, field, remapField, ...bys);
     }
 
     // ========================================
@@ -370,7 +375,7 @@ class Vaporous {
      * @private
      */
     _checkpoint(operation, name, data, options) {
-        return checkpoints._checkpoint?.call(this, operation, name, data, options) || this;
+        return checkpoints._checkpoint.call(this, operation, name, data, options);
     }
 
     /**
@@ -381,7 +386,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     checkpoint(operation, name, options) {
-        return checkpoints.checkpoint?.call(this, operation, name, options) || this;
+        return checkpoints.checkpoint.call(this, operation, name, options);
     }
 
     /**
@@ -392,7 +397,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     filterIntoCheckpoint(checkpointName, funct, options) {
-        return checkpoints.filterIntoCheckpoint?.call(this, checkpointName, funct, options) || this;
+        return checkpoints.filterIntoCheckpoint.call(this, checkpointName, funct, options);
     }
 
     /**
@@ -402,8 +407,8 @@ class Vaporous {
      * @param {string} partitionBy - Field to partition by
      * @returns {Vaporous} - Returns this instance for chaining
      */
-    async storedCheckpoint(operation, name, partitionBy) {
-        return await checkpoints.storedCheckpoint?.call(this, operation, name, partitionBy) || this;
+    storedCheckpoint(operation, name, partitionBy) {
+        return checkpoints.storedCheckpoint.call(this, operation, name, partitionBy);
     }
 
     // ========================================
@@ -416,7 +421,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     toGraph(...keys) {
-        return visualization.toGraph?.call(this, ...keys) || this;
+        return visualization.toGraph.call(this, ...keys);
     }
 
     /**
@@ -427,16 +432,17 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     build(title, type, options) {
-        return visualization.build?.call(this, title, type, options) || this;
+        return visualization.build.call(this, title, type, options);
     }
 
     /**
      * Render visualizations
+     * @param {string} location - Location to render visualizations
      * @param {Object} [options] - Optional render options
      * @returns {Vaporous} - Returns this instance for chaining
      */
-    render(options) {
-        return visualization.render?.call(this, options) || this;
+    render(location, options) {
+        return visualization.render.call(this, location, options);
     }
 
     // ========================================
@@ -449,7 +455,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     load_http(options) {
-        return http.load_http?.call(this, options) || this;
+        return http.load_http.call(this, options);
     }
 
     // ========================================
@@ -463,7 +469,7 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     interval(callback, ms) {
-        return processing.interval?.call(this, callback, ms) || this;
+        return processing.interval.call(this, callback, ms);
     }
 
     /**
@@ -474,19 +480,35 @@ class Vaporous {
      * @returns {Vaporous} - Returns this instance for chaining
      */
     parallel(concurrency, callback, options) {
-        return processing.parallel?.call(this, concurrency, callback, options) || this;
+        return processing.parallel.call(this, concurrency, callback, options);
     }
 
+    /**
+     * Recursively process events
+     * @param {Function} callback - Function to execute recursively
+     * @returns {Vaporous} - Returns this instance for chaining
+     */
     recurse(callback) {
-        return processing.recurse?.call(this, callback)
+        return processing.recurse.call(this, callback);
     }
 
+    /**
+     * Debug events by executing a callback
+     * @param {Function} callback - Debug callback function
+     * @returns {Vaporous} - Returns this instance for chaining
+     */
     debug(callback) {
-        return core.debug.call(this, callback)
+        return core.debug.call(this, callback);
     }
 
+    /**
+     * Conditionally execute a callback
+     * @param {Function|boolean} condition - Condition to evaluate or boolean value
+     * @param {Function} callback - Function to execute if condition is true
+     * @returns {Vaporous} - Returns this instance for chaining
+     */
     doIf(condition, callback) {
-        return processing.doIf.call(this, condition, callback)
+        return processing.doIf.call(this, condition, callback);
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vaporous",
-  "version": "0.0.10",
+  "version": "0.0.12",
   "description": "Transition data to different structured states for analytical processing",
   "main": "Vaporous.js",
   "scripts": {
@@ -10,6 +10,8 @@
   "license": "ISC",
   "dependencies": {
     "dayjs": "^1.11.18",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
     "papaparse": "^5.5.3",
     "split2": "^4.2.0"
   }

package/src/core.js

@@ -1,23 +1,4 @@
 module.exports = {
-    method(operation, name, options) {
-        const operations = {
-            create: () => {
-                this.savedMethods[name] = options
-            },
-            retrieve: () => {
-                if (!this.savedMethods[name]) throw new Error('Method not found ' + name)
-                this.savedMethods[name](this, options)
-            },
-            delete: () => {
-                delete this.savedMethods[name]
-            }
-        }
-
-        operations[operation]()
-        if (operation !== 'retrieve') return this;
-        return this;
-    },
-
     async debug(callback) {
         await callback(this)
         return this;
@@ -71,20 +52,12 @@ module.exports = {
         const Vaporous = require('../Vaporous').Vaporous;
         const cloneInstance = new Vaporous()
 
-        const excludeStructualClone = ['loggers', 'intervals']
-        const excludeCompletely = ['_isExecuting']
-
-        Object.keys(this).forEach(key => {
-            if (excludeCompletely.includes(key)) return;
-            cloneInstance[key] = (deep && !excludeStructualClone.includes(key)) ? structuredClone(this[key]) : this[key]
-        })
-
-        const purge = ['checkpoints', 'events']
-
-        purge.forEach(purgeItem => {
-            cloneInstance[purgeItem] = []
+        Object.keys(this).forEach(prop => {
+            cloneInstance[prop] = this[prop]
         })
 
+        cloneInstance.processingQueue = []
+        cloneInstance._isExecuting = false
         return cloneInstance
     },
 

package/src/http.js

@@ -3,59 +3,64 @@ const httpsLib = require('https')
 const { HttpProxyAgent } = require('http-proxy-agent')
 const { HttpsProxyAgent } = require('https-proxy-agent')
 
+const request = async ({ uri, method, headers, body, options = {} }) => {
+    const response = await new Promise((resolve, reject) => {
+        const isHTTPs = uri.startsWith('https')
+        const lib = isHTTPs ? httpsLib : httpLib
+        const { hostname, port, pathname, search } = new URL(uri)
+        const path = pathname + (search || '')
 
-async function load_http() {
+        const options = {
+            method,
+            hostname,
+            port,
+            path,
+            headers
+        }
 
-    for (let event of this.events) {
-        const { _http_req_uri,
-            _http_req_method,
-            _http_req_headers,
-            _http_req_body } = event
-
-
-        const task = new Promise((resolve, reject) => {
-            const lib = _http_req_uri.startsWith('https') ? httpsLib : httpLib
-            const { hostname, port, pathname, search } = new URL(_http_req_uri)
-            const path = pathname + (search || '')
-
-            const options = {
-                method: _http_req_method,
-                hostname,
-                port,
-                path,
-                headers: _http_req_headers
-            }
-
-            const proxy = process.env.HTTP_PROXY || process.env.HTTPS_PROXY || process.env.http_proxy || process.env.https_proxy
-
-            if (proxy) {
-                const isHttps = _http_req_uri.startsWith('https')
-                options.agent = isHttps ? new HttpsProxyAgent(proxy) : new HttpProxyAgent(proxy)
-            }
-
-            const req = lib.request(options, res => {
-                let body = '';
-
-                res.on('data', data => {
-                    body += data
-                })
-
-                res.on('end', () => {
-                    event._http_res_body = body
-                    event._http_res_status = res.statusCode
-                    event._http_res_headers = res.headers
-                    resolve(body)
-                })
+        const proxy = isHTTPs ? (process.env.https_proxy || process.env.HTTPS_PROXY) : (process.env.http_proxy || process.env.HTTP_PROXY)
+
+        if (proxy) options.agent = isHTTPs ? new HttpsProxyAgent(proxy) : new HttpProxyAgent(proxy)
+
+        const req = lib.request(options, res => {
+            let body = '';
+
+            res.on('data', data => {
+                body += data
+            })
+
+            res.on('end', () => {
+                resolve({ body, statusCode: res.statusCode, headers: res.headers })
             })
+        })
 
-            if (_http_req_body) req.write(_http_req_body)
-            req.end()
+        if (body) req.write(body)
+        req.end()
+    })
+
+    if (response.statusCode === 307) {
+        return request({ uri: response.headers.location, method, headers, body, options })
+    }
+
+    return response;
+}
+
+async function load_http() {
+
+    for (let event of this.events) {
+        const res = await request({
+            uri: event._http_req_uri,
+            method: event._http_req_method,
+            headers: event._http_req_headers,
+            body: event._http_req_body
         })
 
-        await task
+        event._http_res_status = res.statusCode
+        event._http_res_headers = res.headers
+        event._http_res_body = res.body
     }
 
-    return this;
+    return this
 }
 
 module.exports = {

package/src/processing.js

@@ -31,8 +31,9 @@ async function parallel(target, { multiThread = false } = {}, callbackPath) {
         const instance = new Vaporous({ loggers })
         instance.events = thisEvent
 
-        const task = await funct(instance)
-        tasks.push(task.begin())
+        const task = funct(instance).begin()
+        tasks.push(task)
+        await task
 
         await processSingleThread()
     }
@@ -135,9 +136,11 @@ async function recurse(funct) {
     for (let event of this.events) {
         const target = new Vaporous({ loggers: this.loggers })
         target.events = [event]
+        event._recursion = true
 
         const localRecursion = async (target) => {
-            const val = await funct(target)
+            const executionChain = funct(target)
+            const val = await executionChain.begin()
             if (val.events[0]._recursion) return localRecursion(target)
             return val
         }
@@ -150,9 +153,35 @@ async function recurse(funct) {
     return this;
 }
 
-function doIf(condition, callback) {
-    const proceed = condition(this)
-    if (proceed) callback(this)
+async function doIf(condition, callback) {
+    const proceed = await condition(this)
+    if (proceed) {
+        const clone = this.clone()
+        const task = await callback(clone)
+        await task.begin()
+        this.events = clone.events
+    }
+    return this;
+}
+
+async function method(operation, name, options) {
+    const operations = {
+        create: () => {
+            this.savedMethods[name] = options
+        },
+        retrieve: async () => {
+            if (!this.savedMethods[name]) throw new Error('Method not found ' + name)
+            const clone = this.clone()
+            const task = await clone.savedMethods[name](clone, options)
+            await task.begin()
+            this.events = clone.events
+        },
+        delete: () => {
+            delete this.savedMethods[name]
+        }
+    }
+
+    await operations[operation]()
     return this;
 }
 
@@ -160,5 +189,6 @@ module.exports = {
     parallel,
     interval,
     recurse,
-    doIf
+    doIf,
+    method
 }

package/src/statistics.js

@@ -41,7 +41,9 @@ module.exports = {
                 const reference = map[key]._statsRaw[aggregation.field]
 
                 if (aggregation.sortable) {
-                    sortedCache[aggregation.field] = reference.slice().sort((a, b) => a - b)
+                    if (sortedCache[aggregation.field] === undefined) {
+                        sortedCache[aggregation.field] = reference.slice().sort((a, b) => a - b)
+                    }
                     result[outputField] = aggregation.calculate(sortedCache[aggregation.field])
                 } else {
                     result[outputField] = aggregation.calculate(reference)

package/src/transformations.js

@@ -60,27 +60,28 @@ module.exports = {
         return this;
     },
 
-    mvexpand(target) {
+    mvexpand(targets) {
 
         const arr = []
         this.events.forEach(event => {
             if (event instanceof Array) {
-                if (!!target) throw new Error('Cannot mvexpand an array to a target')
+                if (targets.length !== 0) throw new Error('Cannot mvexpand on a target value when source data is array')
+
                 event.forEach((item, i) => {
-                    item.i = i
+                    item._mvExpand = i
                     arr.push(item)
                 })
             } else {
+                // Identify max iterations
+                const max = targets.reduce((prev, curr) => Math.max(prev, event[curr].length), 0)
 
-
-                if (!event[target]) return arr.push(event)
-                event[target].forEach((item, i) => {
-                    arr.push({
-                        ...event,
-                        [target]: item,
-                        [`_mvExpand_${target}`]: i
+                for (let i = 0; i < max; i++) {
+                    const obj = { ...event, _mvExpand: i }
+                    targets.forEach(target => {
+                        obj[target] = event[target][i]
                     })
-                })
+                    arr.push(obj)
+                }
             }
         })
 

package/src/visualization.js

@@ -372,12 +372,14 @@ module.exports = {
         return this;
     },
 
-    render(location = './Vaporous_generation.html', { tabOrder } = {}) {
+    render(location = './Vaporous_generation.html', options = {}) {
 
 
         const classSafe = (name) => name.replace(/[^a-zA-Z0-9]/g, "_")
 
-        if (tabOrder) this.tabs = tabOrder
+        if (typeof options === 'function') options = options()
+
+        if (options.tabOrder) this.tabs = options.tabOrder
 
         const createElement = (name, type, visualisationOptions, eventData, { trellis, trellisName = "", columnDefinitions }) => {