total5 0.0.7-2 → 0.0.7

package/changelog.txt

@@ -9,6 +9,16 @@
 - added `$.ctrl(ctrl_instance)` method
 - extended `Mail.from(email, [name])` method by adding `name` argument by [Marek Mráz](https://github.com/Mrazbb)
 - added `CONF.mail_from_name {String}` option
+- improved `$.query` parser
+- improved `Total.run()` method
+- added a new method `NEWCOMPONENT(html_or_URL_or_name, [callback])`
+	- it compiles a FlowStream component to a method
+- extended FlowStream by adding support for `input` and `output` Total.js inline schemas
+- fixed memory usage in the Flow in non-worker mode
+- improved Flow error handling
+- improved HTML parser
+- improved `controller.view()` supports calling without arguments
+- extended ViewEngine by adding `view.renderlayout(name, content)` method
 
 ========================
 0.0.6

package/components.js

@@ -0,0 +1,381 @@
+'use strict';
+
+const REG_ARGS = /\{{1,2}[a-z0-9_.-\s]+\}{1,2}/gi;
+
+function variables(str, data, encoding) {
+
+	if (typeof(str) === 'object') {
+		var obj = {};
+		for (var key in str) {
+			var val = str[key];
+			if (typeof(val) === 'string')
+				obj[key] = variables.call(this, val, data, encoding);
+			else
+				obj[key] = val;
+		}
+		return obj;
+	}
+
+	if (typeof(str) !== 'string' || str.indexOf('{') === -1)
+		return str;
+
+	var main = this.instance ? this.instance.module : (this.module || this);
+
+	if (data == null || data == true)
+		data = this;
+
+	return str.replace(REG_ARGS, function(text) {
+
+		var l = text[1] === '{' ? 2 : 1;
+		var key = text.substring(l, text.length - l).trim();
+		var val = null;
+
+		if (main.variables)
+			val = main.variables[key];
+
+		if (!val && main.variables2)
+			val = main.variables2[key];
+
+		if (!val && main.secrets)
+			val = main.secrets[key];
+
+		/*
+		if (!val && key === 'hostname') {
+			val = '';
+			if (val[val.length - 1] === '/')
+				val = val.substring(0, val.length - 1);
+		}*/
+
+		var customencoding = typeof(encoding) === 'function';
+
+		if (!val && data != null && typeof(data) === 'object') {
+			var nested = key.indexOf('.') !== -1;
+			val = nested ? F.TUtils.get(data, key) : data[key];
+		}
+
+		if (customencoding) {
+
+			val = encoding(val, key);
+
+		} else {
+
+			if (encoding !== 'json') {
+				if (val instanceof Date)
+					val = val.format();
+			}
+
+			switch (encoding) {
+				case 'urlencoded':
+				case 'url':
+					val = encodeURIComponent(val);
+					break;
+				case 'json':
+					val = JSON.stringify(val);
+					break;
+				case 'querify':
+					val = F.TUtils.querify(val).substring(1);
+					break;
+			}
+		}
+
+		return val == null ? text : val;
+
+	});
+}
+
+function Component() {
+	let t = this;
+	t.variables = t.variables2 = {};
+	t.secrets = {};
+	t.instances = [];
+	t.debugger = true;
+}
+
+Component.prototype.service = function(counter) {
+	for (let m of this.instances)
+		m.service && m.service(counter);
+};
+
+Component.prototype.status = function(instance, msg) {
+	if (t.debugger)
+		console.log('STATUS', this.name, msg);
+};
+
+Component.prototype.debug = function(instance, msg) {
+	if (t.debugger)
+		console.log('DEBUG', this.name + ':', msg);
+};
+
+Component.prototype.dashboard = function(instance, msg) {
+	if (t.debugger)
+		console.log('DASHBOARD', this.name + ':', msg);
+};
+
+Component.prototype.throw = function(instance, err) {
+	if (t.debugger)
+		console.log('ERROR', this.name + ':', err);
+};
+
+Component.prototype.output = function(instance, response) {
+	if (t.debugger)
+		console.log('OUTPUT', this.name + ' | ' + response.output + ':', response.data);
+};
+
+Component.prototype.create = function(opt, status) {
+
+	let t = this;
+	let instance = new Instance();
+	instance.id = U.random_text(8);
+	instance.config = t.config ? F.TUtils.clone(t.config) : {};
+	instance.module = t;
+
+	if (opt) {
+		for (let key in opt)
+			instance.config[key] = opt[key];
+	}
+
+	t.instances.push(instance);
+	t.onextend && t.onextend(instance);
+	t.make.call(instance, instance, instance.config, status);
+	t.oncreate && setImmediate(() => t.oncreate(instance));
+	return instance;
+};
+
+Component.prototype.remove = function() {
+	let t = this;
+	t.instances.wait(function(instance, next) {
+		instance.remove();
+		setImmediate(next);
+	}, function() {
+		t.uninstall && t.uninstall.call(t, t);
+	});
+};
+
+Component.prototype.save = function(instance) {
+	// save state
+};
+
+function Message() {
+	let t = this;
+	t.repo = {};
+	t.vars = {};
+	t.used = 1;
+	t.main = null;
+	t.processed = 0;
+}
+
+Message.prototype.replace = variables;
+
+Message.prototype.send = function(output, data) {
+	let t = this;
+	if (!t.instance)
+		return;
+	if (data != null)
+		t.data = data;
+	t.output = output;
+	t.callback && t.callback(t);
+	t.instance.output && t.instance.output(t);
+};
+
+Message.prototype.end = Message.prototype.destroy = function() {
+	let t = this;
+	t.data = null;
+	t.fromcomponent = null;
+	t.instance = null;
+	t.vars = null;
+	t.repo = null;
+	t.refs = null;
+}
+
+function Instance() {
+	let t = this;
+	t.id = U.random_text(8);
+	t.cache = {};
+	t.middleware = NOOP;
+	t.transform = NOOP;
+	t.replace = variables;
+	t.instances = EMPTYOBJECT;
+	t.components = EMPTYOBJECT;
+}
+
+Instance.prototype.replace = variables;
+
+Instance.prototype.remove = function() {
+	let t = this;
+	t.close && t.close.call(t, true);
+	t.destroy && t.destroy.call(t);
+	t.module.onremove && t.module.onremove(t);
+	let index = t.module.instances.indexOf(t);
+	t.module.instances.splice(index, 1);
+};
+
+Instance.prototype.input = function(input, data, callback) {
+	let t = this;
+	if (t.message) {
+
+		let msg = t.newmessage(data);
+		msg.input = input;
+		msg.callback = callback;
+
+		var schema = t.module.inputschemas[input];
+		if (schema) {
+			let tmp = schema.transform(msg.data);
+			msg.data = tmp.response;
+			msg.error = tmp.error;
+		}
+
+		t.message(msg);
+		let fn = t['message_' + input];
+		fn && fn(msg);
+	}
+};
+
+Instance.prototype.send = function(output, data) {
+	let msg = data instanceof Message ? data : this.newmessage();
+	msg.output = output;
+	msg.data = data;x
+	msg.send(output);
+};
+
+Instance.prototype.newmessage = function(data) {
+	var t = this;
+	var msg = new Message();
+	msg.from = msg.instance = t;
+	msg.fromid = msg.id;
+	msg.fromcomponent = msg.component;
+	msg.data = data instanceof Message ? data.data : data;
+	return msg;
+};
+
+Instance.prototype.save = function() {
+	this.module.save(this);
+};
+
+Instance.prototype.output = function(response) {
+	this.module.output(this, response);
+};
+
+Instance.prototype.debug = function(msg) {
+	this.module.debug(this, msg);
+};
+
+Instance.prototype.throw = function(err) {
+	this.module.throw(this, error);
+};
+
+Instance.prototype.dashboard = function(msg) {
+	this.module.dashboard(this, msg);
+};
+
+Instance.prototype.reconfigure = function(opt) {
+	let t = this;
+	for (let key in opt)
+		t.config[key] = opt[key];
+	t.configure && t.configure(t.config);
+	t.save();
+};
+
+Instance.prototype.status = function(msg) {
+	this.module.status(this, msg);
+};
+
+Instance.prototype.logger = NOOP;
+Instance.prototype.transform = NOOP;
+Instance.prototype.middleware = NOOP;
+
+exports.compile = function(html, callback) {
+
+	if (callback == null) {
+		return new Promise(function(resolve, reject) {
+			exports.compile(html, function(err, com) {
+				if (err)
+					reject(err);
+				else
+					resolve(com);
+			});
+		})
+	}
+
+	if ((/^http(s)\:\/\//i).test(html)) {
+		let opt = {};
+		opt.url = html;
+		opt.callback = function(err, response) {
+			console.log(err, response);
+			if (err)
+				callback(err);
+			else
+				exports.compile(response.body, callback);
+		};
+		REQUEST(opt);
+		return;
+	}
+
+	if (!html.includes('<')) {
+		F.Fs.readFile(PATH.root('components/' + html + '.html'), 'utf8', function(err, response) {
+			if (err)
+				callback(err);
+			else
+				exports.compile(response, callback);
+		});
+		return;
+	}
+
+	var meta = html.parseComponent({ readme: '<readme>', settings: '<settings>', css: '<style>', be: '<script total>', be2: '<script node>', js: '<script>', html: '<body>', schema: '<schema>', template: '<template>' });
+	var node = (meta.be || meta.be2 || '').trim().replace(/\n\t/g, '\n');
+
+	if (!meta.be && !meta.be2) {
+		var e = new Error('Invalid component content');
+		callback(e);
+		return;
+	}
+
+	var fn = null;
+	var com = new Component();
+
+	delete meta.be;
+	delete meta.be2;
+	com.ui = meta;
+
+	try {
+		fn = new Function('exports', 'require', node);
+		fn(com, F.require);
+	} catch (e) {
+		callback(e);
+		return;
+	}
+
+	var errors = [];
+
+	(com.npm || EMPTYARRAY).wait(function(name, next) {
+		NPMINSTALL(name, function(err) {
+			if (err) {
+				callback(err);
+				next = null;
+			} else
+				next();
+		});
+	}, function() {
+
+		com.inputschemas = {};
+		com.outputschemas = {};
+
+		if (com.inputs) {
+			for (let m of com.inputs) {
+				if (m.schema)
+					com.inputschemas[m.id] = m.schema.toJSONSchema();
+			}
+		}
+
+		if (com.outputs) {
+			for (let m of com.outputs) {
+				if (m.schema)
+					com.outputschemas[m.id] = m.schema.toJSONSchema();
+			}
+		}
+
+		com.install && com.install.call(com, com);
+
+		callback(null, com);
+	});
+
+};

package/controller.js

@@ -113,6 +113,14 @@ function Controller(req, res) {
 
 Controller.prototype = {
 
+	get query() {
+		return this.$query || (this.$query = ctrl.uri.search.parseEncoded());
+	},
+
+	set query(val) {
+		this.$query = val;
+	},
+
 	get mobile() {
 		let ua = this.headers['user-agent'];
 		return ua ? REG_MOBILE.test(ua) : false;
@@ -458,13 +466,17 @@ Controller.prototype.layout = function(name) {
 Controller.prototype.view = function(name, model) {
 
 	var ctrl = this;
+	var view = new F.TViewEngine.View(ctrl);
+
+	if (!name)
+		return view;
 
 	if (ctrl.destroyed)
 		return;
 
-	var view = new F.TViewEngine.View(ctrl);
 	ctrl.response.layout && view.layout(ctrl.response.layout);
 	setImmediate(renderview, view, name, model);
+
 	return view;
 };
 
@@ -1017,12 +1029,16 @@ Controller.prototype.$route = function() {
 
 				ctrl.payload = Buffer.concat(ctrl.payload);
 				F.stats.performance.download += ctrl.payload.length / 1024 / 1024;
+				let val;
+
 				switch (ctrl.datatype) {
 					case 'json':
-						ctrl.body = F.def.parsers.json(ctrl.payload.toString('utf8'));
+						val = ctrl.payload.toString('utf8');
+						ctrl.body = val ? F.def.parsers.json(val) : null;
 						break;
 					case 'urlencoded':
-						ctrl.body = F.def.parsers.urlencoded(ctrl.payload.toString('utf8'));
+						val = ctrl.payload.toString('utf8');
+						ctrl.body = val ? F.def.parsers.urlencoded(val) : {};
 						break;
 				}
 

package/flow-flowstream.js

@@ -1320,7 +1320,7 @@ function init_current(meta, callback, nested) {
 
 			if (instance) {
 				if (typeof(instance) === 'string') {
-					if (source === 'add') {
+					if (source === 'add' || source === 'register') {
 						componentid = instance;
 						F.error(err, 'FlowStream | register component | ' + instance);
 					} else if (meta.design[instance]) {
@@ -1339,9 +1339,6 @@ function init_current(meta, callback, nested) {
 				} else if (source === 'instance_make') {
 					instanceid = instance.id;
 					componentid = instance.component;
-				} else if (source === 'register') {
-					instanceid = '';
-					componentid = instance;
 				} else {
 					instanceid = instance.id;
 					componentid = instance.module.id;
@@ -1449,9 +1446,11 @@ function init_current(meta, callback, nested) {
 				} else if (source === 'register') {
 					instanceid = '';
 					componentid = instance;
+				} else if (source === 'add') {
+					componentid = instance;
 				} else {
 					instanceid = instance.id;
-					componentid = instance.module.id;
+					componentid = instance.module ? instance.module.id : null;
 				}
 			}
 
@@ -2461,6 +2460,8 @@ function MAKEFLOWSTREAM(meta) {
 			minutes = stats.minutes;
 			if (isFLOWSTREAMWORKER)
 				memory = process.memoryUsage().heapUsed;
+			else if (F.consumption.memory)
+				memory = F.consumption.memory * 1024 * 1024;
 		}
 
 		flow.stats.memory = memory;

package/flow.js

@@ -16,7 +16,6 @@ FS.worker = false;
 FS.instances = {};
 FS.onerror = function(err, source, id, componentid, stack) {
 
-throw new Error('FET');
 	var flow = FS.db[this.id];
 	var empty = '---';
 	var output = '';

package/flowstream.js

@@ -305,10 +305,8 @@ function variables(str, data, encoding) {
 
 		var customencoding = typeof(encoding) === 'function';
 
-		if (!val && data != null && typeof(data) === 'object') {
-			var nested = key.indexOf('.') !== -1;
-			val = nested ? F.TUtils.get(data, key) : data[key];
-		}
+		if (!val && data != null && typeof(data) === 'object')
+			val = key.includes('.') ? F.TUtils.get(data, key) : data[key];
 
 		if (customencoding) {
 
@@ -353,7 +351,7 @@ function timeouthandler(msg) {
 
 MP.send = function(outputindex, data, clonedata) {
 
-	var self = this;
+	let self = this;
 
 	if (clonedata == null)
 		clonedata = self.main.cloning;
@@ -364,13 +362,13 @@ MP.send = function(outputindex, data, clonedata) {
 		return 0;
 	}
 
-	var outputs;
-	var count = 0;
+	let outputs;
+	let count = 0;
 
 	if (outputindex == null) {
 
 		if (self.instance.connections) {
-			for (var key in self.instance.connections)
+			for (let key in self.instance.connections)
 				count += self.send(key);
 		}
 
@@ -381,8 +379,8 @@ MP.send = function(outputindex, data, clonedata) {
 	}
 
 
-	var meta = self.main.meta;
-	var now = Date.now();
+	let meta = self.main.meta;
+	let now = Date.now();
 
 	outputs = self.instance.connections ? (self.instance.connections[outputindex] || F.EMPTYARRAY) : F.EMPTYARRAY;
 
@@ -407,7 +405,7 @@ MP.send = function(outputindex, data, clonedata) {
 		return count;
 	}
 
-	var tid = self.toid + D + outputindex + (self.color || '');
+	let tid = self.toid + D + outputindex + (self.color || '');
 
 	if (self.main.stats.traffic[tid]) {
 		self.main.stats.traffic[tid]++;
@@ -419,29 +417,30 @@ MP.send = function(outputindex, data, clonedata) {
 	if (self.transformation === '1')
 		self.transformation = '_' + tid;
 
-	for (var i = 0; i < outputs.length; i++) {
-		var output = outputs[i];
+	for (let i = 0; i < outputs.length; i++) {
+
+		let output = outputs[i];
 
 		if (output.disabled || output.paused)
 			continue;
 
-		var schema = meta.flow[output.id];
+		let schema = meta.flow[output.id];
 		if (schema && (schema.message || schema['message_' + output.index]) && schema.component && schema.ready && self.main.$can(true, output.id, output.index)) {
-			var next = meta.components[schema.component];
+			let next = meta.components[schema.component];
 			if (next && next.connected && !next.isdestroyed && !next.disabled) {
 
 				if (output.color && self.color && self.color !== output.color)
 					continue;
 
-				var inputindex = output.index;
-				var message = self.clone();
+				let inputindex = output.index;
+				let message = self.clone();
 
 				if (data != undefined)
 					message.data = data;
 
 				if (clonedata && message.data && typeof(message.data) === 'object') {
 					if (message.data instanceof Buffer) {
-						var buf = Buffer.alloc(message.data.length);
+						let buf = Buffer.alloc(message.data.length);
 						buf.copy(message.data);
 						message.data = buf;
 					} else
@@ -462,6 +461,13 @@ MP.send = function(outputindex, data, clonedata) {
 				message.ts = now;
 				message.color = output.color;
 
+				let ti = schema.inputschemas[message.input];
+				if (ti && ti.schema) {
+					let tmp = ti.schema.transform(typeof(message.data) === 'object' ? message.data : {});
+					message.data = tmp.response;
+					message.error = tmp.error;
+				}
+
 				if (self.$timeout)
 					message.$timeoutid = setTimeout(timeouthandler, self.$timeout, message);
 
@@ -999,7 +1005,6 @@ function newmessage(data) {
 	msg.data = data instanceof Message ? data.data : data;
 	msg.cloned = 0;
 	msg.count = 0;
-	msg.instance = self;
 	msg.duration = msg.ts = Date.now();
 	msg.used = 1;
 	msg.main = self instanceof FlowStream ? self : self.main;
@@ -1141,6 +1146,13 @@ FP.ontrigger = function(outputindex, data, controller, events) {
 					message.cache = target.cache;
 					message.processed = 0;
 
+					let ti = target.inputschemas[message.input];
+					if (ti && ti.schema) {
+						let tmp = ti.schema.transform(message.data);
+						message.data = tmp.response;
+						message.error = tmp.error;
+					}
+
 					target.stats.pending++;
 					target.stats.input++;
 					schema.stats.output++;
@@ -1194,6 +1206,7 @@ FP.reconfigure = function(id, config, rewrite) {
 		self.onreconfigure && self.onreconfigure(instance);
 		self.$events.configure && self.emit('configure', instance);
 	}
+
 	return !!instance;
 };
 
@@ -1606,6 +1619,28 @@ FP.initcomponent = function(key, component) {
 	instance.replace = variables;
 	instance.instances = self.meta.flow;
 	instance.components = self.meta.components;
+	instance.inputschemas = {};
+	instance.outputschemas = {};
+
+	// Due to inline data schemas
+	if (component.inputs) {
+		for (let m of component.inputs) {
+			let tmp = CLONE(m);
+			instance.inputschemas[m.id] = tmp;
+			if (tmp.schema)
+				tmp.schema = tmp.schema.toJSONSchema();
+		}
+	}
+
+	// Due to inline data schemas
+	if (component.outputs) {
+		for (let m in component.outputs) {
+			let tmp = CLONE(m);
+			instance.outputschemas[m.id] = tmp;
+			if (tmp.schema)
+				tmp.schema = tmp.schema.toJSONSchema();
+		}
+	}
 
 	self.onconnect && self.onconnect(instance);
 	self.$events.connect && self.emit('connect', instance);

package/global.js

@@ -40,6 +40,7 @@ global.DATA = new F.TQueryBuilder.Controller(true);
 global.DB = () => new F.TQueryBuilder.Controller();
 global.CACHE = F.cache;
 global.NEWACTION = F.TBuilders.newaction;
+global.NEWCOMPONENT = F.newcomponent;
 global.NEWSCHEMA = F.TBuilders.newschema;
 global.ACTION = global.EXEC = F.TBuilders.action;
 global.TEMPLATE = F.template;

package/htmlparser.js

@@ -561,7 +561,15 @@ function parseHTML(html, trim, onerror, isxml) {
 		}
 
 		var tag = node;
-		var index = tag.indexOf(' ');
+		var index = -1;
+
+		for (let i = 0; i < tag.length; i++) {
+			let c = tag[i];
+			if (c === '\n' || c === ' ' || c === '/' || c === '>') {
+				index = i;
+				break;
+			}
+		}
 
 		if (index > 0) {
 			tag = tag.substring(0, index);
@@ -597,9 +605,14 @@ function parseHTML(html, trim, onerror, isxml) {
 				case 'BR':
 				case 'HR':
 				case 'IMG':
+				case 'INPUT':
 				case 'META':
 				case 'LINK':
-				case 'INPUT':
+				case 'SOURCE':
+				case 'AREA':
+				case 'COL':
+				case 'EMBED':
+				case 'TRACK':
 					dom.unpair = true;
 					return str;
 			}

package/index.js

@@ -1243,6 +1243,11 @@ F.httpload = function(opt) {
 	F.server = F.Http.createServer(F.THttp.listen);
 	F.server.on('upgrade', F.TWebSocket.listen);
 
+	CONF.$performance && F.server.on('connection', function(socket) {
+		socket.setNoDelay(true);
+		socket.setKeepAlive(true, 10);
+	});
+
 	var unixsocket = opt.unixsocket || F.config.$unixsocket;
 	if (unixsocket) {
 
@@ -2499,7 +2504,7 @@ F.filestorage = function(name) {
 
 F.encryptreq = function(ctrl, val, key, strict) {
 	var obj = {};
-	obj.ua = ctrl.ua;
+	obj.ua = HASH(ctrl.headers['user-agent'] || '').toString(36);
 	if (strict)
 		obj.ip = ctrl.ip;
 	obj.data = val;
@@ -2510,7 +2515,7 @@ F.decryptreq = function(ctrl, val, key) {
 	if (!val)
 		return;
 	var obj = F.decrypt(val, key || '', true);
-	if (!obj || (obj.ip && obj.ip !== ctrl.ip) || (obj.ua !== ctrl.ua))
+	if (!obj || (obj.ip && obj.ip !== ctrl.ip) || (obj.ua !== HASH(ctrl.headers['user-agent'] || '').toString(36)))
 		return;
 	return obj.data;
 };
@@ -2572,6 +2577,8 @@ F.dir = function(val) {
 };
 
 F.run = function(opt) {
+	if (!opt)
+		opt = {};
 	var type = opt.watcher === false ? 'release' : 'debug';
 	opt.watcher = false;
 	require('./' + type)(opt);
@@ -2648,6 +2655,10 @@ F.datauri = function(path) {
 	}));
 };
 
+F.newcomponent = function(html, callback) {
+	return require('./components').compile(html, callback);
+};
+
 F.loadstats = function() {
 
 	var main = {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "total5",
-  "version": "0.0.7-2",
+  "version": "0.0.7",
   "description": "Total.js framework v5",
   "main": "index.js",
   "directories": {

package/viewengine.js

@@ -873,12 +873,21 @@ View.prototype.set = function() {
 	return '';
 };
 
+View.prototype.renderlayout = function(name, content) {
+	let self = this;
+	self.output = content;
+	self.islayout = true;
+	content = self.render(self.repository.layout);
+	self.islayout = false;
+	return content;
+};
+
 View.prototype.render = function(name, model, ispartial = false) {
 
-	var self = this;
-	var key = name + '_' + self.language;
-	var fn = F.temporary.views[key];
-	var content;
+	let self = this;
+	let key = name + '_' + self.language;
+	let fn = F.temporary.views[key];
+	let content;
 
 	self.model = model;
 
@@ -897,6 +906,7 @@ View.prototype.render = function(name, model, ispartial = false) {
 		self.output = content;
 		self.islayout = true;
 		content = self.render(self.repository.layout);
+		self.islayout = false;
 	}
 
 	return content;