total5 0.0.1 → 0.0.2

package/nosql-stream.js

@@ -0,0 +1,617 @@
+// Total.js NoSQL FileStream
+// The MIT License
+// Copyright 2020-2023 (c) Peter Širka <petersirka@gmail.com>
+
+'use strict';
+
+const BUFFERSIZE = 1024 * 32;
+const BUFFERDOCS = 15;
+const NEWLINEBUFFER = Buffer.from('\n', 'utf8');
+const DEFSTATS = { size: 0 };
+
+function NoSQLStream(filename) {
+	this.filename = filename;
+	this.fd = null;
+	this.stats = DEFSTATS;
+	this.type = null;
+	this.bytesread = 0;
+	this.ticks = 0;
+	this.position = 0;
+	this.cache = [null, null];
+	this.buffer = null;
+	this.divider = ',';
+	this.remchar = '-';
+	this.buffercount = BUFFERDOCS;
+	this.buffersize = BUFFERSIZE;
+	this.linesize = 0;
+	this.start = 0;
+	this.indexer = 0;
+	// this.canceled = false;
+	// this.docs = '';
+	// this.docscount = 0;
+}
+
+// Because of performance
+NoSQLStream.prototype.readhelpers = function() {
+
+	var self = this;
+
+	self.cb_read = () => self.read();
+
+	self.cb_readbuffer = function(err, size, chunk) {
+
+		self.position += size;
+
+		var beg = 0;
+
+		if (self.buffer) {
+			self.cache[0] = self.buffer;
+			self.cache[1] = chunk;
+
+			beg = self.buffer.length - 1;
+
+			if (beg < 0)
+				beg = 0;
+
+			self.buffer = Buffer.concat(self.cache);
+
+		} else
+			self.buffer = chunk;
+
+		var index = self.buffer.lastIndexOf(NEWLINEBUFFER);
+		if (index === -1) {
+			self.read();
+			return;
+		}
+
+		var tmp = self.buffer.toString('utf8', 0, index).split('\n');
+		for (var i = 0; i < tmp.length; i++) {
+			if (tmp[i][0] !== self.remchar) {
+				if (self.array)
+					self.docs.push(tmp[i]);
+				else
+					self.docs += (self.docs ? self.divider : '') + tmp[i];
+				self.docscount++;
+				self.indexer++;
+			}
+		}
+
+		self.buffer = self.buffer.slice(index + 1);
+
+		if (self.ondocuments() === false)
+			self.canceled = true;
+
+		self.docs = self.array ? [] : '';
+		self.docscount = 0;
+		self.ticks++;
+
+		if (self.ticks % 5 === 0)
+			setImmediate(self.cb_readticks);
+		else
+			self.read();
+	};
+
+	self.cb_readticks = () => self.read();
+	self.cb_readreverse = () => self.readreverse2();
+
+	self.cb_readreversebuffer = function(err, size, chunk) {
+
+		self.bytesread += size;
+
+		if (self.buffer) {
+			self.cache[0] = chunk;
+			self.cache[1] = self.buffer;
+			self.buffer = Buffer.concat(self.cache);
+		} else
+			self.buffer = chunk;
+
+		var index = self.buffer.indexOf(NEWLINEBUFFER);
+		if (index === -1) {
+			self.readreverse2();
+			return;
+		}
+
+		var tmp = self.buffer.toString('utf8', index).trim().split('\n');
+		for (var i = 0; i < tmp.length; i++) {
+			if (tmp[i][0] !== self.remchar) {
+				if (self.array)
+					self.docs.push(tmp[i]);
+				else
+					self.docs += (self.docs ? self.divider : '') + tmp[i];
+				self.docscount++;
+				self.indexer++;
+			}
+		}
+
+		if (self.ondocuments() === false)
+			self.canceled = true;
+
+		self.buffer = self.buffer.slice(0, index + 1);
+		self.docs = self.array ? [] : '';
+		self.docscount = 0;
+		self.ticks++;
+
+		if (self.ticks % 5 === 0)
+			setImmediate(self.cb_readreverseticks);
+		else
+			self.readreverse2();
+	};
+
+	self.cb_readreverseticks = () => self.readreverse2();
+
+	self.cb_readstream = function(chunk) {
+
+		if (self.canceled)
+			return;
+
+		var beg = 0;
+
+		if (self.buffer) {
+
+			self.cache[0] = self.buffer;
+			self.cache[1] = chunk;
+			self.buffer = Buffer.concat(self.cache);
+			beg = self.cache[0].length - 1;
+
+			if (beg < 0)
+				beg = 0;
+		} else
+			self.buffer = chunk;
+
+		var index = self.buffer.lastIndexOf(NEWLINEBUFFER);
+		if (index === -1)
+			return;
+
+		var tmp = self.buffer.toString('utf8', 0, index).split('\n');
+		for (var i = 0; i < tmp.length; i++) {
+			if (tmp[i][0] !== self.remchar) {
+				if (self.array)
+					self.docs.push(tmp[i]);
+				else
+					self.docs += (self.docs ? self.divider : '') + tmp[i];
+				self.docscount++;
+				self.indexer++;
+			}
+		}
+
+		self.buffer = self.buffer.slice(index + 1);
+
+		if (self.ondocuments() === false)
+			self.canceled = true;
+
+		if (self.canceled) {
+			self.stream.destroy && self.stream.destroy();
+		} else {
+			self.docs = self.array ? [] : '';
+			self.docscount = 0;
+			self.ticks++;
+		}
+	};
+
+};
+
+// Because of performance
+NoSQLStream.prototype.writehelpers = function() {
+
+	var self = this;
+
+	self.cb_writeAddUpdAdd = function(err, size) {
+		if (err) {
+			console.log('ERROR --> TextDBStreamer.writer (add)', err);
+			self.canceled = true;
+			self.bufferstacknew.length = 0;
+			self.bufferstack.length = 0;
+			self.writing = false;
+		} else {
+			self.positionappend += size;
+			var item = self.bufferstack.shift();
+			F.Fs.write(self.fd, item.data, item.position, 'utf8', self.cb_writeAddUpdUpd);
+		}
+	};
+
+	self.cb_writeAddUpdUpd = function(err) {
+
+		self.writing = false;
+
+		if (err) {
+			console.log('ERROR --> TextDBStreamer.writer (upd)', err);
+			self.canceled = true;
+			self.bufferstack.length = 0;
+			self.bufferstacknew.length = 0;
+		} else
+			self.$write();
+	};
+
+	self.cb_writeAdd = function(err, size) {
+
+		self.writing = false;
+		self.positionappend += size;
+
+		if (err) {
+			console.log('ERROR --> TextDBStreamer.writer (add)', err);
+			self.canceled = true;
+		} else
+			self.$write();
+	};
+
+	self.cb_writeUpd = function(err) {
+
+		self.writing = false;
+
+		if (err) {
+			console.log('ERROR --> TextDBStreamer.writer (upd)', err);
+			self.canceled = true;
+		} else
+			self.$write();
+	};
+
+	self.cb_flush = () => self.flush();
+
+	self.cb_readwritebuffer = function(err, size, chunk) {
+
+		self.position += size;
+
+		var beg = 0;
+		var index;
+
+		if (self.buffer) {
+			self.cache[0] = self.buffer;
+			self.cache[1] = chunk;
+			beg = self.buffer.length - 1;
+			if (beg < 0)
+				beg = 0;
+			self.buffer = Buffer.concat(self.cache);
+		} else
+			self.buffer = chunk;
+
+
+		var index = self.buffer.lastIndexOf(NEWLINEBUFFER);
+		if (index === -1) {
+			self.readupdate();
+			return;
+		}
+
+		var tmp = self.buffer.toString('utf8', 0, index).split('\n');
+		for (var i = 0; i < tmp.length; i++) {
+			var size = Buffer.byteLength(tmp[i], 'utf8');
+			if (tmp[i][0] !== self.remchar) {
+				if (self.array)
+					self.docs.push(tmp[i]);
+				else
+					self.docs += (self.docs ? self.divider : '') + tmp[i];
+				self.docsbuffer.push({ length: size, doc: tmp[i], position: self.positionupdate });
+				self.docscount++;
+				self.indexer++;
+			}
+			self.positionupdate += size + 1;
+		}
+
+		if (self.ondocuments() === false)
+			self.canceled = true;
+
+		self.docsbuffer = [];
+		self.docs = self.array ? [] : '';
+		self.buffer = self.buffer.slice(index + 1);
+
+		if (self.bufferstack.length || self.bufferstacknew.length)
+			setImmediate(self.cb_writeticks);
+		else
+			self.readupdate();
+	};
+
+	self.cb_writeticks = function() {
+		if (self.bufferstack.length || self.bufferstacknew.length)
+			setImmediate(self.cb_writeticks);
+		else
+			self.readupdate();
+	};
+};
+
+NoSQLStream.prototype.openread = function() {
+	var self = this;
+	self.type = 'r';
+	self.position = self.start;
+	self.open();
+	return self;
+};
+
+NoSQLStream.prototype.openreadreverse = function() {
+	var self = this;
+	self.type = 'r';
+	self.position = self.start;
+	self.$reverse = true;
+	self.open();
+	return self;
+};
+
+NoSQLStream.prototype.openupdate = function() {
+	var self = this;
+	self.type = 'r+';
+	F.Fs.open(self.filename, self.type, function(err, fd) {
+
+		if (err) {
+			self.$callback(err);
+			return;
+		}
+
+		F.Fs.fstat(fd, function(err, stats) {
+
+			self.docs = self.array ? [] : '';
+			self.docscount = 0;
+
+			if (err) {
+				F.Fs.close(fd, NOOP);
+				self.$callback(err);
+				return;
+			}
+
+			self.docs = self.array ? [] : '';
+			self.docscount = 0;
+			self.fd = fd;
+			self.stats = stats;
+			self.position = self.start;
+			self.positionappend = self.stats.size;
+			self.positionupdate = self.start;
+			self.bufferstack = [];
+			self.bufferstacknew = [];
+			self.docsbuffer = [];
+			self.writehelpers();
+			self.readupdate();
+		});
+	});
+
+	return self;
+};
+
+// For e.g. files on URL address
+NoSQLStream.prototype.openstream = function(stream) {
+
+	var self = this;
+
+	var close = function() {
+		if (self.docscount) {
+			self.ondocuments();
+			self.docscount = 0;
+			self.docs = self.array ? [] : '';
+		}
+		self.$callback && self.$callback();
+		self.$callback = null;
+	};
+
+	self.docs = self.array ? [] : '';
+	self.docscount = 0;
+	self.readhelpers();
+	self.stream = stream;
+	self.stream.on('error', close);
+	self.stream.on('end', close);
+	self.stream.on('data', self.cb_readstream);
+	return self;
+};
+
+NoSQLStream.prototype.open = function() {
+	var self = this;
+	F.Fs.open(self.filename, self.type, function(err, fd) {
+
+		if (err) {
+			self.$callback(err);
+			return;
+		}
+
+		F.Fs.fstat(fd, function(err, stats) {
+			self.docs = self.array ? [] : '';
+			self.docscount = 0;
+			self.fd = fd;
+			self.stats = stats;
+			self.position = self.start;
+
+			if (err) {
+				F.Fs.close(fd, NOOP);
+				self.$callback(err);
+				return;
+			}
+
+			self.readhelpers();
+
+			if (self.$reverse)
+				self.readreverse();
+			else
+				self.read();
+		});
+	});
+};
+
+NoSQLStream.prototype.close = function() {
+
+	var self = this;
+
+	if (self.fd) {
+
+		self.stream = null;
+
+		F.Fs.close(self.fd, function(err) {
+			err && F.error(err);
+			self.$callback && self.$callback();
+		});
+
+		if (self.buffer) {
+			self.buffer = null;
+			self.cache[0] = null;
+			self.cache[1] = null;
+			self.bytesread = 0;
+		}
+
+		self.canceled = false;
+		self.fd = null;
+		self.type = null;
+		self.docscache = null;
+		self.docs = null;
+
+	} else if (self.$callback)
+		self.$callback && self.$callback();
+
+	return self;
+};
+
+NoSQLStream.prototype.replace = function(index, value, divider) {
+
+	var self = this;
+	var rec = self.docsbuffer[index];
+
+	if (rec.doc === value)
+		return;
+
+	var was = true;
+	var sep = (divider || self.divider);
+
+	if (rec.doc.length === value.length) {
+		var b = Buffer.byteLength(value);
+		if (rec.length === b) {
+			self.write(value + sep, rec.position);
+			was = false;
+		}
+	}
+
+	if (was) {
+		var tmp = self.remchar + rec.doc.substring(1) + sep;
+		self.write(tmp, rec.position);
+		self.write2(value + sep);
+	}
+
+	return self;
+};
+
+NoSQLStream.prototype.write = function(doc, position) {
+	var self = this;
+	self.bufferstack.push({ position: position, data: doc });
+	!self.writing && self.$write();
+	return self;
+};
+
+NoSQLStream.prototype.write2 = function(doc) {
+	var self = this;
+	self.bufferstacknew.push(Buffer.from(doc));
+	!self.writing && self.$write();
+	return self;
+};
+
+NoSQLStream.prototype.$write = function() {
+	var self = this;
+	if (self.bufferstacknew.length && self.bufferstack.length) {
+		self.writing = true;
+		var buf = self.bufferstacknew.splice(0, 5);
+		buf = buf.length > 1 ? Buffer.concat(buf) : buf[0];
+		F.Fs.write(self.fd, buf, 0, buf.length, self.positionappend, self.cb_writeAddUpdAdd);
+	} else if (self.bufferstacknew.length) {
+		self.writing = true;
+		var buf = self.bufferstacknew.splice(0, 5);
+		buf = buf.length > 1 ? Buffer.concat(buf) : buf[0];
+		F.Fs.write(self.fd, buf, 0, buf.length, self.positionappend, self.cb_writeAdd);
+	} else if (self.bufferstack.length) {
+		self.writing = true;
+		var item = self.bufferstack.shift();
+		F.Fs.write(self.fd, item.data, item.position, 'utf8', self.cb_writeUpd);
+	}
+};
+
+NoSQLStream.prototype.flush = function() {
+	var self = this;
+	if (self.writing)
+		setTimeout(self.cb_flush, 100);
+	else
+		self.close();
+	return self;
+};
+
+NoSQLStream.prototype.read = function() {
+
+	var self = this;
+	var size = self.stats.size - self.position;
+
+
+	if (!self.fd || size <= 0 || self.canceled) {
+
+		if (!self.canceled && self.buffer && self.buffer.length) {
+			self.cb_readbuffer(null, 1, NEWLINEBUFFER);
+			self.buffer = Buffer.alloc(0);
+			return;
+		}
+
+		if (self.docscount) {
+			self.ondocuments();
+			self.docscount = 0;
+			self.docs = self.array ? [] : '';
+		}
+
+		self.close();
+
+	} else {
+		size = size < self.buffersize ? size : self.buffersize;
+		var buffer = Buffer.alloc(size);
+		F.Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readbuffer);
+	}
+};
+
+NoSQLStream.prototype.readreverse = function() {
+	var self = this;
+	self.position = self.stats.size;
+	self.readreverse2();
+	return self;
+};
+
+NoSQLStream.prototype.readreverse2 = function() {
+	var self = this;
+
+	if (!self.fd || self.position <= self.start || self.canceled) {
+
+		if (!self.canceled && self.buffer && self.buffer.length) {
+			self.cb_readreversebuffer(null, 1, NEWLINEBUFFER);
+			self.buffer = Buffer.alloc(0);
+			return;
+		}
+
+		if (self.docscount) {
+			self.ondocuments();
+			self.docs = self.array ? [] : '';
+			self.docscount = 0;
+		}
+
+		self.close();
+
+	} else {
+		var size = self.stats.size - self.bytesread;
+		size = size < self.buffersize ? size : self.buffersize;
+		self.position -= size;
+		var buffer = Buffer.alloc(size);
+		F.Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readreversebuffer);
+	}
+};
+
+NoSQLStream.prototype.readupdate = function() {
+
+	var self = this;
+	var size = self.stats.size - self.position;
+
+	if (!self.fd || size <= 0 || self.canceled) {
+
+		if (!self.canceled && self.buffer && self.buffer.length) {
+			self.positionappend++;
+			self.cb_readwritebuffer(null, 1, NEWLINEBUFFER);
+			self.buffer = Buffer.alloc(0);
+			return;
+		}
+
+		if (self.docsbuffer.length) {
+			self.ondocuments();
+			self.docsbuffer = [];
+			self.docs = self.array ? [] : '';
+		}
+
+		self.flush();
+	} else {
+		size = size < self.buffersize ? size : self.buffersize;
+		var buffer = Buffer.alloc(size);
+		F.Fs.read(self.fd, buffer, 0, size, self.position, self.cb_readwritebuffer);
+	}
+};
+
+module.exports = NoSQLStream;