ueberdb2 5.0.38 → 5.0.40

package/dist/databases/elasticsearch_db.js

@@ -0,0 +1,269 @@
+const require_runtime = require("../_virtual/_rolldown/runtime.js");
+const require_AbstractDatabase = require("../lib/AbstractDatabase.js");
+let assert = require("assert");
+assert = require_runtime.__toESM(assert);
+let buffer = require("buffer");
+let crypto = require("crypto");
+let _elastic_elasticsearch = require("@elastic/elasticsearch");
+//#region databases/elasticsearch_db.ts
+/**
+* 2015 Visionist, Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS-IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+const schema = "2";
+const keyToId = (key) => {
+	const keyBuf = buffer.Buffer.from(key);
+	return keyBuf.length > 512 ? (0, crypto.createHash)("sha512").update(keyBuf).digest("hex") : key;
+};
+const mappings = { properties: {
+	key: { type: "wildcard" },
+	value: {
+		type: "object",
+		enabled: false
+	}
+} };
+const legacyDocToSchema2Key = (index, id, type, v1BaseIndex) => {
+	const legacyType = typeof type === "string" && type !== "" && type !== "_doc" ? type : null;
+	if (v1BaseIndex && index !== v1BaseIndex) {
+		const parts = index.slice(v1BaseIndex.length + 1).split("-");
+		if (parts.length !== 2) throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
+		if (legacyType != null) return `${parts[0]}:${decodeURIComponent(legacyType)}:${parts[1]}:${id}`;
+		const idParts = id.split(":");
+		if (idParts.length !== 2) throw new Error(`unable to migrate records from index ${index} due to data ambiguity`);
+		return `${parts[0]}:${idParts[0]}:${parts[1]}:${idParts[1]}`;
+	}
+	if (legacyType != null) return `${legacyType}:${id}`;
+	const idParts = id.split(":");
+	if (idParts.length !== 2) throw new Error(`unable to migrate records from index ${index} due to missing legacy type metadata`);
+	return `${idParts[0]}:${idParts[1]}`;
+};
+const migrateToSchema2 = async (client, v1BaseIndex, v2Index, logger) => {
+	let recordsMigratedLastLogged = 0;
+	let recordsMigrated = 0;
+	const totals = /* @__PURE__ */ new Map();
+	logger.info(`Attempting elasticsearch record migration from schema v1 at base index ${v1BaseIndex} to schema v2 at index ${v2Index}...`);
+	const indices = await client.indices.get({ index: [v1BaseIndex, `${v1BaseIndex}-*-*`] });
+	const scrollIds = /* @__PURE__ */ new Map();
+	const q = [];
+	try {
+		for (const index of Object.keys(indices)) {
+			const res = await client.search({
+				index,
+				scroll: "10m"
+			});
+			scrollIds.set(index, res._scroll_id);
+			q.push({
+				index,
+				res
+			});
+		}
+		while (q.length) {
+			const { index, res: { hits: { hits, total: { value: total } } } } = q.shift();
+			if (hits.length === 0) continue;
+			totals.set(index, total);
+			const body = [];
+			for (const { _id, _type, _source: { val } } of hits) {
+				const key = legacyDocToSchema2Key(index, _id, _type, v1BaseIndex);
+				body.push({ index: { _id: keyToId(key) } }, {
+					key,
+					value: JSON.parse(val)
+				});
+			}
+			await client.bulk({
+				index: v2Index,
+				body
+			});
+			recordsMigrated += hits.length;
+			if (Math.floor(recordsMigrated / 100) > Math.floor(recordsMigratedLastLogged / 100)) {
+				const total = [...totals.values()].reduce((a, b) => a + b, 0);
+				logger.info(`Migrated ${recordsMigrated} records out of ${total}`);
+				recordsMigratedLastLogged = recordsMigrated;
+			}
+			q.push({
+				index,
+				res: await client.scroll({
+					scroll: "5m",
+					scroll_id: scrollIds.get(index)
+				})
+			});
+		}
+		logger.info(`Finished migrating ${recordsMigrated} records`);
+	} finally {
+		await Promise.all([...scrollIds.values()].map((scrollId) => client.clearScroll({ scroll_id: scrollId })));
+	}
+};
+var elasticsearch_db_default = class extends require_AbstractDatabase.default {
+	_client;
+	_index;
+	_indexClean;
+	_q;
+	constructor(settings) {
+		super(settings);
+		this._client = null;
+		this.settings = {
+			host: "127.0.0.1",
+			port: "9200",
+			base_index: "ueberes",
+			migrate_to_newer_schema: false,
+			api: "7.6",
+			...settings || {},
+			json: false
+		};
+		this._index = `${this.settings.base_index}_s${schema}`;
+		this._q = { index: this._index };
+		this._indexClean = true;
+	}
+	get isAsync() {
+		return true;
+	}
+	async _refreshIndex() {
+		if (this._indexClean) return;
+		this._indexClean = true;
+		await this._client.indices.refresh(this._q);
+	}
+	/**
+	* Initialize the elasticsearch client, then ping the server to ensure that a
+	* connection was made.
+	*/
+	async init() {
+		const client = new _elastic_elasticsearch.Client({ node: `http://${this.settings.host}:${this.settings.port}` });
+		await client.ping();
+		if (!await client.indices.exists({ index: this._index })) {
+			let tmpIndex;
+			const exists = await client.indices.exists({ index: this.settings.base_index });
+			if (exists && !this.settings.migrate_to_newer_schema) throw new Error(`Data exists under the legacy index (schema) named ${this.settings.base_index}. Set migrate_to_newer_schema to true to copy the existing data to a new index named ${this._index}.`);
+			let attempt = 0;
+			while (true) {
+				tmpIndex = `${this._index}_${exists ? "migrate_attempt_" : "i"}${attempt++}`;
+				if (!await client.indices.exists({ index: tmpIndex })) break;
+			}
+			await client.indices.create({
+				index: tmpIndex,
+				mappings
+			});
+			if (exists) await migrateToSchema2(client, this.settings.base_index, tmpIndex, this.logger);
+			await client.indices.putAlias({
+				index: tmpIndex,
+				name: this._index
+			});
+		}
+		const indices = Object.values(await client.indices.get({ index: this._index }));
+		(0, assert.equal)(indices.length, 1);
+		try {
+			assert.default.deepEqual(indices[0].mappings, mappings);
+		} catch (err) {
+			this.logger.warn(`Index ${this._index} mappings does not match expected; attempting to use index anyway. Details: ${err}`);
+		}
+		this._client = client;
+	}
+	/**
+	*  This function provides read functionality to the database.
+	*
+	*  @param {String} key Key
+	*/
+	async get(key) {
+		const res = await this._client.get({
+			...this._q,
+			id: keyToId(key)
+		}, { ignore: [404] });
+		if (!res.found) return null;
+		return res._source.value;
+	}
+	/**
+	*  @param key Search key, which uses an asterisk (*) as the wild card.
+	*  @param notKey Used to filter the result set
+	*/
+	async findKeys(key, notKey) {
+		await this._refreshIndex();
+		const q = {
+			...this._q,
+			body: { query: { bool: {
+				filter: { wildcard: { key: { value: key } } },
+				...notKey == null ? {} : { must_not: { wildcard: { key: { value: notKey } } } }
+			} } }
+		};
+		const { hits } = await this._client.search(q);
+		return hits.hits.map((h) => h._source.key);
+	}
+	/**
+	*  This function provides write functionality to the database.
+	*
+	*  @param {String} key Record identifier.
+	*  @param {JSON|String} value The value to store in the database.
+	*/
+	async set(key, value) {
+		this._indexClean = false;
+		await this._client.index({
+			...this._q,
+			id: keyToId(key),
+			body: {
+				key,
+				value
+			}
+		});
+	}
+	/**
+	*  This function provides delete functionality to the database.
+	*
+	*  The index, type, and ID will be parsed from the key, and this document will
+	*  be deleted from the database.
+	*
+	*  @param {String} key Record identifier.
+	*/
+	async remove(key) {
+		this._indexClean = false;
+		await this._client.delete({
+			...this._q,
+			id: keyToId(key)
+		}, { ignore: [404] });
+	}
+	/**
+	*  This uses the bulk upload functionality of elasticsearch (url:port/_bulk).
+	*
+	*  The CacheAndBufferLayer will periodically (every this.settings.writeInterval)
+	*  flush writes that have already been done in the local cache out to the database.
+	*
+	*  @param {Array} bulk An array of JSON data in the format:
+	*      {"type":type, "key":key, "value":value}
+	*/
+	async doBulk(bulk) {
+		const operations = [];
+		for (const { type, key, value } of bulk) {
+			this._indexClean = false;
+			switch (type) {
+				case "set":
+					operations.push({ index: { _id: keyToId(key) } });
+					operations.push({
+						key,
+						value
+					});
+					break;
+				case "remove":
+					operations.push({ delete: { _id: keyToId(key) } });
+					break;
+				default:
+			}
+		}
+		await this._client.bulk({
+			...this._q,
+			body: operations
+		});
+	}
+	async close() {
+		if (this._client != null) this._client.close();
+		this._client = null;
+	}
+};
+//#endregion
+exports.default = elasticsearch_db_default;

package/dist/databases/memory_db.js

@@ -0,0 +1,37 @@
+const require_AbstractDatabase = require("../lib/AbstractDatabase.js");
+//#region databases/memory_db.ts
+var MemoryDB = class extends require_AbstractDatabase.default {
+	_data;
+	constructor(settings) {
+		super(settings);
+		this.settings = settings;
+		settings.json = false;
+		settings.cache = 0;
+		settings.writeInterval = 0;
+		this._data = null;
+	}
+	get isAsync() {
+		return true;
+	}
+	close() {
+		this._data = null;
+	}
+	findKeys(key, notKey) {
+		const regex = this.createFindRegex(key, notKey);
+		return [...this._data.keys()].filter((k) => regex.test(k));
+	}
+	get(key) {
+		return this._data.get(key);
+	}
+	init() {
+		this._data = this.settings.data || /* @__PURE__ */ new Map();
+	}
+	remove(key) {
+		this._data.delete(key);
+	}
+	set(key, value) {
+		this._data.set(key, value);
+	}
+};
+//#endregion
+exports.default = MemoryDB;

package/dist/databases/mock_db.js

@@ -0,0 +1,40 @@
+const require_runtime = require("../_virtual/_rolldown/runtime.js");
+let events = require("events");
+events = require_runtime.__toESM(events);
+//#region databases/mock_db.ts
+var mock_db_default = class extends events.default.EventEmitter {
+	settings;
+	mock;
+	constructor(settings) {
+		super();
+		this.settings = {
+			writeInterval: 1,
+			...settings
+		};
+		settings.mock = this;
+		this.settings = settings;
+	}
+	close(cb) {
+		this.emit("close", cb);
+	}
+	doBulk(ops, cb) {
+		this.emit("doBulk", ops, cb);
+	}
+	findKeys(key, notKey, cb) {
+		this.emit("findKeys", key, notKey, cb);
+	}
+	get(key, cb) {
+		this.emit("get", key, cb);
+	}
+	async init(cb) {
+		this.emit("init", cb());
+	}
+	remove(key, cb) {
+		this.emit("remove", key, cb);
+	}
+	set(key, value, cb) {
+		this.emit("set", key, value, cb);
+	}
+};
+//#endregion
+exports.default = mock_db_default;

package/dist/databases/mongodb_db.js

@@ -0,0 +1,95 @@
+require("../_virtual/_rolldown/runtime.js");
+const require_AbstractDatabase = require("../lib/AbstractDatabase.js");
+let mongodb = require("mongodb");
+//#region databases/mongodb_db.ts
+/**
+* 2020 Sylchauf
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS-IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+var mongodb_db_default = class extends require_AbstractDatabase.default {
+	interval;
+	database;
+	client;
+	collection;
+	constructor(settings) {
+		super(settings);
+		this.settings = settings;
+		if (!this.settings.url) throw new Error("You must specify a mongodb url");
+		if (this.settings.database == null) this.settings.database = this.settings.dbName;
+		if (!this.settings.collection) this.settings.collection = "ueberdb";
+	}
+	clearPing() {
+		if (this.interval) clearInterval(this.interval[Symbol.toPrimitive]());
+	}
+	schedulePing() {
+		this.clearPing();
+		this.interval = setInterval(() => {
+			this.database.command({ ping: 1 });
+		}, 1e4);
+	}
+	init(callback) {
+		mongodb.MongoClient.connect(this.settings.url).then((v) => {
+			this.client = v;
+			this.database = v.db(this.settings.database);
+			this.schedulePing();
+			this.collection = this.database.collection(this.settings.collection);
+			callback(null);
+		}).catch((v) => {
+			callback(v);
+		});
+	}
+	get(key, callback) {
+		this.collection.findOne({ _id: key }).then((v) => {
+			callback(null, v && v.value);
+		}).catch((v) => {
+			console.log(v);
+			callback(v);
+		});
+		this.schedulePing();
+	}
+	findKeys(key, notKey, callback) {
+		const selector = { $and: [{ _id: { $regex: `${key.replace(/\*/g, "")}` } }] };
+		if (notKey) selector.$and.push({ _id: { $not: { $regex: `${notKey.replace(/\*/g, "")}` } } });
+		this.collection.find(selector).map((i) => i._id).toArray().then((r) => {
+			callback(null, r);
+		}).catch((v) => callback(v));
+		this.schedulePing();
+	}
+	set(key, value, callback) {
+		if (key.length > 100) callback("Your Key can only be 100 chars");
+		else this.collection.updateMany({ _id: key }, { $set: { value } }, { upsert: true }).then(() => callback(null)).catch((v) => callback(v));
+		this.schedulePing();
+	}
+	remove(key, callback) {
+		this.collection.deleteOne({ _id: key }).then((r) => callback(null, r)).catch((v) => callback(v));
+		this.schedulePing();
+	}
+	doBulk(bulk, callback) {
+		const bulkMongo = this.collection.initializeOrderedBulkOp();
+		for (const i in bulk) if (bulk[i].type === "set") bulkMongo.find({ _id: bulk[i].key }).upsert().updateOne({ $set: { value: bulk[i].value } });
+		else if (bulk[i].type === "remove") bulkMongo.find({ _id: bulk[i].key }).deleteOne();
+		bulkMongo.execute().then((res) => {
+			callback(null, res);
+		}).catch((error) => {
+			callback(error);
+		});
+		this.schedulePing();
+	}
+	close(callback) {
+		this.clearPing();
+		this.client.close().then((r) => callback(r));
+	}
+};
+//#endregion
+exports.default = mongodb_db_default;

package/dist/databases/mssql_db.js

@@ -0,0 +1,132 @@
+const require_runtime = require("../_virtual/_rolldown/runtime.js");
+const require_AbstractDatabase = require("../lib/AbstractDatabase.js");
+let async = require("async");
+async = require_runtime.__toESM(async);
+let mssql = require("mssql");
+mssql = require_runtime.__toESM(mssql);
+//#region databases/mssql_db.ts
+/**
+* 2019 - exspecto@gmail.com
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS-IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*
+* Note: This requires MS SQL Server >= 2008 due to the usage of the MERGE statement
+*
+*/
+var MSSQL = class extends require_AbstractDatabase.default {
+	db;
+	constructor(settings) {
+		super(settings);
+		settings = settings || {};
+		if (settings.json != null) settings.parseJSON = settings.json;
+		settings.requestTimeout = 3e5;
+		settings.server = settings.host;
+		this.settings = settings;
+		this.settings.cache = 0;
+		this.settings.writeInterval = 0;
+	}
+	init(callback) {
+		const sqlCreate = "IF OBJECT_ID(N'dbo.store', N'U') IS NULL  BEGIN    CREATE TABLE [store] (      [key] NVARCHAR(100) PRIMARY KEY,      [value] NTEXT NOT NULL    );  END";
+		new mssql.default.ConnectionPool(this.settings).connect().then((pool) => {
+			this.db = pool;
+			new mssql.default.Request(this.db).query(sqlCreate, (err) => {
+				callback(err);
+			});
+			this.db.on("error", (err) => {
+				console.log(err);
+			});
+		});
+	}
+	get(key, callback) {
+		const request = new mssql.default.Request(this.db);
+		request.input("key", mssql.default.NVarChar(100), key);
+		request.query("SELECT [value] FROM [store] WHERE [key] = @key", (err, results) => {
+			let value = null;
+			if (!err && results && results.rowsAffected[0] === 1) value = results.recordset[0].value;
+			callback(err, value);
+		});
+	}
+	findKeys(key, notKey, callback) {
+		const request = new mssql.default.Request(this.db);
+		let query = "SELECT [key] FROM [store] WHERE [key] LIKE @key";
+		key = key.replace(/\*/g, "%");
+		request.input("key", mssql.default.NVarChar(100), key);
+		if (notKey != null) {
+			notKey = notKey.replace(/\*/g, "%");
+			request.input("notkey", mssql.default.NVarChar(100), notKey);
+			query += " AND [key] NOT LIKE @notkey";
+		}
+		request.query(query, (err, results) => {
+			const value = [];
+			if (!err && results && results.rowsAffected[0] > 0) for (let i = 0; i < results.recordset.length; i++) value.push(results.recordset[i].key);
+			callback(err, value);
+		});
+	}
+	set(key, value, callback) {
+		const request = new mssql.default.Request(this.db);
+		if (key.length > 100) callback("Your Key can only be 100 chars");
+		else {
+			const query = "MERGE [store] t USING (SELECT @key [key], @value [value]) s ON t.[key] = s.[key] WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value] WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);";
+			request.input("key", mssql.default.NVarChar(100), key);
+			request.input("value", mssql.default.NText, value);
+			request.query(query, (err, info) => {
+				callback(err ? err.toString() : "");
+			});
+		}
+	}
+	remove(key, callback) {
+		const request = new mssql.default.Request(this.db);
+		request.input("key", mssql.default.NVarChar(100), key);
+		request.query("DELETE FROM [store] WHERE [key] = @key", callback);
+	}
+	doBulk(bulk, callback) {
+		const maxInserts = 100;
+		const request = new mssql.default.Request(this.db);
+		let firstReplace = true;
+		let firstRemove = true;
+		const replacements = [];
+		let removeSQL = "DELETE FROM [store] WHERE [key] IN (";
+		for (const i in bulk) if (bulk[i].type === "set") {
+			if (firstReplace) {
+				replacements.push("BEGIN TRANSACTION;");
+				firstReplace = false;
+			} else if (Number(i) % maxInserts === 0) replacements.push("\nCOMMIT TRANSACTION;\nBEGIN TRANSACTION;\n");
+			replacements.push(`MERGE [store] t USING (SELECT '${bulk[i].key}' [key], '${bulk[i].value}' [value]) s`, "ON t.[key] = s.[key]", "WHEN MATCHED AND s.[value] IS NOT NULL THEN UPDATE SET t.[value] = s.[value]", "WHEN NOT MATCHED THEN INSERT ([key], [value]) VALUES (s.[key], s.[value]);");
+		} else if (bulk[i].type === "remove") {
+			if (!firstRemove) removeSQL += ",";
+			firstRemove = false;
+			removeSQL += `'${bulk[i].key}'`;
+		}
+		removeSQL += ");";
+		replacements.push("COMMIT TRANSACTION;");
+		async.default.parallel([(callback) => {
+			if (!firstReplace) request.batch(replacements.join("\n"), (err, results) => {
+				if (err) callback(err);
+				callback(err, results);
+			});
+			else callback();
+		}, (callback) => {
+			if (!firstRemove) request.query(removeSQL, callback);
+			else callback();
+		}], (err, results) => {
+			if (err) callback(err);
+			callback(err, results);
+		});
+	}
+	close(callback) {
+		this.db && this.db.close(callback);
+	}
+};
+//#endregion
+exports.default = MSSQL;