utilitas 1989.8.52

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Leask Wong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,3 @@
+# utilitas
+
+Just another common utility for Node.js.

package/index.js

@@ -0,0 +1,40 @@
+'use strict';
+
+module.exports = {
+    // dependencies
+    base64url: require('base64url'),
+    colors: require('colors/safe'),
+    fetch: require('node-fetch').default,
+    fileType: require('file-type'),
+    geoIp: require('fast-geoip'),
+    jwt: require('jsonwebtoken'),
+    mailgun: require('mailgun-js'),
+    mailjet: require('node-mailjet'),
+    math: require('mathjs'),
+    mysql: require('mysql2/promise'),
+    ping: require('ping'),
+    publicIp: require('public-ip'),
+    qs: require('qs'),
+    redis: require('ioredis'),
+    sentry: require('@sentry/node'),
+    telesign: require('telesignsdk'),
+    twilio: require('twilio'),
+    uuid: require('uuid'),
+    winston: require('winston'),
+    winstonPapertrail: require('winston-papertrail-mproved'),
+    // features
+    cache: require('./lib/cache'),
+    dbio: require('./lib/dbio'),
+    email: require('./lib/email'),
+    encryption: require('./lib/encryption'),
+    event: require('./lib/event'),
+    network: require('./lib/network'),
+    sentinel: require('./lib/sentinel'),
+    shell: require('./lib/shell'),
+    shot: require('./lib/shot'),
+    sms: require('./lib/sms'),
+    storage: require('./lib/storage'),
+    tape: require('./lib/tape'),
+    uoid: require('./lib/uoid'),
+    utilitas: require('./lib/utilitas'),
+};

package/lib/cache.js

@@ -0,0 +1,281 @@
+'use strict';
+
+const defaultOptions = {
+    port: 6379, host: '127.0.0.1', family: 4, db: 0, connectTimeout: 1000 * 3,
+};
+
+let ioredis, redis;
+
+const log = (content) => { return utilitas.modLog(content, 'cache'); };
+
+const init = async (options) => {
+    if ((options = options === '@' ? defaultOptions : options)) {
+        ioredis = require('ioredis');
+        redis = new ioredis(options);
+        if (!options.silent) {
+            log(`Initialized: redis://${redis.options.host}`
+                + `:${redis.options.port}/${redis.options.db} .`);
+        }
+    }
+    utilitas.assert(redis, 'Redis has not been initialized.', 501);
+    return redis;
+};
+
+module.exports = {
+    init,
+    //     set,
+    //     get,
+    //     mGet,
+    //     lPush,
+    //     rPush,
+    //     lRange,
+    //     getList,
+    //     del,
+    //     lrem,
+    //     smartDel,
+    //     setWithExpired,
+    //     tryLock,
+    //     lock,
+    //     unLock,
+    //     init,
+    //     pTryLock,
+    //     pUnLock
+};
+
+const utilitas = require('./utilitas');
+
+// https://github.com/luin/ioredis
+
+// var getKey = function(type, id, callback) {
+//     if (!type || !id) {
+//         return callback ? callback('Error cache type or id.') : null;
+//     }
+//     var key = 'PRSCACHE_' + type.toUpperCase() + '_' + String(id);
+//     return callback ? callback(null, key) : key;
+// };
+
+// var getKeys = function(type, ids, callback) {
+//     if (!type || !ids || !ids.length) {
+//         return callback ? callback('Error cache type or ids.') : null;
+//     }
+//     var result = [];
+//     for (var i in ids) {
+//         var id = getKey(type, ids[i]);
+//         if (id) {
+//             result.push(id);
+//         }
+//     }
+//     return callback ? callback(null, result) : result;
+// };
+
+// var rawSet = function(type, id, data, options, callback) {
+//     options = options || {};
+//     getKey(type, id, function(err, key) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         var value = JSON.stringify(data);
+//         if (verbose && global.config.debug) {
+//             console.log('Set cache: ' + key + ' = ' + value);
+//         }
+//         var args = [key, value];
+//         for (var i in options) {
+//             if (['EX', 'PX'].indexOf(i) !== -1 && options[i]) {
+//                 args.push(i, options[i]);
+//             }
+//             options['NX'] && args.push('NX');
+//             options['XX'] && args.push('XX');
+//         }
+//         global.redis.set(args, callback || function() { });
+//     });
+// };
+
+// var set = function(type, id, data, callback) {
+//     rawSet(type, id, data, {}, callback);
+// };
+
+// var setWithExpired = function(type, id, data, ex, nx, callback) {
+//     rawSet(type, id, data, { EX: ex, NX: nx }, callback);
+// };
+
+// var del = function(type, id, callback) {
+//     getKey(type, id, function(err, key) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         if (verbose && global.config.debug) {
+//             console.log('Del cache: ' + key);
+//         }
+//         global.redis.del(key, callback || function() { });
+//     });
+// };
+
+// let lrem = function(type, id, data, callback) {
+//     getKey(type, id, function(err, key) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         if (global.config.debug) {
+//             console.log('Del cache by value: ' + data);
+//         }
+//         global.redis.lrem(key, 0, data, callback || function() { });
+//     });
+// };
+
+// var findKeys = function(type, preStr, callback) {
+//     getKey(type, preStr, function(err, key) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         redis.keys(key, callback);
+//     });
+// };
+
+// var findAndDel = function(type, preStr, callback) {
+//     findKeys(type, preStr, function(err, data) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         async.eachSeries(data, function(item, cbf) {
+//             // console.log('Clean ' + item);
+//             redis.del(item, function(err, result) {
+//                 if (err) {
+//                     console.log(err);
+//                 }
+//                 cbf();
+//             });
+//         }, function(result) {
+//             callback(null, data.length);
+//         });
+//     });
+// };
+
+// var smartDel = function(type, id, callback) {
+//     return /^.*\*$/.test(id)
+//         ? findAndDel(type, id, callback) : del(type, id, callback);
+// };
+
+// var get = function(type, id, callback) {
+//     getKey(type, id, function(err, key) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         if (global.argv.ignore_cache) {
+//             return callback(null, null);
+//         }
+//         global.redis.get(key, function(err, reply) {
+//             if (err) {
+//                 return callback(err);
+//             }
+//             var value = null;
+//             try {
+//                 value = JSON.parse(reply);
+//             } catch (err) {
+//                 console.log(err);
+//                 return callback(err);
+//             }
+//             if (verbose && global.config.debug) {
+//                 console.log('Get cache: ' + key + ' = ' + reply);
+//             }
+//             callback(null, value);
+//         });
+//     });
+// };
+
+// var mGet = function(type, ids, callback) {
+//     getKeys(type, ids, function(err, keys) {
+//         if (err) {
+//             return callback(err);
+//         }
+//         if (global.argv.ignore_cache) {
+//             return callback(null, null);
+//         }
+//         global.redis.mget(keys, function(err, reply) {
+//             if (err) {
+//                 return callback(err);
+//             }
+//             var values = [];
+//             for (var i in reply) {
+//                 var value = null;
+//                 try {
+//                     value = JSON.parse(reply[i]);
+//                 } catch (err) {
+//                     return callback(err);
+//                 }
+//                 if (value) {
+//                     values.push(value);
+//                 }
+//             }
+//             if (verbose && global.config.debug) {
+//                 console.log('Get cache: ' + keys + ' = ' + reply);
+//             }
+//             callback(null, values);
+//         });
+//     });
+// };
+
+// let rawPush = (type, id, left, values, callback) => {
+//     getKey(type, id, (err, key) => {
+//         if (err) {
+//             return callback(err);
+//         }
+//         if (verbose && global.config.debug) {
+//             console.log(`RPUSH ${value} to ${key}`);
+//         }
+//         callback = callback || function() { };
+//         left ? global.redis.lpush(key, values, callback)
+//             : global.redis.rpush(key, values, callback);
+//     });
+// };
+
+// let lPush = (type, id, values, callback) => {
+//     rawPush(type, id, true, values, callback);
+// };
+
+// let rPush = (type, id, values, callback) => {
+//     rawPush(type, id, false, values, callback);
+// };
+
+// let lRange = (type, id, start, stop, callback) => {
+//     getKey(type, id, (err, key) => {
+//         if (err) {
+//             return callback(err);
+//         }
+//         if (global.argv.ignore_cache) {
+//             return callback(null, null);
+//         }
+//         global.redis.lrange(key, start, stop, (err, reply) => {
+//             if (verbose && global.config.debug) {
+//                 console.log('Get list: ' + key + ' = ' + reply);
+//             }
+//             callback(err, reply);
+//         });
+//     });
+// };
+
+// let getList = (type, id, callback) => {
+//     lRange(type, id, 0, -1, callback);
+// };
+
+// let tryLock = (key, timeout, callback) => { // 有返回值表示已经被锁，需要等待。
+//     get('lock', key, (err, data) => {
+//         if (err || data) {
+//             return callback(err, true);
+//         }
+//         lock(key, timeout, (err, data) => {
+//             if (err || !data) {
+//                 callback(err, true);
+//             }
+//             callback(err, false);
+//         });
+//     });
+// };
+
+// let lock = (key, timeout, callback) => {
+//     timeout = ~~timeout;
+//     setWithExpired('lock', key, new Date(), timeout || 30, true, callback);
+// };
+
+// let unLock = (key, callback) => {
+//     del('lock', key, callback);
+// };

package/lib/dbio.js

@@ -0,0 +1,291 @@
+'use strict';
+
+const defaultPort = 3306;
+const orders = { '+': 'ASC', '-': 'DESC' };
+const fieldId = 'id';
+const fieldAny = '*';
+const fieldCount = `COUNT(${fieldAny})`;
+const comparison = ['=', '>', '<', '<>', '!='];
+const fieldNoQuote = [fieldAny, fieldCount];
+const log = (content) => { return utilitas.modLog(content, 'dbio'); };
+
+let mysql, pool;
+
+const init = async (options) => {
+    if (options) {
+        mysql = require('mysql2/promise');
+        pool = mysql.createPool(options);
+        if (!options.silent) {
+            log(`Initialized: mysql://${options.user}@${options.host}`
+                + `:${options.port || defaultPort}/${options.database} .`);
+        }
+    }
+    utilitas.assert(pool, 'Database has not been initialized.', 501);
+    return pool;
+};
+
+const end = async (options) => {
+    pool && pool.end();
+    log('Terminated.');
+};
+
+const logCommand = (args) => {
+    if (~~global.debug > 0 && args && args[0]) { log(`SQL: ${args[0]}`); }
+    if (~~global.debug > 1 && args && args[1]) { console.log(args[1]); };
+};
+
+const getComparison = (val) => {
+    return utilitas.isObject(val) && comparison.includes(Object.keys(val)[0])
+        ? Object.keys(val)[0] : null;
+};
+
+const getOrder = (ord) => {
+    utilitas.assert(orders[ord], `Invalid order expression: '${ord}'.`, 400);
+    return orders[ord];
+};
+
+const distillArguments = (args) => {
+    Array.isArray(args?.['1']) && args['1'].map((v, k) => {
+        const cps = getComparison(v); cps && (args['1'][k] = v[cps]);
+    });
+    return args;
+};
+
+const rawQuery = async function() {
+    const conn = await init();
+    logCommand(arguments);
+    return await conn.query.apply(conn, [...arguments]);
+};
+
+const rawExecute = async function() {
+    const conn = await init();
+    logCommand(arguments);
+    return await conn.execute.apply(conn, [...arguments]);
+};
+
+const query = async function() {
+    return (await rawQuery.apply(null, [...distillArguments(arguments)]))[0];
+};
+
+const execute = async function() {
+    return (await rawExecute.apply(null, [...distillArguments(arguments)]))[0];
+};
+
+const defaultKey = (options) => {
+    return options && options.key ? options.key : fieldId;
+};
+
+const assertTable = (table, message = 'Table is required.', status = 500) => {
+    utilitas.assert(table, message, status);
+};
+
+const assertKeyValue = (key, value) => {
+    utilitas.assert(key, 'Key is required.', 500);
+    utilitas.assertSet(value, 'Value is required.', 500);
+};
+
+// const assertTableKeyValue = (table, key, value) => {
+//     assertTable(table);
+//     assertKeyValue(key, value);
+// };
+
+const assembleQuery = (table, options) => {
+    options = options || {};
+    assertTable(table);
+    const fields = [];
+    utilitas.ensureArray(options.fields).map((field) => {
+        fields.push(fieldNoQuote.includes(field) || options.noQuote
+            ? field : `\`${field}\``);
+    });
+    if (!fields.length) { fields.push(fieldAny); }
+    return `SELECT ${fields.join(', ')} FROM \`${table}\``;
+};
+
+const rawAssembleKeyValue = (key, value, options) => {
+    options = options || {};
+    assertKeyValue(key, value);
+    let express = `${getComparison(value) || '='} ?`;
+    if (Array.isArray(value)) { express = 'IN (?)'; }
+    else if (value === null) { express = 'IS ?'; }
+    return `${options.prefix || ''}\`${key}\` ${express}`;
+};
+
+const assembleKeyValue = (key, value, options) => {
+    options = options || {};
+    options.prefix = ' WHERE ';
+    return rawAssembleKeyValue(key, value, options);
+};
+
+const assembleSet = (data, options) => {
+    options = options || {};
+    const [isArray, result] = [options.asArray || Array.isArray(data), []];
+    utilitas.ensureArray(data).map((item) => {
+        utilitas.assert(Object.keys(item).length, 'Fields are required.', 500);
+        let [sql, values, dupSql] = [[], [], []];
+        for (let k in item) { sql.push(`\`${k}\` = ?`); values.push(item[k]); }
+        if (options.upsert) {
+            for (let k in item) {
+                dupSql.push(`\`${k}\` = ?`); values.push(item[k]);
+            }
+            dupSql = ` ON DUPLICATE KEY UPDATE ${dupSql.join(', ')}`;
+        } else { dupSql = ''; }
+        result.push({
+            sql: `${options.prefix || ''}SET ${sql.join(', ')}${dupSql}`,
+            values, object: item,
+        });
+    });
+    return isArray ? result : result[0];
+};
+
+const assembleInsert = (table, data, options) => {
+    options = options || {};
+    assertTable(table);
+    options.prefix = `INSERT INTO \`${table}\` `;
+    return assembleSet(data, options);
+};
+
+const assembleUpdate = (table, data, options) => {
+    options = options || {};
+    assertTable(table);
+    options.prefix = `UPDATE \`${table}\` `;
+    return assembleSet(data, options);
+};
+
+const assembleDelete = (table) => {
+    assertTable(table);
+    return `DELETE FROM \`${table}\``;
+};
+
+const assembleTail = (options) => {
+    let sort = [];
+    utilitas.ensureArray(options?.order || []).map?.(x => {
+        const ord = utilitas.isObject(x) ? x : { [x]: '+' };
+        const key = Object.keys(ord)[0];
+        sort.push(`\`${key}\` ${getOrder(ord[key])}`);
+    });
+    return (sort.length ? ` ORDER BY ${sort.join(', ')}` : '')
+        + (~~options?.limit ? ` LIMIT ${~~options.limit}` : '');
+};
+
+const queryAll = async (table, opts) => {
+    return await query(`${assembleQuery(table, opts)}${assembleTail(opts)}`);
+};
+
+const queryByKeyValue = async (table, key, value, options) => {
+    options = options || {};
+    const sql = assembleQuery(table, options)
+        + assembleKeyValue(key, value)
+        + assembleTail(options);
+    const resp = await query(sql, [value]);
+    return options.unique ? (resp && resp.length ? resp[0] : null) : resp;
+};
+
+const queryById = async (table, id, options) => {
+    options = options || {};
+    options.unique = !Array.isArray(id);
+    return await queryByKeyValue(table, defaultKey(options), id, options);
+};
+
+const insert = async (table, fields, options) => {
+    options = options || {};
+    let [isArray, key, ids, error, result]
+        = [Array.isArray(fields), defaultKey(options), [], [], []];
+    for (let item of assembleInsert(
+        table, fields, { ...options, asArray: true }
+    )) {
+        try {
+            const resp = await execute(item.sql, item.values);
+            resp.key = key;
+            resp.insertId = !resp.insertId && item.object[key]
+                ? item.object[key] : resp.insertId;
+            result.push(resp);
+            ids.push(resp.insertId);
+        } catch (err) { error.push(err); }
+    }
+    if (!options.skipEcho && ids.length) {
+        result = await queryById(table, ids, options);
+    }
+    if (!isArray) {
+        if (error.length) { throw error[0]; }
+        return result.length ? result[0] : null;
+    }
+    return { error, result };
+};
+
+const upsert = async (table, fields, options) => {
+    return await insert(table, fields, { ...options || {}, upsert: true });
+};
+
+const countAll = async (table) => {
+    const sql = assembleQuery(table, { fields: fieldCount });
+    return (await query(sql))[0][fieldCount];
+};
+
+const countByKeyValue = async (table, key, value) => {
+    const sql = assembleQuery(table, { fields: fieldCount })
+        + assembleKeyValue(key, value);
+    return (await query(sql, [value]))[0][fieldCount];
+};
+
+const updateByKeyValue = async (table, key, value, fields, options) => {
+    options = options || {};
+    assertTable(table);
+    let { sql, values } = assembleUpdate(table, fields);
+    sql += assembleKeyValue(key, value) + assembleTail(options);
+    const resp = await query(sql, [...values, value]);
+    return options.skipEcho
+        ? resp : await queryByKeyValue(table, key, value, options);
+};
+
+const updateById = async (table, id, fields, options) => {
+    const resp = await updateByKeyValue(
+        table, defaultKey(options), id, fields, options
+    );
+    return Array.isArray(id) ? resp : (resp && resp.length ? resp[0] : null);
+};
+
+const deleteByKeyValue = async (table, key, value, options) => {
+    const sql = assembleDelete(table)
+        + assembleKeyValue(key, value)
+        + assembleTail(options);
+    return await query(sql, [value]);
+};
+
+const deleteById = async (table, id, options) => {
+    return await deleteByKeyValue(table, defaultKey(options), id);
+};
+
+const deleteAll = async (table, options) => {
+    utilitas.assert(options && options.force,
+        "Option 'force' is required.", 500);
+    return await execute(assembleDelete(table));
+};
+
+module.exports = {
+    assembleInsert,
+    assembleQuery,
+    assembleSet,
+    assembleTail,
+    assembleUpdate,
+    countAll,
+    countByKeyValue,
+    deleteAll,
+    deleteById,
+    deleteByKeyValue,
+    end,
+    execute,
+    init,
+    insert,
+    query,
+    queryAll,
+    queryById,
+    queryByKeyValue,
+    rawAssembleKeyValue,
+    rawExecute,
+    rawQuery,
+    updateById,
+    updateByKeyValue,
+    upsert,
+};
+
+const utilitas = require('./utilitas');