utilitas 1995.2.7 → 1995.2.9

package/lib/dbio.mjs

@@ -1,8 +1,11 @@
-import { assertSet, ensureArray, log as _log, need } from './utilitas.mjs';
+import {
+    assertSet, ensureArray, ensureString, log as _log, need,
+} from './utilitas.mjs';
+
 import { isPrimary } from './callosum.mjs';
 
-const _NEED = ['mysql2'];
-const defaultPort = 3306;
+const _NEED = ['mysql2', 'pg'];
+const [mysqlDefaultPort, postgresqlDefaultPort] = [3306, 5432];
 const orders = { '+': 'ASC', '-': 'DESC' };
 const [fieldId, fieldAny] = ['id', '*'];
 const fieldCount = `COUNT(${fieldAny})`;
@@ -12,16 +15,47 @@ const log = (content) => _log(content, import.meta.url);
 const defaultKey = (options) => options && options.key ? options.key : fieldId;
 const queryOne = async (...args) => (await query(...args))[0];
 const assertForce = op => assert(op?.force, "Option 'force' is required.", 500);
+const [MYSQL, POSTGRESQL] = ['MYSQL', 'POSTGRESQL'];
+const quote = name => `${bracket}${name}${bracket}`;
+const join = elements => elements.join(', ');
 
-let pool;
+let [provider, pool, actExecute, bracket, actDuplicate] = [];
 
 const init = async (options) => {
     if (options) {
-        const mysql = await need('mysql2/promise');
-        pool = mysql.createPool(options);
+        const _provider = ensureString(options?.provider, { case: 'UP' });
+        let port;
+        switch (_provider) {
+            case MYSQL:
+            case 'MARIA':
+            case 'MARIADB':
+                const mysql = await need('mysql2/promise');
+                delete options.provider;
+                [provider, pool, port, actExecute, bracket, actDuplicate] = [
+                    MYSQL, mysql.createPool(options), mysqlDefaultPort,
+                    'execute', '`', 'ON DUPLICATE KEY',
+                ];
+                break;
+            case 'PG':
+            case 'POSTGRE':
+            case 'PSQL':
+            case POSTGRESQL:
+                // https://node-postgres.com/apis/pool
+                const { Pool } = await need('pg');
+                [provider, pool, port, actExecute, bracket, actDuplicate] = [
+                    POSTGRESQL, new Pool(options), postgresqlDefaultPort,
+                    'query', '', 'ON CONFLICT DO',
+                ];
+                break;
+            default:
+                assert(
+                    !options?.provider,
+                    `Invalid database provider: '${options.provider}'.`, 400
+                );
+        }
         isPrimary && !options.silent && log(
-            `Initialized: mysql://${options.user}@${options.host}`
-            + `:${options.port || defaultPort}/${options.database}`
+            `Initialized: ${_provider.toLowerCase()}://${options.user}@`
+            + `${options.host}:${options.port || port}/${options.database}`
         );
     }
     assert(pool, 'Database has not been initialized.', 501);
@@ -34,8 +68,9 @@ const end = async (options) => {
 };
 
 const logCommand = (args) => {
-    if (~~globalThis.debug > 0 && args && args[0]) { log(`SQL: ${args[0]}`); }
-    if (~~globalThis.debug > 1 && args && args[1]) { console.log(args[1]); };
+    const _args = ensureArray(args);
+    if (~~globalThis.debug > 0 && _args && _args[0]) { log(`SQL: ${_args[0]}`); }
+    if (~~globalThis.debug > 1 && _args && _args[1]) { console.log(_args[1]); };
 };
 
 const getComparison = (val) => Object.isObject(val)
@@ -61,16 +96,28 @@ const rawQuery = async (...args) => {
 
 const rawExecute = async (...args) => {
     const conn = await init();
+    console.log(args);
     logCommand(...args);
-    return await conn.execute(...args);
+    return await conn[actExecute](...args);
 };
 
-const query = async (...args) => {
-    return (await rawQuery(...distillArguments(...args)))[0];
+const handleResult = result => {
+    switch (provider) {
+        case MYSQL: return result[0];
+        case POSTGRESQL: return result.rows;
+    }
 };
 
+const query = async (...args) => handleResult(
+    await rawQuery(...distillArguments(...args))
+);
+
 const execute = async (...args) => {
-    return (await rawExecute(...distillArguments(...args)))[0];
+    const resp = await rawExecute(...distillArguments(...args));
+    switch (provider) {
+        case MYSQL: return resp[0];
+        case POSTGRESQL: return resp;
+    }
 };
 
 const assertTable = (table, message, status) =>
@@ -92,10 +139,10 @@ const assembleQuery = (table, options) => {
     const fields = [];
     ensureArray(options.fields).map((field) => {
         fields.push(fieldNoQuote.includes(field) || options.noQuote
-            ? field : `\`${field}\``);
+            ? field : quote(field));
     });
     if (!fields.length) { fields.push(fieldAny); }
-    return `SELECT ${fields.join(', ')} FROM \`${table}\``;
+    return `SELECT ${join(fields)} FROM ${quote(table)}`;
 };
 
 const rawAssembleKeyValue = (key, value, options) => {
@@ -106,7 +153,7 @@ const rawAssembleKeyValue = (key, value, options) => {
         assert(value.length, 'Invalid array value.', 500);
         express = 'IN (?)';
     } else if (value === null) { express = 'IS ?'; }
-    return `${options.prefix || ''}\`${key}\` ${express}`;
+    return `${options.prefix || ''}${quote(key)} ${express}`;
 };
 
 const assembleKeyValue = (key, value, options) => {
@@ -124,22 +171,26 @@ const assembleSet = (data, options) => {
     const [isArray, result] = [options.asArray || Array.isArray(data), []];
     ensureArray(data).map((item) => {
         assert(Object.keys(item).length, 'Fields are required.', 500);
-        let [sql, values, dupSql] = [[], [], []];
+        let [keys, vals, values, dupSql] = [[], [], [], []];
         for (let k in item) {
-            sql.push(`\`${k}\` = ?`);
+            keys.push(k);
+            switch (provider) {
+                case MYSQL: vals.push('?'); break;
+                case POSTGRESQL: vals.push(`$${vals.length + 1}`); break;
+            }
             pushValue(values, item[k]);
         }
         if (options.upsert) {
             for (let k in item) {
-                dupSql.push(`\`${k}\` = ?`);
+                dupSql.push(`${quote(k)} = ?`);
                 pushValue(values, item[k]);
             }
-            dupSql = ` ON DUPLICATE KEY UPDATE ${dupSql.join(', ')}`;
+            dupSql = ` ${actDuplicate} UPDATE ${join(dupSql)}`;
         } else { dupSql = ''; }
-        result.push({
-            sql: `${options.prefix || ''}SET ${sql.join(', ')}${dupSql}`,
-            values, object: item,
-        });
+        const sql = `${options.prefix || ''}`
+            + `(${join(keys.map(quote))}) VALUES (${join(vals)})${dupSql}`
+            + `${options.subfix || ''}`
+        result.push({ sql, values, object: item });
     });
     return isArray ? result : result[0];
 };
@@ -147,20 +198,20 @@ const assembleSet = (data, options) => {
 const assembleInsert = (table, data, options) => {
     options = options || {};
     assertTable(table);
-    options.prefix = `INSERT INTO \`${table}\` `;
+    options.prefix = `INSERT INTO ${quote(table)} `;
     return assembleSet(data, options);
 };
 
 const assembleUpdate = (table, data, options) => {
     options = options || {};
     assertTable(table);
-    options.prefix = `UPDATE \`${table}\` `;
+    options.prefix = `UPDATE ${quote(table)} `;
     return assembleSet(data, options);
 };
 
 const assembleDelete = (table) => {
     assertTable(table);
-    return `DELETE FROM \`${table}\``;
+    return `DELETE FROM ${quote(table)}`;
 };
 
 const assembleTail = (options) => {
@@ -168,15 +219,28 @@ const assembleTail = (options) => {
     ensureArray(options?.order || []).map?.(x => {
         const ord = Object.isObject(x) ? x : { [x]: '+' };
         const key = Object.keys(ord)[0];
-        sort.push(`\`${key}\` ${getOrder(ord[key])}`);
+        sort.push(`${quote(key)} ${getOrder(ord[key])}`);
     });
-    return (sort.length ? ` ORDER BY ${sort.join(', ')}` : '')
+    return (sort.length ? ` ORDER BY ${join(sort)}` : '')
         + (~~options?.limit ? ` LIMIT ${~~options.limit}` : '');
 };
 
 const tables = async (options) => {
-    const resp = await query(`SHOW TABLES`);
-    return options?.raw ? resp : resp.map(x => Object.values(x)[0]);
+    let sql = '';
+    switch (provider) {
+        case MYSQL:
+            sql = 'SHOW TABLES';
+            break;
+        case POSTGRESQL:
+            sql = "SELECT * FROM information_schema.tables WHERE table_schema = 'public'";
+    }
+    const resp = await query(sql);
+    return options?.raw ? resp : resp.map(x => {
+        switch (provider) {
+            case MYSQL: return Object.values(x)[0];
+            case POSTGRESQL: return x.table_name;
+        }
+    });
 };
 
 const desc = async (table, options) => {
@@ -198,7 +262,7 @@ const indexes = async (table, options) => {
 const drop = async (table, options) => {
     assertTable(table);
     assertForce(options);
-    return await query(`DROP TABLE IF EXISTS ??`, [table]);
+    return await query('DROP TABLE IF EXISTS ??', [table]);
 };
 
 const queryAll = (table, options) =>
@@ -223,20 +287,32 @@ const insert = async (table, fields, options) => {
     options = options || {};
     let [isArray, key, ids, error, result]
         = [Array.isArray(fields), defaultKey(options), [], [], []];
+    const subfix = provider === POSTGRESQL ? (options.skipEcho ? (
+        options.key ? ` RETURNING ${key}` : ''
+    ) : ` RETURNING *`) : '';
     for (let item of assembleInsert(
-        table, fields, { ...options, asArray: true }
-    )) {
+        table, fields, { ...options, asArray: true, subfix })
+    ) {
         try {
             const resp = await execute(item.sql, item.values);
+            if (provider === POSTGRESQL) {
+                resp.affectedRows = resp.rowCount;
+            }
             resp.key = key;
-            resp.insertId = !resp.insertId && item.object[key]
-                ? item.object[key] : resp.insertId;
+            !resp.insertId && item.object[key]
+                && (resp.insertId = item.object[key]);
             result.push(resp);
             ids.push(resp.insertId);
         } catch (err) { error.push(err); }
     }
     if (!options.skipEcho && ids.length) {
-        result = await queryById(table, ids, options);
+        switch (provider) {
+            case MYSQL:
+                result = await queryById(table, ids, options);
+                break;
+            case POSTGRESQL:
+                result = result.map(x => x.rows).flat();
+        }
     }
     if (!isArray) {
         if (error.length) { throw error[0]; }

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.2.7",
+    "version": "1995.2.9",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -28,7 +28,7 @@ const manifest = {
         "@google-cloud/text-to-speech": "^5.0.1",
         "@google-cloud/vision": "^4.0.2",
         "@mozilla/readability": "^0.4.4",
-        "@sentry/node": "^7.80.1",
+        "@sentry/node": "^7.83.0",
         "@waylaidwanderer/chatgpt-api": "^1.37.3",
         "acme-client": "^5.0.0",
         "browserify-fs": "^1.0.0",
@@ -37,24 +37,26 @@ const manifest = {
         "fluent-ffmpeg": "^2.1.2",
         "form-data": "^4.0.0",
         "ioredis": "^5.3.2",
-        "jsdom": "^22.1.0",
+        "jsdom": "^23.0.0",
         "lorem-ipsum": "^2.0.8",
         "mailgun.js": "^9.3.0",
         "mime-types": "^2.1.35",
-        "mysql2": "^3.6.3",
-        "node-mailjet": "^6.0.4",
+        "mysql2": "^3.6.5",
+        "node-mailjet": "^6.0.5",
         "node-polyfill-webpack-plugin": "^2.0.1",
         "office-text-extractor": "^3.0.2",
-        "openai": "^4.19.0",
+        "openai": "^4.20.1",
+        "pdfjs-dist": "^4.0.269",
+        "pg": "^8.11.3",
         "ping": "^0.4.4",
         "say": "^0.16.0",
-        "telegraf": "^4.15.0",
+        "telegraf": "^4.15.2",
         "telesignsdk": "^2.2.3",
         "tesseract.js": "^5.0.3",
         "twilio": "^4.19.0",
         "url": "github:Leask/node-url",
         "webpack-cli": "^5.1.4",
-        "whisper-node": "^0.2.12",
+        "whisper-node": "^1.1.1",
         "youtube-transcript": "^1.0.6"
     }
 };

package/lib/shot.mjs

@@ -122,14 +122,12 @@ const get = async (url, options) => {
             case _JSON:
                 content = parseJson(buf2utf(buffer), null);
                 break;
-            case 'TEXT':
-                content = buf2utf(buffer);
-                break;
             case _PARSED:
                 content = await distillHtml(buf2utf(buffer));
                 break;
             default:
                 assert(!options.encode, 'Invalid encoding.', 400);
+            case 'TEXT':
                 content = buf2utf(buffer);
         }
     }

package/lib/vision.mjs

@@ -1,6 +1,3 @@
-import { getApiKeyCredentials } from './encryption.mjs';
-import get from './shot.mjs';
-
 import {
     convert, deleteOnCloud, downloadFromCloud, getIdByGs, uploadToCloud,
 } from './storage.mjs';
@@ -9,7 +6,9 @@ import {
     ensureArray, ignoreErrFunc, log as _log, need, throwError, trim,
 } from './utilitas.mjs';
 
-const _NEED = ['@google-cloud/vision', 'tesseract.js'];
+import { getApiKeyCredentials } from './encryption.mjs';
+
+const _NEED = ['@google-cloud/vision', 'pdfjs-dist', 'tesseract.js'];
 const [BASE64, BUFFER, FILE, DEFAULT_LANG] = ['BASE64', 'BUFFER', 'FILE', 'eng'];
 const ceil = num => num.toFixed(4);
 const errorMessage = 'Invalid image data.';
@@ -128,6 +127,12 @@ const see = async (image, options) => {
 
 const read = async (image, options) => {
     assert(client, 'Vision API has not been initialized.', 500);
+    if (options?.allPages) {
+        assert(options?.input === FILE, 'Only file input is supported.', 400);
+        if ((await getPdfInfo(image)).numPages > pages.length) {
+            return await readAll(image, options);
+        }
+    }
     const content = await convert(image, {
         input: options?.input, expected: BASE64, errorMessage,
     });
@@ -163,9 +168,45 @@ const readAll = async (image, options) => {
     ).flat();
 };
 
+const getPdfPage = async (doc, pageNum) => {
+    const page = await doc.getPage(pageNum);
+    const viewport = page.getViewport({ scale: 1.0 });
+    const result = {
+        pageNum: pageNum,
+        width: viewport.width,
+        height: viewport.height,
+        content: (await page.getTextContent()).items.map(x => x.str).join(' '),
+    };
+    page.cleanup();
+    return result
+};
+
+const getPdfPages = async (doc) => {
+    const result = [];
+    for (let i = 1; i <= doc.numPages; i++) { result.push(getPdfPage(doc, i)); }
+    return await Promise.all(result);
+};
+
+// https://github.com/mozilla/pdf.js/blob/master/examples/node/getinfo.mjs
+const getPdfInfo = async (file, options) => {
+    const { getDocument } = await need('pdfjs-dist');
+    const doc = await getDocument(file).promise;
+    const data = await doc.getMetadata();
+    const result = {
+        numPages: doc.numPages,
+        info: data.info,
+        metadata: { ...data.metadata.getAll() },
+        pages: options?.withPages ? await getPdfPages(doc) : null,
+    };
+    return result;
+};
+
 export {
     _NEED,
     annotateImage,
+    getPdfInfo,
+    getPdfPage,
+    getPdfPages,
     init,
     ocrImage,
     ocrImageGoogle,

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.2.7",
+    "version": "1995.2.9",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -39,7 +39,7 @@
         "@google-cloud/text-to-speech": "^5.0.1",
         "@google-cloud/vision": "^4.0.2",
         "@mozilla/readability": "^0.4.4",
-        "@sentry/node": "^7.80.1",
+        "@sentry/node": "^7.83.0",
         "@waylaidwanderer/chatgpt-api": "^1.37.3",
         "acme-client": "^5.0.0",
         "browserify-fs": "^1.0.0",
@@ -48,24 +48,26 @@
         "fluent-ffmpeg": "^2.1.2",
         "form-data": "^4.0.0",
         "ioredis": "^5.3.2",
-        "jsdom": "^22.1.0",
+        "jsdom": "^23.0.0",
         "lorem-ipsum": "^2.0.8",
         "mailgun.js": "^9.3.0",
         "mime-types": "^2.1.35",
-        "mysql2": "^3.6.3",
-        "node-mailjet": "^6.0.4",
+        "mysql2": "^3.6.5",
+        "node-mailjet": "^6.0.5",
         "node-polyfill-webpack-plugin": "^2.0.1",
         "office-text-extractor": "^3.0.2",
-        "openai": "^4.19.0",
+        "openai": "^4.20.1",
+        "pdfjs-dist": "^4.0.269",
+        "pg": "^8.11.3",
         "ping": "^0.4.4",
         "say": "^0.16.0",
-        "telegraf": "^4.15.0",
+        "telegraf": "^4.15.2",
         "telesignsdk": "^2.2.3",
         "tesseract.js": "^5.0.3",
         "twilio": "^4.19.0",
         "url": "github:Leask/node-url",
         "webpack-cli": "^5.1.4",
-        "whisper-node": "^0.2.12",
+        "whisper-node": "^1.1.1",
         "youtube-transcript": "^1.0.6"
     }
 }