utilitas 1995.2.6 → 1995.2.8

package/lib/hal.mjs

@@ -13,7 +13,10 @@ const li = (id, text, url) => `\n${id}. ` + (url ? link(text, url) : text);
 const cardReg = /^\[\d*\]:\ ([^\ ]*)\ "(.*)"$/ig;
 // https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
 const countTokens = txt => Math.ceil(txt.split(/[^a-z0-9]/i).length * 100 / 75);
-const MAX_CONTEXT_TOKENS = 8192;
+// Keep this for GPT4 {
+// const MAX_CONTEXT_TOKENS = 8192;
+// }
+const MAX_CONTEXT_TOKENS = 4096;
 const MAX_PROMPT_TOKENS = Math.floor(MAX_CONTEXT_TOKENS * 0.6);
 const MAX_RESPONSE_TOKENS = MAX_CONTEXT_TOKENS - MAX_PROMPT_TOKENS;
 
@@ -35,10 +38,15 @@ const init = async options => {
             engine = (await import('@waylaidwanderer/chatgpt-api')).ChatGPTClient;
             client = new engine(options?.clientOptions?.apiKey, {
                 keepNecessaryMessagesOnly: true,
-                maxContextTokens: MAX_CONTEXT_TOKENS,
+                // Keep this for GPT4 {
+                // maxContextTokens: MAX_CONTEXT_TOKENS,
+                // }
                 modelOptions: {
-                    model: options?.model || 'gpt-4',
-                    max_tokens: MAX_RESPONSE_TOKENS,
+                    model: options?.model || 'gpt-3.5-turbo',
+                    // Keep this for GPT4 {
+                    // model: options?.model || 'gpt-4',
+                    // max_tokens: MAX_RESPONSE_TOKENS,
+                    // }
                     ...options?.clientOptions?.modelOptions || {}
                 }, ...options?.clientOptions || {},
             }, options?.cacheOptions);

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.2.6",
+    "version": "1995.2.8",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -28,7 +28,7 @@ const manifest = {
         "@google-cloud/text-to-speech": "^5.0.1",
         "@google-cloud/vision": "^4.0.2",
         "@mozilla/readability": "^0.4.4",
-        "@sentry/node": "^7.80.1",
+        "@sentry/node": "^7.83.0",
         "@waylaidwanderer/chatgpt-api": "^1.37.3",
         "acme-client": "^5.0.0",
         "browserify-fs": "^1.0.0",
@@ -37,24 +37,25 @@ const manifest = {
         "fluent-ffmpeg": "^2.1.2",
         "form-data": "^4.0.0",
         "ioredis": "^5.3.2",
-        "jsdom": "^22.1.0",
+        "jsdom": "^23.0.0",
         "lorem-ipsum": "^2.0.8",
         "mailgun.js": "^9.3.0",
         "mime-types": "^2.1.35",
-        "mysql2": "^3.6.3",
-        "node-mailjet": "^6.0.4",
+        "mysql2": "^3.6.5",
+        "node-mailjet": "^6.0.5",
         "node-polyfill-webpack-plugin": "^2.0.1",
         "office-text-extractor": "^3.0.2",
-        "openai": "^4.19.0",
+        "openai": "^4.20.1",
+        "pdfjs-dist": "^4.0.269",
         "ping": "^0.4.4",
         "say": "^0.16.0",
-        "telegraf": "^4.15.0",
+        "telegraf": "^4.15.2",
         "telesignsdk": "^2.2.3",
         "tesseract.js": "^5.0.3",
         "twilio": "^4.19.0",
         "url": "github:Leask/node-url",
         "webpack-cli": "^5.1.4",
-        "whisper-node": "^0.2.12",
+        "whisper-node": "^1.1.1",
         "youtube-transcript": "^1.0.6"
     }
 };

package/lib/shot.mjs

@@ -122,14 +122,12 @@ const get = async (url, options) => {
             case _JSON:
                 content = parseJson(buf2utf(buffer), null);
                 break;
-            case 'TEXT':
-                content = buf2utf(buffer);
-                break;
             case _PARSED:
                 content = await distillHtml(buf2utf(buffer));
                 break;
             default:
                 assert(!options.encode, 'Invalid encoding.', 400);
+            case 'TEXT':
                 content = buf2utf(buffer);
         }
     }

package/lib/vision.mjs

@@ -1,6 +1,3 @@
-import { getApiKeyCredentials } from './encryption.mjs';
-import get from './shot.mjs';
-
 import {
     convert, deleteOnCloud, downloadFromCloud, getIdByGs, uploadToCloud,
 } from './storage.mjs';
@@ -9,7 +6,9 @@ import {
     ensureArray, ignoreErrFunc, log as _log, need, throwError, trim,
 } from './utilitas.mjs';
 
-const _NEED = ['@google-cloud/vision', 'tesseract.js'];
+import { getApiKeyCredentials } from './encryption.mjs';
+
+const _NEED = ['@google-cloud/vision', 'pdfjs-dist', 'tesseract.js'];
 const [BASE64, BUFFER, FILE, DEFAULT_LANG] = ['BASE64', 'BUFFER', 'FILE', 'eng'];
 const ceil = num => num.toFixed(4);
 const errorMessage = 'Invalid image data.';
@@ -128,6 +127,12 @@ const see = async (image, options) => {
 
 const read = async (image, options) => {
     assert(client, 'Vision API has not been initialized.', 500);
+    if (options?.allPages) {
+        assert(options?.input === FILE, 'Only file input is supported.', 400);
+        if ((await getPdfInfo(image)).numPages > pages.length) {
+            return await readAll(image, options);
+        }
+    }
     const content = await convert(image, {
         input: options?.input, expected: BASE64, errorMessage,
     });
@@ -163,9 +168,45 @@ const readAll = async (image, options) => {
     ).flat();
 };
 
+const getPdfPage = async (doc, pageNum) => {
+    const page = await doc.getPage(pageNum);
+    const viewport = page.getViewport({ scale: 1.0 });
+    const result = {
+        pageNum: pageNum,
+        width: viewport.width,
+        height: viewport.height,
+        content: (await page.getTextContent()).items.map(x => x.str).join(' '),
+    };
+    page.cleanup();
+    return result
+};
+
+const getPdfPages = async (doc) => {
+    const result = [];
+    for (let i = 1; i <= doc.numPages; i++) { result.push(getPdfPage(doc, i)); }
+    return await Promise.all(result);
+};
+
+// https://github.com/mozilla/pdf.js/blob/master/examples/node/getinfo.mjs
+const getPdfInfo = async (file, options) => {
+    const { getDocument } = await need('pdfjs-dist');
+    const doc = await getDocument(file).promise;
+    const data = await doc.getMetadata();
+    const result = {
+        numPages: doc.numPages,
+        info: data.info,
+        metadata: { ...data.metadata.getAll() },
+        pages: options?.withPages ? await getPdfPages(doc) : null,
+    };
+    return result;
+};
+
 export {
     _NEED,
     annotateImage,
+    getPdfInfo,
+    getPdfPage,
+    getPdfPages,
     init,
     ocrImage,
     ocrImageGoogle,

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.2.6",
+    "version": "1995.2.8",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -39,7 +39,7 @@
         "@google-cloud/text-to-speech": "^5.0.1",
         "@google-cloud/vision": "^4.0.2",
         "@mozilla/readability": "^0.4.4",
-        "@sentry/node": "^7.80.1",
+        "@sentry/node": "^7.83.0",
         "@waylaidwanderer/chatgpt-api": "^1.37.3",
         "acme-client": "^5.0.0",
         "browserify-fs": "^1.0.0",
@@ -48,24 +48,25 @@
         "fluent-ffmpeg": "^2.1.2",
         "form-data": "^4.0.0",
         "ioredis": "^5.3.2",
-        "jsdom": "^22.1.0",
+        "jsdom": "^23.0.0",
         "lorem-ipsum": "^2.0.8",
         "mailgun.js": "^9.3.0",
         "mime-types": "^2.1.35",
-        "mysql2": "^3.6.3",
-        "node-mailjet": "^6.0.4",
+        "mysql2": "^3.6.5",
+        "node-mailjet": "^6.0.5",
         "node-polyfill-webpack-plugin": "^2.0.1",
         "office-text-extractor": "^3.0.2",
-        "openai": "^4.19.0",
+        "openai": "^4.20.1",
+        "pdfjs-dist": "^4.0.269",
         "ping": "^0.4.4",
         "say": "^0.16.0",
-        "telegraf": "^4.15.0",
+        "telegraf": "^4.15.2",
         "telesignsdk": "^2.2.3",
         "tesseract.js": "^5.0.3",
         "twilio": "^4.19.0",
         "url": "github:Leask/node-url",
         "webpack-cli": "^5.1.4",
-        "whisper-node": "^0.2.12",
+        "whisper-node": "^1.1.1",
         "youtube-transcript": "^1.0.6"
     }
 }