utilitas 1995.3.21 → 1995.3.23

package/lib/alan.mjs

@@ -53,7 +53,7 @@ const trimTailing = text => text.replace(/[\.\s]*$/, '');
 const newSessionId = () => createUoid({ type: sessionType });
 const renderText = (t, o) => _renderText(t, { extraCodeBlock: 0, ...o || {} });
 const log = (cnt, opt) => _log(cnt, import.meta.url, { time: 1, ...opt || {} });
-const buildGeminiParts = (text, atcmt) => [{ text }, ...atcmt ? [atcmt] : []];
+const buildGeminiParts = (text, atcmts) => [{ text }, ...atcmts || []];
 
 const [
     png, jpeg, mov, mpeg, mp4, mpg, avi, wmv, mpegps, flv, gif, webp, pdf, aac,
@@ -108,6 +108,7 @@ const MODELS = {
     },
     [GPT_4O]: {
         contextWindow: 128000,
+        imageCostTokens: 1105,
         maxOutputTokens: 4096,
         requestLimitsRPM: 5000,
         tokenLimitsTPD: 40000000,
@@ -127,7 +128,7 @@ const MODELS = {
         // https://ai.google.dev/gemini-api/docs/models/gemini
         // https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts?hl=en#gemini-send-multimodal-samples-pdf-nodejs
         contextWindow: 1048576,
-        imageUseTokens: 258,
+        imageCostTokens: 258,
         maxAudioLength: 60 * 60 * 9.5, // 9.5 hours
         maxAudioPerPrompt: 1,
         maxFileSize: 20 * 1024 * 1024, // 20 MB
@@ -201,6 +202,9 @@ for (const n in MODELS) {
     }
 }
 
+const MAX_INPUT_TOKENS = MODELS[GPT_35_TURBO].maxInputTokens;
+const ATTACHMENT_TOKEN_COST = MODELS[GPT_4O].imageCostTokens;
+
 const unifyType = (type, name) => {
     const TYPE = ensureString(type, { case: 'UP' });
     assert(TYPE, `${name} is required.`);
@@ -313,23 +317,20 @@ const buildGptMessage = (content, options) => {
 
 const buildVertexMessage = (text, options) => {
     assert(text, 'Text is required.');
-    // only 1 attachment is allowed while using inline_data:
-    const attachment = (options?.attachments || []).map(x => ({
+    const attachments = (options?.attachments || []).map(x => ({
         inline_data: { mime_type: x.mime_type, data: x.data }
-    }))?.[0];
+    }));
     return String.isString(text) ? {
-        role: options?.role || user, parts: buildGeminiParts(text, attachment),
+        role: options?.role || user, parts: buildGeminiParts(text, attachments),
     } : text;
 };
 
 const buildGeminiMessage = (text, options) => {
     assert(text, 'Text is required.');
-    // @todo: check this issue similar to Vertex AI:
-    // only 1 attachment is allowed while using inline_data?
-    const attachment = (options?.attachments || []).map(x => ({
+    const attachments = (options?.attachments || []).map(x => ({
         inlineData: { mimeType: x.mime_type, data: x.data }
     }))?.[0];
-    return String.isString(text) ? buildGeminiParts(text, attachment) : text;
+    return String.isString(text) ? buildGeminiParts(text, attachments) : text;
 };
 
 const [getOpenAIClient, getVertexClient, getGeminiClient, getOllamaClient]
@@ -947,12 +948,13 @@ const getMaxChatPromptLimit = (options) => {
         resp = resp ? Math.min(resp, maxInputTokens) : maxInputTokens;
     }
     assert(resp > 0, 'Chat engine has not been initialized.');
-    return resp;
+    return options?.raw ? resp : Math.min(resp, MAX_INPUT_TOKENS);
 };
 
 export default init;
 export {
     _NEED,
+    ATTACHMENT_TOKEN_COST,
     CODE_INTERPRETER,
     DEFAULT_MODELS,
     EMBEDDING_001,
@@ -963,6 +965,8 @@ export {
     GEMINI_15_FLASH_VERTEX,
     GEMINI_15_FLASH,
     GPT_35_TURBO,
+    GPT_4O,
+    MAX_INPUT_TOKENS,
     MISTRAL,
     MODELS,
     RETRIEVAL,

package/lib/bot.mjs

@@ -612,67 +612,59 @@ const subconscious = [{
     },
 }, {
     run: true, priority: -8860, name: 'vision', func: async (ctx, next) => {
-        let fileId, type, file_name, mime_type, ocrFunc, asPrompt = false;
-        if ('application/pdf' === ctx.m.document?.mime_type) {
-            ocrFunc = ctx._.vision?.read;
-            fileId = ctx.m.document.file_id;
-            file_name = ctx.m.document.file_name;
-            mime_type = ctx.m.document.mime_type;
-            type = 'DOCUMENT';
-        } else if (/^image\/.*$/ig.test(ctx.m.document?.mime_type)) {
-            asPrompt = bot._.supportedMimeTypes.has(ctx.m.document.mime_type);
-            ocrFunc = ctx._.vision?.see;
-            fileId = ctx.m.document.file_id;
-            file_name = ctx.m.document.file_name;
-            mime_type = ctx.m.document.mime_type;
-            type = 'IMAGE';
-        } else if (/^.*\.(docx|xlsx|pptx)$/.test(ctx.m.document?.file_name)) {
-            ocrFunc = officeParser;
-            fileId = ctx.m.document.file_id;
-            file_name = ctx.m.document.file_name;
-            mime_type = ctx.m.document.mime_type;
-            type = 'DOCUMENT';
-        } else if (ctx.m.document) {
-            ocrFunc = async file => (await isTextFile(file)) && file.toString();
-            fileId = ctx.m.document.file_id;
-            file_name = ctx.m.document.file_name;
-            mime_type = ctx.m.document.mime_type;
-            type = 'FILE';
-        } else if (ctx.m.photo) {
-            asPrompt = true;
-            ocrFunc = ctx._.vision?.see;
-            fileId = ctx.m.photo[ctx.m.photo.length - 1]?.file_id;
-            mime_type = 'image';
-            type = 'PHOTO';
+        const files = [];
+        if (ctx.m.document) {
+            let file = {
+                asPrompt: bot._.supportedMimeTypes.has(ctx.m.document.mime_type),
+                file_name: ctx.m.document.file_name, fileId: ctx.m.document.file_id,
+                mime_type: ctx.m.document.mime_type, type: 'FILE',
+                ocrFunc: async f => (await isTextFile(f)) && f.toString(),
+            };
+            if ('application/pdf' === ctx.m.document?.mime_type) {
+                file = { ...file, ocrFunc: ctx._.vision?.read, type: 'DOCUMENT' };
+            } else if (/^image\/.*$/ig.test(ctx.m.document?.mime_type)) {
+                file = { ...file, ocrFunc: ctx._.vision?.see, type: 'IMAGE' };
+            } else if (/^.*\.(docx|xlsx|pptx)$/.test(ctx.m.document?.file_name)) {
+                file = { ...file, ocrFunc: officeParser, type: 'DOCUMENT' };
+            }
+            files.push(file);
         }
-        if (fileId && (asPrompt || ocrFunc)) {
+        ctx.m.photo?.map(p => files.push({
+            asPrompt: true, file_name: `${p.file_id}.jpg`, fileId: p.file_id,
+            mime_type: 'image/jpeg', type: 'PHOTO', ocrFunc: ctx._.vision?.see,
+        }));
+        if (files.length) {
             await ctx.ok(EMOJI_LOOK);
-            try {
-                const image_url = await getFileUrl(fileId);
-                const file = (await get(image_url, BUFFER_ENCODE)).content;
-                if (asPrompt) {
-                    ctx.collect(ctx.m.caption || '');
-                    ctx.collect({
-                        mime_type: mime_type === 'image' ? 'image/jpeg' : mime_type,
-                        image_url, data: base64Encode(file, true),
-                    }, 'PROMPT');
-                }
-                if (ocrFunc) {
-                    const content = trim(ensureArray(
-                        await ignoreErrFunc(async () => await ocrFunc(
-                            file, BUFFER_ENCODE
-                        ), logOptions)
-                    ).filter(x => x).join('\n'));
-                    if (content) {
+            for (const f of files) {
+                if (!f.asPrompt && !f.ocrFunc) { continue; }
+                try {
+                    const image_url = await getFileUrl(f.fileId);
+                    const file = (await get(image_url, BUFFER_ENCODE)).content;
+                    if (f.asPrompt) {
                         ctx.collect(ctx.m.caption || '');
-                        ctx.collect(lines([
-                            '---', ...file_name ? [`file_name: ${file_name}`] : [],
-                            `mime_type: ${mime_type}`, `type: ${type}`, '---',
-                            content
-                        ]), 'VISION');
+                        ctx.collect({
+                            mime_type: f.mime_type, image_url,
+                            data: base64Encode(file, true),
+                        }, 'PROMPT');
                     }
-                }
-            } catch (err) { return await ctx.er(err); }
+                    if (f.ocrFunc) {
+                        const content = trim(ensureArray(
+                            await ignoreErrFunc(async () => await f.ocrFunc(
+                                file, BUFFER_ENCODE
+                            ), logOptions)
+                        ).filter(x => x).join('\n'));
+                        if (content) {
+                            ctx.collect(ctx.m.caption || '');
+                            ctx.collect(lines([
+                                '---', `file_name: ${f.file_name}`,
+                                `mime_type: ${f.mime_type}`, `type: ${f.type}`,
+                                '---',
+                                content
+                            ]), 'VISION');
+                        }
+                    }
+                } catch (err) { return await ctx.er(err); }
+            }
         }
         await next();
     },

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.3.21",
+    "version": "1995.3.23",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1995.3.21",
+    "version": "1995.3.23",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",