utilitas 2000.3.31 → 2000.3.33

package/lib/alan.mjs

@@ -1,20 +1,19 @@
 import { checkSearch, distill, search } from './web.mjs';
 import { create as createUoid } from './uoid.mjs';
-import { fileTypeFromBuffer } from 'file-type';
 import { packPcmToWav } from './media.mjs';
 import { v4 as uuidv4 } from 'uuid';
 
 import {
-    STREAM, FILE, BASE64, BUFFER, DATAURL, MIME_BINARY, MIME_TEXT, MIME_PNG,
-    MIME_JPEG, MIME_MOV, MIME_MPEG, MIME_MP4, MIME_MPG, MIME_AVI, MIME_WMV,
-    MIME_MPEGPS, MIME_FLV, MIME_GIF, MIME_WEBP, MIME_PDF, MIME_AAC, MIME_FLAC,
-    MIME_MP3, MIME_MPEGA, MIME_M4A, MIME_MPGA, MIME_OPUS, MIME_PCM, MIME_WAV,
-    MIME_WEBM, MIME_TGPP, MIME_PCM16, MIME_OGG, convert, formatDataURL,
-    getTempPath, decodeBase64DataURL,
+    STREAM, FILE, BASE64, BUFFER, DATAURL, MIME_TEXT, MIME_PNG, MIME_JPEG,
+    MIME_MOV, MIME_MPEG, MIME_MP4, MIME_MPG, MIME_AVI, MIME_WMV, MIME_MPEGPS,
+    MIME_FLV, MIME_GIF, MIME_WEBP, MIME_PDF, MIME_AAC, MIME_FLAC, MIME_MP3,
+    MIME_MPEGA, MIME_M4A, MIME_MPGA, MIME_OPUS, MIME_PCM, MIME_WAV, MIME_WEBM,
+    MIME_TGPP, MIME_PCM16, MIME_OGG, convert, getTempPath, decodeBase64DataURL,
+    getMime,
 } from './storage.mjs';
 
 import {
-    log as _log, renderText as _renderText, ensureArray, ensureString, extract,
+    log as _log, renderText as _renderText, ensureArray, ensureString,
     ignoreErrFunc, insensitiveCompare, isSet, need, parseJson, throwError,
     tryUntil, timeout, mergeAtoB,
 } from './utilitas.mjs';
@@ -225,8 +224,6 @@ let ATTACHMENT_TOKEN_COST = 0;
 for (const n in MODELS) {
     MODELS[n]['name'] = n;
     MODELS[n].supportedMimeTypes = MODELS[n].supportedMimeTypes || [];
-    MODELS[n].supportedDocTypes = MODELS[n].supportedDocTypes || [];
-    MODELS[n].supportedAudioTypes = MODELS[n].supportedAudioTypes || [];
     MODELS[n].maxOutputTokens = MODELS[n].maxOutputTokens
         || Math.ceil(MODELS[n].contextWindow * 0.4);
     MODELS[n].maxInputTokens = MODELS[n].maxInputTokens
@@ -258,9 +255,7 @@ for (const n in MODELS) {
 //         );
 //     }
 //     // combine supported types
-//     for (const key of [
-//         'supportedAudioTypes', 'supportedDocTypes', 'supportedMimeTypes',
-//     ]) {
+//     for (const key of ['supportedMimeTypes']) {
 //         MODELS[AUTO][key] = [...new Set(
 //             [...MODELS[AUTO][key] || [], ...MODELS[n][key] || []]
 //         )];
@@ -588,55 +583,51 @@ const isOpenrouter = (provider, model) => insensitiveCompare(
     provider, OPENROUTER
 ) && (model ? model?.source : true);
 
-const selectVisionModel = options => {
-    assert(
-        MODELS[options.model]?.vision,
-        `Vision modality is not supported by model: ${options.model}`
-    );
-    return String.isString(MODELS[options.model].vision)
-        ? MODELS[options.model].vision : null;
-};
-
-const selectAudioModel = options => {
-    assert(
-        MODELS[options.model]?.audio,
-        `Audio modality is not supported by model: ${options.model}`
-    );
-    return String.isString(MODELS[options.model]?.audio)
-        ? MODELS[options.model]?.audio : null;
-};
-
-const buildMessage = (content, options) => {
+const buildMessage = async (content, options) => {
     content = content || '';
-    let alterModel = options?.audioMode && selectAudioModel(options);
-    const attachments = (options?.attachments || []).map(x => {
-        assert(MODELS[options?.model], 'Model is required.');
-        if (MODELS[options.model]?.supportedMimeTypes?.includes?.(x.mime_type)) {
-            alterModel = selectVisionModel(options); // URL or Base64URL
+    const attachments = await Promise.all((options?.attachments || []).map(async x => {
+        assert(
+            MODELS[options?.model]?.supportedMimeTypes?.includes?.(x.mime_type),
+            `Unsupported mime type: '${x.mime_type}'.`
+        );
+        if ([
+            MIME_PNG, MIME_JPEG, MIME_GIF, MIME_WEBP
+        ].includes?.(x.mime_type)) {
             return {
                 type: 'image_url',
-                image_url: { url: x.url, detail: 'high' },
-            };
-        } else if (MODELS[options.model]?.supportedDocTypes?.includes?.(x.mime_type)) {
-            alterModel = selectVisionModel(options);
-            return {
-                type: 'file',
-                file: {
-                    file_data: formatDataURL(x.mime_type, x.data),
-                    filename: x.file_name
-                        || `${uuidv4()}.${x.mime_type.split('/')[1]}`,
+                image_url: {
+                    url: x.url || await convert(x.data, {
+                        input: BUFFER, expected: DATAURL,
+                    }), detail: 'high',
                 },
             };
-        } else if (MODELS[options.model]?.supportedAudioTypes?.includes?.(x.mime_type)) {
-            alterModel = selectAudioModel(options);
+        } else if ([
+            MIME_AAC, MIME_FLAC, MIME_MP3, MIME_MPEGA, MIME_M4A, MIME_MPGA,
+            MIME_OPUS, MIME_PCM, MIME_WAV, MIME_TGPP, MIME_PCM16, MIME_OGG,
+        ].includes?.(x.mime_type)) {
             return {
                 type: 'input_audio',
-                input_audio: { data: x.data, format: WAV },
+                input_audio: {
+                    data: x.url || await convert(x.data, {
+                        input: BUFFER, expected: BASE64,
+                    }), format: WAV,
+                },
             };
         }
-        throwError(`Unsupported mime type: '${x.mime_type}'.`);
-    });
-    alterModel && (options.model = alterModel);
+        [
+            MIME_TEXT, MIME_MOV, MIME_MPEG, MIME_WEBM, MIME_MP4, MIME_MPG,
+            MIME_AVI, MIME_WMV, MIME_MPEGPS, MIME_FLV, MIME_PDF,
+        ].includes?.(x.mime_type)
+            || log(`Unknown mime type: '${x.mime_type}', fallbacked.`);
+        return {
+            type: 'file', file: {
+                file_data: await convert(x.data, {
+                    input: BUFFER, expected: DATAURL,
+                }), filename: x.file_name
+                    || `${uuidv4()}.${x.mime_type.split('/')[1]}`,
+            },
+        };
+    }));
     const message = String.isString(content) ? {
         role: options?.role || user,
         content: content.length ? [{ type: TEXT, text: content }] : [],
@@ -786,10 +777,10 @@ const packModelLabel = (model_reference) => {
 const buildPrompts = async (model, input, options = {}) => {
     assert(!(
         options.jsonMode && !model?.json
-    ), `This model does not support JSON output: ${options.model}`);
+    ), `This model does not support JSON output: ${model.name}`);
     assert(!(
         options.reasoning && !model?.reasoning
-    ), `This model does not support reasoning: ${options.model}`);
+    ), `This model does not support reasoning: ${model.name}`);
     let [history, content, prompt, _model, _assistant, _history]
         = [null, input, null, { role: MODEL }, { role: assistant }, null];
     options.systemPrompt = options.systemPrompt || INSTRUCTIONS;
@@ -797,42 +788,31 @@ const buildPrompts = async (model, input, options = {}) => {
         options.attachments?.length ? options.attachments : []
     ).map(async x => {
         if (String.isString(x)) {
-            var convResp = await convert(x, { input: FILE, expected: DATAURL, meta: true });
-            return {
-                url: convResp.content,
-                mime_type: convResp.mime,
-            }
+            var convResp = await convert(x, { input: FILE, expected: BUFFER, meta: true });
+            return { data: convResp.content, mime_type: convResp.mime };
         } else if (Buffer.isBuffer(x)) {
-            var convResp = await convert(x, { input: BUFFER, expected: DATAURL, meta: true });
-            return {
-                url: convResp.content,
-                mime_type: convResp.mime,
-            }
+            return { data: x, mime_type: (await getMime(x))?.mime }
         } else if (Object.isObject(x)) { return x; } else { return null; }
-    }))).filter(x => x && [
-        ...model?.supportedMimeTypes,
-        ...model?.supportedDocTypes,
-        ...model?.supportedAudioTypes,
-    ].includes(x.mime_type));
-    const systemPrompt = buildMessage(options.systemPrompt, system);
-    const msgBuilder = () => {
+    }))).filter(x => (model?.supportedMimeTypes || []).includes(x.mime_type));
+    const systemPrompt = await buildMessage(options.systemPrompt, system);
+    const msgBuilder = async () => {
         [history, _history] = [[], []];
-        (options.messages?.length ? options.messages : []).map((x, i) => {
-            _history.push(buildMessage(x.request));
-            _history.push(buildMessage(x.response, _assistant));
-        });
+        await Promise.all((options.messages?.length ? options.messages : []).map(async (x, i) => {
+            _history.push(await buildMessage(x.request));
+            _history.push(await buildMessage(x.response, _assistant));
+        }));
         history = messages([
-            systemPrompt, ..._history, buildMessage(content, options),
+            systemPrompt, ..._history, await buildMessage(content, options),
             ...options.toolsResult?.length ? options.toolsResult : []
         ]);
     };
-    msgBuilder();
+    await msgBuilder();
     await trimPrompt(() => [
         systemPrompt, _history, content, options.toolsResult
-    ], () => {
+    ], async () => {
         if (options.messages?.length) {
             options.messages?.shift();
-            msgBuilder();
+            await msgBuilder();
         } else if (options.trimBeginning) {
             content = '...' + trimBeginning(trimBeginning(content).slice(1));
         } else {
@@ -1251,7 +1231,7 @@ const initChat = async (options = {}) => {
     }
     options.instructions && (chatConfig.systemPrompt = options.instructions);
     // Use Gemini instead of ChatGPT because of the longer package.
-    const [spTokens, ais] = await Promise.all([countTokens([buildMessage(
+    const [spTokens, ais] = await Promise.all([countTokens([await buildMessage(
         chatConfig.systemPrompt, system
     )]), getAi(null, { all: true })]);
     for (const ai of ais.filter(x => ![
@@ -1348,15 +1328,9 @@ const distillFile = async (attachments, o) => {
         o?.keepPaging ? '' : '- If the document has multiple pages, merge them into one page. Please do not return any paging information.',
         o?.keepDecoration ? '' : '- If the document has side notes, headers, footers, or watermarks, please ignore them.',
     ].filter(x => x).join('\n');
-    attachments = await Promise.all(ensureArray(attachments).map(async x => {
-        const convResp = await convert(
-            x, { expected: DATAURL, ...o || {}, meta: true }
-        );
-        return { url: convResp.content, mime_type: convResp.mime };
-    }));
     return await prompt(strPmt, {
         select: { vision: true, hearing: true, fast: true },
-        simple: true, ...o, attachments,
+        simple: true, ...o, attachments: ensureArray(attachments),
     });
 };
 

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.31",
+    "version": "2000.3.33",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/lib/storage.mjs

@@ -538,6 +538,7 @@ export {
     getConfigFilename,
     getGcUrlByBucket,
     getIdByGs,
+    getMime,
     getTempPath,
     handleError,
     init,

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "2000.3.31",
+    "version": "2000.3.33",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",