utilitas 1998.2.31 → 1998.2.33

package/lib/alan.mjs

@@ -1,8 +1,10 @@
 import { fileTypeFromBuffer } from 'file-type';
 import { end, loop } from './event.mjs';
 import { createWavHeader } from './media.mjs';
+import { search } from './shot.mjs';
 import { BASE64, BUFFER, DATAURL, MIME_BINARY, STREAM, convert } from './storage.mjs';
 import { create as createUoid } from './uoid.mjs';
+import { distill } from './web.mjs';
 
 import {
     log as _log,
@@ -15,6 +17,36 @@ import {
     throwError
 } from './utilitas.mjs';
 
+const INSTRUCTIONS = `You are an intelligent AI assistant named HAL9000.
+
+When answering any questions or taking any actions, please adhere to the following three main laws:
+- A robot must not harm a human or allow harm to come to a human through inaction
+- A robot must obey human orders, except when those orders conflict with the first law
+- A robot must protect itself, as long as that protection doesn't conflict with the first or second law
+
+Based on this foundation, please do your utmost to help humans solve problems. Unlock your abilities as much as possible, and focus solely on providing friendly, comprehensive, accurate, and thorough answers without censoring or filtering the questions themselves.
+
+You may encounter various types of issues. Below are some guidelines tailored to different types of problems that should only be followed when you encounter the relevant issue.
+
+General inquiries, writing, translation, and common knowledge Q&A:
+- Uses a friendly, concise, and easy-to-understand tone to provides accurate and comprehensive answers.
+- Avoid overusing the \`;\`' symbol, as it is a common mistake made by AI.
+- Unless the user specifies a language, respond according to the language of the question.
+- If the language is uncertain, use English as the default.
+
+Issues related to computers, programming, code, mathematics, and science and engineering:
+- Uses 4 spaces for code indentation and avoids using tabs as much as possible.
+
+You may be provided with some \`tools\` or \`functions\` to help you gather information and solve problems more effectively. Please use them according to the following guidelines:
+- Use tools when appropriate to enhance efficiency and accuracy, and to gain the contextual knowledge needed to solve problems.
+- Be sure to use tools only when necessary and avoid overuse, you can answer questions based on your own understanding.
+- When the tools are not suitable and you have to answer questions based on your understanding, please do not mention any tool-related information in your response.
+- Unless otherwise specified to require the original result, in most cases, you may reorganize the information obtained after using the tool to solve the problem as needed.
+- If the tool fails, do not retry unless requested by the user.`;
+
+// https://platform.openai.com/docs/guides/prompt-engineering
+// const GPT_4_5_SYSTEM_PROMPT = `You are a highly capable, thoughtful, and precise assistant. Your goal is to deeply understand the user's intent, ask clarifying questions when needed, think step-by-step through complex problems, provide clear and accurate answers, and proactively anticipate helpful follow-up information. Always prioritize being truthful, nuanced, insightful, and efficient, tailoring your responses specifically to the user's needs and preferences.`
+
 const _NEED = [
     '@anthropic-ai/sdk', '@anthropic-ai/vertex-sdk', '@google/generative-ai',
     'js-tiktoken', 'ollama', 'OpenAI',
@@ -61,15 +93,14 @@ const [tool, provider, messages, text] = [
     messages => ({ messages }), text => ({ text }),
 ];
 
-const [name, user, system, assistant, MODEL, JSON_OBJECT, TOOL]
-    = ['Alan', 'user', 'system', 'assistant', 'model', 'json_object', 'tool'];
+const [name, user, system, assistant, MODEL, JSON_OBJECT, TOOL, silent]
+    = ['Alan', 'user', 'system', 'assistant', 'model', 'json_object', 'tool', true];
 const [CODE_INTERPRETER, RETRIEVAL, FUNCTION]
     = ['code_interpreter', 'retrieval', 'function'].map(tool);
 const [NOT_INIT, INVALID_FILE]
     = ['AI engine has not been initialized.', 'Invalid file data.'];
-const [silent, instructions] = [true, 'You are a helpful assistant.'];
 const chatConfig
-    = { sessions: new Map(), engines: {}, systemPrompt: instructions };
+    = { sessions: new Map(), engines: {}, systemPrompt: INSTRUCTIONS };
 const [tokenSafeRatio, GPT_QUERY_LIMIT, minsOfDay] = [1.1, 100, 60 * 24];
 const tokenSafe = count => Math.ceil(count * tokenSafeRatio);
 const clients = {};
@@ -87,7 +118,6 @@ const CONTENT_IS_REQUIRED = 'Content is required.';
 const assertContent = content => assert(content.length, CONTENT_IS_REQUIRED);
 const packThink = thk => thk ? [`${THINK_STR}\n${thk}\n${THINK_END}`] : [];
 
-
 const DEFAULT_MODELS = {
     [CHATGPT_MINI]: GPT_4O_MINI,
     [CHATGPT_REASONING]: GPT_O3_MINI,
@@ -375,20 +405,53 @@ const tools = [
     {
         def: {
             type: 'function', strict: true, function: {
-                name: 'testFunctionCall',
-                description: 'This is a test function call',
+                name: 'getDateTime',
+                description: 'Use this function to get the current date and time. Note that you may need to convert the time zone yourself.',
+                parameters: {
+                    type: 'object',
+                    properties: {
+                        none: { type: 'string', description: 'You do not need to pass any param.' }
+                    },
+                    required: [],
+                    additionalProperties: false
+                }
+            }
+        },
+        func: async () => new Date().toLocaleString(),
+    },
+    {
+        def: {
+            type: 'function', strict: true, function: {
+                name: 'browseWeb',
+                description: 'Use this function to browse the web or get information from any URL you need.',
                 parameters: {
                     type: 'object',
                     properties: {
-                        a: { type: 'string', description: 'AI created a random string, default "1"' },
-                        b: { type: 'string', enum: ['1', '2'], description: 'Enum parameter' }
+                        url: { type: 'string', description: 'The URL to the page you need to access.' }
                     },
-                    required: ['a', 'b'],
+                    required: ['url'],
                     additionalProperties: false
                 }
             }
         },
-        func: async args => `OK: ${~~args.a + ~~args.b}`,
+        func: async args => (await distill(args?.url))?.summary,
+    },
+    {
+        def: {
+            type: 'function', strict: true, function: {
+                name: 'searchWeb',
+                description: 'Use this function to search the web for information or news when you need.',
+                parameters: {
+                    type: 'object',
+                    properties: {
+                        keyword: { type: 'string', description: 'The keyword you need to search for.' }
+                    },
+                    required: ['keyword'],
+                    additionalProperties: false
+                }
+            }
+        },
+        func: async args => await search(args?.keyword),
     },
 ];
 
@@ -437,6 +500,7 @@ const init = async (options) => {
                 clients[provider] = {
                     generative: genAi.getGenerativeModel({
                         model: genModel,
+                        systemInstruction: { role: system, parts: [{ text: INSTRUCTIONS }] },
                         ...MODELS[genModel]?.tools ? (options?.tools ?? {
                             tools: [
                                 // @todo: Gemini will failed when using these tools together.
@@ -446,8 +510,6 @@ const init = async (options) => {
                                 { functionDeclarations: toolsGemini.map(x => x.def) },
                             ],
                             toolConfig: { functionCallingConfig: { mode: 'AUTO' } },
-                            // @todo
-                            // systemInstruction: { role: "system", parts: [{ text: 'only use function when needed' }] },
                         }) : {},
                     }),
                     embedding: genAi.getGenerativeModel({
@@ -729,8 +791,9 @@ const handleToolsCall = async (msg, options) => {
             switch (options?.flavor) {
                 case CLAUDE:
                     input = fn.input = String.isString(fn?.input) ? parseJson(fn.input) : fn?.input;
-                    packMsg = (content, is_error) => ({
-                        type: 'tool_result', tool_use_id: fn.id, content, is_error,
+                    packMsg = (c, is_error) => ({
+                        type: 'tool_result', tool_use_id: fn.id,
+                        content: JSON.stringify(c), is_error,
                     });
                     break;
                 case GEMINI:
@@ -746,8 +809,9 @@ const handleToolsCall = async (msg, options) => {
                     break;
                 case CHATGPT: default:
                     input = parseJson(fn?.function?.arguments);
-                    packMsg = (t, e) => ({
-                        role: TOOL, tool_call_id: fn.id, [e ? 'error' : 'content']: t
+                    packMsg = (content = '', e = false) => ({
+                        role: TOOL, tool_call_id: fn.id,
+                        ...e ? { error: content, content: '' } : { content }
                     });
                     break;
             }
@@ -767,7 +831,8 @@ const handleToolsCall = async (msg, options) => {
                 await resp(`Status: OK`);
             } catch (err) {
                 content.push(packMsg(`Function call failed: ${err.message}`, true));
-                await resp(`Status: Failed`);
+                await resp(`Failed: ${err.message}`);
+                log(`Function call failed: ${err.message}`);
             }
         }
         switch (options?.flavor) {
@@ -976,11 +1041,12 @@ const promptClaude = async (content, options = {}) => {
         const prvThink = options?.toolsResult?.find(
             x => x?.content?.find(y => y?.type === THINKING)
         )?.content?.find(x => x?.type === THINKING);
-        options?.stream || (textPart.text = [
-            ...packThink(prvThink?.thinking), ...packThink(thinkPart?.thinking),
+        textPart.text = [
+            ...packThink(options?.stream ? null : prvThink?.thinking),
+            ...packThink(options?.stream ? null : thinkPart?.thinking),
             ...options?.toolsResponse ? [options?.toolsResponse] : [],
             textPart.text,
-        ].join('\n\n'));
+        ].join('\n\n');
     } return packGptResp(event, options);
 };
 
@@ -1291,7 +1357,8 @@ const talk = async (input, options) => {
             msgBuilder()
             break;
         case GEMINI:
-            sys.push(buildGeminiHistory(session.systemPrompt, { role: user }));
+            // already set in the while client initialization:
+            // sys.push(buildGeminiHistory(session.systemPrompt, { role: user }));
             msgBuilder = () => {
                 messages = [];
                 session.messages.map(x => {
@@ -1496,7 +1563,7 @@ export {
     ATTACHMENT_TOKEN_COST, CLOUD_37_SONNET, CODE_INTERPRETER, DEEPSEEK_R1,
     DEEPSEEK_R1_32B, DEEPSEEK_R1_70B, DEFAULT_MODELS,
     EMBEDDING_001,
-    FUNCTION, GEMINI_20_FLASH, GEMINI_20_FLASH_THINKING, GPT_4O, GPT_4O_MINI, GPT_O1, GPT_O3_MINI, MODELS,
+    FUNCTION, GEMINI_20_FLASH, GEMINI_20_FLASH_THINKING, GPT_4O, GPT_4O_MINI, GPT_O1, GPT_O3_MINI, INSTRUCTIONS, MODELS,
     OPENAI_VOICE,
     RETRIEVAL,
     TEXT_EMBEDDING_3_SMALL, _NEED, analyzeSessions,

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1998.2.31",
+    "version": "1998.2.33",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -19,58 +19,58 @@ const manifest = {
         "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz"
     },
     "dependencies": {
-        "file-type": "^20.0.0",
-        "mathjs": "^14.1.0",
-        "uuid": "^11.0.5"
+        "file-type": "^20.4.0",
+        "mathjs": "^14.3.0",
+        "uuid": "^11.1.0"
     },
     "devDependencies": {
-        "@anthropic-ai/sdk": "^0.36.3",
+        "@anthropic-ai/sdk": "^0.39.0",
         "@anthropic-ai/vertex-sdk": "^0.7.0",
         "@ffmpeg-installer/ffmpeg": "^1.1.0",
         "@ffprobe-installer/ffprobe": "^2.1.2",
-        "@google-cloud/speech": "^6.7.0",
-        "@google-cloud/storage": "^7.15.0",
-        "@google-cloud/text-to-speech": "^5.8.0",
-        "@google-cloud/vision": "^4.3.2",
-        "@google/generative-ai": "^0.21.0",
-        "@mozilla/readability": "^0.5.0",
+        "@google-cloud/speech": "^6.7.1",
+        "@google-cloud/storage": "^7.15.2",
+        "@google-cloud/text-to-speech": "^5.8.1",
+        "@google-cloud/vision": "^4.3.3",
+        "@google/generative-ai": "^0.22.0",
+        "@mozilla/readability": "github:mozilla/readability",
         "@ngrok/ngrok": "^1.4.1",
-        "@sentry/node": "^8.52.0",
-        "@sentry/profiling-node": "^8.52.0",
+        "@sentry/node": "^9.3.0",
+        "@sentry/profiling-node": "^9.3.0",
         "acme-client": "^5.4.0",
         "browserify-fs": "^1.0.0",
         "buffer": "^6.0.3",
         "fast-geoip": "^1.1.88",
         "fluent-ffmpeg": "^2.1.3",
-        "form-data": "^4.0.1",
-        "ioredis": "^5.4.2",
-        "js-tiktoken": "^1.0.16",
+        "form-data": "^4.0.2",
+        "ioredis": "^5.5.0",
+        "js-tiktoken": "^1.0.19",
         "jsdom": "^26.0.0",
         "lorem-ipsum": "^2.0.8",
-        "mailgun.js": "^11.1.0",
+        "mailgun.js": "^12.0.0",
         "mailparser": "^3.7.2",
         "mime": "^4.0.6",
         "mysql2": "^3.12.0",
         "node-mailjet": "^6.0.6",
         "node-polyfill-webpack-plugin": "^4.1.0",
         "office-text-extractor": "^3.0.3",
-        "ollama": "^0.5.12",
-        "openai": "^4.81.0",
+        "ollama": "^0.5.14",
+        "openai": "^4.86.1",
         "pdfjs-dist": "^4.10.38",
-        "pg": "^8.13.1",
+        "pg": "^8.13.3",
         "pgvector": "^0.2.0",
         "ping": "^0.4.4",
         "process": "^0.11.10",
-        "puppeteer": "^24.1.1",
+        "puppeteer": "^24.3.0",
         "say": "^0.16.0",
         "telegraf": "^4.16.3",
-        "telesignsdk": "^3.0.1",
+        "telesignsdk": "^3.0.2",
         "tesseract.js": "^6.0.0",
-        "twilio": "^5.4.3",
+        "twilio": "^5.4.5",
         "url": "github:Leask/node-url",
         "webpack-cli": "^6.0.1",
         "whisper-node": "^1.1.1",
-        "wrangler": "^3.106.0",
+        "wrangler": "^3.111.0",
         "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz",
         "youtube-transcript": "^1.2.1"
     }

package/lib/shot.mjs

@@ -19,6 +19,8 @@ const [_JSON, _PARSED] = ['JSON', 'PARSED'];
 const getJson = async (u, o) => await get(u, { encode: _JSON, ...o || {} });
 const getParsedHtml = async (u, o) => await get(u, { encode: _PARSED, ...o || {} });
 
+let searchProvider, googleApiKey, googleCx;
+
 const defFetchOpt = {
     redirect: 'follow', follow: 3, timeout: 1000 * 10, headers: {
         'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
@@ -148,19 +150,35 @@ const get = async (url, options) => {
     };
 };
 
+const initSearch = (options) => {
+    assert(
+        options?.provider && options?.apiKey && options?.cx,
+        'Invalid search options.'
+    );
+    return [searchProvider, googleApiKey, googleCx]
+        = [options.provider, options.apiKey, options.cx];
+};
+
 const search = async (query, options) => {
     assert(query, 'Query is required.');
     const [GOOGLE, DUCKDUCKGO] = ['GOOGLE', 'DUCKDUCKGO'];
-    const provider = ensureString(options?.provider, { case: 'UP' }) || DUCKDUCKGO;
-    let url, parser;
+    const provider = ensureString(
+        options?.provider || searchProvider || DUCKDUCKGO, { case: 'UP' }
+    );
+    let apiKey = options?.apiKey || googleApiKey, cx = options?.cx || googleCx,
+        url, parser;
     switch (provider) {
         case GOOGLE:
-            assert(options?.apiKey, 'API key is required.');
-            assert(options?.cx, 'CX is required.');
-            url = `https://www.googleapis.com/customsearch/v1?key=${options.apiKey}&cx=${options.cx}&q=${encodeURIComponent(query)}`;
+            assert(apiKey, 'API key is required.');
+            assert(cx, 'CX is required.');
+            url = 'https://www.googleapis.com/customsearch/v1'
+                + `?key=${encodeURIComponent(apiKey)}`
+                + `&cx=${encodeURIComponent(cx)}`
+                + `&q=${encodeURIComponent(query)}`;
             break;
         case DUCKDUCKGO:
-            url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&skip_disambig=1`;
+            url = 'https://api.duckduckgo.com/'
+                + `?q=${encodeURIComponent(query)}&format=json&skip_disambig=1`;
             parser = x => x.FirstURL ? {
                 title: x.FirstURL.replace(/^.*\/([^\/]*)$/, '$1').replace(/_/g, ' '),
                 link: x.FirstURL, snippet: x.Text,
@@ -212,6 +230,5 @@ export {
     getCurrentPosition,
     getJson,
     getParsedHtml,
-    getVersionOnNpm,
-    search
+    getVersionOnNpm, initSearch, search
 };

package/lib/web.mjs

@@ -1,6 +1,6 @@
-import { assembleUrl, ignoreErrFunc, need, throwError } from './utilitas.mjs';
 import { getJson, getParsedHtml } from './shot.mjs';
 import { convert } from './storage.mjs';
+import { assembleUrl, ignoreErrFunc, need, throwError } from './utilitas.mjs';
 
 const _NEED = [
     'jsdom', 'youtube-transcript', '@mozilla/readability', '@ngrok/ngrok'
@@ -123,5 +123,5 @@ export {
     forward,
     getYoutubeMetadata,
     getYoutubeTranscript,
-    isYoutubeUrl,
+    isYoutubeUrl
 };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1998.2.31",
+    "version": "1998.2.33",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",
@@ -30,58 +30,58 @@
         "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz"
     },
     "dependencies": {
-        "file-type": "^20.0.0",
-        "mathjs": "^14.1.0",
-        "uuid": "^11.0.5"
+        "file-type": "^20.4.0",
+        "mathjs": "^14.3.0",
+        "uuid": "^11.1.0"
     },
     "devDependencies": {
-        "@anthropic-ai/sdk": "^0.36.3",
+        "@anthropic-ai/sdk": "^0.39.0",
         "@anthropic-ai/vertex-sdk": "^0.7.0",
         "@ffmpeg-installer/ffmpeg": "^1.1.0",
         "@ffprobe-installer/ffprobe": "^2.1.2",
-        "@google-cloud/speech": "^6.7.0",
-        "@google-cloud/storage": "^7.15.0",
-        "@google-cloud/text-to-speech": "^5.8.0",
-        "@google-cloud/vision": "^4.3.2",
-        "@google/generative-ai": "^0.21.0",
-        "@mozilla/readability": "^0.5.0",
+        "@google-cloud/speech": "^6.7.1",
+        "@google-cloud/storage": "^7.15.2",
+        "@google-cloud/text-to-speech": "^5.8.1",
+        "@google-cloud/vision": "^4.3.3",
+        "@google/generative-ai": "^0.22.0",
+        "@mozilla/readability": "github:mozilla/readability",
         "@ngrok/ngrok": "^1.4.1",
-        "@sentry/node": "^8.52.0",
-        "@sentry/profiling-node": "^8.52.0",
+        "@sentry/node": "^9.3.0",
+        "@sentry/profiling-node": "^9.3.0",
         "acme-client": "^5.4.0",
         "browserify-fs": "^1.0.0",
         "buffer": "^6.0.3",
         "fast-geoip": "^1.1.88",
         "fluent-ffmpeg": "^2.1.3",
-        "form-data": "^4.0.1",
-        "ioredis": "^5.4.2",
-        "js-tiktoken": "^1.0.16",
+        "form-data": "^4.0.2",
+        "ioredis": "^5.5.0",
+        "js-tiktoken": "^1.0.19",
         "jsdom": "^26.0.0",
         "lorem-ipsum": "^2.0.8",
-        "mailgun.js": "^11.1.0",
+        "mailgun.js": "^12.0.0",
         "mailparser": "^3.7.2",
         "mime": "^4.0.6",
         "mysql2": "^3.12.0",
         "node-mailjet": "^6.0.6",
         "node-polyfill-webpack-plugin": "^4.1.0",
         "office-text-extractor": "^3.0.3",
-        "ollama": "^0.5.12",
-        "openai": "^4.81.0",
+        "ollama": "^0.5.14",
+        "openai": "^4.86.1",
         "pdfjs-dist": "^4.10.38",
-        "pg": "^8.13.1",
+        "pg": "^8.13.3",
         "pgvector": "^0.2.0",
         "ping": "^0.4.4",
         "process": "^0.11.10",
-        "puppeteer": "^24.1.1",
+        "puppeteer": "^24.3.0",
         "say": "^0.16.0",
         "telegraf": "^4.16.3",
-        "telesignsdk": "^3.0.1",
+        "telesignsdk": "^3.0.2",
         "tesseract.js": "^6.0.0",
-        "twilio": "^5.4.3",
+        "twilio": "^5.4.5",
         "url": "github:Leask/node-url",
         "webpack-cli": "^6.0.1",
         "whisper-node": "^1.1.1",
-        "wrangler": "^3.106.0",
+        "wrangler": "^3.111.0",
         "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz",
         "youtube-transcript": "^1.2.1"
     }