wechaty-web-panel 1.6.40 → 1.6.42

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## 更新日志
 
+### V1.6.42(2024-02-06)
+1、修复图像识别的问题
+
+### V1.6.41(2024-02-06)
+1、添加语音识别及多模态
+
 ### V1.6.40(2024-01-25)
 1、支持 Dify 的智能助手模式，支持生成图片和识图
 2、使用 Dify 可以开启联网模式，网站内容总结

package/dist/cjs/src/botInstance/dify.js

@@ -79,7 +79,6 @@ class DifyAi {
                     return [{ type: 1, content: '上下文已重置' }];
                 }
             }
-            console.log('this.chatOption[uid]', this.chatOption[uid]);
             const { conversationId, text, files } = systemMessage ? await this.difyChat.sendMessage(content, { ...this.chatOption[uid], systemMessage, timeoutMs: this.config.timeoutMs * 1000 || 80 * 1000, user: uid }) : await this.difyChat.sendMessage(content, { ...this.chatOption[uid], timeoutMs: this.config.timeoutMs * 1000 || 80 * 1000, user: uid });
             if (this.config.filter) {
                 const censor = await this.contentCensor.checkText(text);

package/dist/cjs/src/botInstance/sdk/difyClient.js

@@ -179,7 +179,7 @@ class ChatClient extends DifyClient {
                                     res = JSON.parse(stream.substring(6)) || {};
                                 }
                                 catch (e) {
-                                    console.log('json 解析错误，不影响输出', e);
+                                    // console.log('json 解析错误，不影响输出', e)
                                     return;
                                 }
                                 if (!res.event || res.event === 'error' || res.status === 400) {

package/dist/cjs/src/common/index.d.ts

@@ -65,7 +65,7 @@ export function addRoom(that: any, contact: any, roomName: any, replys: any): Pr
  * @param contact
  * @param msg
  * @param isRoom
- *  type 1 文字 2 图片url 3 图片base64 4 url链接 5 小程序  6 名片
+ *  type 1 文字 2 图片url 3 图片base64 4 url链接 5 小程序  6 名片 7 富文本 8 语音
  */
 export function contactSay(contact: any, msg: any, isRoom?: boolean): Promise<void>;
 /**

package/dist/cjs/src/common/index.js

@@ -272,6 +272,15 @@ async function roomSay(room, contact, msg) {
             });
             await room.say(miniProgram);
         }
+        else if (msg.type === 8 && msg.url && msg.voiceLength) {
+            const fileBox = file_box_1.FileBox.fromUrl(msg.url);
+            fileBox.mimeType = "audio/silk";
+            fileBox.mediaType = "audio/silk";
+            fileBox.metadata = {
+                voiceLength: msg.voiceLength,
+            };
+            await room.say(fileBox);
+        }
     }
     catch (e) {
         console.log('群回复错误', e);
@@ -283,7 +292,7 @@ exports.roomSay = roomSay;
  * @param contact
  * @param msg
  * @param isRoom
- *  type 1 文字 2 图片url 3 图片base64 4 url链接 5 小程序  6 名片
+ *  type 1 文字 2 图片url 3 图片base64 4 url链接 5 小程序  6 名片 7 富文本 8 语音
  */
 async function contactSay(contact, msg, isRoom = false) {
     const config = await (0, configDb_js_1.allConfig)();
@@ -343,6 +352,15 @@ async function contactSay(contact, msg, isRoom = false) {
             });
             await contact.say(miniProgram);
         }
+        else if (msg.type === 8 && msg.url && msg.voiceLength) {
+            const fileBox = file_box_1.FileBox.fromUrl(msg.url);
+            fileBox.mimeType = "audio/silk";
+            fileBox.mediaType = "audio/silk";
+            fileBox.metadata = {
+                voiceLength: msg.voiceLength,
+            };
+            await contact.say(fileBox);
+        }
     }
     catch (e) {
         console.log('私聊发送消息失败', e);

package/dist/cjs/src/common/multiReply.d.ts

@@ -4,18 +4,9 @@ export class BotManage {
     userBotDict: {};
     userTimeDict: {};
     maxuser: number;
-    creatBot(username: any, content: any): Promise<{
-        type: number;
-        content: any;
-    }[] | undefined>;
-    updateBot(username: any, content: any, config: any): Promise<{
-        type: number;
-        content: any;
-    }[] | undefined>;
-    talk(username: any, content: any, config: any): Promise<{
-        type: number;
-        content: any;
-    }[] | undefined>;
+    creatBot(username: any, content: any): Promise<any>;
+    updateBot(username: any, content: any, config: any): Promise<any>;
+    talk(username: any, content: any, config: any): Promise<any>;
     removeBot(uid: any): void;
     getBotList(): {};
     /**
@@ -23,17 +14,11 @@ export class BotManage {
      * @param {*} username 用户名
      * @returns
      */
-    generateImage(username: any, question: any, config: any): Promise<{
-        type: number;
-        content: any;
-    }[]>;
+    generateImage(username: any, question: any, config: any): Promise<any>;
     getImage(username: any, content: any, step: any): {
         type: number;
         content: string;
     }[];
-    run(userId: any, content: any, config: any): Promise<{
-        type: number;
-        content: any;
-    }[] | undefined>;
+    run(userId: any, content: any, config: any): Promise<any>;
 }
 //# sourceMappingURL=multiReply.d.ts.map

package/dist/cjs/src/common/multiReply.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BotManage = void 0;
 const gpt4v_js_1 = require("../botInstance/gpt4v.js");
+const multimodal_js_1 = require("../proxy/multimodal.js");
 class MultiReply {
     constructor() {
         this.step = 0; // 当前step
@@ -84,7 +85,7 @@ class BotManage {
             const replys = await (0, gpt4v_js_1.getDify4vReply)(images, question, config, username);
             return replys;
         }
-        else {
+        else if (config.robotType === 6) {
             for (let id of this.userBotDict[username].imageIds) {
                 const msg = await this.Bot.Message.find({ id });
                 const file = await msg.toFileBox();
@@ -94,6 +95,16 @@ class BotManage {
             const replys = await (0, gpt4v_js_1.get4vReply)(images, question, config);
             return replys;
         }
+        else {
+            for (let id of this.userBotDict[username].imageIds) {
+                const msg = await this.Bot.Message.find({ id });
+                const file = await msg.toFileBox();
+                const base = await file.toDataURL();
+                images.push(base);
+            }
+            const replys = await (0, multimodal_js_1.getImageVision)(images, question, config);
+            return replys;
+        }
     }
     getImage(username, content, step) {
         if (this.userBotDict[username].imageIds.length === 5) {

package/dist/cjs/src/handlers/on-heartbeat.js

@@ -3,14 +3,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const aibotk_js_1 = require("../proxy/aibotk.js");
 const index_js_1 = require("../lib/index.js");
 async function onHeartBeat(str) {
-    if (!str) {
-        await (0, aibotk_js_1.sendHeartBeat)('dead');
+    try {
+        if (!str) {
+            await (0, aibotk_js_1.sendHeartBeat)('dead');
+        }
+        else if (str.type === 'scan') {
+            await (0, aibotk_js_1.sendHeartBeat)('scan');
+        }
+        else if (str.includes('heartbeat')) {
+            (0, index_js_1.throttle)((0, aibotk_js_1.sendHeartBeat)('live'), 30000);
+        }
     }
-    else if (str.type === 'scan') {
-        await (0, aibotk_js_1.sendHeartBeat)('scan');
-    }
-    else if (str.includes('heartbeat')) {
-        (0, index_js_1.throttle)((0, aibotk_js_1.sendHeartBeat)('live'), 30000);
+    catch (e) {
+        console.log('心跳更新失败', e);
     }
 }
 exports.default = onHeartBeat;

package/dist/cjs/src/handlers/on-message.js

@@ -10,10 +10,12 @@ const roomDb_js_1 = require("../db/roomDb.js");
 const hook_js_1 = require("../common/hook.js");
 const puppet_type_js_1 = require("../const/puppet-type.js");
 const gpt4vService_js_1 = require("../service/gpt4vService.js");
+const multimodal_js_1 = require("../proxy/multimodal.js");
+const msg_filters_js_1 = require("../service/msg-filters.js");
 const ignoreRecord = [
-    { type: "include", word: "加入了群聊" },
-    { type: "include", word: "与群里其他人都不是朋友关系" },
-    { type: "include", word: "收到一条暂不支持的消息类型" }
+    { type: 'include', word: '加入了群聊' },
+    { type: 'include', word: '与群里其他人都不是朋友关系' },
+    { type: 'include', word: '收到一条暂不支持的消息类型' }
 ];
 /**
  * 检测是否属于忽略的消息
@@ -26,7 +28,7 @@ function checkIgnore(msg, list) {
     for (let item of list) {
         const word = item.word;
         const type = item.type;
-        if ((type === "start" && msg.startsWith(word)) || (type === "end" && msg.endsWith(word)) || (type === "equal" && msg === word) || (type === "include" && msg.includes(word))) {
+        if ((type === 'start' && msg.startsWith(word)) || (type === 'end' && msg.endsWith(word)) || (type === 'equal' && msg === word) || (type === 'include' && msg.includes(word))) {
             return true;
         }
     }
@@ -46,7 +48,7 @@ async function dispatchFriendFilterByMsgType(that, msg) {
         const contact = msg.talker(); // 发消息人
         const name = await contact.name();
         const isOfficial = contact.type() === that.Contact.Type.Official;
-        let content = "";
+        let content = '';
         let replys = [];
         const res = await (0, hook_js_1.privateForward)({ that, msg, name, config });
         if (res) {
@@ -84,7 +86,44 @@ async function dispatchFriendFilterByMsgType(that, msg) {
                     }
                 }
                 else {
-                    console.log("公众号消息");
+                    console.log('公众号消息');
+                }
+                break;
+            case that.Message.Type.Audio:
+                let finalConfig = await (0, msg_filters_js_1.getCustomConfig)({ name, id: contact.id, roomName: '', roomId: '', room: false, type: 'openWhisper' });
+                if (finalConfig) {
+                    const audioFileBox = await msg.toFileBox();
+                    const text = await (0, multimodal_js_1.getVoiceText)(audioFileBox, finalConfig.botConfig.whisperConfig);
+                    console.log('语音解析结果', text);
+                    const keyword = finalConfig.botConfig.whisperConfig?.keywords?.length ? finalConfig.botConfig?.whisperConfig.keywords?.find((item) => text.includes(item)) : true;
+                    const isIgnore = checkIgnore(content.trim(), aibotConfig.ignoreMessages);
+                    if (text.trim() && !isIgnore && keyword) {
+                        const gpt4vReplys = await (0, gpt4vService_js_1.getGpt4vChat)({
+                            that,
+                            room: false,
+                            roomId: '',
+                            uniqueId: contact.id,
+                            id: contact.id,
+                            roomName: '',
+                            isMention: false,
+                            name,
+                            msgContent: { type: 1, content: text }
+                        });
+                        if (gpt4vReplys.length) {
+                            for (let reply of gpt4vReplys) {
+                                await index_js_1.contactSay.call(that, contact, reply);
+                            }
+                            return;
+                        }
+                        replys = await (0, reply_js_1.getContactTextReply)(that, contact, text.trim());
+                        for (let reply of replys) {
+                            await (0, index_js_2.delay)(1000);
+                            await index_js_1.contactSay.call(that, contact, reply);
+                        }
+                    }
+                    else {
+                        console.log('语音解析结果没有匹配到需要回复的关键词');
+                    }
                 }
                 break;
             case that.Message.Type.Emoticon:
@@ -143,7 +182,7 @@ async function dispatchFriendFilterByMsgType(that, msg) {
         }
     }
     catch (error) {
-        console.log("监听消息错误", error);
+        console.log('监听消息错误', error);
     }
 }
 /**
@@ -164,8 +203,8 @@ async function dispatchRoomFilterByMsgType(that, room, msg) {
         const isFriend = contact.friend();
         const type = msg.type();
         const receiver = msg.to();
-        let content = "";
-        let replys = "";
+        let content = '';
+        let replys = '';
         let contactId = contact.id;
         let contactAvatar = await contact.avatar();
         const userSelfName = that.currentUser?.name() || that.userSelf()?.name();
@@ -174,7 +213,7 @@ async function dispatchRoomFilterByMsgType(that, room, msg) {
                 content = msg.text();
                 const mentionSelf = await msg.mentionSelf() || content.includes(`@${userSelfName}`);
                 const receiverName = receiver?.name();
-                content = content.replace('@' + receiverName, "").replace('@' + userSelfName, "").replace(/@[^,，：:\s@]+/g, "").trim();
+                content = content.replace('@' + receiverName, '').replace('@' + userSelfName, '').replace(/@[^,，：:\s@]+/g, '').trim();
                 console.log(`群名: ${roomName} 发消息人: ${contactName} 内容: ${content} | 机器人被@：${mentionSelf ? '是' : '否'}`);
                 // 检测是否需要这条消息
                 const isIgnore = checkIgnore(content, aibotConfig.ignoreMessages);
@@ -215,9 +254,9 @@ async function dispatchRoomFilterByMsgType(that, room, msg) {
                     await index_js_1.roomSay.call(that, room, contact, reply);
                 }
                 const cloudRoom = config.cloudRoom;
-                if (role === "vip" && cloudRoom.includes(roomName) && !checkIgnore(content, ignoreRecord)) {
+                if (role === 'vip' && cloudRoom.includes(roomName) && !checkIgnore(content, ignoreRecord)) {
                     const regex = /(<([^>]+)>)/ig;
-                    content = content.replace(regex, "");
+                    content = content.replace(regex, '');
                     void (0, roomDb_js_1.addRoomRecord)({
                         roomName,
                         roomId: room.id,
@@ -257,6 +296,52 @@ async function dispatchRoomFilterByMsgType(that, room, msg) {
                 break;
             case that.Message.Type.Audio:
                 console.log(`群名: ${roomName} 发消息人: ${contactName} 发了一个语音`);
+                let finalConfig = await (0, msg_filters_js_1.getCustomConfig)({ name: contactName, id: contactId, roomName, roomId: room.id, room, type: 'openWhisper' });
+                if (finalConfig) {
+                    const audioFileBox = await msg.toFileBox();
+                    const text = await (0, multimodal_js_1.getVoiceText)(audioFileBox, finalConfig.botConfig.whisperConfig);
+                    console.log('语音解析结果', text);
+                    const keyword = finalConfig.botConfig.whisperConfig?.keywords?.length ? finalConfig.botConfig?.whisperConfig?.keywords?.find((item) => text.includes(item)) : true;
+                    const isIgnore = checkIgnore(content.trim(), aibotConfig.ignoreMessages);
+                    if (text.trim() && !isIgnore && keyword) {
+                        const gpt4vReplys = await (0, gpt4vService_js_1.getGpt4vChat)({
+                            that,
+                            room,
+                            roomId: room.id,
+                            id: contactId,
+                            uniqueId: `${room.id}-${contactId}`,
+                            roomName,
+                            isMention: true,
+                            name: contactName,
+                            msgContent: { type: 1, content: text }
+                        });
+                        if (gpt4vReplys.length) {
+                            for (let reply of gpt4vReplys) {
+                                await index_js_1.roomSay.call(that, room, contact, reply);
+                            }
+                            return;
+                        }
+                        replys = await await (0, reply_js_1.getRoomTextReply)({
+                            that,
+                            content: text,
+                            isFriend,
+                            name: contactName,
+                            id: contactId,
+                            roomId: room.id,
+                            avatar: contactAvatar,
+                            room,
+                            roomName,
+                            isMention: true
+                        });
+                        for (let reply of replys) {
+                            await (0, index_js_2.delay)(1000);
+                            await index_js_1.roomSay.call(that, room, contact, reply);
+                        }
+                    }
+                    else {
+                        console.log('语音解析结果没有匹配到需要回复的关键词');
+                    }
+                }
                 break;
             case that.Message.Type.MiniProgram:
                 console.log(`群名: ${roomName} 发消息人: ${contactName} 发了一个小程序`);
@@ -285,7 +370,7 @@ async function dispatchRoomFilterByMsgType(that, room, msg) {
         }
     }
     catch (e) {
-        console.log("error", e);
+        console.log('error', e);
     }
 }
 async function onMessage(msg) {
@@ -301,7 +386,7 @@ async function onMessage(msg) {
             const contact = msg.talker(); // 发消息人
             const contactName = contact.name();
             await dispatchRoomFilterByMsgType(this, room, msg);
-            if (role === "vip" && roomName !== contactName) {
+            if (role === 'vip' && roomName !== contactName) {
                 const roomAsyncList = config.roomAsyncList || [];
                 if (roomAsyncList.length) {
                     await (0, room_async_service_js_1.dispatchAsync)(this, msg, roomAsyncList);
@@ -313,7 +398,7 @@ async function onMessage(msg) {
         }
     }
     catch (e) {
-        console.log("监听消息失败", e);
+        console.log('监听消息失败', e);
     }
 }
 exports.default = onMessage;

package/dist/cjs/src/package-json.js

@@ -6,7 +6,7 @@ exports.packageJson = void 0;
  */
 exports.packageJson = {
     "name": "wechaty-web-panel",
-    "version": "1.6.40",
+    "version": "1.6.42",
     "description": "智能微秘书插件",
     "exports": {
         ".": {

package/dist/cjs/src/proxy/mqtt.js

@@ -204,7 +204,7 @@ async function initMqtt(that) {
                                     console.log(`获取到输入的验证码:${global_js_1.default.getVerifyCode()}，正在填入`);
                                     const verifyCode = global_js_1.default.getVerifyCode(); // 通过一些途径输入验证码
                                     try {
-                                        await that.enterVerifyCode(id, verifyCode); // 如果没抛错，则说明输入成功，会推送登录事件
+                                        await that.enterVerifyCode(global_js_1.default.getVerifyId(), verifyCode); // 如果没抛错，则说明输入成功，会推送登录事件
                                     }
                                     catch (e) {
                                         console.log('验证码校验错误：', e.message);

package/dist/cjs/src/proxy/multimodal.d.ts

@@ -0,0 +1,19 @@
+/**
+ * 语音转换文字
+ * @param file
+ * @param aiConfig
+ * @returns {Promise<*|string>}
+ */
+export function getVoiceText(file: any, aiConfig: any): Promise<any | string>;
+/**
+ * 识别图像
+ * @param images
+ * @param question
+ * @param config
+ * @returns {Promise<*|[{type: number, content: string}]>}
+ */
+export function getImageVision(images: any, question: any, config: any): Promise<any | [{
+    type: number;
+    content: string;
+}]>;
+//# sourceMappingURL=multimodal.d.ts.map

package/dist/cjs/src/proxy/multimodal.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getImageVision = exports.getVoiceText = void 0;
+const axios_1 = __importDefault(require("axios"));
+const form_data_1 = __importDefault(require("form-data"));
+const config_js_1 = require("./config.js");
+const aiDb_js_1 = require("../db/aiDb.js");
+/**
+ * 语音转换文字
+ * @param file
+ * @param aiConfig
+ * @returns {Promise<*|string>}
+ */
+async function getVoiceText(file, aiConfig) {
+    try {
+        const env = await (0, aiDb_js_1.getAibotConfig)();
+        const { apiKey } = env;
+        const base64 = await file.toBase64();
+        const readable = Buffer.from(base64, 'base64');
+        const formData = new form_data_1.default();
+        formData.append('file', readable, { contentType: file.mediaType, filename: file.name });
+        formData.append('aiConfig', JSON.stringify(aiConfig));
+        let config = {
+            method: 'post',
+            maxBodyLength: Infinity,
+            timeout: 30000,
+            url: config_js_1.AIBOTK_OUTAPI + '/voice/text',
+            headers: {
+                ...formData.getHeaders(),
+                Authorization: `Bearer ${apiKey}`,
+            },
+            data: formData
+        };
+        const result = await axios_1.default.request(config);
+        if (result.data.code === 200) {
+            return result.data.data;
+        }
+        else {
+            console.log('语音转换出错', result.data.message);
+            return '';
+        }
+    }
+    catch (e) {
+        console.log(`语音转换出错: ${e}`);
+        return '';
+    }
+}
+exports.getVoiceText = getVoiceText;
+/**
+ * 识别图像
+ * @param images
+ * @param question
+ * @param config
+ * @returns {Promise<*|[{type: number, content: string}]>}
+ */
+async function getImageVision(images, question, config) {
+    try {
+        const env = await (0, aiDb_js_1.getAibotConfig)();
+        const { apiKey } = env;
+        const reqConfig = {
+            method: 'post',
+            maxBodyLength: Infinity,
+            timeout: 30000,
+            url: config_js_1.AIBOTK_OUTAPI + '/image/vision',
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+            },
+            data: {
+                images,
+                question,
+                config
+            }
+        };
+        const result = await axios_1.default.request(reqConfig);
+        if (result.data.code === 200) {
+            return result.data.data;
+        }
+        else {
+            console.log('识别图像出错', result.data.message);
+            return [{ type: 1, content: '' }];
+        }
+    }
+    catch (e) {
+        console.log('识别图像出错', e);
+        return [{ type: 1, content: '' }];
+    }
+}
+exports.getImageVision = getImageVision;
+//# sourceMappingURL=multimodal.js.map

package/dist/cjs/src/proxy/superagent.js

@@ -70,19 +70,21 @@ function get({ url, params, contentType = 'application/json', platform = 'tx', a
                 console.log('请求出错', err);
                 reject(err);
             }
-            if (spider) {
-                // 如果是爬取内容，直接返回页面html
-                resolve(res.text);
-            }
             else {
-                // 如果是非爬虫，返回格式化后的内容
-                res = res && res.text && JSON.parse(res.text) || {};
-                if (platform !== 'chuan') {
-                    if ((res.code !== 200 && platform === 'tx') || (res.code !== 200 && platform === 'aibot') || (res.code !== 0 && platform === 'qi') || (res.code !== 100000 && platform === 'tl')) {
-                        console.error(`接口${url}请求失败`, res.msg || res.text);
+                if (spider) {
+                    // 如果是爬取内容，直接返回页面html
+                    resolve(res.text);
+                }
+                else {
+                    // 如果是非爬虫，返回格式化后的内容
+                    res = res && res.text && JSON.parse(res.text) || {};
+                    if (platform !== 'chuan') {
+                        if ((res.code !== 200 && platform === 'tx') || (res.code !== 200 && platform === 'aibot') || (res.code !== 0 && platform === 'qi') || (res.code !== 100000 && platform === 'tl')) {
+                            console.error(`接口${url}请求失败`, res.msg || res.text);
+                        }
                     }
+                    resolve(res);
                 }
-                resolve(res);
             }
         });
     });
@@ -106,19 +108,21 @@ function post({ url, params, contentType = 'application/json', authorization = '
                 console.log('请求出错', err);
                 reject(err);
             }
-            if (spider) {
-                // 如果是爬取内容，直接返回页面html
-                resolve(res.text);
-            }
             else {
-                // 如果是非爬虫，返回格式化后的内容
-                res = res && res.text && JSON.parse(res.text) || {};
-                if (platform !== 'chuan') {
-                    if ((res.code !== 200 && platform === 'tx') || (res.code !== 200 && platform === 'aibot') || (res.code !== 100000 && platform === 'tl')) {
-                        console.error(`接口请求失败${url}`, res.msg || res.text || res.error);
+                if (spider) {
+                    // 如果是爬取内容，直接返回页面html
+                    resolve(res.text);
+                }
+                else {
+                    // 如果是非爬虫，返回格式化后的内容
+                    res = res && res.text && JSON.parse(res.text) || {};
+                    if (platform !== 'chuan') {
+                        if ((res.code !== 200 && platform === 'tx') || (res.code !== 200 && platform === 'aibot') || (res.code !== 100000 && platform === 'tl')) {
+                            console.error(`接口请求失败${url}`, res.msg || res.text || res.error);
+                        }
                     }
+                    resolve(res);
                 }
-                resolve(res);
             }
         });
     });