yt-chat-components 1.5.0 → 1.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yt-chat-components",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "main": "build/static/js/bundle.min.js",
   "module": "build/static/js/bundle.min.js",
   "types": "build/static/js/index.d.ts",

package/public/index.html

@@ -111,7 +111,6 @@
         host-url="https://ai-api.yuntu.cn"
         sign-url="https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/logo.png"
         app-id="5f48c683-38d8-430e-8479-17730a605821"
-        user-info='{"id": "123", "name": "John Doe", "code":"1606451129" }'
         box-style='{"height":"100%","background":"url(https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/system/portal_bg_blue.png) center top / 100% 100% no-repeat"}'
         is-show-side-right=false
         is-show-side-left=true

package/src/YtChatView/chatWidget/chatWindow/callInterface/StreamAudioPlayer.tsx

@@ -0,0 +1,194 @@
+// @ts-ignore
+export class StreamAudioPlayer {
+    private audioContext: AudioContext;
+    private audioQueue: AudioBuffer[] = [];
+    private pendingBuffer: Float32Array[] = [];
+    private pendingBufferSizeSec = 0;
+
+    private isPlaying = false;
+    private lastPlaybackTime = 0;
+    private FADE_MS = 50; // 渐入/渐出时间（毫秒）
+    private FADE_SAMPLES = 50;
+
+    private onPlaybackCompleteCallback: (() => void) | null = null;
+    private source: AudioBufferSourceNode | null = null;
+
+    constructor() {
+        const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext;
+        // 强制创建 16000 Hz 的音频上下文
+        const contextOptions = {
+            sampleRate: 16000
+        };
+        this.audioContext = new AudioContextClass(contextOptions);
+        // 确保 FADE_SAMPLES 正确计算
+        this.FADE_SAMPLES = Math.floor((this.audioContext.sampleRate * this.FADE_MS) / 1000);
+        // console.log(`[音频] 使用采样率: ${this.audioContext.sampleRate} Hz`);
+    }
+
+    public get context(): AudioContext {
+        return this.audioContext;
+    }
+
+    /**
+     * 添加 PCM 音频数据到播放队列中
+     * @param audioData base64 编码的 PCM 数据
+     */
+    public enqueueAudioChunk(audioData: string): void {
+        try {
+            const byteCharacters = atob(audioData);
+            const pcmBytes = new Uint8Array(byteCharacters.length);
+            for (let i = 0; i < byteCharacters.length; i++) {
+                pcmBytes[i] = byteCharacters.charCodeAt(i);
+            }
+
+            const sampleCount = pcmBytes.length / 2;
+            const durationSec = sampleCount / this.audioContext.sampleRate;
+
+            const dataView = new DataView(pcmBytes.buffer);
+            const channelData = new Float32Array(sampleCount);
+
+            for (let i = 0; i < sampleCount; i++) {
+                const int16 = dataView.getInt16(i * 2, true); // 小端读取有符号 16bit
+                let value = int16 / 32768.0;
+
+                // 渐入渐出处理
+                if (i < this.FADE_SAMPLES) {
+                    value *= i / this.FADE_SAMPLES;
+                }
+                if (i >= sampleCount - this.FADE_SAMPLES) {
+                    value *= (sampleCount - i) / this.FADE_SAMPLES;
+                }
+
+                channelData[i] = value;
+            }
+
+            // 合并小块
+            this.pendingBuffer.push(channelData);
+            this.pendingBufferSizeSec += durationSec;
+
+            // 如果累计 ≥ 0.5s 才创建 buffer 并推入队列
+            if (this.pendingBufferSizeSec >= 0.5) {
+                const totalLength = this.pendingBuffer.reduce((sum, b) => sum + b.length, 0);
+                const mergedChannel = new Float32Array(totalLength);
+
+                let offset = 0;
+                for (const ch of this.pendingBuffer) {
+                    mergedChannel.set(ch, offset);
+                    offset += ch.length;
+                }
+
+                const audioBuffer = this.audioContext.createBuffer(
+                    1,
+                    mergedChannel.length,
+                    this.audioContext.sampleRate
+                );
+                audioBuffer.copyToChannel(mergedChannel, 0);
+                this.audioQueue.push(audioBuffer);
+
+                // 清空缓存
+                this.pendingBuffer = [];
+                this.pendingBufferSizeSec = 0;
+            }
+        } catch (e) {
+            console.error('[音频] 解码音频失败:', e);
+        }
+    }
+
+    /**
+     * 开始播放音频队列
+     * @param onComplete 所有音频播放完成后的回调
+     */
+    public play(onComplete?: () => void): void {
+        if (this.isPlaying) return;
+
+        this.onPlaybackCompleteCallback = onComplete || null;
+
+        if (this.audioQueue.length === 0) {
+            this.handlePlaybackComplete();
+            return;
+        }
+
+        this.playNextChunk();
+    }
+
+    private playNextChunk(): void {
+        if (!this.audioContext || this.isPlaying || this.audioQueue.length === 0) {
+            return;
+        }
+
+        const nextBuffer = this.audioQueue.shift();
+        if (!nextBuffer) {
+            this.handlePlaybackComplete();
+            return;
+        }
+
+        this.source = this.audioContext.createBufferSource();
+        this.source.buffer = nextBuffer;
+        this.source.connect(this.audioContext.destination);
+
+        const now = this.audioContext.currentTime;
+        const startTime = Math.max(now, this.lastPlaybackTime);
+        const bufferDuration = nextBuffer.duration;
+
+        this.source.start(startTime);
+
+        this.lastPlaybackTime = startTime + bufferDuration;
+        this.isPlaying = true;
+
+        this.source.onended = () => {
+            this.isPlaying = false;
+            this.source = null;
+
+            // 继续播放下一个 chunk
+            this.playNextChunk();
+        };
+    }
+
+    private handlePlaybackComplete(): void {
+        if (this.onPlaybackCompleteCallback) {
+            this.onPlaybackCompleteCallback();
+            this.onPlaybackCompleteCallback = null;
+        }
+    }
+
+    /**
+     * 停止当前播放并清空队列
+     */
+    public stop(): void {
+        if (this.source) {
+            this.source.stop();
+            this.source.disconnect();
+            this.source = null;
+        }
+        this.isPlaying = false;
+        this.audioQueue = [];
+        this.pendingBuffer = [];
+        this.pendingBufferSizeSec = 0;
+        this.lastPlaybackTime = 0;
+    }
+
+    /**
+     * 暂停当前播放，不清空队列
+     */
+    public pause(): void {
+        this.stop();
+    }
+
+    /**
+     * 恢复播放
+     */
+    public resume(): void {
+        // @ts-ignore
+        this.play(this.onPlaybackCompleteCallback);
+    }
+
+    /**
+     * 销毁音频上下文（页面卸载时调用）
+     */
+    public destroy(): void {
+        this.stop();
+        if (this.audioContext && this.audioContext.close) {
+            this.audioContext.close();
+        }
+    }
+}

package/src/YtChatView/chatWidget/chatWindow/callInterface/index.tsx

@@ -1,14 +1,14 @@
-import React, { useState, useEffect, useRef } from 'react';
-import { Button, Avatar, message } from 'antd';
+import React, {useEffect, useRef, useState} from 'react';
+import {Avatar, Button, message} from 'antd';
 import './index.module.css';
-import { callStyle } from './style';
-import { ChatMessageType } from '../../chatWindow/types/chatWidget';
+import {callStyle} from './style';
 import Recorder from 'recorder-core'
 import 'recorder-core/src/engine/mp3'
 import 'recorder-core/src/engine/mp3-engine'
 // import 'recorder-core/src/extensions/waveview'
 import ai_call_thinking from '../../../../assets/aicenter/ai_call_thinking.png'
 import ai_call_working from '../../../../assets/aicenter/ai_call_working.gif'
+import {StreamAudioPlayer} from "./StreamAudioPlayer";
 
 
 interface CallInterfaceProps {
@@ -19,7 +19,7 @@ interface CallInterfaceProps {
   flowId: string;
   sessionId: React.MutableRefObject<string>;
   hostUrl: string;
-  addMessage: (message: ChatMessageType) => void;
+  addMessage: Function;
   userInfo: {
     code: string;
   };
@@ -63,7 +63,7 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
   const vadProcessorRef = useRef<ScriptProcessorNode | null>(null);
   const audioContextRef = useRef<AudioContext | null>(null);
   const [isSpeakingNow,setIsSpeakingNow] = useState(false);
-
+  const audioPlayerInstance = new StreamAudioPlayer();
 
   // 添加统一的WebSocket消息发送方法
   const sendWebSocketMessage = (type: string, data: any) => {
@@ -120,6 +120,27 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
     });
   };
 
+  const handleAudioChunkData = async (audioData: string) => {
+    try {
+      // 添加音频片段到播放器
+      audioPlayerInstance.enqueueAudioChunk(audioData);
+
+      // 设置状态为“正在回复”
+      setWorkStatus('正在回复');
+
+      // 开始播放，并在所有音频完成后恢复状态
+      audioPlayerInstance.play(() => {
+        console.log("全部音频播放完成！");
+        setWorkStatus('正在聆听');
+      });
+
+    } catch (e) {
+      console.error('[音频] 解码音频失败:', e);
+      setWorkStatus('正在聆听');
+    }
+  };
+
+
   const initWebRTC = async () => {
     try {
       const peerConnection = new RTCPeerConnection({
@@ -139,7 +160,7 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
       console.log('开始连接WebSocket');
       const ws = new WebSocket(`${hostUrl.replace('http', 'ws')}/api/v1/ws/webrtc?flow_id=${flowId}&session_id=${sessionId}&api_key=${api_key}&operator_id=${opeartorId}`);
       wsRef.current = ws;
-      
+
       // 添加连接超时处理
       const connectionTimeout = setTimeout(() => {
         if (ws.readyState === WebSocket.CONNECTING) {
@@ -148,29 +169,29 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
           ws.close();
         }
       }, 60000); // 60秒超时
-      
+
       // 创建和发送offer的函数
       const createAndSendOffer = async (_peerConnection: RTCPeerConnection) => {
         try {
           const offer = await _peerConnection.createOffer();
           await _peerConnection.setLocalDescription(offer);
-          
+
           sendWebSocketMessage('offer', { offer });
         } catch (error) {
           console.error('创建offer失败:', error);
         }
       };
-      
+
       ws.onopen = () => {
         console.log('WebSocket连接已建立');
         clearTimeout(connectionTimeout); // 清除超时定时器
-        
+
         // 只有在WebSocket连接成功后才创建和发送offer
         if (peerConnectionRef.current) {
           createAndSendOffer(peerConnectionRef.current);
         }
       };
-      
+
       const localStream = await navigator.mediaDevices.getUserMedia({
         audio: {
           echoCancellation: true, // 启用回声消除
@@ -180,11 +201,11 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
         video: false
       });
       localStreamRef.current = localStream;
-      
+
       // if (localAudioRef.current) {
       //   localAudioRef.current.srcObject = localStream;
       // }
-      
+
       // 设置VAD(语音活动检测)
       const audioContext = new AudioContext();
       audioContextRef.current = audioContext;
@@ -192,14 +213,20 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
       analyser.fftSize = 256;
       const bufferLength = analyser.frequencyBinCount;
       const dataArray = new Uint8Array(bufferLength);
-      
+
       const source = audioContext.createMediaStreamSource(localStream);
-      source.connect(analyser);
-      
+      // 创建高通滤波器
+      const highpassFilter = audioContext.createBiquadFilter();
+      highpassFilter.type = "highpass";
+      highpassFilter.frequency.setValueAtTime(100, audioContext.currentTime); // 过滤低于100Hz的声音
+      // 连接顺序：source -> highpassFilter -> analyser 和 vadProcessor
+      source.connect(highpassFilter);
+      highpassFilter.connect(analyser);
       // 创建音频处理器用于VAD
       const vadProcessor = audioContext.createScriptProcessor(2048, 1, 1);
+      highpassFilter.connect(vadProcessor);
       vadProcessorRef.current = vadProcessor;
-      
+
       vadProcessor.onaudioprocess = (e) => {
         // 正在听对方说话 or 思考，不处理
         if (workStatus === '正在回复'){
@@ -209,16 +236,16 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
           console.log("--------------------- 正在思考")
           return;
         }
-        
+
         analyser.getByteFrequencyData(dataArray);
-        
+
         // 计算音量平均值
         let sum = 0;
         for (let i = 0; i < bufferLength; i++) {
           sum += dataArray[i];
         }
         const average = sum / bufferLength;
-        
+
         // 检测是否有语音
         const isSpeakingNow = average > SILENCE_THRESHOLD;
 
@@ -288,6 +315,8 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
                         sampleRate:16000,
                         msgId: msgIdRef.current,
                       });
+                      // 暂停当前正在播放的内容
+                      audioPlayerInstance.stop()
                       console.log('发送完毕');
 
                       // 添加用户消息到聊天记录
@@ -326,16 +355,16 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
           }
         }
       };
-      
+
       // 连接VAD处理器
       source.connect(vadProcessor);
       vadProcessor.connect(audioContext.destination);
-      
+
       // 修改WebSocket消息处理
       ws.onmessage = async (event) => {
         const message = JSON.parse(event.data);
-        console.log('收到消息', message);
-        
+        // console.log('收到消息', message);
+
         if (message.type === 'offer') {
           await handleOffer(message);
         } else if (message.type === 'answer') {
@@ -353,16 +382,62 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
           if (message.audioData) {
             handleAudioData(message.audioData);
           }else {
+            console.log('收到后端语音发送结束指令：恢复聆听状态')
             setWorkStatus('正在聆听');
           }
+        }if (message.type === 'audio-chunk') {
+          // 处理从后端返回的音频流响应
+          msgIdRef.current = null
+          if (message.audioData) {
+            handleAudioChunkData(message.audioData);
+          }
+        }if (message.type === 'user-audio-text') {
+          // 处理从后端返回的音频转文字展示用
+          // msgIdRef.current = null
+          if (message.textResponse) {
+            console.log("user-audio-text")
+            console.log(message.textResponse)
+            // 清理当前正在回复的内容
+            audioPlayerInstance.stop()
+            // 插入语音到聊天记录
+            addMessage({
+              messageItemList: [
+                {
+                  id: new Date().getTime() + "",
+                  name: '[语音转文字]',
+                  message: message.textResponse,
+                  thinkMessage: '',
+                }
+              ],
+              isSend: true,
+              receivingMessage: false,
+              host_url: hostUrl,
+              // timestamp: new Date().toISOString()
+            });
+          }
+        }if (message.type === 'audio-chunk-text') {
+          // 处理从后端返回的音频转文字展示用
+          // msgIdRef.current = null
+          if (message.textResponse && addMessage) {
+            // 流式插入消息到聊天记录
+            // console.log(message.textResponse)
+            // handleMessageContent('token',{
+            //   chunk: message.textResponse,
+            //   name:'agent',
+            //   id: '9999999999',
+            //   timestamp: new Date()
+            // })
+          }
+        }if (message.type === 'complete-chunk-audio'){
+          console.log('本次语音流传输完毕')
         }
       };
-      
+
       // 添加本地音频轨道到对等连接
       localStream.getTracks().forEach(track => {
         peerConnection.addTrack(track, localStream);
       });
-      
+
       // 处理远程流
       peerConnection.ontrack = (event) => {
         remoteStreamRef.current = event.streams[0];
@@ -370,7 +445,7 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
           remoteAudioRef.current.srcObject = event.streams[0];
         }
       };
-      
+
       // 处理ICE候选
       peerConnection.onicecandidate = (event) => {
         if (event.candidate && wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
@@ -490,7 +565,8 @@ const CallInterface: React.FC<CallInterfaceProps> = ({
     if (localStreamRef.current) {
       localStreamRef.current.getTracks().forEach(track => track.stop());
     }
-    
+    // 关闭音频播放
+    audioPlayerInstance.destroy();
     onHangup();
   };
   

package/src/YtChatView/chatWidget/chatWindow/chatMessage/index.tsx

@@ -37,7 +37,10 @@ for (let i = 1; i <= 68; i++) {
 }
 
 // 根据name生成固定的颜色索引
-const getAvatarByName = (name: string) => {
+const getAvatar = (name: string, icon: string) => {
+  if(icon){
+    return <Avatar shape="square" src={icon} style={{marginRight: 8}}/>
+  }
   const total = name.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);
   const index = total % avatarImages.length;
   const img = avatarImages[index];
@@ -315,6 +318,35 @@ export default function ChatMessage({
           return renderLoading(MessageType.url, 50);
         }
       }
+      else if (className && className.includes('language-image')) {
+        try {
+          // 解析代码块内容为 JSON 配置
+          const data = JSON.parse(children.toString().trim().replace(/\\n/g, '').replace('\n', ''));
+          const {url} = data;
+          // 渲染组件
+          return <Image
+              height={'auto'}
+              width={'100%'}
+              src={url}
+              alt={''}
+              fallback="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMIAAADDCAYAAADQvc6UAAABRWlDQ1BJQ0MgUHJvZmlsZQAAKJFjYGASSSwoyGFhYGDIzSspCnJ3UoiIjFJgf8LAwSDCIMogwMCcmFxc4BgQ4ANUwgCjUcG3awyMIPqyLsis7PPOq3QdDFcvjV3jOD1boQVTPQrgSkktTgbSf4A4LbmgqISBgTEFyFYuLykAsTuAbJEioKOA7DkgdjqEvQHEToKwj4DVhAQ5A9k3gGyB5IxEoBmML4BsnSQk8XQkNtReEOBxcfXxUQg1Mjc0dyHgXNJBSWpFCYh2zi+oLMpMzyhRcASGUqqCZ16yno6CkYGRAQMDKMwhqj/fAIcloxgHQqxAjIHBEugw5sUIsSQpBobtQPdLciLEVJYzMPBHMDBsayhILEqEO4DxG0txmrERhM29nYGBddr//5/DGRjYNRkY/l7////39v///y4Dmn+LgeHANwDrkl1AuO+pmgAAADhlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAAqACAAQAAAABAAAAwqADAAQAAAABAAAAwwAAAAD9b/HnAAAHlklEQVR4Ae3dP3PTWBSGcbGzM6GCKqlIBRV0dHRJFarQ0eUT8LH4BnRU0NHR0UEFVdIlFRV7TzRksomPY8uykTk/zewQfKw/9znv4yvJynLv4uLiV2dBoDiBf4qP3/ARuCRABEFAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghggQAQZQKAnYEaQBAQaASKIAQJEkAEEegJmBElAoBEgghgg0Aj8i0JO4OzsrPv69Wv+hi2qPHr0qNvf39+iI97soRIh4f3z58/u7du3SXX7Xt7Z2enevHmzfQe+oSN2apSAPj09TSrb+XKI/f379+08+A0cNRE2ANkupk+ACNPvkSPcAAEibACyXUyfABGm3yNHuAECRNgAZLuYPgEirKlHu7u7XdyytGwHAd8jjNyng4OD7vnz51dbPT8/7z58+NB9+/bt6jU/TI+AGWHEnrx48eJ/EsSmHzx40L18+fLyzxF3ZVMjEyDCiEDjMYZZS5wiPXnyZFbJaxMhQIQRGzHvWR7XCyOCXsOmiDAi1HmPMMQjDpbpEiDCiL358eNHurW/5SnWdIBbXiDCiA38/Pnzrce2YyZ4//59F3ePLNMl4PbpiL2J0L979+7yDtHDhw8vtzzvdGnEXdvUigSIsCLAWavHp/+qM0BcXMd/q25n1vF57TYBp0a3mUzilePj4+7k5KSLb6gt6ydAhPUzXnoPR0dHl79WGTNCfBnn1uvSCJdegQhLI1vvCk+fPu2ePXt2tZOYEV6/fn31dz+shwAR1sP1cqvLntbEN9MxA9xcYjsxS1jWR4AIa2Ibzx0tc44fYX/16lV6NDFLXH+YL32jwiACRBiEbf5KcXoTIsQSpzXx4N28Ja4BQoK7rgXiydbHjx/P25TaQAJEGAguWy0+2Q8PD6/Ki4R8EVl+bzBOnZY95fq9rj9zAkTI2SxdidBHqG9+skdw43borCXO/ZcJdraPWdv22uIEiLA4q7nvvCug8WTqzQveOH26fodo7g6uFe/a17W3+nFBAkRYENRdb1vkkz1CH9cPsVy/jrhr27PqMYvENYNlHAIesRiBYwRy0V+8iXP8+/fvX11Mr7L7ECueb/r48eMqm7FuI2BGWDEG8cm+7G3NEOfmdcTQw4h9/55lhm7DekRYKQPZF2ArbXTAyu4kDYB2YxUzwg0gi/41ztHnfQG26HbGel/crVrm7tNY+/1btkOEAZ2M05r4FB7r9GbAIdxaZYrHdOsgJ/wCEQY0J74TmOKnbxxT9n3FgGGWWsVdowHtjt9Nnvf7yQM2aZU/TIAIAxrw6dOnAWtZZcoEnBpNuTuObWMEiLAx1HY0ZQJEmHJ3HNvGCBBhY6jtaMoEiJB0Z29vL6ls58vxPcO8/zfrdo5qvKO+d3Fx8Wu8zf1dW4p/cPzLly/dtv9Ts/EbcvGAHhHyfBIhZ6NSiIBTo0LNNtScABFyNiqFCBChULMNNSdAhJyNSiECRCjUbEPNCRAhZ6NSiAARCjXbUHMCRMjZqBQiQIRCzTbUnAARcjYqhQgQoVCzDTUnQIScjUohAkQo1GxDzQkQIWejUogAEQo121BzAkTI2agUIkCEQs021JwAEXI2KoUIEKFQsw01J0CEnI1KIQJEKNRsQ80JECFno1KIABEKNdtQcwJEyNmoFCJAhELNNtScABFyNiqFCBChULMNNSdAhJyNSiECRCjUbEPNCRAhZ6NSiAARCjXbUHMCRMjZqBQiQIRCzTbUnAARcjYqhQgQoVCzDTUnQIScjUohAkQo1GxDzQkQIWejUogAEQo121BzAkTI2agUIkCEQs021JwAEXI2KoUIEKFQsw01J0CEnI1KIQJEKNRsQ80JECFno1KIABEKNdtQcwJEyNmoFCJAhELNNtScABFyNiqFCBChULMNNSdAhJyNSiECRCjUbEPNCRAhZ6NSiAARCjXbUHMCRMjZqBQiQIRCzTbUnAARcjYqhQgQoVCzDTUnQIScjUohAkQo1GxDzQkQIWejUogAEQo121BzAkTI2agUIkCEQs021JwAEXI2KoUIEKFQsw01J0CEnI1KIQJEKNRsQ80JECFno1KIABEKNdtQcwJEyNmoFCJAhELNNtScABFyNiqFCBChULMNNSdAhJyNSiEC/wGgKKC4YMA4TAAAAABJRU5ErkJggg=="
+          />
+        } catch (error) {
+          return renderLoading(MessageType.url, 50);
+        }
+      }
+      else if (className && className.includes('language-video')) {
+        try {
+          // 解析代码块内容为 JSON 配置
+          const data = JSON.parse(children.toString().trim().replace(/\\n/g, '').replace('\n', ''));
+          const {url} = data;
+
+          // 渲染组件
+          return <video src={url} autoPlay={true} controls={false}/>
+        } catch (error) {
+          return renderLoading(MessageType.url, 50);
+        }
+      }
       // 默认渲染其他代码块
       return <code className={className} {...props}>{children}</code>;
     },
@@ -357,7 +389,7 @@ export default function ChatMessage({
 
   const renderBotTextMessage = () => {
     const items = messageItemList.map((item, index) => {
-      const {id, thinkMessage, message, loadingMessage, type, name, toolCallInfo, rawInfo} = item;
+      const {id, thinkMessage, message, loadingMessage, type, name, icon, toolCallInfo, rawInfo} = item||{};
       const isLatestItem = index === messageItemList.length - 1;
       let status = 'success'
       // 只有在接收信息的时候有 pending 状态，最后的信息才为 pending
@@ -367,7 +399,7 @@ export default function ChatMessage({
 
       const resultItem: CustomThoughtChainItem = {
         title: <div style={{display:"flex", flexDirection: 'row', alignItems: 'center'}}>
-          {getAvatarByName(name)}
+          {getAvatar(name, icon)}
           <span>{name}</span>
         </div>,
         description: toolCallInfo,

package/src/YtChatView/chatWidget/chatWindow/controllers/index.ts

@@ -247,4 +247,13 @@ export async function getFlowInfo(baseUrl: string, flowId: string, api_key: stri
     timeout:600000
   };
   return axios.get(`${baseUrl}/api/t1/flow/detail/${flowId}`,{headers});
+}
+
+export async function getKnowledgeInfo(baseUrl: string, flowId: string, api_key: string) {
+  const headers = {
+    'Content-Type': 'multipart/form-data',
+    'x-api-key': api_key,
+    timeout:600000
+  };
+  return axios.get(`${baseUrl}/api/t1/knowledge/detail/${flowId}`,{headers});
 }

package/src/YtChatView/chatWidget/chatWindow/index.module.css

@@ -1,8 +1,9 @@
 .w_send_file_box {
-  height: 23px;
+  height: 100%;
+  align-items: end;
+  padding-bottom: 16px;
   display: flex;
   background: transparent;
-  align-items: center;
   justify-content: center;
   border-radius: 24px;
   font-size: 12px;

package/src/YtChatView/chatWidget/chatWindow/index.tsx

@@ -164,7 +164,7 @@ export default function ChatWindow({
   let content_ns: string = null
   let event_latest: string = null
   const nowAIContentListRef = useRef(nowAIContentList);
-  const {registerChatMethods} = React.useContext(MethodContext);
+  const {registerChatMethods} = React.useContext(MethodContext) || {};
   let voiceChunks = []; // 临时存储录制的语音片段
   // 滚动事件处理，选择文件时，文件内容超出显示框时，显示左右箭头
   const handleScroll = () => {
@@ -206,7 +206,7 @@ export default function ChatWindow({
     }
   }
   
-  const addMessageItem = ({chunk, id, name, status = null, isThinkChunk = false, loadingMessage = null}) => {
+  const addMessageItem = ({chunk, id, name, icon = null, status = null, isThinkChunk = false, loadingMessage = null}) => {
     // console.log("--- addMessageItem", chunk, status, loadingMessage)
     setNowAIContentList((prevState) => {
       const content = {}
@@ -221,6 +221,7 @@ export default function ChatWindow({
       const messageItem : MessageItem = {
         id,
         name,
+        icon,
         type: MessageType.text,
         toolCallInfo: status,
         loadingMessage,
@@ -251,6 +252,7 @@ export default function ChatWindow({
       const newMessageItem: MessageItem = {
         id: latestMessageItem.id === 'WAIT' ? content_id : latestMessageItem.id,
         name: latestMessageItem.name,
+        icon: latestMessageItem.icon,
         type: latestMessageItem.type,
         toolCallInfo: status,
         loadingMessage,
@@ -259,7 +261,6 @@ export default function ChatWindow({
 
       const newState = [...prevState]
       newState[newState.length - 1] = newMessageItem
-
       nowAIContentListRef.current = newState
       return newState
     })
@@ -273,7 +274,7 @@ export default function ChatWindow({
       getHistoryList();
     }
     else if (event == 'token' || event == 't_full_token') {
-      let { chunk, id, r_id, ns, name, loading_message } = data
+      let { chunk, id, r_id, ns, name, icon, loading_message } = data
       if (chunk.includes('```') && !chunk.startsWith('\n')) {
         // 确保 ``` 前有换行
         chunk = '\n' + chunk;
@@ -290,7 +291,7 @@ export default function ChatWindow({
         // 新建信息
         else{
           content_id = id
-          addMessageItem({chunk, id, name, loadingMessage: loading_message || ''})
+          addMessageItem({chunk, id, name, icon, loadingMessage: loading_message || ''})
         }
       }
       // 输出主体没有变化
@@ -309,7 +310,7 @@ export default function ChatWindow({
         //   if (event_latest === 'token' && event === 't_full_token'){
         //     updateMessageItem({chunk, loadingMessage: loading_message || ''})
         //   } else {
-        //     addMessageItem({chunk, id, name, loadingMessage:loading_message})
+        //     addMessageItem({chunk, id, name, icon, loadingMessage:loading_message})
         //   }
         // }else{
           updateMessageItem({chunk, loadingMessage: loading_message || ''})
@@ -328,7 +329,7 @@ export default function ChatWindow({
         content_ns = ns
         // 新建信息
         content_id = id
-        addMessageItem({chunk, id, name, status:null, isThinkChunk:true, loadingMessage: loading_message || ''})
+        addMessageItem({chunk, id, name, icon, status:null, isThinkChunk:true, loadingMessage: loading_message || ''})
       }else {
         updateMessageItem({chunk, status:null, isThinkChunk:true, loadingMessage: loading_message || ''})
       }
@@ -350,7 +351,7 @@ export default function ChatWindow({
         }else{
           // 这个时候还没有信息的id，所以 content_id 给特殊值
           content_id = "WAIT"
-          addMessageItem({chunk:"", id: "WAIT", name, status})
+          addMessageItem({chunk:"", id: "WAIT", name, icon, status})
         }
       }
       // 当前智能体在调用工具
@@ -633,6 +634,15 @@ export default function ChatWindow({
   // 处理挂断电话
   const handleHangup = () => {
     setShowCallInterface(false);
+
+    fetchChatHistory().then();
+
+    setTimeout(() => {
+      if (lastMessage.current) {
+        console.error('开始滚动最后一条消息到可视区域')
+        lastMessage.current.scrollIntoView({ behavior: 'smooth' });
+      }
+    }, 600);
   };
 
   /**
@@ -957,7 +967,6 @@ export default function ChatWindow({
   }, [sessionId]);
 
   useEffect(() => {
-    console.log("registerChatMethods",registerChatMethods)
     if (registerChatMethods) {
       registerChatMethods({
         isActiveMessage,
@@ -1360,8 +1369,11 @@ export default function ChatWindow({
                 flowId={flowId}
                 sessionId={sessionId}
                 hostUrl={hostUrl}
-                onMessage={addMessage}
                 userInfo={userInfo}
+                addMessage={e => {
+                    // addMessage(e)
+                    setDropDownList([])
+                }}
             />
             :
             <div className="cl-middle-container">

package/src/YtChatView/chatWidget/chatWindow/types/chatWidget/index.ts

@@ -8,6 +8,7 @@ export enum MessageType {
 export type MessageItem = {
   id:string,
   name: string;
+  icon: string;
   thinkMessage: string;
   message: string;
   loadingMessage?: string;

package/src/YtChatView/chatWidget/style.js

@@ -2204,10 +2204,11 @@ export const yt_style = `
 }
 
 .w_send_file_box {
-  height: 23px;
+  height: 100%;
+  align-items: end;
+  padding-bottom: 16px;
   display: flex;
   background: transparent;
-  align-items: center;
   justify-content: center;
   border-radius: 24px;
   font-size: 12px;

package/src/YtChatView/components/TabSelector/index.jsx

@@ -16,19 +16,17 @@ const defaultMap = {
     '就业实习': ['是否提供实习机会？', '就业指导服务如何？', '有哪些合作企业？', '实习是否计入学分？', '就业率有多高？', '毕业生去向有哪些？', '能否推荐实习单位？', '是否有职业规划课？', '能考研继续深造吗？', '就业双选会多不多？'],
 };
 
-const TabSelector = ({ map = defaultMap, customUrl, handleRowClick, welcomeWords, isSimple }) => {
-    const [activeKey, setActiveKey] = useState(Object.keys(map)[0]);
+const TabSelector = ({ customUrl, handleRowClick, welcomeWords, isSimple, dataList = [] }) => {
+    const [activeKey, setActiveKey] = useState(dataList[0]?.category);
     const [valuePage, setValuePage] = useState(0);
     const [tabLocation, setTabLocation] = useState(0);
     const valuesPerPage = 5; // Number of values to show per page
     const [pageLoading, setPageLoading] = useState(false);
     const keyShowSize = isSimple ? 2 : 4;
 
-    const keys = Object.keys(map);
-    const values = map[activeKey] || [];
-
+    const keys = dataList.map(e=>e.category)||[];
+    const values = dataList.filter(e => activeKey === e.category).map(e => e.data)[0] || [];
     const visibleValues = isSimple ? values : values.slice(valuePage * valuesPerPage, (valuePage + 1) * valuesPerPage);
-
     const handleTabChange = (key) => {
         setActiveKey(key);
         setValuePage(0); // Reset value page when changing tabs
@@ -136,22 +134,21 @@ const TabSelector = ({ map = defaultMap, customUrl, handleRowClick, welcomeWords
                         />
                     </div>
                     <div className={styles.valueList}>
-                        {visibleValues.map((item, index) => (
-                            <div className={styles.row} key={index} onClick={() => handleRowClick(item)}>
+                        {visibleValues.map((item, index) => {
+                            return (<div className={styles.row} key={index} onClick={() => handleRowClick(item.q)}>
                                 <Typography.Paragraph
                                     className={styles.text}
                                     style={{ maxWidth: isSimple ? '210px' : '300px' }}
-                                    ellipsis={{ rows: 1, tooltip: `${item}`, }}><span
-                                    style={{ marginRight: '12px' }}>·</span>{item}
+                                    ellipsis={{ rows: 1, tooltip: `${item.q}`, }}>
+                                    {isSimple && <span style={{ marginRight: '12px' }}>·</span>}
+                                    {item.q}
                                 </Typography.Paragraph>
-                                {
-                                    !isSimple && <RightOutlined
-                                        className={styles.rightArrow}
-                                        style={{ float: 'right' }}
-                                    />
-                                }
-                            </div>
-                        ))}
+                                { !isSimple && <RightOutlined
+                                    className={styles.rightArrow}
+                                    style={{ float: 'right' }}
+                                />}
+                            </div>)
+                        })}
                     </div>
                 </div>
             </div>

package/src/YtChatView/previewDialogV2/index.jsx

@@ -1,13 +1,13 @@
 import React from 'react';
 import ChatWidget from '../chatWidget/index';
 import aiAvatarPng from '../../assets/aicenter/aiavatar.png';
-import { getFlowInfo, getHistoryList } from '../chatWidget/chatWindow/controllers/index';
+import { getFlowInfo, getHistoryList, getKnowledgeInfo } from '../chatWidget/chatWindow/controllers/index';
 import './index.module.css'
 import { v4 as uuidv4 } from 'uuid';
 import historyListEmptyPng from '../../assets/aicenter/history-list-empty.png';
 import { isEmpty, isFunction } from "lodash";
-import { Carousel, message, message as messageTip, Progress, Skeleton, Tooltip } from 'antd'
-import Icon, { HistoryOutlined, PlusOutlined } from "@ant-design/icons";
+import { Carousel, message, message as messageTip, Progress, Skeleton, Tooltip, Image } from 'antd'
+import Icon, { EyeOutlined, HistoryOutlined, PlusOutlined } from "@ant-design/icons";
 import TabSelector from "../components/TabSelector";
 
 
@@ -23,6 +23,28 @@ const CommentIcon = (props) => (
     <Icon component={commentSvg} {...props} />
 );
 
+const knowledgeInfo = {
+  media: [
+    {
+      a: "https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/smartSchool/appCreator/school/bcyz/user/ai/banner.png",
+      a_type: "image"
+    },
+    {
+      a: "https://trans-from-yuntu-resourse.oss-cn-beijing.aliyuncs.com/temp/test.mp4",
+      a_type: "video"
+    }
+  ],
+  question: [
+    {category:'学校文化',data: [{q:'学校最有特色的传统是什么？'}, {q:'有哪些值得传承的校训？'}, {q:'学校重视学生个性发展吗？'}, {q:'有什么知名校友吗？'}, {q:'新生入学是否有迎新活动？'}, {q:'学校支持学生创业吗？'}, {q:'学校节日有哪些特色？'}, {q:'学校的学习氛围怎么样？'}, {q:'学生组织有哪些？'}, {q:'是否有艺术或文化类社团？'}]},
+    {category:'校园风景',data: [{q:'校园内有哪些打卡景点？'}, {q:'校园绿化做得好吗？'}, {q:'有湖泊或花园吗？'}, {q:'有特色建筑吗？'}, {q:'校园大不大？适合散步吗？'}, {q:'有哪些适合拍照的地方？'}, {q:'有室外自习空间吗？'}, {q:'校园晚上灯光如何？'}, {q:'有没有露天表演广场？'}, {q:'有樱花或银杏树吗？'}]},
+    {category:'招生流程',data: [{q:'什么时候开始填志愿？'}, {q:'招生简章在哪查看？'}, {q:'是否有预估分数线？'}, {q:'如何获取往年录取线？'}, {q:'能否跨省报考？'}, {q:'有哪些专项招生计划？'}, {q:'报志愿前可参观校园吗？'}, {q:'有无强基计划？'}, {q:'是否有面试环节？'}, {q:'报名后多久公布录取？'}]},
+    {category:'住宿生活',data: [{q:'几人一间宿舍？'}, {q:'宿舍有空调吗？'}, {q:'洗衣机是否自助？'}, {q:'有独立卫生间吗？'}, {q:'能否申请调换宿舍？'}, {q:'有晚归门禁吗？'}, {q:'网络信号好吗？'}, {q:'有宿舍文化活动吗？'}, {q:'宿舍能使用电器吗？'}, {q:'有快递柜或驿站吗？'}]},
+    {category:'餐饮服务',data: [{q:'食堂饭菜贵不贵？'}, {q:'食堂口味多样吗？'}, {q:'有无清真窗口？'}, {q:'学校周边吃饭方便吗？'}, {q:'是否支持刷脸就餐？'}, {q:'饭卡能否线上充值？'}, {q:'有没有饮品店？'}, {q:'食堂有无特色菜？'}, {q:'吃饭时间是否固定？'}, {q:'是否有外卖平台？'}]},
+    {category:'教学资源',data: [{q:'师资力量如何？'}, {q:'有外教课程吗？'}, {q:'是否提供选修课？'}, {q:'教材是否统一购买？'}, {q:'有开放实验室吗？'}, {q:'是否配有学习中心？'}, {q:'有哪些在线课程平台？'}, {q:'图书馆开放时间？'}, {q:'能否跨专业听课？'}, {q:'有无名师公开课？'}]},
+    {category:'就业实习',data: [{q:'是否提供实习机会？'}, {q:'就业指导服务如何？'}, {q:'有哪些合作企业？'}, {q:'实习是否计入学分？'}, {q:'就业率有多高？'}, {q:'毕业生去向有哪些？'}, {q:'能否推荐实习单位？'}, {q:'是否有职业规划课？'}, {q:'能考研继续深造吗？'}, {q:'就业双选会多不多？'}]},
+  ]
+}
+
 const contentStyle = {
   width:'100%',
   color: '#fff',
@@ -299,25 +321,35 @@ const api_key = 'sk-mniUFDpcWwvNF3iFzssXa6etN-S_Y4AApyHYcBU44L0';
 export const MethodContext = React.createContext();
 
 export class ToolDialogV2 extends React.Component {
-
+  sessionUserInfo = {};
   state = {
     historyList: [], // 历史对话列表
     sessionId: uuidv4(), // 当前激活的对话对应的sessionId
     dropDownList: [],
     currentFlow: {},
+    knowledgeInfo: {...knowledgeInfo},
   };
 
+  componentWillMount() {
+    const { userInfo = {} } = this.props;
+    const { sessionId } = this.state;
+    if (isEmpty(userInfo.code)) {
+      const code = sessionId.replaceAll("-", "").slice(2);
+      this.sessionUserInfo.code = code;
+      this.sessionUserInfo.id = code;
+      this.sessionUserInfo.name = code;
+    }else{
+      this.sessionUserInfo = userInfo;
+    }
+  }
+
   async componentDidMount() {
-    this.getHistoryList({},true);
+    this.getHistoryList(true);
   }
 
   getCurrentFlowHistory=(flowId)=>{
-    let { userInfo = {} } = this.props;
     const { sessionId } = this.state;
-    if(isEmpty(userInfo.code)) {
-      userInfo.code = sessionId;
-    }
-    const operationId = userInfo['code'] || '';
+    const operationId = this.sessionUserInfo['code'] || '';
     getHistoryList(this.props.hostUrl, flowId, operationId).then((res) => {
       if(res.status === 200  && typeof res.data !== "string" && typeof res.data !== "string"){
         this.setState({ historyList: res.data });
@@ -343,6 +375,18 @@ export class ToolDialogV2 extends React.Component {
   getFlowInfo = async () => {
     const res = await getFlowInfo(this.props.hostUrl, this.props.appId, api_key);
     if(res.status === 200 && typeof res.data !== "string") {
+      //  查询知识库相关信息用作预置问题
+      // getKnowledgeInfo(this.props.hostUrl, this.props.appId, api_key).then(res => {
+      //   if (res.status === 200 && typeof res.data !== "string") {
+      //     if (!isEmpty(knowledgeInfo)) {
+      //       this.setState({ knowledgeInfo: res.data })
+      //     } else if (!isEmpty(this.props.knowledgeInfo)) {
+      //       this.setState({ knowledgeInfo: this.props.knowledgeInfo })
+      //     }
+      //   }
+      // }).catch(e => {
+      //   console.log(e);
+      // });
       this.setState({ currentFlow: res.data });
     }
     return res.data;
@@ -392,7 +436,8 @@ export class ToolDialogV2 extends React.Component {
   renderCustomDropDown=(dropDownList)=>{
     const { title } = this.props;
     const { currentFlow } = this.state;
-    return <TabSelector handleRowClick={(word)=> {
+    return <TabSelector
+        handleRowClick={(word)=> {
       if(!this.isActiveMessage()){
         this.setState({ dropDownList: [] });
         this.handleSendMessage(word);
@@ -400,7 +445,10 @@ export class ToolDialogV2 extends React.Component {
         message.destroy();
         message.info("请等待回复结束后再发送")
       }
-    }} welcomeWords={`Hi，欢迎使用${currentFlow.name||title}，您可以这样问我:`}/>
+    }}
+        welcomeWords={`Hi，欢迎使用${currentFlow.name||title}，您可以这样问我:`}
+        dataList={this.state.knowledgeInfo?.question || []}
+    />
   }
 
   render() {
@@ -426,7 +474,7 @@ export class ToolDialogV2 extends React.Component {
       modalWidth,
       isShowReadIcon = false,
       signUrl,
-      bannerMap,
+      knowledgeInfo,
     } = this.props;
     const { currentFlow = {} } = this.state;
     return (
@@ -464,7 +512,7 @@ export class ToolDialogV2 extends React.Component {
                       <div className="p_historyDialog">
                         <div className="p_historyTitle">
                           <div className="p_dialogTitle">对话列表</div>
-                          <HistoryOutlined style={{color:'#A4A4A4'}}/>
+                          <Tooltip title={isEmpty(userInfo)?'当前历史记录将会在页面刷新后清除':'历史对话记录'}><HistoryOutlined style={{color:'#A4A4A4'}}/></Tooltip>
                         </div>
                         {this.state.historyList.length > 0 ? (
                             <div className="p_historyList">
@@ -508,7 +556,7 @@ export class ToolDialogV2 extends React.Component {
                       host_url={hostUrl}
                       api_key={api_key}
                       session_id={this.state.sessionId}
-                      userInfo={userInfo}
+                      userInfo={this.sessionUserInfo}
                       getHistoryList={this.getHistoryList}
                       setDropDownList={(list) => this.setState({ dropDownList: list })}
                       dropDownList={this.state.dropDownList}
@@ -527,11 +575,22 @@ export class ToolDialogV2 extends React.Component {
                 <div className={'p_toolRightToRight'}>
                   <Carousel autoplay={{ dotDuration: true }} autoplaySpeed={4000} className={'p_carousel'}>
                     {
-                      Object.values(bannerMap).map(item => (
-                        <div key={item}>
-                          <img style={contentStyle} src={item}/>
-                        </div>
-                      ))
+                      (this.state.knowledgeInfo?.media||[]).map(item => {
+                        if (item.a_type==='image') {
+                          return <div key={item.a}>
+                            <Image
+                                preview={{
+                                  mask: <div><EyeOutlined style={{marginRight:'4px'}}/>预览</div>
+                                }}
+                                style={contentStyle} src={item.a}
+                            />
+                          </div>
+                        }else if(item.a_type==='video'){
+                          return <div key={item.a}>
+                            <video style={contentStyle} src={item.a} controls={true}/>
+                          </div>
+                        }
+                      })
                     }
                   </Carousel>
                   <div style={{ height: '1px', background: '#E5E5E5',marginTop: '25px' }}/>
@@ -544,6 +603,7 @@ export class ToolDialogV2 extends React.Component {
                         handleRowClick={(word) => {
                           this.handleClick(word)
                         }}
+                        dataList={this.state.knowledgeInfo?.question || []}
                     />
                 </div>
               </div>