react_hsbc_teller 2.0.25 → 2.0.27

package/lib/hsbc.js.LICENSE.txt

@@ -4,6 +4,17 @@
   http://jedwatson.github.io/classnames
 */
 
+/*!
+ * 
+ * js-audio-recorder - js audio recorder plugin
+ * 
+ * @version v0.5.7
+ * @homepage https://github.com/2fps/recorder
+ * @author 2fps <echoweb@126.com> (https://www.zhuyuntao.cn)
+ * @license MIT
+ *         
+ */
+
 /*!
  * Determine if an object is a Buffer
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react_hsbc_teller",
-  "version": "2.0.25",
+  "version": "2.0.27",
   "description": "React",
   "private": false,
   "main": "lib/hsbc.js",
@@ -89,6 +89,7 @@
     "axios": "0.19.0",
     "crypto-js": "^4.1.1",
     "html2canvas": "^1.4.0",
+    "js-audio-recorder": "^0.5.7",
     "qrcode.react": "^3.1.0",
     "react": "^17.0.1",
     "react-canvas-draw": "^1.1.1",

package/packages/demo/demo.js

@@ -192,6 +192,10 @@ userExit =(val)=>{
             
           })
     }
+    componentDidCatch(error, info) {
+        console.log('componentDidCatch')
+        console.error(error, info)
+    }
   render() {
     return (
       <div className="test">

package/packages/pages/video/video.jsx

@@ -7,7 +7,7 @@ import API from '../../api/api';
 import { BoardOperate } from '../../common/index.esm.js';
 import './video.less'
 import { compressImage, isLight } from '../../utils/utils'
-import Recorder from "../../utils/recorder";
+import Recorder from "js-audio-recorder";
 import MyAsrController from '../../utils/asrController'
 import Header from '../header/header.jsx'
 import Foot from '../foot/foot.jsx'
@@ -58,7 +58,7 @@ import MultiModule from '../multiModule/multiModule'//签字、抄录
 import axios from 'axios';
 import CryptoJS from "crypto-js";
 
-const SDK_VERISON = '2.0.25'
+const SDK_VERISON = '2.0.27'
 const IDtypeFront = '请客户在其设备后置摄像头下展示证件正面（如：身份证照片页面）'
 const IDtypeBack = '请在后置摄像头下展示证件反面（如：身份证国徽页）'
 const { Option } = Select;
@@ -93,6 +93,7 @@ let muteJson = new Map()
 // let dateTime = 0
 let messageValue = ''
 let messageValueMap = new Map()
+let videoMessageMap = new Map() // 画中画消息数据
 let third_id_Map = new Map()
 // let beautyType = false
 // let beautyNum = 0  // 0-关闭  1--弱  2--中 5---高
@@ -138,6 +139,7 @@ function clearStreamRemain() {
     beautyDiv.beauty_canvas = null;
   }
   if (!(beautyDiv.beauty_player == null || beautyDiv.beauty_player == undefined)) {
+    console.log('beautyDiv destroy')
     beautyDiv.beauty_player.destroy();
     beautyDiv.beauty_player = null;
   }
@@ -590,6 +592,8 @@ class Video extends Component {
     this.test_controller.InitRoomConfig(config_param)
     this.saveLog('mrtc InitRoomConfig start')
     // 预热摄像头
+    config_param.videoSource = this.state.cameraValue
+    config_param.audioSource = this.state.microphoneValue
     this.test_controller.PreOpenLocalMedia(config_param)
   };
   rateAll = async () => {
@@ -609,15 +613,16 @@ class Video extends Component {
   // 开启录制
   enableServerRecording = (record_business_id) => {
     const that = this
+    const SCALE = 1.5
     this.rateAll().then((res) => {
       console.log('退出房间者', res)
       let list = [
         {
           tag: 'projectionWhiteboard',
           xPosition: 0,
-          yPosition: 175,
-          width: 960,
-          height: 540
+          yPosition: 175 * SCALE,
+          width: 1280,
+          height: 720
         },
         // {
         //   tag: 'VIDEO_SOURCE_SCREEN_ONE',
@@ -637,50 +642,50 @@ class Video extends Component {
           tag: 'tag1',
           xPosition: 0,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         {
           tag: 'customer1',
-          xPosition: 213,
+          xPosition: 213 * SCALE,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         {
           tag: 'customer2',
-          xPosition: 426,
+          xPosition: 426 * SCALE,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         {
           tag: 'customer3',
-          xPosition: 639,
+          xPosition: 639 * SCALE,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         {
           tag: 'customer4',
-          xPosition: 852,
+          xPosition: 852 * SCALE,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         {
           tag: 'customer5',
-          xPosition: 1065,
+          xPosition: 1065 * SCALE,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         {
           tag: 'customer6',
-          xPosition: 1278,
+          xPosition: 1278 * SCALE,
           yPosition: 0,
-          width: 213,
-          height: 175
+          width: 213 * SCALE,
+          height: 175 * SCALE
         },
         // {
         //   tag: 'sharedScreen', // 一期暂时不需要录制
@@ -694,18 +699,18 @@ class Video extends Component {
         res.map((item) => {
           list.push({
             tag: item.tag,
-            xPosition: item.xaxis,
-            yPosition: item.yaxis,
-            width: item.width,
-            height: item.height
+            xPosition: (item.xaxis * SCALE),
+            yPosition: (item.yaxis * SCALE),
+            width: (item.width * SCALE),
+            height: (item.height * SCALE)
           })
         }) :
         list.push({
           tag: 'VIDEO_SOURCE_SCREEN',
-          xPosition: 960,
-          yPosition: 535,
-          width: 320,
-          height: 180
+          xPosition: 960 * SCALE,
+          yPosition: 535 * SCALE,
+          width: 320 * SCALE,
+          height: 180 * SCALE
         })
       // list.push({
       //   tag: 'VIDEO_SOURCE_SCREEN_19_9',
@@ -716,8 +721,8 @@ class Video extends Component {
       // })
       const filePath = 'recordId_' + new Date().valueOf();
       const recordParam = {};
-      recordParam.width = 1280;
-      recordParam.height = 720;
+      recordParam.width = 1280 * SCALE;
+      recordParam.height = 720 * SCALE;
       recordParam.recordTotalStream = 0;
       recordParam.startTimeout = 10;
       recordParam.splitType = 0;
@@ -729,10 +734,10 @@ class Video extends Component {
           type: 2, // 1 为时间戳⽔印；2 为⽂字⽔印；3 为图⽚⽔印
           id: 2, // ⽔印 ID
           enable: true,
-          xPosition: 640, // x 轴位置
-          yPosition: 10, // y 轴位置
+          xPosition: 640 * SCALE, // x 轴位置
+          yPosition: 10 * SCALE, // y 轴位置
           text: `${this.props.recordMode ==1 ?'远程录制' :'网点录制'} ${this.props.salesBranchCode || ''} ${this.state.branchName || ''} ${this.props.financialOffice || ''}`,
-          fontSize: 16, // 字体⼤⼩
+          fontSize: 16 * SCALE, // 字体⼤⼩
           url: '' // ⽔印图⽚ HTTP 地址
         },
         {
@@ -740,10 +745,10 @@ class Video extends Component {
           type: 1, // 1 为时间戳⽔印；2 为⽂字⽔印；3 为图⽚⽔印
           id: 1, // ⽔印 ID
           enable: true,
-          xPosition: 1080, // x 轴位置
-          yPosition: 10, // y 轴位置
+          xPosition: 1080 * SCALE, // x 轴位置
+          yPosition: 10 * SCALE, // y 轴位置
           text: '', // ⽔印⽂字
-          fontSize: 16, // 字体⼤⼩
+          fontSize: 16 * SCALE, // 字体⼤⼩
           url: '' // ⽔印图⽚ HTTP 地址
         },
       ];
@@ -1002,13 +1007,17 @@ class Video extends Component {
       this.messageClick('当前暂无客户', 'error')
     }
   }
-  messageClick = (value, tipType) => {
+  messageClick = (value, tipType, userId) => {
     console.log('messageClick', value, tipType)
+
+    if (userId) {
+      // 画中画实时消息
+      this.addPictureMessage(value, userId)
+    }
     const intervalSec = 5 // 相同提示触发间隔
     const durationSec = 4.5 // 提示持续时间
     if (messageValueMap.get(value) == undefined) {
       // map中没有相同消息
-      messageValue = value
       messageValueMap.set(value, setTimeout(() => {
         messageValueMap.delete(value) 
       }, intervalSec * 1000))
@@ -1030,12 +1039,45 @@ class Video extends Component {
           duration: durationSec
         })
       }
-      clearTimeout(this.messageClearTimer)
-      this.messageClearTimer = setTimeout(() => {
-        messageValue = ''
-      }, durationSec * 1000);
+      if (!userId) {
+        // 坐席消息
+        messageValue = value
+
+        clearTimeout(this.messageClearTimer)
+        this.messageClearTimer = setTimeout(() => {
+          messageValue = ''
+        }, durationSec * 1000);
+      } 
+    }
+  }
+  // 画中画内消息
+  addPictureMessage = (value, userId) => {
+    console.log('addPictureMessage', value, userId)
+    
+    const intervalSec = 5 // 相同提示触发间隔
+    const durationSec = 4.5 // 提示持续时间
+    let mapItem = videoMessageMap.get(userId)
+    if (!mapItem) {
+      mapItem = {
+        messageValue: value,
+        clearTimer: null,
+        valueMap: new Map()
+      }
     }
+    const valueMap = mapItem.valueMap
+    if (valueMap.get(value) == undefined) {
+      // map中没有相同消息
+      valueMap.set(value, setTimeout(() => {
+        valueMap.delete(value) 
+      }, intervalSec * 1000))
 
+      mapItem.messageValue = value
+      clearTimeout(mapItem.clearTimer)
+      mapItem.clearTimer = setTimeout(() => {
+        mapItem.messageValue = ''
+      }, durationSec * 1000)
+      videoMessageMap.set(userId, mapItem)
+    }
   }
   // 人脸识别
   facialRecognition = () => {
@@ -1201,7 +1243,8 @@ class Video extends Component {
             name: 'video' + sortedlist[i-1].idIndex,
             title: sortedlist[i-1].videoName,
             mute: sortedlist[i-1].mute,
-            noVideo: sortedlist[i-1].noVideo
+            noVideo: sortedlist[i-1].noVideo,
+            userId: sortedlist[i-1].userId,
           })
         }
     }
@@ -1243,6 +1286,10 @@ class Video extends Component {
       cobj.clearRect(0, 0, 640, 360 * this.state.listVideoPicture.length + baseStartHeight);
       canvas.width = 640;
       canvas.height = 360 * this.state.listVideoPicture.length + baseStartHeight;
+      if (this.state.listVideoPicture.length == 0) {
+        // 画中画设置最小高度，避免没有人时 宽度被拉得很大
+        canvas.height = 360 + baseStartHeight
+      }
       for (let i = 0; i < this.state.listVideoPicture.length; i++) {
 
         let videoId = this.state.listVideoPicture[i].name;
@@ -1378,6 +1425,23 @@ class Video extends Component {
 
         }
 
+        // 绘制消息
+        if (videoMessageMap.get(this.state.listVideoPicture[i].userId)) {
+          let str = videoMessageMap.get(this.state.listVideoPicture[i].userId).messageValue
+          if (str) {
+            cobj.fillStyle = '#F8F2F3';
+            cobj.font = "normal lighter 30px sans-serif";
+            // const textWidth = Math.min(cobj.measureText(str).width, 610)
+            const startY = 360 * i + baseStartHeight
+            cobj.fillRect(0, startY, 640, messageBoxHeight)
+            cobj.textAlign = 'center';
+            cobj.strokeStyle = 'black';
+            cobj.lineWidth = 2
+            cobj.strokeText(str, 320, startY + messageBoxHeight - 15);
+            cobj.stroke();
+          }
+        }
+
       }
       cobj.fillStyle = '#333333';
       cobj.fillRect(0, 0, 640, baseStartHeight)
@@ -1570,7 +1634,7 @@ class Video extends Component {
           }, ()=>{
             this.pictureInPicture('Refresh')
           })
-          this.messageClick('客户切换其他软件', 'error')
+          this.messageClick('客户切换其他软件', 'error', Mival.data.userId)
         }
       }
       else if (Mival.typeId == 3003) { // app进入前台
@@ -1592,7 +1656,7 @@ class Video extends Component {
         if (Mival.data && Mival.data.userId) {
           TITLE = this.getUserTitle(Mival.data.userId)
         }
-        this.messageClick(TITLE+'人脸已离框', 'error')
+        this.messageClick(TITLE+'人脸已离框', 'error', Mival.data.userId)
         // this.saveVideoPoint('ffd', '客户人脸离框');
       }
       else if (Mival.typeId == 5002) { // 客户端背光
@@ -1600,7 +1664,7 @@ class Video extends Component {
         if (Mival.data && Mival.data.userId) {
           TITLE = this.getUserTitle(Mival.data.userId)
         }
-        this.messageClick(TITLE+'背光、曝光过度', 'error')
+        this.messageClick(TITLE+'背光、曝光过度', 'error', Mival.data.userId)
         // this.saveVideoPoint('env', '客户端背光、曝光过度');
       }
       else if (Mival.typeId == 5003) { // 客户端弱网
@@ -1608,11 +1672,11 @@ class Video extends Component {
         if (Mival.data && Mival.data.userId) {
           TITLE = this.getUserTitle(Mival.data.userId)
         }
-        this.messageClick(TITLE+'当前网络较弱', 'error')
+        this.messageClick(TITLE+'当前网络较弱', 'error', Mival.data.userId)
         // this.saveVideoPoint('wnd', '客户端弱网');
       }
       else if (Mival.typeId == 5004) { // ipad低电量
-        this.messageClick('IPAD电量过低', 'error')
+        this.messageClick('IPAD电量过低', 'error', Mival.data.userId)
         
         this.setState({
           ipadLowPowerErrorModalVisible: true,
@@ -1628,6 +1692,13 @@ class Video extends Component {
         }
         setTimeout(this.ipadLowPowerPlay, 0);
       }
+      else if (Mival.typeId == 5005) { // 客户端人脸数量变化
+        let TITLE = '客户'
+        if (Mival.data && Mival.data.userId) {
+          TITLE = this.getUserTitle(Mival.data.userId)
+        }
+        this.messageClick(TITLE+'人脸数量发生变化', 'error', Mival.data.userId)
+      }
       else if (Mival.typeId == 1220) {
         // 一炒多的图片  1214
         if (Mival.sessionId == this.state.sessionId) {
@@ -1912,7 +1983,7 @@ class Video extends Component {
       img.onload = () => {
         const origin = cv.imread(img)
         // let start = new Date().getTime()
-        const res = isLight(origin)
+        const res = isLight(origin, this.props.lightSensitivity)
         if (res==1 || res== -1){
           // 背景曝光过度
           this.messageClick('检测到当前背光、曝光过度', 'error')
@@ -1937,7 +2008,7 @@ class Video extends Component {
         img.onload = () => {
           const origin = cv.imread(img)
           // let start = new Date().getTime()
-          const res = isLight(origin)
+          const res = isLight(origin, this.props.lightSensitivity)
           if (res==1 || res==-1){
             // 背景曝光过度
             this.state.envInfo.lightResult = '不合格'
@@ -2708,6 +2779,7 @@ class Video extends Component {
 
       }
       if (sid == document.getElementById('publish_streamId1').name || !document.getElementById('publish_streamId1').name) {
+        console.log('OnUnPublishSucc clearStreamRemain!!!')
         clearStreamRemain()
       }
     };
@@ -3496,10 +3568,15 @@ class Video extends Component {
       }
     }
   }
+  componentDidCatch(error, info) {
+    console.log('componentDidCatch')
+    console.error(error, info)
+  }
+
   componentWillUnmount() {
     this.saveLog('Exit meeting')
     if (this.test_controller && this.state.sessionType) {
-      clearStreamRemain()
+      // clearStreamRemain() // 谨慎使用，有页面crash问题
       this.test_controller.LeaveRoom()
       this.test_controller.Disconnect()
       try {
@@ -3798,7 +3875,7 @@ class Video extends Component {
     })
     this.asr_controller = new MyAsrController({
       token: this.state.asrToken,
-      appkey: this.state.asrAppKey,
+      appKey: this.state.asrAppKey,
       url: this.state.asrServerUrl,
       recorder: this.state.recorder
     })
@@ -5627,7 +5704,7 @@ class Video extends Component {
     this.setState({
       isModalVisibleEnvironment: false,
     })
-    if (!this.state.isPictureInPicture && this.props.recordMode != 2) {
+    if (!this.state.isPictureInPicture) {
       this.pictureInPicture()
     }
   }
@@ -7050,6 +7127,7 @@ Video.defaultProps = {
   whetherNeedAsr: false, // 是否在双录时开启违禁词检测
   faceDetectInterval: 6, // 人脸检测间隔（秒）
   lightDetectInterval: 6, // 背光检测间隔（秒）
+  lightSensitivity: 1, // 光线敏感度系数，默认1
   userSide: 2,
   meetingDuration: null, // 会议时长，单位小时
   recordMode: 1, // 录制模式 1远程录制 2网点录制

package/packages/utils/asrController.js

@@ -2,7 +2,6 @@
 // import Recorder from "js-audio-recorder"; 
 // import Recorder from "./recorder"; 
 
-import header from "../pages/header/header";
 
 // console.log(Recorder)
 //生成32位随机数UUID
@@ -38,8 +37,9 @@ let websocket_audio2txt_complete_time_end = 100;  //websocket 语音转文本
 
 export default class MyAsrController {
   constructor(headers) {
+    console.log('MyAsrController', headers, headers.appKey)
     this.token = headers.token || 'default';
-    this.appKey = headers.appKey || 'E6DAPlpMun5L2dJ7';
+    this.appKey = headers.appKey;
     this.url = headers.url;
 
     //初始化录音实例
@@ -98,7 +98,11 @@ export default class MyAsrController {
   }
   
   Disconnect() {
-    this.recorderInstance.stop();
+    try { 
+      this.recorderInstance.stop();
+    } catch(err){
+      console.log(err)
+    }
     clearInterval(this.interval);
     this.websocketStopTranscription()
   }
@@ -213,7 +217,7 @@ export default class MyAsrController {
 getPCMAndSend() {
   //获取音频信息
   let NextData = this.recorderInstance.getNextData();
-  let blob = new Blob([NextData])
+  let blob = new Blob(NextData)
   let blob_size = blob.size;
   // console.log("获取音频信息，并发送,blob_size:"+blob_size , blob);
 

package/packages/utils/utils.js

@@ -49,7 +49,7 @@ function debounce(fn, delay = 500) {
      },delay)
  }
 }
-function isLight(src) {
+function isLight(src, sensitivity = 1) {
   let gray = new cv.Mat() 
   cv.cvtColor(src, gray, cv.COLOR_BGR2GRAY)
   let sum = 0;
@@ -84,8 +84,16 @@ function isLight(src) {
   gray.delete();
 
   console.log('亮度值:', cast)
+  if (typeof sensitivity == 'number') {
+    console.log('灵敏系数:', sensitivity)
+    console.log('计算值(<1正常):', cast * sensitivity)
+  } else {
+    console.error('sensitivity param should be type of number')
+    sensitivity = 1
+  }
+
   //std::cout <<  "亮度异常值:" << cast << std::endl;
-  if (cast > 1)
+  if (cast * sensitivity > 1)
   {
       if (avg > 0)
       {