ttp-agent-sdk 2.33.5 → 2.34.3

package/dist/audio-processor.js

@@ -1 +1,321 @@
-class AudioProcessor extends AudioWorkletProcessor{constructor(e){super(),this.config=e.processorOptions||{},this.sampleRate="undefined"!=typeof sampleRate?sampleRate:this.config.sampleRate||this.config.outputSampleRate||44100,this.bufferSize=128,this.buffer=new Float32Array(this.bufferSize),this.bufferIndex=0,this.silenceThreshold=.02,this.VOICE_FRAMES_REQUIRED=2,this.minVoiceDuration=100,this.pauseThreshold=3e3,this.isVoiceActive=!1,this.voiceStartTime=0,this.lastVoiceTime=0,this.consecutiveSilenceFrames=0,this.silenceFramesThreshold=5,this.voiceFrameCount=0,this.frameCount=0,this.lastLogTime=0,this.continuousMode=!0,this.forceContinuous=!0,this.isCurrentlyStreaming=!1,this.sendBuffer=null,this.sendBufferBytes=0,this.port.onmessage=e=>{const{type:t,data:s}=e.data;switch(t){case"start":this.isProcessing=!0,this.isCurrentlyStreaming=!0;break;case"stop":this.isProcessing=!1,this.isCurrentlyStreaming=!1,this.isVoiceActive=!1,this.forceContinuous=!1,this.voiceFrameCount=0,this.flushBuffer();break;case"setForceContinuous":this.forceContinuous=s.enabled,this.isProcessing=!0,this.isCurrentlyStreaming=!1;break;case"flush":this.flushBuffer();break;case"config":Object.assign(this.config,s)}}}process(e,t,s){const i=e[0],o=t[0];return i.length>0&&o.length>0&&o[0].set(i[0]),i.length>0&&i[0].length>0&&this.processAudioData(i[0]),!0}processAudioData(e){this.frameCount++;for(let t=0;t<e.length;t+=this.bufferSize){const s=Math.min(this.bufferSize,e.length-t);for(let i=0;i<s;i++)this.buffer[i]=e[t+i];for(let e=s;e<this.bufferSize;e++)this.buffer[e]=0;let i=0;for(let e=0;e<this.bufferSize;e++)i+=this.buffer[e]*this.buffer[e];const o=Math.sqrt(i/this.bufferSize);let r=0,n=0;for(let e=1;e<this.bufferSize;e++){const t=Math.abs(this.buffer[e]-this.buffer[e-1]);r+=t,t>.1&&n++}r/=this.bufferSize,this.bufferSize;const h=Date.now();let f=o>this.silenceThreshold;const a=h-this.lastVoiceTime;if(f)this.consecutiveSilenceFrames=0,this.voiceFrameCount++,this.voiceFrameCount>=this.VOICE_FRAMES_REQUIRED&&(this.isVoiceActive||(this.isVoiceActive=!0,this.voiceStartTime=h,this.isCurrentlyStreaming=!0,this.frameCount%50==0&&console.log(`🎤 VAD: VOICE DETECTED (RMS: ${o.toFixed(4)}, frames: ${this.voiceFrameCount})`))),this.lastVoiceTime=h;else{this.voiceFrameCount=0,this.consecutiveSilenceFrames++;const e=this.forceContinuous?3e3:200;!f&&(this.isVoiceActive||this.isCurrentlyStreaming)&&a>=e&&(this.isVoiceActive=!1,this.isCurrentlyStreaming=!1,this.voiceStartTime=0,this.lastVoiceTime=0,this.consecutiveSilenceFrames=0,this.voiceFrameCount=0,console.log(`🔇 VAD: SILENCE DETECTED (${a}ms silence, RMS: ${o.toFixed(4)})`))}this.isCurrentlyStreaming&&this.isProcessing?(this.frameCount%200==0&&console.log(`📤 VAD: Sending audio (isVoiceActive: ${this.isVoiceActive}, RMS: ${o.toFixed(4)})`),this.sendPCMAudioData(this.buffer)):this.frameCount%200==0&&this.isProcessing&&console.log(`🚫 VAD: Blocking audio (isCurrentlyStreaming: ${this.isCurrentlyStreaming}, RMS: ${o.toFixed(4)})`)}}sendPCMAudioData(e){const t=new Int16Array(e.length);for(let s=0;s<e.length;s++){const i=Math.max(-1,Math.min(1,e[s]));t[s]=Math.round(32767*i)}for(this.sendBuffer||(this.sendBuffer=[],this.sendBufferBytes=0),this.sendBuffer.push(t),this.sendBufferBytes+=t.byteLength;this.sendBufferBytes>=4096;){let e=0,t=0;for(let s=0;s<this.sendBuffer.length;s++){const i=this.sendBuffer[s].byteLength;if(!(t+i<=4096))break;e++,t+=i}const s=this.sendBuffer.slice(0,e),i=s.reduce((e,t)=>e+t.length,0),o=new Int16Array(i);let r=0;for(const e of s)o.set(e,r),r+=e.length;this.port.postMessage({type:"pcm_audio_data",data:o,sampleRate:this.sampleRate,channelCount:1,frameCount:this.frameCount,batchSize:e,totalBytes:o.byteLength}),this.sendBuffer=this.sendBuffer.slice(e),this.sendBufferBytes-=t}}flushBuffer(){if(this.sendBuffer&&this.sendBuffer.length>0){const e=this.sendBuffer.reduce((e,t)=>e+t.length,0),t=new Int16Array(e);let s=0;for(const e of this.sendBuffer)t.set(e,s),s+=e.length;this.port.postMessage({type:"pcm_audio_data",data:t,sampleRate:this.sampleRate,channelCount:1,frameCount:this.frameCount,batchSize:this.sendBuffer.length,totalBytes:t.byteLength,isFlush:!0}),this.sendBuffer=[],this.sendBufferBytes=0}}}registerProcessor("audio-processor",AudioProcessor);
+/**
+ * AudioProcessor - AudioWorklet for real-time audio processing
+ * 
+ * This AudioWorklet processes audio data in real-time and sends it to the main thread
+ * for transmission to the WebSocket server.
+ */
+
+class AudioProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    
+    // Configuration
+    this.config = options.processorOptions || {};
+    // Use AudioContext sampleRate (available as global 'sampleRate' in AudioWorkletProcessor)
+    // Fall back to config if sampleRate not available, default to 44100 Hz to match server output
+    this.sampleRate = typeof sampleRate !== 'undefined' ? sampleRate : (this.config.sampleRate || this.config.outputSampleRate || 44100);
+    this.bufferSize = 128; // Process 128 samples at a time (256 bytes = 8ms at 16kHz)
+    this.buffer = new Float32Array(this.bufferSize);
+    this.bufferIndex = 0;
+    
+    // VAD (Voice Activity Detection) parameters
+    this.silenceThreshold = 0.02; // RMS threshold for voice detection (increased to reduce ambient noise sensitivity)
+    this.VOICE_FRAMES_REQUIRED = 2; // Require 2 consecutive frames above threshold before activating
+    this.minVoiceDuration = 100; // ms - minimum speech duration
+    this.pauseThreshold = 3000; // ms - longer pause before processing
+    
+    // VAD state
+    this.isVoiceActive = false;
+    this.voiceStartTime = 0;
+    this.lastVoiceTime = 0;
+    this.consecutiveSilenceFrames = 0;
+    this.silenceFramesThreshold = 5; // More frames needed for silence detection
+    this.voiceFrameCount = 0; // Count consecutive frames above threshold
+    
+    // Audio quality tracking
+    this.frameCount = 0;
+    this.lastLogTime = 0;
+    
+    // Continuous recording mode
+    this.continuousMode = true; // Always send audio when voice is detected
+    this.forceContinuous = true; // Force continuous for toggle button behavior
+    this.isCurrentlyStreaming = false; // Track if we're currently sending audio
+    
+    // Batching buffer
+    this.sendBuffer = null;
+    this.sendBufferBytes = 0;
+    
+    // Handle messages from main thread
+    this.port.onmessage = (event) => {
+      const { type, data } = event.data;
+      
+      switch (type) {
+        case 'start':
+          this.isProcessing = true;
+          this.isCurrentlyStreaming = true;
+          break;
+          
+        case 'stop':
+          this.isProcessing = false;
+          this.isCurrentlyStreaming = false;
+          this.isVoiceActive = false;
+          this.forceContinuous = false;
+          this.voiceFrameCount = 0; // Reset voice frame count
+          // Flush any remaining data
+          this.flushBuffer();
+          break;
+          
+        case 'setForceContinuous':
+          this.forceContinuous = data.enabled;
+          this.isProcessing = true;
+          // FIXED: Don't set isCurrentlyStreaming = true here - let VAD control it
+          // VAD will set isCurrentlyStreaming = true when voice is detected
+          this.isCurrentlyStreaming = false;
+          break;
+          
+        case 'flush':
+          this.flushBuffer();
+          break;
+          
+        case 'config':
+          Object.assign(this.config, data);
+          break;
+      }
+    };
+  }
+  
+  /**
+   * Process audio data
+   */
+  process(inputs, outputs, parameters) {
+    // CRITICAL: If processing is stopped, terminate the processor
+    if (!this.isProcessing) {
+      // Return false to terminate the AudioWorklet processor
+      // This stops all VAD processing immediately
+      return false;
+    }
+    
+    const input = inputs[0];
+    const output = outputs[0];
+    
+    // Copy input to output (pass-through)
+    if (input.length > 0 && output.length > 0) {
+      output[0].set(input[0]);
+    }
+    
+    // Process audio for PCM recording and VAD
+    if (input.length > 0 && input[0].length > 0) {
+      this.processAudioData(input[0]);
+    }
+    
+    // Keep the processor alive
+    return true;
+  }
+  
+  processAudioData(audioData) {
+    // CRITICAL: Early return if processing is stopped
+    // This prevents VAD calculations and logging when stopped
+    if (!this.isProcessing) {
+      return;
+    }
+    
+    this.frameCount++;
+    
+    // Process audio in consistent 128-sample chunks (256 bytes)
+    for (let i = 0; i < audioData.length; i += this.bufferSize) {
+      const chunkSize = Math.min(this.bufferSize, audioData.length - i);
+      
+      // Copy chunk to buffer
+      for (let j = 0; j < chunkSize; j++) {
+        this.buffer[j] = audioData[i + j];
+      }
+      
+      // Pad with zeros if needed
+      for (let j = chunkSize; j < this.bufferSize; j++) {
+        this.buffer[j] = 0;
+      }
+      
+      // Calculate RMS for VAD on this chunk
+      let sum = 0;
+      for (let j = 0; j < this.bufferSize; j++) {
+        sum += this.buffer[j] * this.buffer[j];
+      }
+      const rms = Math.sqrt(sum / this.bufferSize);
+      
+      // Calculate additional features for better VAD
+      let variation = 0;
+      let highFreqCount = 0;
+      for (let j = 1; j < this.bufferSize; j++) {
+        const diff = Math.abs(this.buffer[j] - this.buffer[j-1]);
+        variation += diff;
+        if (diff > 0.1) highFreqCount++;
+      }
+      variation = variation / this.bufferSize;
+      const highFreqRatio = highFreqCount / this.bufferSize;
+      
+      const currentTime = Date.now();
+      
+      // VAD with reduced sensitivity - require consecutive frames above threshold
+      let hasVoice = rms > this.silenceThreshold;
+      // Calculate time since last voice detection
+      const timeSinceLastVoice = currentTime - this.lastVoiceTime;
+
+      // Voice detection logic - require consecutive frames above threshold
+      if (hasVoice) {
+        this.consecutiveSilenceFrames = 0;
+        this.voiceFrameCount++; // Increment consecutive voice frame count
+
+        // Only activate streaming after required consecutive frames
+        if (this.voiceFrameCount >= this.VOICE_FRAMES_REQUIRED) {
+          // Start voice if needed
+          if (!this.isVoiceActive) {
+            this.isVoiceActive = true;
+            this.voiceStartTime = currentTime;
+            this.isCurrentlyStreaming = true;
+            // Log voice detection (every 50 frames = ~400ms to avoid spam)
+            if (this.frameCount % 50 === 0) {
+              console.log(`🎤 VAD: VOICE DETECTED (RMS: ${rms.toFixed(4)}, frames: ${this.voiceFrameCount})`);
+            }
+          }
+        }
+
+        this.lastVoiceTime = currentTime;
+      } else {
+        // Silence detected - reset voice frame count
+        this.voiceFrameCount = 0;
+        this.consecutiveSilenceFrames++;
+        
+        // In continuous mode, we still use VAD but require longer silence before stopping
+        // In non-continuous mode, stop quickly
+        const silenceThreshold = this.forceContinuous ? 3000 : 200; // 3s for continuous, 200ms otherwise
+        
+        // FIXED: Stop condition - also check isCurrentlyStreaming (not just isVoiceActive)
+        // This handles case where setForceContinuous set streaming=true but no voice was detected yet
+        if (!hasVoice && (this.isVoiceActive || this.isCurrentlyStreaming) && timeSinceLastVoice >= silenceThreshold) {
+          this.isVoiceActive = false;
+          this.isCurrentlyStreaming = false;
+          this.voiceStartTime = 0;
+          this.lastVoiceTime = 0;
+          this.consecutiveSilenceFrames = 0;
+          this.voiceFrameCount = 0; // Reset voice frame count
+          // Log silence detection
+          console.log(`🔇 VAD: SILENCE DETECTED (${timeSinceLastVoice}ms silence, RMS: ${rms.toFixed(4)})`);
+        }
+      }
+
+      // Send PCM **only if streaming and processing** - hard gate
+      // This ensures we only send audio when voice is detected, even in continuous mode
+      if (this.isCurrentlyStreaming && this.isProcessing) {
+        // Log occasionally when sending (every 200 frames = ~1.6 seconds to avoid spam)
+        if (this.frameCount % 200 === 0) {
+          console.log(`📤 VAD: Sending audio (isVoiceActive: ${this.isVoiceActive}, RMS: ${rms.toFixed(4)})`);
+        }
+        this.sendPCMAudioData(this.buffer);
+      } else {
+        // Log occasionally when blocking (every 200 frames)
+        if (this.frameCount % 200 === 0 && this.isProcessing) {
+          console.log(`🚫 VAD: Blocking audio (isCurrentlyStreaming: ${this.isCurrentlyStreaming}, RMS: ${rms.toFixed(4)})`);
+        }
+      }
+    }
+  }
+  
+  sendPCMAudioData(float32Data) {
+    // Convert Float32Array (-1.0 to 1.0) to Int16Array (-32768 to 32767)
+    const pcmData = new Int16Array(float32Data.length);
+    
+    for (let i = 0; i < float32Data.length; i++) {
+      // Clamp and convert to 16-bit PCM
+      const sample = Math.max(-1.0, Math.min(1.0, float32Data[i]));
+      pcmData[i] = Math.round(sample * 32767);
+    }
+    
+    // Initialize send buffer if not exists
+    if (!this.sendBuffer) {
+      this.sendBuffer = [];
+      this.sendBufferBytes = 0;
+    }
+    
+    // Accumulate chunks in buffer
+    this.sendBuffer.push(pcmData);
+    this.sendBufferBytes += pcmData.byteLength;
+    
+    // Send in ~4 KB batches (≈128 ms of audio at 16kHz)
+    // Use sliding window approach to maintain continuous flow
+    while (this.sendBufferBytes >= 4096) {
+      // Calculate how many chunks we need for ~4KB
+      let chunksToSend = 0;
+      let bytesToSend = 0;
+      
+      for (let i = 0; i < this.sendBuffer.length; i++) {
+        const chunkBytes = this.sendBuffer[i].byteLength;
+        if (bytesToSend + chunkBytes <= 4096) {
+          chunksToSend++;
+          bytesToSend += chunkBytes;
+        } else {
+          break;
+        }
+      }
+
+      // Create merged buffer from selected chunks
+      const chunksForBatch = this.sendBuffer.slice(0, chunksToSend);
+      const totalSamples = chunksForBatch.reduce((a, b) => a + b.length, 0);
+      const merged = new Int16Array(totalSamples);
+      let offset = 0;
+      
+      for (const chunk of chunksForBatch) {
+        merged.set(chunk, offset);
+        offset += chunk.length;
+      }
+      
+      // Send batched PCM data to main thread
+      this.port.postMessage({
+        type: 'pcm_audio_data',
+        data: merged, // Send the Int16Array directly, not the buffer
+        sampleRate: this.sampleRate,
+        channelCount: 1,
+        frameCount: this.frameCount,
+        batchSize: chunksToSend,
+        totalBytes: merged.byteLength
+      });
+      
+      // Remove sent chunks from buffer (sliding window)
+      this.sendBuffer = this.sendBuffer.slice(chunksToSend);
+      this.sendBufferBytes -= bytesToSend;
+    }
+  }
+  
+  // Flush any remaining buffered data
+  flushBuffer() {
+    if (this.sendBuffer && this.sendBuffer.length > 0) {
+      // Merge remaining chunks
+      const totalSamples = this.sendBuffer.reduce((a, b) => a + b.length, 0);
+      const merged = new Int16Array(totalSamples);
+      let offset = 0;
+      
+      for (const chunk of this.sendBuffer) {
+        merged.set(chunk, offset);
+        offset += chunk.length;
+      }
+      
+      // Send remaining data
+      this.port.postMessage({
+        type: 'pcm_audio_data',
+        data: merged, // Send the Int16Array directly, not the buffer
+        sampleRate: this.sampleRate,
+        channelCount: 1,
+        frameCount: this.frameCount,
+        batchSize: this.sendBuffer.length,
+        totalBytes: merged.byteLength,
+        isFlush: true
+      });
+      
+      // Reset buffer
+      this.sendBuffer = [];
+      this.sendBufferBytes = 0;
+    }
+  }
+}
+
+// Register the processor
+registerProcessor('audio-processor', AudioProcessor);

package/dist/demos/index.html

@@ -271,6 +271,21 @@
             <span class="badge badge--new">Demo</span>
           </div>
         </a>
+
+        <a href="widget-customization.html" class="test-card">
+          <span class="test-card__icon">🎨</span>
+          <h3 class="test-card__title">Widget Live Customization</h3>
+          <p class="test-card__description">
+            Interactive customization tool for the TTP Chat Widget. Click on any element 
+            to customize colors, sizes, text, and icons. See all available designs and 
+            get instant code output for your configuration.
+          </p>
+          <div class="test-card__badges">
+            <span class="badge badge--widget">Widget</span>
+            <span class="badge badge--sdk">Customization</span>
+            <span class="badge badge--new">New</span>
+          </div>
+        </a>
       </div>
     </div>