pygpt-net 2.1.37__py3-none-any.whl → 2.4.48__py3-none-any.whl

pygpt_net/plugin/audio_input/config.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.18 19:00:00                  #
+# ================================================== #
+
+from pygpt_net.plugin.base.config import BaseConfig, BasePlugin
+
+
+class Config(BaseConfig):
+    def __init__(self, plugin: BasePlugin = None, *args, **kwargs):
+        super(Config, self).__init__(plugin)
+        self.plugin = plugin
+
+    def from_defaults(self, plugin: BasePlugin = None):
+        """
+        Set default options for plugin
+
+        :param plugin: plugin instance
+        """
+        plugin.add_option(
+            "provider",
+            type="combo",
+            value="openai_whisper",
+            label="Provider",
+            description="Select audio transcribe provider, default: OpenAI Whisper",
+            tooltip="Select audio transcribe provider",
+            keys=plugin.get_provider_options(),
+        )
+        plugin.add_option(
+            "auto_send",
+            type="bool",
+            value=True,
+            label="Auto send",
+            description="Automatically send input when voice is detected. Default: True",
+        )
+        plugin.add_option(
+            "advanced",
+            type="bool",
+            value=False,
+            label="Enable advanced mode",
+            description="Enable only if you want to use advanced mode and settings below, "
+                        "do not enable this option if you just want to use simple mode (default)",
+        )
+        plugin.add_option(
+            "timeout",
+            type="int",
+            value=5,
+            label="Timeout",
+            description="Speech recognition timeout. Default: 5",
+            min=0,
+            max=30,
+            slider=True,
+            tooltip="Timeout, default: 5",
+            advanced=True,
+        )
+        plugin.add_option(
+            "phrase_length",
+            type="int",
+            value=10,
+            label="Phrase max length",
+            description="Speech recognition phrase length. Default: 10",
+            min=0,
+            max=30,
+            slider=True,
+            tooltip="Phrase max length, default: 10",
+            advanced=True,
+        )
+        plugin.add_option(
+            "min_energy",
+            type="float",
+            value=1.3,
+            label="Min. energy",
+            description="Minimum threshold multiplier above the noise level to begin recording;"
+                        " 1 = disabled. Default: 1.3",
+            min=1,
+            max=50,
+            slider=True,
+            tooltip="Min. energy, default: 1.3, 1 = disabled, adjust for your microphone",
+            multiplier=10,
+            advanced=True,
+        )
+        plugin.add_option(
+            "adjust_noise",
+            type="bool",
+            value=True,
+            label="Adjust ambient noise",
+            description="Adjust for ambient noise. Default: True",
+            advanced=True,
+        )
+        plugin.add_option(
+            "continuous_listen",
+            type="bool",
+            value=False,
+            label="Continuous listening",
+            description="EXPERIMENTAL: continuous listening - do not stop listening after a single input.\n"
+                        "Warning: This feature may lead to unexpected results and requires fine-tuning "
+                        "with the rest of the options!",
+            advanced=True,
+        )
+        plugin.add_option(
+            "wait_response",
+            type="bool",
+            value=True,
+            label="Wait for response",
+            description="Wait for a response before listening for the next input. Default: True",
+            advanced=True,
+        )
+        plugin.add_option(
+            "magic_word",
+            type="bool",
+            value=False,
+            label="Magic word",
+            description="Activate listening only after the magic word is provided, "
+                        "like 'Hey GPT' or 'OK GPT'. Default: False",
+            advanced=True,
+        )
+        plugin.add_option(
+            "magic_word_reset",
+            type="bool",
+            value=True,
+            label="Reset Magic word",
+            description="Reset the magic word status after it is received "
+                        "(the magic word will need to be provided again). Default: True",
+            advanced=True,
+        )
+        plugin.add_option(
+            "magic_words",
+            type="text",
+            value="OK, Okay, Hey GPT, OK GPT",
+            label="Magic words",
+            description="Specify magic words for 'Magic word' option: if received this word then "
+                        "start listening, put words separated by comma. Magic word option must be enabled, "
+                        "examples: \"Hey GPT, OK GPT\"",
+            advanced=True,
+        )
+        plugin.add_option(
+            "magic_word_timeout",
+            type="int",
+            value=1,
+            label="Magic word timeout",
+            description="Magic word recognition timeout. Default: 1",
+            min=0,
+            max=30,
+            slider=True,
+            tooltip="Timeout, default: 1",
+            advanced=True,
+        )
+        plugin.add_option(
+            "magic_word_phrase_length",
+            type="int",
+            value=2,
+            label="Magic word phrase max length",
+            description="Magic word phrase length. Default: 2",
+            min=0,
+            max=30,
+            slider=True,
+            tooltip="Phrase length, default: 2",
+            advanced=True,
+        )
+        plugin.add_option(
+            "prefix_words",
+            type="text",
+            value="",
+            label="Prefix words",
+            description="Specify prefix words: if defined, only phrases starting with these words "
+                        "will be transmitted, and the remainder will be ignored. Separate the words with "
+                        "a comma., eg. 'OK, Okay, GPT'. Leave empty to disable",
+            advanced=True,
+        )
+        plugin.add_option(
+            "stop_words",
+            type="text",
+            value="stop, exit, quit, end, finish, close, terminate, kill, halt, abort",
+            label="Stop words",
+            description="Specify stop words: if any of these words are received, then stop listening. "
+                        "Separate the words with a comma, or leave it empty to disable the feature, "
+                        "default: stop, exit, quit, end, finish, close, terminate, kill, halt, abort",
+            advanced=True,
+        )
+
+        # advanced options
+        plugin.add_option(
+            "recognition_energy_threshold",
+            type="int",
+            value=300,
+            label="energy_threshold",
+            description="Represents the energy level threshold for sounds. Default: 300",
+            min=0,
+            max=10000,
+            slider=True,
+            advanced=True,
+        )
+        plugin.add_option(
+            "recognition_dynamic_energy_threshold",
+            type="bool",
+            value=True,
+            label="dynamic_energy_threshold",
+            description="Represents whether the energy level threshold for sounds should be automatically "
+                        "adjusted based on the currently ambient noise level while listening. "
+                        "Default: True",
+            advanced=True,
+        )
+        plugin.add_option(
+            "recognition_dynamic_energy_adjustment_damping",
+            type="float",
+            value=0.15,
+            label="dynamic_energy_adjustment_damping",
+            description="Represents approximately the fraction of the current energy threshold that is "
+                        "retained after one second of dynamic threshold adjustment. Default: 0.15",
+            min=0,
+            max=100,
+            slider=True,
+            multiplier=100,
+            advanced=True,
+        )
+        plugin.add_option(
+            "recognition_pause_threshold",
+            type="float",
+            value=0.8,
+            label="pause_threshold",
+            description="Represents the minimum length of silence (in seconds) that will register as the end "
+                        "of a phrase.\nDefault: 0.8",
+            min=0,
+            max=100,
+            slider=True,
+            multiplier=10,
+            advanced=True,
+        )
+        plugin.add_option(
+            "recognition_adjust_for_ambient_noise_duration",
+            type="float",
+            value=1,
+            label="adjust_for_ambient_noise: duration",
+            description="The duration parameter is the maximum number of seconds that it will dynamically "
+                        "adjust the threshold for before returning. Default: 1",
+            min=0,
+            max=100,
+            slider=True,
+            multiplier=10,
+            advanced=True,
+        )

pygpt_net/plugin/audio_input/simple.py

@@ -6,17 +6,24 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.15 10:00:00                  #
+# Updated Date: 2024.12.08 00:00:00                  #
 # ================================================== #
 
 import pyaudio
 import wave
 import os
 
+from PySide6.QtCore import QTimer
+
+from pygpt_net.core.events import AppEvent
 from pygpt_net.utils import trans
 
 
 class Simple:
+
+    TIMEOUT_SECONDS = 120  # 2 minutes, max recording time before timeout
+    MIN_FRAMES = 25  # minimum frames to start transcription
+
     def __init__(self, plugin=None):
         """
         Simple audio input handler
@@ -28,6 +35,7 @@ class Simple:
         self.frames = []
         self.p = None
         self.stream = None
+        self.timer = None
 
     def toggle_recording(self):
         """Toggle recording"""
@@ -47,6 +55,10 @@ class Simple:
         self.plugin.window.ui.plugin_addon['audio.input.btn'].btn_toggle.setText(trans('audio.speak.btn'))
         self.plugin.window.ui.plugin_addon['audio.input.btn'].btn_toggle.setToolTip(trans('audio.speak.btn.tooltip'))
 
+    def stop_timeout(self):
+        """Stop timeout"""
+        self.stop_recording(timeout=True)
+
     def start_recording(self):
         """Start recording"""
         self.frames = []  # clear audio frames
@@ -61,15 +73,41 @@ class Simple:
         try:
             self.is_recording = True
             self.switch_btn_stop()
+
+            # stop audio output if playing
+            if self.plugin.window.controller.audio.is_playing():
+                self.plugin.window.controller.audio.stop_output()
+
+            # start timeout timer to prevent infinite recording
+            if self.timer is None:
+                self.timer = QTimer()
+                self.timer.timeout.connect(self.stop_timeout)
+                self.timer.start(self.TIMEOUT_SECONDS * 1000)
+
+            # select audio input device
+            device_id = int(self.plugin.window.core.config.get('audio.input.device', 0))
+            rate = int(self.plugin.window.core.config.get('audio.input.rate', 44100))
+            channels = int(self.plugin.window.core.config.get('audio.input.channels', 1))
+            if not self.plugin.window.core.audio.is_device_compatible(device_id):
+                err = self.plugin.window.core.audio.get_last_error()
+                message = "Selected audio input device is not compatible. Please select another one. ERROR: " + str(err)
+                self.is_recording = False
+                self.plugin.window.core.debug.log(message)
+                self.plugin.window.ui.dialogs.alert(message)
+                self.switch_btn_start()  # switch button to start
+                return
+
             self.p = pyaudio.PyAudio()
             self.stream = self.p.open(format=pyaudio.paInt16,
-                                      channels=1,
-                                      rate=44100,
+                                      channels=channels,
+                                      rate=rate,
                                       input=True,
+                                      input_device_index=device_id,
                                       frames_per_buffer=1024,
                                       stream_callback=callback)
 
-            self.plugin.window.ui.status(trans('audio.speak.now'))
+            self.plugin.window.update_status(trans('audio.speak.now'))
+            self.plugin.window.dispatch(AppEvent(AppEvent.INPUT_VOICE_LISTEN_STARTED))  # app event
             self.stream.start_stream()
         except Exception as e:
             self.is_recording = False
@@ -83,10 +121,18 @@ class Simple:
                 )
             self.switch_btn_start()  # switch button to start
 
-    def stop_recording(self):
-        """Stop recording"""
+    def stop_recording(self, timeout: bool = False):
+        """
+        Stop recording
+
+        :param timeout: True if stopped due to timeout
+        """
         self.is_recording = False
-        self.switch_btn_start()
+        self.plugin.window.dispatch(AppEvent(AppEvent.INPUT_VOICE_LISTEN_STOPPED))  # app event
+        if self.timer:
+            self.timer.stop()
+            self.timer = None
+        self.switch_btn_start()  # switch button to start
         path = os.path.join(self.plugin.window.core.config.path, self.plugin.input_file)
 
         if self.stream is not None:
@@ -94,11 +140,24 @@ class Simple:
             self.stream.close()
             self.p.terminate()
 
+            # abort if timeout
+            if timeout:
+                self.plugin.window.update_status("Aborted.".format(self.TIMEOUT_SECONDS))
+                return
+
             if self.frames:
+                if len(self.frames) < self.MIN_FRAMES:
+                    self.plugin.window.update_status(trans("status.audio.too_short"))
+                    self.plugin.window.dispatch(AppEvent(AppEvent.VOICE_CONTROL_STOPPED))  # app event
+                    return
                 wf = wave.open(path, 'wb')
-                wf.setnchannels(1)
+                channels = int(self.plugin.window.core.config.get('audio.input.channels', 1))
+                rate = int(self.plugin.window.core.config.get('audio.input.rate', 44100))
+                wf.setnchannels(channels)
                 wf.setsampwidth(self.p.get_sample_size(pyaudio.paInt16))
-                wf.setframerate(44100)
+                wf.setframerate(rate)
                 wf.writeframes(b''.join(self.frames))
                 wf.close()
                 self.plugin.handle_thread(True)  # handle transcription in simple mode
+        else:
+            self.plugin.window.update_status("")

pygpt_net/plugin/audio_input/worker.py

@@ -6,7 +6,7 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.19 06:00:00                  #
+# Updated Date: 2024.11.26 19:00:00                  #
 # ================================================== #
 
 import os.path
@@ -17,11 +17,12 @@ import audioop
 from PySide6.QtCore import Slot, Signal
 
 from pygpt_net.utils import trans
-from pygpt_net.plugin.base import BaseWorker, BaseSignals
+from pygpt_net.plugin.base.worker import BaseWorker, BaseSignals
 
 
 class WorkerSignals(BaseSignals):
     transcribed = Signal(str, str)
+    on_realtime = Signal(str)
 
 
 class Worker(BaseWorker):
@@ -73,8 +74,16 @@ class Worker(BaseWorker):
             if os.path.exists(self.path):
                 # set status
                 self.status(trans('audio.speak.wait'))
+
+                # if multimodal audio, then only return path to audio file and do not transcribe
+                if self.plugin.window.controller.chat.audio.enabled():
+                    self.signals.on_realtime.emit(self.path)
+                    self.status('')
+                    return
+
                 # transcribe audio
                 transcript = self.plugin.get_provider().transcribe(self.path)
+                self.status('')
 
                 # handle transcript
                 if transcript is not None and transcript.strip() != '':
@@ -254,3 +263,35 @@ class Worker(BaseWorker):
             self.error(e)
             self.destroyed()
             print("Audio input thread error: {}".format(str(e)))
+
+
+class ControlWorker(BaseWorker):
+    def __init__(self, *args, **kwargs):
+        super(ControlWorker, self).__init__()
+        self.signals = WorkerSignals()
+        self.window = None
+        self.args = args
+        self.kwargs = kwargs
+        self.path = None
+        self.transcribe = False
+
+    @Slot()
+    def run(self):
+        """Handle mic simple mode."""
+        try:
+            # do transcribe
+            if os.path.exists(self.path):
+                # set status
+                self.status(trans('audio.speak.wait'))
+                # transcribe audio
+                transcript = self.window.core.plugins.get('audio_input').get_provider().transcribe(self.path)
+                self.status('')
+
+                # handle transcript
+                if transcript is not None and transcript.strip() != '':
+                    self.response(transcript)
+
+        except Exception as e:
+            self.error(e)
+            self.stopped()
+            self.status('Error: {}'.format(e))

pygpt_net/plugin/audio_output/__init__.py

@@ -6,16 +6,18 @@
 # GitHub:  https://github.com/szczyglis-dev/py-gpt   #
 # MIT License                                        #
 # Created By  : Marcin Szczygliński                  #
-# Updated Date: 2024.03.09 10:00:00                  #
+# Updated Date: 2024.12.14 18:00:00                  #
 # ================================================== #
 
 from PySide6.QtCore import Slot
 
-from pygpt_net.plugin.base import BasePlugin
+from pygpt_net.core.types import MODE_AUDIO
+from pygpt_net.plugin.base.plugin import BasePlugin
 from pygpt_net.provider.audio_output.base import BaseProvider
-from pygpt_net.core.dispatcher import Event
+from pygpt_net.core.events import Event
 from pygpt_net.item.ctx import CtxItem
 
+from .config import Config
 from .worker import Worker
 
 
@@ -26,24 +28,17 @@ class Plugin(BasePlugin):
         self.name = "Audio Output"
         self.type = ['audio.output']
         self.description = "Enables audio/voice output (speech synthesis)"
+        self.prefix = "Audio Output"
         self.input_text = None
         self.playback = None
         self.order = 1
         self.use_locale = True
         self.output_file = "output.mp3"
+        self.config = Config(self)
 
     def init_options(self):
         """Initialize options"""
-        self.add_option(
-            "provider",
-            type="combo",
-            value="openai_tts",
-            label="Provider",
-            description="Select audio output provider, default: OpenAI TTS",
-            tooltip="Select audio output provider",
-            keys=self.get_provider_options(),
-        )
-        # register provider options
+        self.config.from_defaults(self)
         self.init_provider()
 
     def init_provider(self):
@@ -81,14 +76,6 @@ class Plugin(BasePlugin):
             tabs[id] = providers[id].name
         return tabs
 
-    def setup(self) -> dict:
-        """
-        Return available config options
-
-        :return: config options
-        """
-        return self.options
-
     def setup_ui(self):
         """
         Setup UI
@@ -132,15 +119,24 @@ class Plugin(BasePlugin):
         name = event.name
         data = event.data
         ctx = event.ctx
+        mode = self.window.core.config.get("mode")
 
         if name == Event.INPUT_BEFORE:
             self.on_input_before(data['value'])
 
-        elif name in [
-            Event.CTX_AFTER,
-            Event.AUDIO_READ_TEXT
-        ]:
-            self.on_ctx_after(ctx, name)
+        elif name == Event.CTX_AFTER:
+            if mode == MODE_AUDIO:
+                return  # skip if audio mode
+            self.stop_audio()
+            self.on_generate(ctx, event)
+
+        elif name == Event.AUDIO_READ_TEXT:
+            self.stop_audio()
+            self.on_generate(ctx, event)
+
+        elif name == Event.AUDIO_PLAYBACK:
+            self.stop_audio()
+            self.on_playback(ctx, event)
 
         elif name == Event.AUDIO_OUTPUT_STOP:
             self.stop_audio()
@@ -153,12 +149,12 @@ class Plugin(BasePlugin):
         """
         self.input_text = text
 
-    def on_ctx_after(self, ctx: CtxItem, name: str):
+    def on_generate(self, ctx: CtxItem, event: Event):
         """
         Events: CTX_AFTER, AUDIO_READ_TEXT
 
         :param ctx: CtxItem
-        :param name: event name
+        :param event: Event
         """
         # check if provider is configured
         if not self.get_provider().is_configured():
@@ -166,23 +162,32 @@ class Plugin(BasePlugin):
             self.window.ui.dialogs.alert(msg)
             return
 
+        name = event.name
         text = ctx.output
+        cache_file = None
+        if event.data is not None and isinstance(event.data, dict) and "cache_file" in event.data:
+            cache_file = event.data["cache_file"]
+
+        # check for audio read allowed. Prevents reading audio in commands, results, etc.
+        if name == Event.CTX_AFTER:
+            if not ctx.audio_read_allowed():
+                return # abort if audio read is not allowed (commands, results, etc.)
+        
         try:
             if text is not None and len(text) > 0:
-                # worker
                 worker = Worker()
-                worker.plugin = self
+                worker.from_defaults(self)
+                worker.ctx = ctx
                 worker.event = name
+                worker.cache_file = cache_file
                 worker.text = self.window.core.audio.clean_text(text)
+                worker.mode = "generate"
 
                 # signals
                 worker.signals.playback.connect(self.handle_playback)
                 worker.signals.stop.connect(self.handle_stop)
-                worker.signals.status.connect(self.handle_status)  # base handler
-                worker.signals.error.connect(self.handle_error)  # base handler
 
-                # start
-                self.window.threadpool.start(worker)
+                worker.run_async()
 
                 # only for manual reading
                 if name == Event.AUDIO_READ_TEXT:
@@ -191,6 +196,28 @@ class Plugin(BasePlugin):
         except Exception as e:
             self.error(e)
 
+    def on_playback(self, ctx: CtxItem, event: Event):
+        """
+        Events: AUDIO_PLAYBACK
+
+        :param ctx: CtxItem
+        :param event: Event
+        """     
+        try:
+            worker = Worker()
+            worker.from_defaults(self)
+            worker.audio_file = event.data["audio_file"]
+            worker.mode = "playback"
+
+            # signals
+            worker.signals.playback.connect(self.handle_playback)
+            worker.signals.stop.connect(self.handle_stop)
+
+            worker.run_async()
+
+        except Exception as e:
+            self.error(e)
+
     def destroy(self):
         """Destroy thread"""
         pass

pygpt_net/plugin/audio_output/config.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# ================================================== #
+# This file is a part of PYGPT package               #
+# Website: https://pygpt.net                         #
+# GitHub:  https://github.com/szczyglis-dev/py-gpt   #
+# MIT License                                        #
+# Created By  : Marcin Szczygliński                  #
+# Updated Date: 2024.11.18 19:00:00                  #
+# ================================================== #
+
+from pygpt_net.plugin.base.config import BaseConfig, BasePlugin
+
+
+class Config(BaseConfig):
+    def __init__(self, plugin: BasePlugin = None, *args, **kwargs):
+        super(Config, self).__init__(plugin)
+        self.plugin = plugin
+
+    def from_defaults(self, plugin: BasePlugin = None):
+        """
+        Set default options for plugin
+
+        :param plugin: plugin instance
+        """
+        plugin.add_option(
+            "provider",
+            type="combo",
+            value="openai_tts",
+            label="Provider",
+            description="Select audio output provider, default: OpenAI TTS",
+            tooltip="Select audio output provider",
+            keys=plugin.get_provider_options(),
+        )